From df975abae42664bbd5fd56436eb2947e2a6f46f9 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 4 Mar 2015 10:10:11 -0500 Subject: add snappy module --- cloudinit/config/cc_snappy.py | 133 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 cloudinit/config/cc_snappy.py (limited to 'cloudinit/config/cc_snappy.py') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py new file mode 100644 index 00000000..1588443f --- /dev/null +++ b/cloudinit/config/cc_snappy.py @@ -0,0 +1,133 @@ +# vi: ts=4 expandtab +# + +from cloudinit import log as logging +from cloudinit import templater +from cloudinit import util +from cloudinit.settings import PER_INSTANCE + +import glob +import os + +LOG = logging.getLogger(__name__) + +frequency = PER_INSTANCE +SNAPPY_ENV_PATH = "/writable/system-data/etc/snappy.env" + +CI_SNAPPY_CFG = { + 'env_file_path': SNAPPY_ENV_PATH, + 'packages': [], + 'packages_dir': '/writable/user-data/cloud-init/click_packages', + 'ssh_enabled': False +} + +""" +snappy: + ssh_enabled: True + packages: + - etcd + - {'name': 'pkg1', 'config': "wark"} +""" + + +def flatten(data, fill=None, tok="_", prefix='', recurse=True): + if fill is None: + fill = {} + for key, val in data.items(): + key = key.replace("-", "_") + if isinstance(val, dict) and recurse: + flatten(val, fill, tok=tok, prefix=prefix + key + tok, + recurse=recurse) + elif isinstance(key, str): + fill[prefix + key] = val + return fill + + +def render2env(data, tok="_", prefix=''): + flat = flatten(data, tok=tok, prefix=prefix) + ret = ["%s='%s'" % (key, val) for key, val in flat.items()] + return '\n'.join(ret) + '\n' + + +def install_package(pkg_name, config=None): + cmd = ["snappy", "install"] + if config: + if os.path.isfile(config): + cmd.append("--config-file=" + config) + else: + cmd.append("--config=" + config) + cmd.append(pkg_name) + util.subp(cmd) + + +def install_packages(package_dir, packages): + local_pkgs = glob.glob(os.path.sep.join([package_dir, '*.click'])) + LOG.debug("installing local packages %s" % local_pkgs) + if local_pkgs: + for pkg in local_pkgs: + cfg = pkg.replace(".click", ".config") + if not os.path.isfile(cfg): + cfg = None + install_package(pkg, config=cfg) + + LOG.debug("installing click packages") + if packages: + for pkg in packages: + if not pkg: + continue + if isinstance(pkg, str): + name = pkg + config = None + elif pkg: + name = pkg.get('name', pkg) + config = pkg.get('config') + install_package(pkg_name=name, config=config) + + +def disable_enable_ssh(enabled): + LOG.debug("setting enablement of ssh to: %s", enabled) + # do something here that would enable or disable + not_to_be_run = "/etc/ssh/sshd_not_to_be_run" + if enabled: + util.del_file(not_to_be_run) + # this is an indempotent operation + util.subp(["systemctl", "start", "ssh"]) + else: + # this is an indempotent operation + util.subp(["systemctl", "stop", "ssh"]) + util.write_file(not_to_be_run, "cloud-init\n") + + +def handle(name, cfg, cloud, log, args): + mycfg = cfg.get('snappy', {'ssh_enabled': False}) + + if not mycfg: + LOG.debug("%s: no top level found", name) + return + + # take out of 'cfg' the cfg keys that cloud-init uses, so + # mycfg has only content external to cloud-init. + ci_cfg = CI_SNAPPY_CFG.copy() + for i in ci_cfg: + if i in mycfg: + ci_cfg[i] = mycfg[i] + del mycfg[i] + + # render the flattened environment variable style file to a path + # this was useful for systemd config environment files. given: + # snappy: + # foo: + # bar: wark + # cfg1: + # key1: value + # you get the following in env_file_path. + # foo_bar=wark + # foo_cfg1_key1=value + contents = render2env(mycfg) + header = '# for internal use only, not a guaranteed interface\n' + util.write_file(ci_cfg['env_file_path'], header + render2env(mycfg)) + + install_packages(ci_cfg['packages_dir'], + ci_cfg['packages']) + + disable_enable_ssh(ci_cfg.get('ssh_enabled', False)) -- cgit v1.2.3 From 46da1b83fba8d1e70dc58dbbf18697216b1eb1e3 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 5 Mar 2015 11:18:45 -0500 Subject: fixes bug: https://launchpad.net/bugs/1428495 snappy: disable by default this does 2 things actually a.) disables snappy by default, and adds checks to filesystem to enable it b.) removes the 'render2env' that was mostly spike code. --- cloudinit/config/cc_snappy.py | 74 ++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 47 deletions(-) (limited to 'cloudinit/config/cc_snappy.py') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index 1588443f..32fbc9f6 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -14,15 +14,16 @@ LOG = logging.getLogger(__name__) frequency = PER_INSTANCE SNAPPY_ENV_PATH = "/writable/system-data/etc/snappy.env" -CI_SNAPPY_CFG = { - 'env_file_path': SNAPPY_ENV_PATH, +BUILTIN_CFG = { 'packages': [], 'packages_dir': '/writable/user-data/cloud-init/click_packages', - 'ssh_enabled': False + 'ssh_enabled': False, + 'system_snappy': "auto" } """ snappy: + system_snappy: auto ssh_enabled: True packages: - etcd @@ -30,25 +31,6 @@ snappy: """ -def flatten(data, fill=None, tok="_", prefix='', recurse=True): - if fill is None: - fill = {} - for key, val in data.items(): - key = key.replace("-", "_") - if isinstance(val, dict) and recurse: - flatten(val, fill, tok=tok, prefix=prefix + key + tok, - recurse=recurse) - elif isinstance(key, str): - fill[prefix + key] = val - return fill - - -def render2env(data, tok="_", prefix=''): - flat = flatten(data, tok=tok, prefix=prefix) - ret = ["%s='%s'" % (key, val) for key, val in flat.items()] - return '\n'.join(ret) + '\n' - - def install_package(pkg_name, config=None): cmd = ["snappy", "install"] if config: @@ -98,34 +80,32 @@ def disable_enable_ssh(enabled): util.write_file(not_to_be_run, "cloud-init\n") -def handle(name, cfg, cloud, log, args): - mycfg = cfg.get('snappy', {'ssh_enabled': False}) +def system_is_snappy(): + # channel.ini is configparser loadable. + # snappy will move to using /etc/system-image/config.d/*.ini + # this is certainly not a perfect test, but good enough for now. + content = util.load_file("/etc/system-image/channel.ini") + if 'ubuntu-core' in content.lower(): + return True + if os.path.isdir("/etc/system-image/config.d/"): + return True + return False + - if not mycfg: - LOG.debug("%s: no top level found", name) +def handle(name, cfg, cloud, log, args): + cfgin = cfg.get('snappy') + if not cfgin: + cfgin = {} + mycfg = util.mergemanydict([BUILTIN_CFG, cfgin]) + + sys_snappy = mycfg.get("system_snappy", "auto") + if util.is_false(sys_snappy): + LOG.debug("%s: System is not snappy. disabling", name) return - # take out of 'cfg' the cfg keys that cloud-init uses, so - # mycfg has only content external to cloud-init. - ci_cfg = CI_SNAPPY_CFG.copy() - for i in ci_cfg: - if i in mycfg: - ci_cfg[i] = mycfg[i] - del mycfg[i] - - # render the flattened environment variable style file to a path - # this was useful for systemd config environment files. given: - # snappy: - # foo: - # bar: wark - # cfg1: - # key1: value - # you get the following in env_file_path. - # foo_bar=wark - # foo_cfg1_key1=value - contents = render2env(mycfg) - header = '# for internal use only, not a guaranteed interface\n' - util.write_file(ci_cfg['env_file_path'], header + render2env(mycfg)) + if sys_snappy.lower() == "auto" and not(system_is_snappy()): + LOG.debug("%s: 'auto' mode, and system not snappy", name) + return install_packages(ci_cfg['packages_dir'], ci_cfg['packages']) -- cgit v1.2.3 From d05f1b00e2498343c03ba2de543990fffde8a02f Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 5 Mar 2015 12:26:26 -0500 Subject: do not raise exception on non-existant channel.ini file --- cloudinit/config/cc_snappy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/config/cc_snappy.py') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index 32fbc9f6..8d73dca3 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -84,7 +84,7 @@ def system_is_snappy(): # channel.ini is configparser loadable. # snappy will move to using /etc/system-image/config.d/*.ini # this is certainly not a perfect test, but good enough for now. - content = util.load_file("/etc/system-image/channel.ini") + content = util.load_file("/etc/system-image/channel.ini", quiet=True) if 'ubuntu-core' in content.lower(): return True if os.path.isdir("/etc/system-image/config.d/"): -- cgit v1.2.3 From c501a37e94b9601740fd7b3dcbcc4cce9136d7f4 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 5 Mar 2015 13:16:28 -0500 Subject: fixes from testing --- cloudinit/config/cc_snappy.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'cloudinit/config/cc_snappy.py') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index 8d73dca3..133336d4 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -96,9 +96,9 @@ def handle(name, cfg, cloud, log, args): cfgin = cfg.get('snappy') if not cfgin: cfgin = {} - mycfg = util.mergemanydict([BUILTIN_CFG, cfgin]) + mycfg = util.mergemanydict([cfgin, BUILTIN_CFG]) - sys_snappy = mycfg.get("system_snappy", "auto") + sys_snappy = str(mycfg.get("system_snappy", "auto")) if util.is_false(sys_snappy): LOG.debug("%s: System is not snappy. disabling", name) return @@ -107,7 +107,7 @@ def handle(name, cfg, cloud, log, args): LOG.debug("%s: 'auto' mode, and system not snappy", name) return - install_packages(ci_cfg['packages_dir'], - ci_cfg['packages']) + install_packages(mycfg['packages_dir'], + mycfg['packages']) - disable_enable_ssh(ci_cfg.get('ssh_enabled', False)) + disable_enable_ssh(mycfg.get('ssh_enabled', False)) -- cgit v1.2.3 From a373e1097f6be460914e6cbbc897c6aa8e4aaefe Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 Mar 2015 20:39:25 -0400 Subject: commit work in progress. tests pass. --- cloudinit/config/cc_snappy.py | 159 +++++++++++++++----- .../unittests/test_handler/test_handler_snappy.py | 163 +++++++++++++++++++++ 2 files changed, 285 insertions(+), 37 deletions(-) create mode 100644 tests/unittests/test_handler/test_handler_snappy.py (limited to 'cloudinit/config/cc_snappy.py') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index 133336d4..bef8c170 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -7,18 +7,21 @@ from cloudinit import util from cloudinit.settings import PER_INSTANCE import glob +import six +import tempfile import os LOG = logging.getLogger(__name__) frequency = PER_INSTANCE -SNAPPY_ENV_PATH = "/writable/system-data/etc/snappy.env" +SNAPPY_CMD = "snappy" BUILTIN_CFG = { 'packages': [], 'packages_dir': '/writable/user-data/cloud-init/click_packages', 'ssh_enabled': False, - 'system_snappy': "auto" + 'system_snappy': "auto", + 'configs': {}, } """ @@ -27,43 +30,111 @@ snappy: ssh_enabled: True packages: - etcd - - {'name': 'pkg1', 'config': "wark"} + - pkg2 + configs: + pkgname: config-blob + pkgname2: config-blob """ -def install_package(pkg_name, config=None): - cmd = ["snappy", "install"] - if config: - if os.path.isfile(config): - cmd.append("--config-file=" + config) +def get_fs_package_ops(fspath): + if not fspath: + return [] + ops = [] + for snapfile in glob.glob(os.path.sep.join([fspath, '*.snap'])): + cfg = snapfile.rpartition(".")[0] + ".config" + name = os.path.basename(snapfile).rpartition(".")[0] + if not os.path.isfile(cfg): + cfg = None + ops.append(makeop('install', name, config=None, + path=snapfile, cfgfile=cfg)) + return ops + + +def makeop(op, name, config=None, path=None, cfgfile=None): + return({'op': op, 'name': name, 'config': config, 'path': path, + 'cfgfile': cfgfile}) + + +def get_package_ops(packages, configs, installed=None, fspath=None): + # get the install an config operations that should be done + if installed is None: + installed = read_installed_packages() + + if not packages: + packages = [] + if not configs: + configs = {} + + ops = [] + ops += get_fs_package_ops(fspath) + + for name in packages: + ops.append(makeop('install', name, configs.get('name'))) + + to_install = [f['name'] for f in ops] + + for name in configs: + if name in installed and name not in to_install: + ops.append(makeop('config', name, config=configs[name])) + + # prefer config entries to filepath entries + for op in ops: + name = op['name'] + if name in configs and op['op'] == 'install' and 'cfgfile' in op: + LOG.debug("preferring configs[%s] over '%s'", name, op['cfgfile']) + op['cfgfile'] = None + op['config'] = configs[op['name']] + + return ops + + +def render_snap_op(op, name, path=None, cfgfile=None, config=None): + if op not in ('install', 'config'): + raise ValueError("cannot render op '%s'" % op) + + try: + cfg_tmpf = None + if config is not None: + if isinstance(config, six.binary_type): + cfg_bytes = config + elif isinstance(config, six.text_type): + cfg_bytes = config_data.encode() + else: + cfg_bytes = yaml.safe_dump(config).encode() + + (fd, cfg_tmpf) = tempfile.mkstemp() + os.write(fd, config_data) + os.close(fd) + cfgfile = cfg_tmpf + + cmd = [SNAPPY_CMD, op] + if op == 'install' and cfgfile: + cmd.append('--config=' + cfgfile) + elif op == 'config': + cmd.append(cfgfile) + + util.subp(cmd) + + finally: + if tmpfile: + os.unlink(tmpfile) + + +def read_installed_packages(): + return [p[0] for p in read_pkg_data()] + + +def read_pkg_data(): + out, err = util.subp([SNAPPY_CMD, "list"]) + for line in out.splitlines()[1:]: + toks = line.split(sep=None, maxsplit=3) + if len(toks) == 3: + (name, date, version) = toks + dev = None else: - cmd.append("--config=" + config) - cmd.append(pkg_name) - util.subp(cmd) - - -def install_packages(package_dir, packages): - local_pkgs = glob.glob(os.path.sep.join([package_dir, '*.click'])) - LOG.debug("installing local packages %s" % local_pkgs) - if local_pkgs: - for pkg in local_pkgs: - cfg = pkg.replace(".click", ".config") - if not os.path.isfile(cfg): - cfg = None - install_package(pkg, config=cfg) - - LOG.debug("installing click packages") - if packages: - for pkg in packages: - if not pkg: - continue - if isinstance(pkg, str): - name = pkg - config = None - elif pkg: - name = pkg.get('name', pkg) - config = pkg.get('config') - install_package(pkg_name=name, config=config) + (name, date, version, dev) = toks + pkgs.append((name, date, version, dev,)) def disable_enable_ssh(enabled): @@ -107,7 +178,21 @@ def handle(name, cfg, cloud, log, args): LOG.debug("%s: 'auto' mode, and system not snappy", name) return - install_packages(mycfg['packages_dir'], - mycfg['packages']) + pkg_ops = get_package_ops(packages=mycfg['packages'], + configs=mycfg['configs'], + fspath=mycfg['packages_dir']) + + fails = [] + for pkg_op in pkg_ops: + try: + render_snap_op(op=pkg_op['op'], name=pkg_op['name'], + cfgfile=pkg_op['cfgfile'], config=pkg_op['config']) + except Exception as e: + fails.append((pkg_op, e,)) + LOG.warn("'%s' failed for '%s': %s", + pkg_op['op'], pkg_op['name'], e) disable_enable_ssh(mycfg.get('ssh_enabled', False)) + + if fails: + raise Exception("failed to install/configure snaps") diff --git a/tests/unittests/test_handler/test_handler_snappy.py b/tests/unittests/test_handler/test_handler_snappy.py new file mode 100644 index 00000000..6b6d3584 --- /dev/null +++ b/tests/unittests/test_handler/test_handler_snappy.py @@ -0,0 +1,163 @@ +from cloudinit.config.cc_snappy import (makeop, get_package_ops) +from cloudinit import util +from .. import helpers as t_help + +import os +import tempfile + +class TestInstallPackages(t_help.TestCase): + def setUp(self): + super(TestInstallPackages, self).setUp() + self.unapply = [] + + # by default 'which' has nothing in its path + self.apply_patches([(util, 'subp', self._subp)]) + self.subp_called = [] + self.snapcmds = [] + self.tmp = tempfile.mkdtemp() + + def tearDown(self): + apply_patches([i for i in reversed(self.unapply)]) + + def apply_patches(self, patches): + ret = apply_patches(patches) + self.unapply += ret + + def _subp(self, *args, **kwargs): + # supports subp calling with cmd as args or kwargs + if 'args' not in kwargs: + kwargs['args'] = args[0] + self.subp_called.append(kwargs) + snap_cmds = [] + args = kwargs['args'] + if args[0:2] == ['snappy', 'config']: + if args[3] == "-": + config = kwargs.get('data', '') + else: + with open(args[3], "rb") as fp: + config = fp.read() + snap_cmds.append(('config', args[2], config,)) + elif args[0:2] == ['snappy', 'install']: + # basically parse the snappy command and add + # to snap_installs a tuple (pkg, config) + config = None + pkg = None + for arg in args[2:]: + if arg.startswith("--config="): + cfgfile = arg.partition("=")[2] + if cfgfile == "-": + config = kwargs.get('data', '') + elif cfgfile: + with open(cfgfile, "rb") as fp: + config = fp.read() + elif not pkg and not arg.startswith("-"): + pkg = os.path.basename(arg) + self.snap_installs.append(('install', pkg, config,)) + + def test_package_ops_1(self): + ret = get_package_ops( + packages=['pkg1', 'pkg2', 'pkg3'], + configs={'pkg2': b'mycfg2'}, installed=[]) + self.assertEqual(ret, + [makeop('install', 'pkg1', None, None), + makeop('install', 'pkg2', b'mycfg2', None), + makeop('install', 'pkg3', None, None)]) + + def test_package_ops_config_only(self): + ret = get_package_ops( + packages=None, + configs={'pkg2': b'mycfg2'}, installed=['pkg1', 'pkg2']) + self.assertEqual(ret, + [makeop('config', 'pkg2', b'mycfg2')]) + + def test_package_ops_install_and_config(self): + ret = get_package_ops( + packages=['pkg3', 'pkg2'], + configs={'pkg2': b'mycfg2', 'xinstalled': b'xcfg'}, + installed=['xinstalled']) + self.assertEqual(ret, + [makeop('install', 'pkg3'), + makeop('install', 'pkg2', b'mycfg2'), + makeop('config', 'xinstalled', b'xcfg')]) + + def test_package_ops_with_file(self): + t_help.populate_dir(self.tmp, + {"snapf1.snap": b"foo1", "snapf1.config": b"snapf1cfg", + "snapf2.snap": b"foo2", "foo.bar": "ignored"}) + ret = get_package_ops( + packages=['pkg1'], configs={}, installed=[], fspath=self.tmp) + self.assertEqual(ret, + [makeop_tmpd(self.tmp, 'install', 'snapf1', path="snapf1.snap", + cfgfile="snapf1.config"), + makeop_tmpd(self.tmp, 'install', 'snapf2', path="snapf2.snap"), + makeop('install', 'pkg1')]) + + +def makeop_tmpd(tmpd, op, name, config=None, path=None, cfgfile=None): + if cfgfile: + cfgfile = os.path.sep.join([tmpd, cfgfile]) + if path: + path = os.path.sep.join([tmpd, path]) + return(makeop(op=op, name=name, config=config, path=path, cfgfile=cfgfile)) + +# def test_local_snaps_no_config(self): +# t_help.populate_dir(self.tmp, +# {"snap1.snap": b"foo", "snap2.snap": b"foo", "foosnap.txt": b"foo"}) +# cc_snappy.install_packages(self.tmp, None) +# self.assertEqual(self.snap_installs, +# [("snap1.snap", None), ("snap2.snap", None)]) +# +# def test_local_snaps_mixed_config(self): +# t_help.populate_dir(self.tmp, +# {"snap1.snap": b"foo", "snap2.snap": b"snap2", +# "snap1.config": b"snap1config"}) +# cc_snappy.install_packages(self.tmp, None) +# self.assertEqual(self.snap_installs, +# [("snap1.snap", b"snap1config"), ("snap2.snap", None)]) +# +# def test_local_snaps_all_config(self): +# t_help.populate_dir(self.tmp, +# {"snap1.snap": "foo", "snap1.config": b"snap1config", +# "snap2.snap": "snap2", "snap2.config": b"snap2config"}) +# cc_snappy.install_packages(self.tmp, None) +# self.assertEqual(self.snap_installs, +# [("snap1.snap", b"snap1config"), ("snap2.snap", b"snap2config")]) +# +# def test_local_snaps_and_packages(self): +# t_help.populate_dir(self.tmp, +# {"snap1.snap": "foo", "snap1.config": b"snap1config"}) +# cc_snappy.install_packages(self.tmp, ["snap-in-store"]) +# self.assertEqual(self.snap_installs, +# [("snap1.snap", b"snap1config"), ("snap-in-store", None)]) +# +# def test_packages_no_config(self): +# cc_snappy.install_packages(self.tmp, ["snap-in-store"]) +# self.assertEqual(self.snap_installs, +# [("snap-in-store", None)]) +# +# def test_packages_mixed_config(self): +# cc_snappy.install_packages(self.tmp, +# ["snap-in-store", +# {'name': 'snap2-in-store', 'config': b"foo"}]) +# self.assertEqual(self.snap_installs, +# [("snap-in-store", None), ("snap2-in-store", b"foo")]) +# +# def test_packages_all_config(self): +# cc_snappy.install_packages(self.tmp, +# [{'name': 'snap1-in-store', 'config': b"boo"}, +# {'name': 'snap2-in-store', 'config': b"wark"}]) +# self.assertEqual(self.snap_installs, +# [("snap1-in-store", b"boo"), ("snap2-in-store", b"wark")]) +# +# + +def apply_patches(patches): + ret = [] + for (ref, name, replace) in patches: + if replace is None: + continue + orig = getattr(ref, name) + setattr(ref, name, replace) + ret.append((ref, name, orig)) + return ret + -- cgit v1.2.3 From bd7165dd67338f742f999fb2c53ec5f67fc66477 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 Mar 2015 21:14:17 -0400 Subject: start of snap_op tests --- cloudinit/config/cc_snappy.py | 10 ++++-- .../unittests/test_handler/test_handler_snappy.py | 40 +++++++++++++++++++--- 2 files changed, 44 insertions(+), 6 deletions(-) (limited to 'cloudinit/config/cc_snappy.py') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index bef8c170..cf441c92 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -114,11 +114,17 @@ def render_snap_op(op, name, path=None, cfgfile=None, config=None): elif op == 'config': cmd.append(cfgfile) + if op == 'install': + if path: + cmd.append(path) + else: + cmd.append(name) + util.subp(cmd) finally: - if tmpfile: - os.unlink(tmpfile) + if cfg_tmpf: + os.unlink(cfg_tmpf) def read_installed_packages(): diff --git a/tests/unittests/test_handler/test_handler_snappy.py b/tests/unittests/test_handler/test_handler_snappy.py index 6b6d3584..7dc77970 100644 --- a/tests/unittests/test_handler/test_handler_snappy.py +++ b/tests/unittests/test_handler/test_handler_snappy.py @@ -1,4 +1,5 @@ -from cloudinit.config.cc_snappy import (makeop, get_package_ops) +from cloudinit.config.cc_snappy import ( + makeop, get_package_ops, render_snap_op) from cloudinit import util from .. import helpers as t_help @@ -36,7 +37,7 @@ class TestInstallPackages(t_help.TestCase): else: with open(args[3], "rb") as fp: config = fp.read() - snap_cmds.append(('config', args[2], config,)) + self.snapcmds.append(['config', args[2], config]) elif args[0:2] == ['snappy', 'install']: # basically parse the snappy command and add # to snap_installs a tuple (pkg, config) @@ -51,8 +52,8 @@ class TestInstallPackages(t_help.TestCase): with open(cfgfile, "rb") as fp: config = fp.read() elif not pkg and not arg.startswith("-"): - pkg = os.path.basename(arg) - self.snap_installs.append(('install', pkg, config,)) + pkg = arg + self.snapcmds.append(['install', pkg, config]) def test_package_ops_1(self): ret = get_package_ops( @@ -92,6 +93,37 @@ class TestInstallPackages(t_help.TestCase): makeop_tmpd(self.tmp, 'install', 'snapf2', path="snapf2.snap"), makeop('install', 'pkg1')]) + #def render_snap_op(op, name, path=None, cfgfile=None, config=None): + def test_render_op_localsnap(self): + t_help.populate_dir(self.tmp, {"snapf1.snap": b"foo1"}) + op = makeop_tmpd(self.tmp, 'install', 'snapf1', + path='snapf1.snap') + render_snap_op(**op) + self.assertEqual(self.snapcmds, + [['install', op['path'], None]]) + + def test_render_op_localsnap_localconfig(self): + t_help.populate_dir(self.tmp, + {"snapf1.snap": b"foo1", 'snapf1.config': b'snapf1cfg'}) + op = makeop_tmpd(self.tmp, 'install', 'snapf1', + path='snapf1.snap', cfgfile='snapf1.config') + render_snap_op(**op) + self.assertEqual(self.snapcmds, + [['install', op['path'], b'snapf1cfg']]) + + def test_render_op_localsnap_config(self): + pass + + def test_render_op_snap(self): + pass + + def test_render_op_snap_config(self): + pass + + def test_render_op_config(self): + pass + + def makeop_tmpd(tmpd, op, name, config=None, path=None, cfgfile=None): if cfgfile: -- cgit v1.2.3 From df43c6bd3726c9a34b9f8ff4bbf75957aa751011 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 Mar 2015 21:55:26 -0400 Subject: pep8, and some more tests --- cloudinit/config/cc_snappy.py | 13 +- .../unittests/test_handler/test_handler_snappy.py | 131 ++++++++------------- 2 files changed, 57 insertions(+), 87 deletions(-) (limited to 'cloudinit/config/cc_snappy.py') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index cf441c92..c926ae0a 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -99,26 +99,25 @@ def render_snap_op(op, name, path=None, cfgfile=None, config=None): if isinstance(config, six.binary_type): cfg_bytes = config elif isinstance(config, six.text_type): - cfg_bytes = config_data.encode() + cfg_bytes = config.encode() else: cfg_bytes = yaml.safe_dump(config).encode() (fd, cfg_tmpf) = tempfile.mkstemp() - os.write(fd, config_data) + os.write(fd, cfg_bytes) os.close(fd) cfgfile = cfg_tmpf cmd = [SNAPPY_CMD, op] - if op == 'install' and cfgfile: - cmd.append('--config=' + cfgfile) - elif op == 'config': - cmd.append(cfgfile) - if op == 'install': + if cfgfile: + cmd.append('--config=' + cfgfile) if path: cmd.append(path) else: cmd.append(name) + elif op == 'config': + cmd += [name, cfgfile] util.subp(cmd) diff --git a/tests/unittests/test_handler/test_handler_snappy.py b/tests/unittests/test_handler/test_handler_snappy.py index 7dc77970..8759a07d 100644 --- a/tests/unittests/test_handler/test_handler_snappy.py +++ b/tests/unittests/test_handler/test_handler_snappy.py @@ -4,8 +4,10 @@ from cloudinit import util from .. import helpers as t_help import os +import shutil import tempfile + class TestInstallPackages(t_help.TestCase): def setUp(self): super(TestInstallPackages, self).setUp() @@ -15,15 +17,19 @@ class TestInstallPackages(t_help.TestCase): self.apply_patches([(util, 'subp', self._subp)]) self.subp_called = [] self.snapcmds = [] - self.tmp = tempfile.mkdtemp() + self.tmp = tempfile.mkdtemp(prefix="TestInstallPackages") def tearDown(self): apply_patches([i for i in reversed(self.unapply)]) + shutil.rmtree(self.tmp) def apply_patches(self, patches): ret = apply_patches(patches) self.unapply += ret + def populate_tmp(self, files): + return t_help.populate_dir(self.tmp, files) + def _subp(self, *args, **kwargs): # supports subp calling with cmd as args or kwargs if 'args' not in kwargs: @@ -31,6 +37,8 @@ class TestInstallPackages(t_help.TestCase): self.subp_called.append(kwargs) snap_cmds = [] args = kwargs['args'] + # here we basically parse the snappy command invoked + # and append to snapcmds a list of (mode, pkg, config) if args[0:2] == ['snappy', 'config']: if args[3] == "-": config = kwargs.get('data', '') @@ -39,8 +47,6 @@ class TestInstallPackages(t_help.TestCase): config = fp.read() self.snapcmds.append(['config', args[2], config]) elif args[0:2] == ['snappy', 'install']: - # basically parse the snappy command and add - # to snap_installs a tuple (pkg, config) config = None pkg = None for arg in args[2:]: @@ -59,72 +65,88 @@ class TestInstallPackages(t_help.TestCase): ret = get_package_ops( packages=['pkg1', 'pkg2', 'pkg3'], configs={'pkg2': b'mycfg2'}, installed=[]) - self.assertEqual(ret, - [makeop('install', 'pkg1', None, None), - makeop('install', 'pkg2', b'mycfg2', None), - makeop('install', 'pkg3', None, None)]) + self.assertEqual( + ret, [makeop('install', 'pkg1', None, None), + makeop('install', 'pkg2', b'mycfg2', None), + makeop('install', 'pkg3', None, None)]) def test_package_ops_config_only(self): ret = get_package_ops( packages=None, configs={'pkg2': b'mycfg2'}, installed=['pkg1', 'pkg2']) - self.assertEqual(ret, - [makeop('config', 'pkg2', b'mycfg2')]) + self.assertEqual( + ret, [makeop('config', 'pkg2', b'mycfg2')]) def test_package_ops_install_and_config(self): ret = get_package_ops( packages=['pkg3', 'pkg2'], configs={'pkg2': b'mycfg2', 'xinstalled': b'xcfg'}, installed=['xinstalled']) - self.assertEqual(ret, - [makeop('install', 'pkg3'), - makeop('install', 'pkg2', b'mycfg2'), - makeop('config', 'xinstalled', b'xcfg')]) + self.assertEqual( + ret, [makeop('install', 'pkg3'), + makeop('install', 'pkg2', b'mycfg2'), + makeop('config', 'xinstalled', b'xcfg')]) def test_package_ops_with_file(self): - t_help.populate_dir(self.tmp, + self.populate_tmp( {"snapf1.snap": b"foo1", "snapf1.config": b"snapf1cfg", "snapf2.snap": b"foo2", "foo.bar": "ignored"}) ret = get_package_ops( packages=['pkg1'], configs={}, installed=[], fspath=self.tmp) - self.assertEqual(ret, + self.assertEqual( + ret, [makeop_tmpd(self.tmp, 'install', 'snapf1', path="snapf1.snap", cfgfile="snapf1.config"), makeop_tmpd(self.tmp, 'install', 'snapf2', path="snapf2.snap"), makeop('install', 'pkg1')]) - #def render_snap_op(op, name, path=None, cfgfile=None, config=None): + def test_package_ops_config_overrides_file(self): + # config data overrides local file .config + self.populate_tmp( + {"snapf1.snap": b"foo1", "snapf1.config": b"snapf1cfg"}) + ret = get_package_ops( + packages=[], configs={'snapf1': 'snapf1cfg-config'}, + installed=[], fspath=self.tmp) + self.assertEqual( + ret, [makeop_tmpd(self.tmp, 'install', 'snapf1', + path="snapf1.snap", config="snapf1cfg-config")]) + def test_render_op_localsnap(self): - t_help.populate_dir(self.tmp, {"snapf1.snap": b"foo1"}) + self.populate_tmp({"snapf1.snap": b"foo1"}) op = makeop_tmpd(self.tmp, 'install', 'snapf1', path='snapf1.snap') render_snap_op(**op) - self.assertEqual(self.snapcmds, - [['install', op['path'], None]]) + self.assertEqual( + self.snapcmds, [['install', op['path'], None]]) def test_render_op_localsnap_localconfig(self): - t_help.populate_dir(self.tmp, + self.populate_tmp( {"snapf1.snap": b"foo1", 'snapf1.config': b'snapf1cfg'}) op = makeop_tmpd(self.tmp, 'install', 'snapf1', path='snapf1.snap', cfgfile='snapf1.config') render_snap_op(**op) - self.assertEqual(self.snapcmds, - [['install', op['path'], b'snapf1cfg']]) - - def test_render_op_localsnap_config(self): - pass + self.assertEqual( + self.snapcmds, [['install', op['path'], b'snapf1cfg']]) def test_render_op_snap(self): - pass + op = makeop('install', 'snapf1') + render_snap_op(**op) + self.assertEqual( + self.snapcmds, [['install', 'snapf1', None]]) def test_render_op_snap_config(self): - pass + op = makeop('install', 'snapf1', config=b'myconfig') + render_snap_op(**op) + self.assertEqual( + self.snapcmds, [['install', 'snapf1', b'myconfig']]) def test_render_op_config(self): - pass + op = makeop('config', 'snapf1', config=b'myconfig') + render_snap_op(**op) + self.assertEqual( + self.snapcmds, [['config', 'snapf1', b'myconfig']]) - def makeop_tmpd(tmpd, op, name, config=None, path=None, cfgfile=None): if cfgfile: cfgfile = os.path.sep.join([tmpd, cfgfile]) @@ -132,56 +154,6 @@ def makeop_tmpd(tmpd, op, name, config=None, path=None, cfgfile=None): path = os.path.sep.join([tmpd, path]) return(makeop(op=op, name=name, config=config, path=path, cfgfile=cfgfile)) -# def test_local_snaps_no_config(self): -# t_help.populate_dir(self.tmp, -# {"snap1.snap": b"foo", "snap2.snap": b"foo", "foosnap.txt": b"foo"}) -# cc_snappy.install_packages(self.tmp, None) -# self.assertEqual(self.snap_installs, -# [("snap1.snap", None), ("snap2.snap", None)]) -# -# def test_local_snaps_mixed_config(self): -# t_help.populate_dir(self.tmp, -# {"snap1.snap": b"foo", "snap2.snap": b"snap2", -# "snap1.config": b"snap1config"}) -# cc_snappy.install_packages(self.tmp, None) -# self.assertEqual(self.snap_installs, -# [("snap1.snap", b"snap1config"), ("snap2.snap", None)]) -# -# def test_local_snaps_all_config(self): -# t_help.populate_dir(self.tmp, -# {"snap1.snap": "foo", "snap1.config": b"snap1config", -# "snap2.snap": "snap2", "snap2.config": b"snap2config"}) -# cc_snappy.install_packages(self.tmp, None) -# self.assertEqual(self.snap_installs, -# [("snap1.snap", b"snap1config"), ("snap2.snap", b"snap2config")]) -# -# def test_local_snaps_and_packages(self): -# t_help.populate_dir(self.tmp, -# {"snap1.snap": "foo", "snap1.config": b"snap1config"}) -# cc_snappy.install_packages(self.tmp, ["snap-in-store"]) -# self.assertEqual(self.snap_installs, -# [("snap1.snap", b"snap1config"), ("snap-in-store", None)]) -# -# def test_packages_no_config(self): -# cc_snappy.install_packages(self.tmp, ["snap-in-store"]) -# self.assertEqual(self.snap_installs, -# [("snap-in-store", None)]) -# -# def test_packages_mixed_config(self): -# cc_snappy.install_packages(self.tmp, -# ["snap-in-store", -# {'name': 'snap2-in-store', 'config': b"foo"}]) -# self.assertEqual(self.snap_installs, -# [("snap-in-store", None), ("snap2-in-store", b"foo")]) -# -# def test_packages_all_config(self): -# cc_snappy.install_packages(self.tmp, -# [{'name': 'snap1-in-store', 'config': b"boo"}, -# {'name': 'snap2-in-store', 'config': b"wark"}]) -# self.assertEqual(self.snap_installs, -# [("snap1-in-store", b"boo"), ("snap2-in-store", b"wark")]) -# -# def apply_patches(patches): ret = [] @@ -192,4 +164,3 @@ def apply_patches(patches): setattr(ref, name, replace) ret.append((ref, name, orig)) return ret - -- cgit v1.2.3 From 973c8b05358fe6ad1ce7adb25cb743ef4d38d792 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 Mar 2015 21:55:45 -0400 Subject: pep8 --- cloudinit/config/cc_snappy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/config/cc_snappy.py') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index c926ae0a..d1447fe5 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -47,7 +47,7 @@ def get_fs_package_ops(fspath): if not os.path.isfile(cfg): cfg = None ops.append(makeop('install', name, config=None, - path=snapfile, cfgfile=cfg)) + path=snapfile, cfgfile=cfg)) return ops -- cgit v1.2.3 From 4c341a87d4b0804565e74e6335a0293dab6c0c7b Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 Mar 2015 22:10:01 -0400 Subject: add tests for data types --- cloudinit/config/cc_snappy.py | 2 +- .../unittests/test_handler/test_handler_snappy.py | 35 +++++++++++++++++++++- 2 files changed, 35 insertions(+), 2 deletions(-) (limited to 'cloudinit/config/cc_snappy.py') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index d1447fe5..bd928e54 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -101,7 +101,7 @@ def render_snap_op(op, name, path=None, cfgfile=None, config=None): elif isinstance(config, six.text_type): cfg_bytes = config.encode() else: - cfg_bytes = yaml.safe_dump(config).encode() + cfg_bytes = util.yaml_dumps(config) (fd, cfg_tmpf) = tempfile.mkstemp() os.write(fd, cfg_bytes) diff --git a/tests/unittests/test_handler/test_handler_snappy.py b/tests/unittests/test_handler/test_handler_snappy.py index 8759a07d..81d891d2 100644 --- a/tests/unittests/test_handler/test_handler_snappy.py +++ b/tests/unittests/test_handler/test_handler_snappy.py @@ -140,12 +140,45 @@ class TestInstallPackages(t_help.TestCase): self.assertEqual( self.snapcmds, [['install', 'snapf1', b'myconfig']]) - def test_render_op_config(self): + def test_render_op_config_bytes(self): op = makeop('config', 'snapf1', config=b'myconfig') render_snap_op(**op) self.assertEqual( self.snapcmds, [['config', 'snapf1', b'myconfig']]) + def test_render_op_config_string(self): + mycfg = 'myconfig: foo\nhisconfig: bar\n' + op = makeop('config', 'snapf1', config=mycfg) + render_snap_op(**op) + self.assertEqual( + self.snapcmds, [['config', 'snapf1', mycfg.encode()]]) + + def test_render_op_config_dict(self): + # config entry for package can be a dict, not a string blob + mycfg = {'foo': 'bar'} + op = makeop('config', 'snapf1', config=mycfg) + render_snap_op(**op) + # snapcmds is a list of 3-entry lists. data_found will be the + # blob of data in the file in 'snappy install --config=' + data_found = self.snapcmds[0][2] + self.assertEqual(mycfg, util.load_yaml(data_found)) + + def test_render_op_config_list(self): + # config entry for package can be a list, not a string blob + mycfg = ['foo', 'bar', 'wark', {'f1': 'b1'}] + op = makeop('config', 'snapf1', config=mycfg) + render_snap_op(**op) + data_found = self.snapcmds[0][2] + self.assertEqual(mycfg, util.load_yaml(data_found, allowed=(list,))) + + def test_render_op_config_int(self): + # config entry for package can be a list, not a string blob + mycfg = 1 + op = makeop('config', 'snapf1', config=mycfg) + render_snap_op(**op) + data_found = self.snapcmds[0][2] + self.assertEqual(mycfg, util.load_yaml(data_found, allowed=(int,))) + def makeop_tmpd(tmpd, op, name, config=None, path=None, cfgfile=None): if cfgfile: -- cgit v1.2.3 From 5e012b1e5f51f82e503a760c8c9c0e2c66aedfee Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 Mar 2015 22:16:13 -0400 Subject: prefer snappy-go to snappy --- cloudinit/config/cc_snappy.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'cloudinit/config/cc_snappy.py') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index bd928e54..dbdc402c 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -168,6 +168,14 @@ def system_is_snappy(): return False +def set_snappy_command(): + if util.which("snappy-go"): + SNAPPY_COMMAND = "snappy-go" + else: + SNAPPY_COMMAND = "snappy" + LOG.debug("snappy command is '%s'", SNAPPY_COMMAND) + + def handle(name, cfg, cloud, log, args): cfgin = cfg.get('snappy') if not cfgin: @@ -187,11 +195,12 @@ def handle(name, cfg, cloud, log, args): configs=mycfg['configs'], fspath=mycfg['packages_dir']) + set_snappy_command() + fails = [] for pkg_op in pkg_ops: try: - render_snap_op(op=pkg_op['op'], name=pkg_op['name'], - cfgfile=pkg_op['cfgfile'], config=pkg_op['config']) + render_snap_op(**pkg_op) except Exception as e: fails.append((pkg_op, e,)) LOG.warn("'%s' failed for '%s': %s", -- cgit v1.2.3 From d0eacf97c72b2613a3f1ce179e284d5aa98744dc Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 Mar 2015 22:18:16 -0400 Subject: encode needed for yaml_dumps --- cloudinit/config/cc_snappy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/config/cc_snappy.py') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index dbdc402c..adb25bc2 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -101,7 +101,7 @@ def render_snap_op(op, name, path=None, cfgfile=None, config=None): elif isinstance(config, six.text_type): cfg_bytes = config.encode() else: - cfg_bytes = util.yaml_dumps(config) + cfg_bytes = util.yaml_dumps(config).encode() (fd, cfg_tmpf) = tempfile.mkstemp() os.write(fd, cfg_bytes) -- cgit v1.2.3 From ef4a19658f354f1cb52b59c093d38d8448e26a70 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 Mar 2015 22:42:26 -0400 Subject: rad_pkg_data: return data, fix undefined variable --- cloudinit/config/cc_snappy.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'cloudinit/config/cc_snappy.py') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index adb25bc2..61c70f03 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -132,6 +132,7 @@ def read_installed_packages(): def read_pkg_data(): out, err = util.subp([SNAPPY_CMD, "list"]) + pkg_data = [] for line in out.splitlines()[1:]: toks = line.split(sep=None, maxsplit=3) if len(toks) == 3: @@ -139,7 +140,8 @@ def read_pkg_data(): dev = None else: (name, date, version, dev) = toks - pkgs.append((name, date, version, dev,)) + pkg_data.append((name, date, version, dev,)) + return pkg_data def disable_enable_ssh(enabled): -- cgit v1.2.3 From f32a0c32081a4c38b9738bd65a2efc35f26ee983 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 Mar 2015 22:51:59 -0400 Subject: improve doc, change 'click_packages' path to be 'snaps' --- cloudinit/config/cc_snappy.py | 45 ++++++++++++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 13 deletions(-) (limited to 'cloudinit/config/cc_snappy.py') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index 61c70f03..09a8d239 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -1,5 +1,36 @@ # vi: ts=4 expandtab # +""" +snappy modules allows configuration of snappy. +Example config: + #cloud-config + snappy: + system_snappy: auto + ssh_enabled: False + packages: [etcd, pkg2] + configs: + pkgname: pkgname-config-blob + pkg2: config-blob + packages_dir: '/writable/user-data/cloud-init/snaps' + + - ssh_enabled: + This defaults to 'False'. Set to a non-false value to enable ssh service + - snap installation and config + The above would install 'etcd', and then install 'pkg2' with a + '--config=' argument where 'file' as 'config-blob' inside it. + If 'pkgname' is installed already, then 'snappy config pkgname ' + will be called where 'file' has 'pkgname-config-blob' as its content. + + If 'packages_dir' has files in it that end in '.snap', then they are + installed. Given 3 files: + /foo.snap + /foo.config + /bar.snap + cloud-init will invoke: + snappy install "--config=/foo.config" \ + /foo.snap + snappy install /bar.snap +""" from cloudinit import log as logging from cloudinit import templater @@ -18,24 +49,12 @@ SNAPPY_CMD = "snappy" BUILTIN_CFG = { 'packages': [], - 'packages_dir': '/writable/user-data/cloud-init/click_packages', + 'packages_dir': '/writable/user-data/cloud-init/snaps', 'ssh_enabled': False, 'system_snappy': "auto", 'configs': {}, } -""" -snappy: - system_snappy: auto - ssh_enabled: True - packages: - - etcd - - pkg2 - configs: - pkgname: config-blob - pkgname2: config-blob -""" - def get_fs_package_ops(fspath): if not fspath: -- cgit v1.2.3 From feab0f913b2c3e98cf5200ea2dd7c19aed347395 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 Mar 2015 22:56:43 -0400 Subject: mention ubuntu-core --- cloudinit/config/cc_snappy.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'cloudinit/config/cc_snappy.py') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index 09a8d239..de6fae4b 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -30,6 +30,10 @@ Example config: snappy install "--config=/foo.config" \ /foo.snap snappy install /bar.snap + + Note, that if provided a 'configs' entry for 'ubuntu-core', then + cloud-init will invoke: snappy config ubuntu-core + Allowing you to configure ubuntu-core in this way. """ from cloudinit import log as logging -- cgit v1.2.3 From b9cdcb6a8ef499c6e3be178fb5f59d369eb3b169 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 Mar 2015 23:22:12 -0400 Subject: fix scope so that SNAPPY_CMD is affected by set_snappy_command --- cloudinit/config/cc_snappy.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'cloudinit/config/cc_snappy.py') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index de6fae4b..e664234a 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -194,11 +194,12 @@ def system_is_snappy(): def set_snappy_command(): + global SNAPPY_CMD if util.which("snappy-go"): - SNAPPY_COMMAND = "snappy-go" + SNAPPY_CMD = "snappy-go" else: - SNAPPY_COMMAND = "snappy" - LOG.debug("snappy command is '%s'", SNAPPY_COMMAND) + SNAPPY_CMD = "snappy" + LOG.debug("snappy command is '%s'", SNAPPY_CMD) def handle(name, cfg, cloud, log, args): -- cgit v1.2.3 From b7b8004bc58f1243d023092e67f3b78743086ff2 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 Mar 2015 11:11:05 -0400 Subject: change 'configs' to 'config', and namespace input to 'snappy config' the input to 'snappy config ' is expected to have config: : content: So here we pad that input correctly. Note, that a .config file on disk is not modified. Also, we change 'configs' to just be 'config', to be possibly compatible with the a future 'snappy config /' that dumped: config: pkg1: data1 pkg2: data2 --- cloudinit/config/cc_snappy.py | 29 +++++----- .../unittests/test_handler/test_handler_snappy.py | 61 ++++++++++++++++------ 2 files changed, 61 insertions(+), 29 deletions(-) (limited to 'cloudinit/config/cc_snappy.py') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index e664234a..a3af98a6 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -8,9 +8,11 @@ Example config: system_snappy: auto ssh_enabled: False packages: [etcd, pkg2] - configs: - pkgname: pkgname-config-blob - pkg2: config-blob + config: + pkgname: + key2: value2 + pkg2: + key1: value1 packages_dir: '/writable/user-data/cloud-init/snaps' - ssh_enabled: @@ -31,7 +33,7 @@ Example config: /foo.snap snappy install /bar.snap - Note, that if provided a 'configs' entry for 'ubuntu-core', then + Note, that if provided a 'config' entry for 'ubuntu-core', then cloud-init will invoke: snappy config ubuntu-core Allowing you to configure ubuntu-core in this way. """ @@ -56,7 +58,7 @@ BUILTIN_CFG = { 'packages_dir': '/writable/user-data/cloud-init/snaps', 'ssh_enabled': False, 'system_snappy': "auto", - 'configs': {}, + 'config': {}, } @@ -119,15 +121,14 @@ def render_snap_op(op, name, path=None, cfgfile=None, config=None): try: cfg_tmpf = None if config is not None: - if isinstance(config, six.binary_type): - cfg_bytes = config - elif isinstance(config, six.text_type): - cfg_bytes = config.encode() - else: - cfg_bytes = util.yaml_dumps(config).encode() - + # input to 'snappy config packagename' must have nested data. odd. + # config: + # packagename: + # config + # Note, however, we do not touch config files on disk. + nested_cfg = {'config': {name: config}} (fd, cfg_tmpf) = tempfile.mkstemp() - os.write(fd, cfg_bytes) + os.write(fd, util.yaml_dumps(nested_cfg).encode()) os.close(fd) cfgfile = cfg_tmpf @@ -218,7 +219,7 @@ def handle(name, cfg, cloud, log, args): return pkg_ops = get_package_ops(packages=mycfg['packages'], - configs=mycfg['configs'], + configs=mycfg['config'], fspath=mycfg['packages_dir']) set_snappy_command() diff --git a/tests/unittests/test_handler/test_handler_snappy.py b/tests/unittests/test_handler/test_handler_snappy.py index 81d891d2..f56a22f7 100644 --- a/tests/unittests/test_handler/test_handler_snappy.py +++ b/tests/unittests/test_handler/test_handler_snappy.py @@ -6,6 +6,9 @@ from .. import helpers as t_help import os import shutil import tempfile +import yaml + +ALLOWED = (dict, list, int, str) class TestInstallPackages(t_help.TestCase): @@ -44,7 +47,7 @@ class TestInstallPackages(t_help.TestCase): config = kwargs.get('data', '') else: with open(args[3], "rb") as fp: - config = fp.read() + config = yaml.safe_load(fp.read()) self.snapcmds.append(['config', args[2], config]) elif args[0:2] == ['snappy', 'install']: config = None @@ -56,7 +59,7 @@ class TestInstallPackages(t_help.TestCase): config = kwargs.get('data', '') elif cfgfile: with open(cfgfile, "rb") as fp: - config = fp.read() + config = yaml.safe_load(fp.read()) elif not pkg and not arg.startswith("-"): pkg = arg self.snapcmds.append(['install', pkg, config]) @@ -126,7 +129,7 @@ class TestInstallPackages(t_help.TestCase): path='snapf1.snap', cfgfile='snapf1.config') render_snap_op(**op) self.assertEqual( - self.snapcmds, [['install', op['path'], b'snapf1cfg']]) + self.snapcmds, [['install', op['path'], 'snapf1cfg']]) def test_render_op_snap(self): op = makeop('install', 'snapf1') @@ -135,49 +138,77 @@ class TestInstallPackages(t_help.TestCase): self.snapcmds, [['install', 'snapf1', None]]) def test_render_op_snap_config(self): - op = makeop('install', 'snapf1', config=b'myconfig') + mycfg = {'key1': 'value1'} + name = "snapf1" + op = makeop('install', name, config=mycfg) render_snap_op(**op) self.assertEqual( - self.snapcmds, [['install', 'snapf1', b'myconfig']]) + self.snapcmds, [['install', name, {'config': {name: mycfg}}]]) def test_render_op_config_bytes(self): - op = makeop('config', 'snapf1', config=b'myconfig') + name = "snapf1" + mycfg = b'myconfig' + op = makeop('config', name, config=mycfg) render_snap_op(**op) self.assertEqual( - self.snapcmds, [['config', 'snapf1', b'myconfig']]) + self.snapcmds, [['config', 'snapf1', {'config': {name: mycfg}}]]) def test_render_op_config_string(self): + name = 'snapf1' mycfg = 'myconfig: foo\nhisconfig: bar\n' - op = makeop('config', 'snapf1', config=mycfg) + op = makeop('config', name, config=mycfg) render_snap_op(**op) self.assertEqual( - self.snapcmds, [['config', 'snapf1', mycfg.encode()]]) + self.snapcmds, [['config', 'snapf1', {'config': {name: mycfg}}]]) def test_render_op_config_dict(self): # config entry for package can be a dict, not a string blob mycfg = {'foo': 'bar'} - op = makeop('config', 'snapf1', config=mycfg) + name = 'snapf1' + op = makeop('config', name, config=mycfg) render_snap_op(**op) # snapcmds is a list of 3-entry lists. data_found will be the # blob of data in the file in 'snappy install --config=' data_found = self.snapcmds[0][2] - self.assertEqual(mycfg, util.load_yaml(data_found)) + self.assertEqual(mycfg, data_found['config'][name]) def test_render_op_config_list(self): # config entry for package can be a list, not a string blob mycfg = ['foo', 'bar', 'wark', {'f1': 'b1'}] - op = makeop('config', 'snapf1', config=mycfg) + name = "snapf1" + op = makeop('config', name, config=mycfg) render_snap_op(**op) data_found = self.snapcmds[0][2] - self.assertEqual(mycfg, util.load_yaml(data_found, allowed=(list,))) + self.assertEqual(mycfg, data_found['config'][name]) def test_render_op_config_int(self): # config entry for package can be a list, not a string blob mycfg = 1 - op = makeop('config', 'snapf1', config=mycfg) + name = 'snapf1' + op = makeop('config', name, config=mycfg) render_snap_op(**op) data_found = self.snapcmds[0][2] - self.assertEqual(mycfg, util.load_yaml(data_found, allowed=(int,))) + self.assertEqual(mycfg, data_found['config'][name]) + + def test_render_does_not_pad_cfgfile(self): + # package_ops with cfgfile should not modify --file= content. + mydata = "foo1: bar1\nk: [l1, l2, l3]\n" + self.populate_tmp( + {"snapf1.snap": b"foo1", "snapf1.config": mydata.encode()}) + ret = get_package_ops( + packages=[], configs={}, installed=[], fspath=self.tmp) + self.assertEqual( + ret, + [makeop_tmpd(self.tmp, 'install', 'snapf1', path="snapf1.snap", + cfgfile="snapf1.config")]) + + # now the op was ok, but test that render didn't mess it up. + render_snap_op(**ret[0]) + data_found = self.snapcmds[0][2] + # the data found gets loaded in the snapcmd interpretation + # so this comparison is a bit lossy, but input to snappy config + # is expected to be yaml loadable, so it should be OK. + self.assertEqual(yaml.safe_load(mydata), data_found) def makeop_tmpd(tmpd, op, name, config=None, path=None, cfgfile=None): -- cgit v1.2.3 From 6c48673245225c5530c7cc08f5ab82794c708f71 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 Mar 2015 11:33:58 -0400 Subject: set snappy command earlier --- cloudinit/config/cc_snappy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cloudinit/config/cc_snappy.py') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index a3af98a6..f237feef 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -218,12 +218,12 @@ def handle(name, cfg, cloud, log, args): LOG.debug("%s: 'auto' mode, and system not snappy", name) return + set_snappy_command() + pkg_ops = get_package_ops(packages=mycfg['packages'], configs=mycfg['config'], fspath=mycfg['packages_dir']) - set_snappy_command() - fails = [] for pkg_op in pkg_ops: try: -- cgit v1.2.3 From b4989280d7285f214c1016efa36a20ad57821d6b Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 Mar 2015 14:19:56 -0400 Subject: address namespacing --- cloudinit/config/cc_snappy.py | 52 +++++++++++++++++----- .../unittests/test_handler/test_handler_snappy.py | 46 +++++++++++++++++++ 2 files changed, 88 insertions(+), 10 deletions(-) (limited to 'cloudinit/config/cc_snappy.py') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index f237feef..f8f67e1f 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -7,7 +7,7 @@ Example config: snappy: system_snappy: auto ssh_enabled: False - packages: [etcd, pkg2] + packages: [etcd, pkg2.smoser] config: pkgname: key2: value2 @@ -18,11 +18,15 @@ Example config: - ssh_enabled: This defaults to 'False'. Set to a non-false value to enable ssh service - snap installation and config - The above would install 'etcd', and then install 'pkg2' with a + The above would install 'etcd', and then install 'pkg2.smoser' with a '--config=' argument where 'file' as 'config-blob' inside it. If 'pkgname' is installed already, then 'snappy config pkgname ' will be called where 'file' has 'pkgname-config-blob' as its content. + Entries in 'config' can be namespaced or non-namespaced for a package. + In either case, the config provided to snappy command is non-namespaced. + The package name is provided as it appears. + If 'packages_dir' has files in it that end in '.snap', then they are installed. Given 3 files: /foo.snap @@ -52,6 +56,7 @@ LOG = logging.getLogger(__name__) frequency = PER_INSTANCE SNAPPY_CMD = "snappy" +NAMESPACE_DELIM = '.' BUILTIN_CFG = { 'packages': [], @@ -81,10 +86,20 @@ def makeop(op, name, config=None, path=None, cfgfile=None): 'cfgfile': cfgfile}) +def get_package_config(configs, name): + # load the package's config from the configs dict. + # prefer full-name entry (config-example.canonical) + # over short name entry (config-example) + if name in configs: + return configs[name] + return configs.get(name.partition(NAMESPACE_DELIM)[0]) + + def get_package_ops(packages, configs, installed=None, fspath=None): # get the install an config operations that should be done if installed is None: installed = read_installed_packages() + short_installed = [p.partition(NAMESPACE_DELIM)[0] for p in installed] if not packages: packages = [] @@ -95,21 +110,31 @@ def get_package_ops(packages, configs, installed=None, fspath=None): ops += get_fs_package_ops(fspath) for name in packages: - ops.append(makeop('install', name, configs.get('name'))) + ops.append(makeop('install', name, get_package_config(configs, name))) to_install = [f['name'] for f in ops] + short_to_install = [f['name'].partition(NAMESPACE_DELIM)[0] for f in ops] for name in configs: - if name in installed and name not in to_install: - ops.append(makeop('config', name, config=configs[name])) + if name in to_install: + continue + shortname = name.partition(NAMESPACE_DELIM)[0] + if shortname in short_to_install: + continue + if name in installed or shortname in short_installed: + ops.append(makeop('config', name, + config=get_package_config(configs, name))) # prefer config entries to filepath entries for op in ops: + if op['op'] != 'install' or not op['cfgfile']: + continue name = op['name'] - if name in configs and op['op'] == 'install' and 'cfgfile' in op: - LOG.debug("preferring configs[%s] over '%s'", name, op['cfgfile']) + fromcfg = get_package_config(configs, op['name']) + if fromcfg: + LOG.debug("preferring configs[%(name)s] over '%(cfgfile)s'", op) op['cfgfile'] = None - op['config'] = configs[op['name']] + op['config'] = fromcfg return ops @@ -118,6 +143,7 @@ def render_snap_op(op, name, path=None, cfgfile=None, config=None): if op not in ('install', 'config'): raise ValueError("cannot render op '%s'" % op) + shortname = name.partition(NAMESPACE_DELIM)[0] try: cfg_tmpf = None if config is not None: @@ -126,7 +152,7 @@ def render_snap_op(op, name, path=None, cfgfile=None, config=None): # packagename: # config # Note, however, we do not touch config files on disk. - nested_cfg = {'config': {name: config}} + nested_cfg = {'config': {shortname: config}} (fd, cfg_tmpf) = tempfile.mkstemp() os.write(fd, util.yaml_dumps(nested_cfg).encode()) os.close(fd) @@ -151,7 +177,13 @@ def render_snap_op(op, name, path=None, cfgfile=None, config=None): def read_installed_packages(): - return [p[0] for p in read_pkg_data()] + ret = [] + for (name, date, version, dev) in read_pkg_data(): + if dev: + ret.append(NAMESPACE_DELIM.join(name, dev)) + else: + ret.append(name) + return ret def read_pkg_data(): diff --git a/tests/unittests/test_handler/test_handler_snappy.py b/tests/unittests/test_handler/test_handler_snappy.py index f56a22f7..f0776259 100644 --- a/tests/unittests/test_handler/test_handler_snappy.py +++ b/tests/unittests/test_handler/test_handler_snappy.py @@ -90,6 +90,15 @@ class TestInstallPackages(t_help.TestCase): makeop('install', 'pkg2', b'mycfg2'), makeop('config', 'xinstalled', b'xcfg')]) + def test_package_ops_install_long_config_short(self): + # a package can be installed by full name, but have config by short + cfg = {'k1': 'k2'} + ret = get_package_ops( + packages=['config-example.canonical'], + configs={'config-example': cfg}, installed=[]) + self.assertEqual( + ret, [makeop('install', 'config-example.canonical', cfg)]) + def test_package_ops_with_file(self): self.populate_tmp( {"snapf1.snap": b"foo1", "snapf1.config": b"snapf1cfg", @@ -114,6 +123,34 @@ class TestInstallPackages(t_help.TestCase): ret, [makeop_tmpd(self.tmp, 'install', 'snapf1', path="snapf1.snap", config="snapf1cfg-config")]) + def test_package_ops_namespacing(self): + cfgs = { + 'config-example': {'k1': 'v1'}, + 'pkg1': {'p1': 'p2'}, + 'ubuntu-core': {'c1': 'c2'}, + 'notinstalled.smoser': {'s1': 's2'}, + } + cfg = {'config-example-k1': 'config-example-k2'} + ret = get_package_ops( + packages=['config-example.canonical'], configs=cfgs, + installed=['config-example.smoser', 'pkg1.canonical', + 'ubuntu-core']) + + expected_configs = [ + makeop('config', 'pkg1', config=cfgs['pkg1']), + makeop('config', 'ubuntu-core', config=cfgs['ubuntu-core'])] + expected_installs = [ + makeop('install', 'config-example.canonical', + config=cfgs['config-example'])] + + installs = [i for i in ret if i['op'] == 'install'] + configs = [c for c in ret if c['op'] == 'config'] + + self.assertEqual(installs, expected_installs) + # configs are not ordered + self.assertEqual(len(configs), len(expected_configs)) + self.assertTrue(all(found in expected_configs for found in configs)) + def test_render_op_localsnap(self): self.populate_tmp({"snapf1.snap": b"foo1"}) op = makeop_tmpd(self.tmp, 'install', 'snapf1', @@ -190,6 +227,15 @@ class TestInstallPackages(t_help.TestCase): data_found = self.snapcmds[0][2] self.assertEqual(mycfg, data_found['config'][name]) + def test_render_long_configs_short(self): + # install a namespaced package should have un-namespaced config + mycfg = {'k1': 'k2'} + name = 'snapf1' + op = makeop('install', name + ".smoser", config=mycfg) + render_snap_op(**op) + data_found = self.snapcmds[0][2] + self.assertEqual(mycfg, data_found['config'][name]) + def test_render_does_not_pad_cfgfile(self): # package_ops with cfgfile should not modify --file= content. mydata = "foo1: bar1\nk: [l1, l2, l3]\n" -- cgit v1.2.3 From 25a05c3367e024fcee5da0b4f15b5ca599dd92f2 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 Mar 2015 14:31:06 -0400 Subject: fix read_install --- cloudinit/config/cc_snappy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/config/cc_snappy.py') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index f8f67e1f..d9dd9771 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -180,7 +180,7 @@ def read_installed_packages(): ret = [] for (name, date, version, dev) in read_pkg_data(): if dev: - ret.append(NAMESPACE_DELIM.join(name, dev)) + ret.append(NAMESPACE_DELIM.join([name, dev])) else: ret.append(name) return ret -- cgit v1.2.3 From 09cc5909e3d69c03622b7dc2c4cb35fd378743cb Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 Mar 2015 16:20:05 -0400 Subject: be more user-friendly when looking for matching .config On fspath installs, look for .config files harder. Given a file named: pkg.namespace_0.version_arch.snap We'll search for config files named: pkg.namespace_0.version_arch.config pkg.namespace.config pkg.config --- cloudinit/config/cc_snappy.py | 21 +++++++++---- .../unittests/test_handler/test_handler_snappy.py | 34 ++++++++++++++++++++++ 2 files changed, 50 insertions(+), 5 deletions(-) (limited to 'cloudinit/config/cc_snappy.py') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index d9dd9771..74ae3ac0 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -67,15 +67,26 @@ BUILTIN_CFG = { } +def parse_filename(fname): + fname = os.path.basename(fname) + fname_noext = fname.rpartition(".")[0] + name = fname_noext.partition("_")[0] + shortname = name.partition(".")[0] + return(name, shortname, fname_noext) + + def get_fs_package_ops(fspath): if not fspath: return [] ops = [] - for snapfile in glob.glob(os.path.sep.join([fspath, '*.snap'])): - cfg = snapfile.rpartition(".")[0] + ".config" - name = os.path.basename(snapfile).rpartition(".")[0] - if not os.path.isfile(cfg): - cfg = None + for snapfile in sorted(glob.glob(os.path.sep.join([fspath, '*.snap']))): + (name, shortname, fname_noext) = parse_filename(snapfile) + cfg = None + for cand in set((fname_noext, name, shortname,)): + fpcand = os.path.sep.join([fspath, cand]) + ".config" + if os.path.isfile(fpcand): + cfg = fpcand + break ops.append(makeop('install', name, config=None, path=snapfile, cfgfile=cfg)) return ops diff --git a/tests/unittests/test_handler/test_handler_snappy.py b/tests/unittests/test_handler/test_handler_snappy.py index f0776259..84512846 100644 --- a/tests/unittests/test_handler/test_handler_snappy.py +++ b/tests/unittests/test_handler/test_handler_snappy.py @@ -112,6 +112,40 @@ class TestInstallPackages(t_help.TestCase): makeop_tmpd(self.tmp, 'install', 'snapf2', path="snapf2.snap"), makeop('install', 'pkg1')]) + def test_package_ops_common_filename(self): + # fish package name from filename + # package names likely look like: pkgname.namespace_version_arch.snap + fname = "xkcd-webserver.canonical_0.3.4_all.snap" + name = "xkcd-webserver.canonical" + shortname = "xkcd-webserver" + + # find filenames + self.populate_tmp( + {"pkg-ws.smoser_0.3.4_all.snap": "pkg-ws-snapdata", + "pkg-ws.config": "pkg-ws-config", + "pkg1.smoser_1.2.3_all.snap": "pkg1.snapdata", + "pkg1.smoser.config": "pkg1.smoser.config-data", + "pkg1.config": "pkg1.config-data", + "pkg2.smoser_0.0_amd64.snap": "pkg2-snapdata", + "pkg2.smoser_0.0_amd64.config": "pkg2.config", + }) + + ret = get_package_ops( + packages=[], configs={}, installed=[], fspath=self.tmp) + raise Exception("ret: %s" % ret) + self.assertEqual( + ret, + [makeop_tmpd(self.tmp, 'install', 'pkg-ws.smoser', + path="pkg-ws.smoser_0.3.4_all.snap", + cfgfile="pkg-ws.config"), + makeop_tmpd(self.tmp, 'install', 'pkg1.smoser', + path="pkg1.smoser_1.2.3_all.snap", + cfgfile="pkg1.smoser.config"), + makeop_tmpd(self.tmp, 'install', 'pkg2.smoser', + path="pkg2.smoser_0.0_amd64.snap", + cfgfile="pkg2.smoser_0.0_amd64.config"), + ]) + def test_package_ops_config_overrides_file(self): # config data overrides local file .config self.populate_tmp( -- cgit v1.2.3 From 6f738bea5d2aa29cdf14d0dc2a6e880517ab2bc2 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 Mar 2015 16:27:47 -0400 Subject: do not use set --- cloudinit/config/cc_snappy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/config/cc_snappy.py') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index 74ae3ac0..05676321 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -82,7 +82,7 @@ def get_fs_package_ops(fspath): for snapfile in sorted(glob.glob(os.path.sep.join([fspath, '*.snap']))): (name, shortname, fname_noext) = parse_filename(snapfile) cfg = None - for cand in set((fname_noext, name, shortname,)): + for cand in (fname_noext, name, shortname): fpcand = os.path.sep.join([fspath, cand]) + ".config" if os.path.isfile(fpcand): cfg = fpcand -- cgit v1.2.3 From 522a146eadcdb30e68acaaf792c391a7f1da3151 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 Mar 2015 17:03:59 -0400 Subject: allow-unauthenticated when done from local file --- cloudinit/config/cc_snappy.py | 1 + 1 file changed, 1 insertion(+) (limited to 'cloudinit/config/cc_snappy.py') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index 05676321..131ee7ea 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -174,6 +174,7 @@ def render_snap_op(op, name, path=None, cfgfile=None, config=None): if cfgfile: cmd.append('--config=' + cfgfile) if path: + cmd.append("--allow-unauthenticated") cmd.append(path) else: cmd.append(name) -- cgit v1.2.3 From 8165000c3975db07cb5b8b29410635dd6c9345bd Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 31 Mar 2015 14:20:00 -0400 Subject: adjust cc_snappy for snappy install package with config. It was believed that to install a package with config the command was: snappy install --config=config-file Instead, what was implemented in snappy was: snappy install [] This modifies cloud-init to invoke the latter and changes the tests appropriately. LP: #1438836 --- cloudinit/config/cc_snappy.py | 9 ++++----- tests/unittests/test_handler/test_handler_snappy.py | 10 ++++++---- 2 files changed, 10 insertions(+), 9 deletions(-) (limited to 'cloudinit/config/cc_snappy.py') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index 131ee7ea..6a7ae09b 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -19,7 +19,7 @@ Example config: This defaults to 'False'. Set to a non-false value to enable ssh service - snap installation and config The above would install 'etcd', and then install 'pkg2.smoser' with a - '--config=' argument where 'file' as 'config-blob' inside it. + '' argument where 'config-file' has 'config-blob' inside it. If 'pkgname' is installed already, then 'snappy config pkgname ' will be called where 'file' has 'pkgname-config-blob' as its content. @@ -33,8 +33,7 @@ Example config: /foo.config /bar.snap cloud-init will invoke: - snappy install "--config=/foo.config" \ - /foo.snap + snappy install /foo.snap /foo.config snappy install /bar.snap Note, that if provided a 'config' entry for 'ubuntu-core', then @@ -171,13 +170,13 @@ def render_snap_op(op, name, path=None, cfgfile=None, config=None): cmd = [SNAPPY_CMD, op] if op == 'install': - if cfgfile: - cmd.append('--config=' + cfgfile) if path: cmd.append("--allow-unauthenticated") cmd.append(path) else: cmd.append(name) + if cfgfile: + cmd.append(cfgfile) elif op == 'config': cmd += [name, cfgfile] diff --git a/tests/unittests/test_handler/test_handler_snappy.py b/tests/unittests/test_handler/test_handler_snappy.py index 8effd99d..f3109bac 100644 --- a/tests/unittests/test_handler/test_handler_snappy.py +++ b/tests/unittests/test_handler/test_handler_snappy.py @@ -53,15 +53,17 @@ class TestInstallPackages(t_help.TestCase): config = None pkg = None for arg in args[2:]: - if arg.startswith("--config="): - cfgfile = arg.partition("=")[2] + if arg.startswith("-"): + continue + if not pkg: + pkg = arg + elif not config: + cfgfile = arg if cfgfile == "-": config = kwargs.get('data', '') elif cfgfile: with open(cfgfile, "rb") as fp: config = yaml.safe_load(fp.read()) - elif not pkg and not arg.startswith("-"): - pkg = arg self.snapcmds.append(['install', pkg, config]) def test_package_ops_1(self): -- cgit v1.2.3 From dcd4b2b371059bd6249b4e43af371ee1162273e8 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 16 Apr 2015 16:41:06 -0400 Subject: pep8 fixes --- cloudinit/config/cc_snappy.py | 4 ++-- cloudinit/handlers/__init__.py | 6 +++--- tests/unittests/test_data.py | 5 +++-- 3 files changed, 8 insertions(+), 7 deletions(-) (limited to 'cloudinit/config/cc_snappy.py') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index 6a7ae09b..bfe76558 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -72,7 +72,7 @@ def parse_filename(fname): name = fname_noext.partition("_")[0] shortname = name.partition(".")[0] return(name, shortname, fname_noext) - + def get_fs_package_ops(fspath): if not fspath: @@ -98,7 +98,7 @@ def makeop(op, name, config=None, path=None, cfgfile=None): def get_package_config(configs, name): # load the package's config from the configs dict. - # prefer full-name entry (config-example.canonical) + # prefer full-name entry (config-example.canonical) # over short name entry (config-example) if name in configs: return configs[name] diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index d62fcd19..52defe66 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -170,12 +170,12 @@ def _extract_first_or_bytes(blob, size): start = blob.split("\n", 1)[0] else: # We want to avoid decoding the whole blob (it might be huge) - # By taking 4*size bytes we have a guarantee to decode size utf8 chars - start = blob[:4*size].decode(errors='ignore').split("\n", 1)[0] + # By taking 4*size bytes we guarantee to decode size utf8 chars + start = blob[:4 * size].decode(errors='ignore').split("\n", 1)[0] if len(start) >= size: start = start[:size] except UnicodeDecodeError: - # Bytes array doesn't contain a text object -- return chunk of raw bytes + # Bytes array doesn't contain text so return chunk of raw bytes start = blob[0:size] return start diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py index 4f24e2dd..b950c9a5 100644 --- a/tests/unittests/test_data.py +++ b/tests/unittests/test_data.py @@ -494,10 +494,10 @@ c: 4 ]) def test_mime_application_octet_stream(self): - """Mime message of type application/octet-stream is ignored but shows warning.""" + """Mime type application/octet-stream is ignored but shows warning.""" ci = stages.Init() message = MIMEBase("application", "octet-stream") - message.set_payload(b'\xbf\xe6\xb2\xc3\xd3\xba\x13\xa4\xd8\xa1\xcc\xbf') + message.set_payload(b'\xbf\xe6\xb2\xc3\xd3\xba\x13\xa4\xd8\xa1\xcc') encoders.encode_base64(message) ci.datasource = FakeDataSource(message.as_string().encode()) @@ -511,6 +511,7 @@ c: 4 mockobj.assert_called_once_with( ci.paths.get_ipath("cloud_config"), "", 0o600) + class TestUDProcess(helpers.ResourceUsingTestCase): def test_bytes_in_userdata(self): -- cgit v1.2.3 From 96854d720d4bd356181acfa093744599a807ea8e Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 1 May 2015 05:38:56 -0400 Subject: fix 'Make pyflakes' --- Makefile | 2 +- cloudinit/config/cc_apt_pipelining.py | 2 +- cloudinit/config/cc_snappy.py | 2 -- cloudinit/sources/DataSourceOpenNebula.py | 1 - tests/unittests/test_datasource/test_smartos.py | 2 -- tests/unittests/test_handler/test_handler_apt_configure.py | 1 - tests/unittests/test_handler/test_handler_snappy.py | 5 ----- tests/unittests/test_templating.py | 5 +---- tools/hacking.py | 2 +- tools/validate-yaml.py | 3 +-- 10 files changed, 5 insertions(+), 20 deletions(-) (limited to 'cloudinit/config/cc_snappy.py') diff --git a/Makefile b/Makefile index 009257ca..bb0c5253 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ pep8: @$(CWD)/tools/run-pep8 $(PY_FILES) pyflakes: - pyflakes $(PY_FILES) + @$(CWD)/tools/tox-venv py34 pyflakes $(PY_FILES) pip-requirements: @echo "Installing cloud-init dependencies..." diff --git a/cloudinit/config/cc_apt_pipelining.py b/cloudinit/config/cc_apt_pipelining.py index e5629175..40c32c84 100644 --- a/cloudinit/config/cc_apt_pipelining.py +++ b/cloudinit/config/cc_apt_pipelining.py @@ -43,7 +43,7 @@ def handle(_name, cfg, _cloud, log, _args): write_apt_snippet("0", log, DEFAULT_FILE) elif apt_pipe_value_s in ("none", "unchanged", "os"): return - elif apt_pipe_value_s in [str(b) for b in xrange(0, 6)]: + elif apt_pipe_value_s in [str(b) for b in range(0, 6)]: write_apt_snippet(apt_pipe_value_s, log, DEFAULT_FILE) else: log.warn("Invalid option for apt_pipeling: %s", apt_pipe_value) diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index bfe76558..7aaec94a 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -42,12 +42,10 @@ Example config: """ from cloudinit import log as logging -from cloudinit import templater from cloudinit import util from cloudinit.settings import PER_INSTANCE import glob -import six import tempfile import os diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 61709c1b..ac2c3b45 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -24,7 +24,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import base64 import os import pwd import re diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 28b41eaf..adee9019 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -36,8 +36,6 @@ from binascii import crc32 import serial import six -import six - from cloudinit import helpers as c_helpers from cloudinit.sources import DataSourceSmartOS from cloudinit.util import b64e diff --git a/tests/unittests/test_handler/test_handler_apt_configure.py b/tests/unittests/test_handler/test_handler_apt_configure.py index 02cad8b2..895728b3 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure.py +++ b/tests/unittests/test_handler/test_handler_apt_configure.py @@ -7,7 +7,6 @@ import os import re import shutil import tempfile -import unittest class TestAptProxyConfig(TestCase): diff --git a/tests/unittests/test_handler/test_handler_snappy.py b/tests/unittests/test_handler/test_handler_snappy.py index f3109bac..eceb14d9 100644 --- a/tests/unittests/test_handler/test_handler_snappy.py +++ b/tests/unittests/test_handler/test_handler_snappy.py @@ -38,7 +38,6 @@ class TestInstallPackages(t_help.TestCase): if 'args' not in kwargs: kwargs['args'] = args[0] self.subp_called.append(kwargs) - snap_cmds = [] args = kwargs['args'] # here we basically parse the snappy command invoked # and append to snapcmds a list of (mode, pkg, config) @@ -117,9 +116,6 @@ class TestInstallPackages(t_help.TestCase): def test_package_ops_common_filename(self): # fish package name from filename # package names likely look like: pkgname.namespace_version_arch.snap - fname = "xkcd-webserver.canonical_0.3.4_all.snap" - name = "xkcd-webserver.canonical" - shortname = "xkcd-webserver" # find filenames self.populate_tmp( @@ -165,7 +161,6 @@ class TestInstallPackages(t_help.TestCase): 'ubuntu-core': {'c1': 'c2'}, 'notinstalled.smoser': {'s1': 's2'}, } - cfg = {'config-example-k1': 'config-example-k2'} ret = get_package_ops( packages=['config-example.canonical'], configs=cfgs, installed=['config-example.smoser', 'pkg1.canonical', diff --git a/tests/unittests/test_templating.py b/tests/unittests/test_templating.py index cf7c03b0..0c19a2c2 100644 --- a/tests/unittests/test_templating.py +++ b/tests/unittests/test_templating.py @@ -18,10 +18,6 @@ from __future__ import print_function -import sys -import six -import unittest - from . import helpers as test_helpers import textwrap @@ -30,6 +26,7 @@ from cloudinit import templater try: import Cheetah HAS_CHEETAH = True + Cheetah # make pyflakes happy, as Cheetah is not used here except ImportError: HAS_CHEETAH = False diff --git a/tools/hacking.py b/tools/hacking.py index e7797564..3175df38 100755 --- a/tools/hacking.py +++ b/tools/hacking.py @@ -128,7 +128,7 @@ def cloud_docstring_multiline_end(physical_line): """ pos = max([physical_line.find(i) for i in DOCSTRING_TRIPLE]) # start if (pos != -1 and len(physical_line) == pos): - print physical_line + print(physical_line) if (physical_line[pos + 3] == ' '): return (pos, "N403: multi line docstring end on new line") diff --git a/tools/validate-yaml.py b/tools/validate-yaml.py index eda59cb8..6e164590 100755 --- a/tools/validate-yaml.py +++ b/tools/validate-yaml.py @@ -4,7 +4,6 @@ """ import sys - import yaml @@ -17,7 +16,7 @@ if __name__ == "__main__": yaml.safe_load(fh.read()) fh.close() sys.stdout.write(" - ok\n") - except Exception, e: + except Exception as e: sys.stdout.write(" - bad (%s)\n" % (e)) bads += 1 if bads > 0: -- cgit v1.2.3 From 6f2b8551e72596adfc685357d8471c454bd96d63 Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Fri, 11 Sep 2015 13:38:14 -0600 Subject: Ubuntu Snappy: conditionally enable SSH on Snappy When a user provides authentication tokens, enable SSH unless SSH has been explicitly disabled (LP: #1494816). --- cloudinit/config/cc_snappy.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'cloudinit/config/cc_snappy.py') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index 7aaec94a..e36542bf 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -274,7 +274,20 @@ def handle(name, cfg, cloud, log, args): LOG.warn("'%s' failed for '%s': %s", pkg_op['op'], pkg_op['name'], e) - disable_enable_ssh(mycfg.get('ssh_enabled', False)) + # Default to disabling SSH + ssh_enabled = mycfg.get('ssh_enabled', False) + + # If the user has not explicitly enabled or disabled SSH, then enable it + # when password SSH authentication is requested or there are SSH keys + if mycfg.get('ssh_enabled', None) is not False: + if len(mycfg.get('public-keys', [])) > 0: + LOG.debug("Enabling SSH, user SSH keys provided") + ssh_enabled = True + elif mycfg.get('ssh_pwauth', False): + LOG.debug("Enabling SSH, password authentication requested") + ssh_enabled = True + + disable_enable_ssh(ssh_enabled) if fails: raise Exception("failed to install/configure snaps") -- cgit v1.2.3 From fd6b08c4d03b07be67398450e40e7e2f91e8db51 Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Fri, 11 Sep 2015 14:04:52 -0600 Subject: Refinements on SSH enablement --- cloudinit/config/cc_snappy.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'cloudinit/config/cc_snappy.py') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index e36542bf..899df10c 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -280,10 +280,12 @@ def handle(name, cfg, cloud, log, args): # If the user has not explicitly enabled or disabled SSH, then enable it # when password SSH authentication is requested or there are SSH keys if mycfg.get('ssh_enabled', None) is not False: - if len(mycfg.get('public-keys', [])) > 0: + user_ssh_keys = cloud.get_public_ssh_keys() or None + password_auth_enabled = cfg.get('ssh_pwauth', False) + if user_ssh_keys: LOG.debug("Enabling SSH, user SSH keys provided") ssh_enabled = True - elif mycfg.get('ssh_pwauth', False): + elif password_auth_enabled: LOG.debug("Enabling SSH, password authentication requested") ssh_enabled = True -- cgit v1.2.3 From 988174dca9e4e5593b357c6def82c857f718282d Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 11 Sep 2015 16:52:26 -0400 Subject: cc_snappy: update doc string, change default to 'auto' --- cloudinit/config/cc_snappy.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) (limited to 'cloudinit/config/cc_snappy.py') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index 899df10c..124452c0 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -6,7 +6,7 @@ Example config: #cloud-config snappy: system_snappy: auto - ssh_enabled: False + ssh_enabled: auto packages: [etcd, pkg2.smoser] config: pkgname: @@ -16,7 +16,12 @@ Example config: packages_dir: '/writable/user-data/cloud-init/snaps' - ssh_enabled: - This defaults to 'False'. Set to a non-false value to enable ssh service + This controls the system's ssh service. The default value is 'auto'. + True: enable ssh service + False: disable ssh service + auto: enable ssh service if either ssh keys have been provided + or user has requested password authentication (ssh_pwauth). + - snap installation and config The above would install 'etcd', and then install 'pkg2.smoser' with a '' argument where 'config-file' has 'config-blob' inside it. @@ -275,19 +280,23 @@ def handle(name, cfg, cloud, log, args): pkg_op['op'], pkg_op['name'], e) # Default to disabling SSH - ssh_enabled = mycfg.get('ssh_enabled', False) + ssh_enabled = mycfg.get('ssh_enabled', "auto") # If the user has not explicitly enabled or disabled SSH, then enable it # when password SSH authentication is requested or there are SSH keys - if mycfg.get('ssh_enabled', None) is not False: + if ssh_enabled == "auto": user_ssh_keys = cloud.get_public_ssh_keys() or None password_auth_enabled = cfg.get('ssh_pwauth', False) if user_ssh_keys: - LOG.debug("Enabling SSH, user SSH keys provided") + LOG.debug("Enabling SSH, ssh keys found in datasource") ssh_enabled = True + elif cfg.get('ssh_authorized_keys'): + LOG.debug("Enabling SSH, ssh keys found in config") elif password_auth_enabled: LOG.debug("Enabling SSH, password authentication requested") ssh_enabled = True + elif ssh_enabled not in (True, False): + LOG.warn("Unknown value '%s' in ssh_enabled", ssh_enabled) disable_enable_ssh(ssh_enabled) -- cgit v1.2.3 From 03b5cac37154476b89e67b231c2888a9cfdc92ca Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Tue, 15 Sep 2015 11:53:36 -0600 Subject: Change Snappy SSH enabled default from false to 'auto' (LP: #1494816) --- cloudinit/config/cc_snappy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/config/cc_snappy.py') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index 124452c0..fa9d54a0 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -63,7 +63,7 @@ NAMESPACE_DELIM = '.' BUILTIN_CFG = { 'packages': [], 'packages_dir': '/writable/user-data/cloud-init/snaps', - 'ssh_enabled': False, + 'ssh_enabled': "auto", 'system_snappy': "auto", 'config': {}, } -- cgit v1.2.3