diff options
| -rw-r--r-- | cloudinit/config/cc_snappy.py | 248 | ||||
| -rw-r--r-- | tests/unittests/test_handler/test_handler_snappy.py | 276 |
2 files changed, 479 insertions, 45 deletions
diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index 133336d4..d9dd9771 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -1,5 +1,46 @@ # vi: ts=4 expandtab # +""" +snappy modules allows configuration of snappy. +Example config: + #cloud-config + snappy: + system_snappy: auto + ssh_enabled: False + packages: [etcd, pkg2.smoser] + config: + pkgname: + key2: value2 + pkg2: + key1: value1 + packages_dir: '/writable/user-data/cloud-init/snaps' + + - ssh_enabled: + This defaults to 'False'. Set to a non-false value to enable ssh service + - snap installation and config + The above would install 'etcd', and then install 'pkg2.smoser' with a + '--config=<file>' argument where 'file' as 'config-blob' inside it. + If 'pkgname' is installed already, then 'snappy config pkgname <file>' + will be called where 'file' has 'pkgname-config-blob' as its content. + + Entries in 'config' can be namespaced or non-namespaced for a package. + In either case, the config provided to snappy command is non-namespaced. + The package name is provided as it appears. + + If 'packages_dir' has files in it that end in '.snap', then they are + installed. Given 3 files: + <packages_dir>/foo.snap + <packages_dir>/foo.config + <packages_dir>/bar.snap + cloud-init will invoke: + snappy install "--config=<packages_dir>/foo.config" \ + <packages_dir>/foo.snap + snappy install <packages_dir>/bar.snap + + Note, that if provided a 'config' entry for 'ubuntu-core', then + cloud-init will invoke: snappy config ubuntu-core <config> + Allowing you to configure ubuntu-core in this way. +""" from cloudinit import log as logging from cloudinit import templater @@ -7,63 +48,156 @@ from cloudinit import util from cloudinit.settings import PER_INSTANCE import glob +import six +import tempfile import os LOG = logging.getLogger(__name__) frequency = PER_INSTANCE -SNAPPY_ENV_PATH = "/writable/system-data/etc/snappy.env" +SNAPPY_CMD = "snappy" +NAMESPACE_DELIM = '.' BUILTIN_CFG = { 'packages': [], - 'packages_dir': '/writable/user-data/cloud-init/click_packages', + 'packages_dir': '/writable/user-data/cloud-init/snaps', 'ssh_enabled': False, - 'system_snappy': "auto" + 'system_snappy': "auto", + 'config': {}, } -""" -snappy: - system_snappy: auto - ssh_enabled: True - packages: - - etcd - - {'name': 'pkg1', 'config': "wark"} -""" +def get_fs_package_ops(fspath): + if not fspath: + return [] + ops = [] + for snapfile in glob.glob(os.path.sep.join([fspath, '*.snap'])): + cfg = snapfile.rpartition(".")[0] + ".config" + name = os.path.basename(snapfile).rpartition(".")[0] + if not os.path.isfile(cfg): + cfg = None + ops.append(makeop('install', name, config=None, + path=snapfile, cfgfile=cfg)) + return ops + + +def makeop(op, name, config=None, path=None, cfgfile=None): + return({'op': op, 'name': name, 'config': config, 'path': path, + 'cfgfile': cfgfile}) + + +def get_package_config(configs, name): + # load the package's config from the configs dict. + # prefer full-name entry (config-example.canonical) + # over short name entry (config-example) + if name in configs: + return configs[name] + return configs.get(name.partition(NAMESPACE_DELIM)[0]) + + +def get_package_ops(packages, configs, installed=None, fspath=None): + # get the install an config operations that should be done + if installed is None: + installed = read_installed_packages() + short_installed = [p.partition(NAMESPACE_DELIM)[0] for p in installed] + + if not packages: + packages = [] + if not configs: + configs = {} + + ops = [] + ops += get_fs_package_ops(fspath) + + for name in packages: + ops.append(makeop('install', name, get_package_config(configs, name))) + + to_install = [f['name'] for f in ops] + short_to_install = [f['name'].partition(NAMESPACE_DELIM)[0] for f in ops] + + for name in configs: + if name in to_install: + continue + shortname = name.partition(NAMESPACE_DELIM)[0] + if shortname in short_to_install: + continue + if name in installed or shortname in short_installed: + ops.append(makeop('config', name, + config=get_package_config(configs, name))) -def install_package(pkg_name, config=None): - cmd = ["snappy", "install"] - if config: - if os.path.isfile(config): - cmd.append("--config-file=" + config) + # prefer config entries to filepath entries + for op in ops: + if op['op'] != 'install' or not op['cfgfile']: + continue + name = op['name'] + fromcfg = get_package_config(configs, op['name']) + if fromcfg: + LOG.debug("preferring configs[%(name)s] over '%(cfgfile)s'", op) + op['cfgfile'] = None + op['config'] = fromcfg + + return ops + + +def render_snap_op(op, name, path=None, cfgfile=None, config=None): + if op not in ('install', 'config'): + raise ValueError("cannot render op '%s'" % op) + + shortname = name.partition(NAMESPACE_DELIM)[0] + try: + cfg_tmpf = None + if config is not None: + # input to 'snappy config packagename' must have nested data. odd. + # config: + # packagename: + # config + # Note, however, we do not touch config files on disk. + nested_cfg = {'config': {shortname: config}} + (fd, cfg_tmpf) = tempfile.mkstemp() + os.write(fd, util.yaml_dumps(nested_cfg).encode()) + os.close(fd) + cfgfile = cfg_tmpf + + cmd = [SNAPPY_CMD, op] + if op == 'install': + if cfgfile: + cmd.append('--config=' + cfgfile) + if path: + cmd.append(path) + else: + cmd.append(name) + elif op == 'config': + cmd += [name, cfgfile] + + util.subp(cmd) + + finally: + if cfg_tmpf: + os.unlink(cfg_tmpf) + + +def read_installed_packages(): + ret = [] + for (name, date, version, dev) in read_pkg_data(): + if dev: + ret.append(NAMESPACE_DELIM.join([name, dev])) else: - cmd.append("--config=" + config) - cmd.append(pkg_name) - util.subp(cmd) - - -def install_packages(package_dir, packages): - local_pkgs = glob.glob(os.path.sep.join([package_dir, '*.click'])) - LOG.debug("installing local packages %s" % local_pkgs) - if local_pkgs: - for pkg in local_pkgs: - cfg = pkg.replace(".click", ".config") - if not os.path.isfile(cfg): - cfg = None - install_package(pkg, config=cfg) - - LOG.debug("installing click packages") - if packages: - for pkg in packages: - if not pkg: - continue - if isinstance(pkg, str): - name = pkg - config = None - elif pkg: - name = pkg.get('name', pkg) - config = pkg.get('config') - install_package(pkg_name=name, config=config) + ret.append(name) + return ret + + +def read_pkg_data(): + out, err = util.subp([SNAPPY_CMD, "list"]) + pkg_data = [] + for line in out.splitlines()[1:]: + toks = line.split(sep=None, maxsplit=3) + if len(toks) == 3: + (name, date, version) = toks + dev = None + else: + (name, date, version, dev) = toks + pkg_data.append((name, date, version, dev,)) + return pkg_data def disable_enable_ssh(enabled): @@ -92,6 +226,15 @@ def system_is_snappy(): return False +def set_snappy_command(): + global SNAPPY_CMD + if util.which("snappy-go"): + SNAPPY_CMD = "snappy-go" + else: + SNAPPY_CMD = "snappy" + LOG.debug("snappy command is '%s'", SNAPPY_CMD) + + def handle(name, cfg, cloud, log, args): cfgin = cfg.get('snappy') if not cfgin: @@ -107,7 +250,22 @@ def handle(name, cfg, cloud, log, args): LOG.debug("%s: 'auto' mode, and system not snappy", name) return - install_packages(mycfg['packages_dir'], - mycfg['packages']) + set_snappy_command() + + pkg_ops = get_package_ops(packages=mycfg['packages'], + configs=mycfg['config'], + fspath=mycfg['packages_dir']) + + fails = [] + for pkg_op in pkg_ops: + try: + render_snap_op(**pkg_op) + except Exception as e: + fails.append((pkg_op, e,)) + LOG.warn("'%s' failed for '%s': %s", + pkg_op['op'], pkg_op['name'], e) disable_enable_ssh(mycfg.get('ssh_enabled', False)) + + if fails: + raise Exception("failed to install/configure snaps") diff --git a/tests/unittests/test_handler/test_handler_snappy.py b/tests/unittests/test_handler/test_handler_snappy.py new file mode 100644 index 00000000..f0776259 --- /dev/null +++ b/tests/unittests/test_handler/test_handler_snappy.py @@ -0,0 +1,276 @@ +from cloudinit.config.cc_snappy import ( + makeop, get_package_ops, render_snap_op) +from cloudinit import util +from .. import helpers as t_help + +import os +import shutil +import tempfile +import yaml + +ALLOWED = (dict, list, int, str) + + +class TestInstallPackages(t_help.TestCase): + def setUp(self): + super(TestInstallPackages, self).setUp() + self.unapply = [] + + # by default 'which' has nothing in its path + self.apply_patches([(util, 'subp', self._subp)]) + self.subp_called = [] + self.snapcmds = [] + self.tmp = tempfile.mkdtemp(prefix="TestInstallPackages") + + def tearDown(self): + apply_patches([i for i in reversed(self.unapply)]) + shutil.rmtree(self.tmp) + + def apply_patches(self, patches): + ret = apply_patches(patches) + self.unapply += ret + + def populate_tmp(self, files): + return t_help.populate_dir(self.tmp, files) + + def _subp(self, *args, **kwargs): + # supports subp calling with cmd as args or kwargs + if 'args' not in kwargs: + kwargs['args'] = args[0] + self.subp_called.append(kwargs) + snap_cmds = [] + args = kwargs['args'] + # here we basically parse the snappy command invoked + # and append to snapcmds a list of (mode, pkg, config) + if args[0:2] == ['snappy', 'config']: + if args[3] == "-": + config = kwargs.get('data', '') + else: + with open(args[3], "rb") as fp: + config = yaml.safe_load(fp.read()) + self.snapcmds.append(['config', args[2], config]) + elif args[0:2] == ['snappy', 'install']: + config = None + pkg = None + for arg in args[2:]: + if arg.startswith("--config="): + cfgfile = arg.partition("=")[2] + if cfgfile == "-": + config = kwargs.get('data', '') + elif cfgfile: + with open(cfgfile, "rb") as fp: + config = yaml.safe_load(fp.read()) + elif not pkg and not arg.startswith("-"): + pkg = arg + self.snapcmds.append(['install', pkg, config]) + + def test_package_ops_1(self): + ret = get_package_ops( + packages=['pkg1', 'pkg2', 'pkg3'], + configs={'pkg2': b'mycfg2'}, installed=[]) + self.assertEqual( + ret, [makeop('install', 'pkg1', None, None), + makeop('install', 'pkg2', b'mycfg2', None), + makeop('install', 'pkg3', None, None)]) + + def test_package_ops_config_only(self): + ret = get_package_ops( + packages=None, + configs={'pkg2': b'mycfg2'}, installed=['pkg1', 'pkg2']) + self.assertEqual( + ret, [makeop('config', 'pkg2', b'mycfg2')]) + + def test_package_ops_install_and_config(self): + ret = get_package_ops( + packages=['pkg3', 'pkg2'], + configs={'pkg2': b'mycfg2', 'xinstalled': b'xcfg'}, + installed=['xinstalled']) + self.assertEqual( + ret, [makeop('install', 'pkg3'), + makeop('install', 'pkg2', b'mycfg2'), + makeop('config', 'xinstalled', b'xcfg')]) + + def test_package_ops_install_long_config_short(self): + # a package can be installed by full name, but have config by short + cfg = {'k1': 'k2'} + ret = get_package_ops( + packages=['config-example.canonical'], + configs={'config-example': cfg}, installed=[]) + self.assertEqual( + ret, [makeop('install', 'config-example.canonical', cfg)]) + + def test_package_ops_with_file(self): + self.populate_tmp( + {"snapf1.snap": b"foo1", "snapf1.config": b"snapf1cfg", + "snapf2.snap": b"foo2", "foo.bar": "ignored"}) + ret = get_package_ops( + packages=['pkg1'], configs={}, installed=[], fspath=self.tmp) + self.assertEqual( + ret, + [makeop_tmpd(self.tmp, 'install', 'snapf1', path="snapf1.snap", + cfgfile="snapf1.config"), + makeop_tmpd(self.tmp, 'install', 'snapf2', path="snapf2.snap"), + makeop('install', 'pkg1')]) + + def test_package_ops_config_overrides_file(self): + # config data overrides local file .config + self.populate_tmp( + {"snapf1.snap": b"foo1", "snapf1.config": b"snapf1cfg"}) + ret = get_package_ops( + packages=[], configs={'snapf1': 'snapf1cfg-config'}, + installed=[], fspath=self.tmp) + self.assertEqual( + ret, [makeop_tmpd(self.tmp, 'install', 'snapf1', + path="snapf1.snap", config="snapf1cfg-config")]) + + def test_package_ops_namespacing(self): + cfgs = { + 'config-example': {'k1': 'v1'}, + 'pkg1': {'p1': 'p2'}, + 'ubuntu-core': {'c1': 'c2'}, + 'notinstalled.smoser': {'s1': 's2'}, + } + cfg = {'config-example-k1': 'config-example-k2'} + ret = get_package_ops( + packages=['config-example.canonical'], configs=cfgs, + installed=['config-example.smoser', 'pkg1.canonical', + 'ubuntu-core']) + + expected_configs = [ + makeop('config', 'pkg1', config=cfgs['pkg1']), + makeop('config', 'ubuntu-core', config=cfgs['ubuntu-core'])] + expected_installs = [ + makeop('install', 'config-example.canonical', + config=cfgs['config-example'])] + + installs = [i for i in ret if i['op'] == 'install'] + configs = [c for c in ret if c['op'] == 'config'] + + self.assertEqual(installs, expected_installs) + # configs are not ordered + self.assertEqual(len(configs), len(expected_configs)) + self.assertTrue(all(found in expected_configs for found in configs)) + + def test_render_op_localsnap(self): + self.populate_tmp({"snapf1.snap": b"foo1"}) + op = makeop_tmpd(self.tmp, 'install', 'snapf1', + path='snapf1.snap') + render_snap_op(**op) + self.assertEqual( + self.snapcmds, [['install', op['path'], None]]) + + def test_render_op_localsnap_localconfig(self): + self.populate_tmp( + {"snapf1.snap": b"foo1", 'snapf1.config': b'snapf1cfg'}) + op = makeop_tmpd(self.tmp, 'install', 'snapf1', + path='snapf1.snap', cfgfile='snapf1.config') + render_snap_op(**op) + self.assertEqual( + self.snapcmds, [['install', op['path'], 'snapf1cfg']]) + + def test_render_op_snap(self): + op = makeop('install', 'snapf1') + render_snap_op(**op) + self.assertEqual( + self.snapcmds, [['install', 'snapf1', None]]) + + def test_render_op_snap_config(self): + mycfg = {'key1': 'value1'} + name = "snapf1" + op = makeop('install', name, config=mycfg) + render_snap_op(**op) + self.assertEqual( + self.snapcmds, [['install', name, {'config': {name: mycfg}}]]) + + def test_render_op_config_bytes(self): + name = "snapf1" + mycfg = b'myconfig' + op = makeop('config', name, config=mycfg) + render_snap_op(**op) + self.assertEqual( + self.snapcmds, [['config', 'snapf1', {'config': {name: mycfg}}]]) + + def test_render_op_config_string(self): + name = 'snapf1' + mycfg = 'myconfig: foo\nhisconfig: bar\n' + op = makeop('config', name, config=mycfg) + render_snap_op(**op) + self.assertEqual( + self.snapcmds, [['config', 'snapf1', {'config': {name: mycfg}}]]) + + def test_render_op_config_dict(self): + # config entry for package can be a dict, not a string blob + mycfg = {'foo': 'bar'} + name = 'snapf1' + op = makeop('config', name, config=mycfg) + render_snap_op(**op) + # snapcmds is a list of 3-entry lists. data_found will be the + # blob of data in the file in 'snappy install --config=<file>' + data_found = self.snapcmds[0][2] + self.assertEqual(mycfg, data_found['config'][name]) + + def test_render_op_config_list(self): + # config entry for package can be a list, not a string blob + mycfg = ['foo', 'bar', 'wark', {'f1': 'b1'}] + name = "snapf1" + op = makeop('config', name, config=mycfg) + render_snap_op(**op) + data_found = self.snapcmds[0][2] + self.assertEqual(mycfg, data_found['config'][name]) + + def test_render_op_config_int(self): + # config entry for package can be a list, not a string blob + mycfg = 1 + name = 'snapf1' + op = makeop('config', name, config=mycfg) + render_snap_op(**op) + data_found = self.snapcmds[0][2] + self.assertEqual(mycfg, data_found['config'][name]) + + def test_render_long_configs_short(self): + # install a namespaced package should have un-namespaced config + mycfg = {'k1': 'k2'} + name = 'snapf1' + op = makeop('install', name + ".smoser", config=mycfg) + render_snap_op(**op) + data_found = self.snapcmds[0][2] + self.assertEqual(mycfg, data_found['config'][name]) + + def test_render_does_not_pad_cfgfile(self): + # package_ops with cfgfile should not modify --file= content. + mydata = "foo1: bar1\nk: [l1, l2, l3]\n" + self.populate_tmp( + {"snapf1.snap": b"foo1", "snapf1.config": mydata.encode()}) + ret = get_package_ops( + packages=[], configs={}, installed=[], fspath=self.tmp) + self.assertEqual( + ret, + [makeop_tmpd(self.tmp, 'install', 'snapf1', path="snapf1.snap", + cfgfile="snapf1.config")]) + + # now the op was ok, but test that render didn't mess it up. + render_snap_op(**ret[0]) + data_found = self.snapcmds[0][2] + # the data found gets loaded in the snapcmd interpretation + # so this comparison is a bit lossy, but input to snappy config + # is expected to be yaml loadable, so it should be OK. + self.assertEqual(yaml.safe_load(mydata), data_found) + + +def makeop_tmpd(tmpd, op, name, config=None, path=None, cfgfile=None): + if cfgfile: + cfgfile = os.path.sep.join([tmpd, cfgfile]) + if path: + path = os.path.sep.join([tmpd, path]) + return(makeop(op=op, name=name, config=config, path=path, cfgfile=cfgfile)) + + +def apply_patches(patches): + ret = [] + for (ref, name, replace) in patches: + if replace is None: + continue + orig = getattr(ref, name) + setattr(ref, name, replace) + ret.append((ref, name, orig)) + return ret |