fix snappy package installation.

Some tests added here to make sure this works.
The install '--config' will only start to work with the next
version of snappy-go.

LP: #1437137

3 files changed, 525 insertions, 45 deletions

diff --git a/ChangeLog b/ChangeLog
index 70ba9ae3..6651b8eb 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -29,6 +29,7 @@
  - readurl, read_file_or_url returns bytes, user must convert as necessary
  - SmartOS: use v2 metadata service (LP: #1436417) [Daniel Watkins]
  - NoCloud: fix local datasource claiming found without explicit dsmode
+ - Snappy: add support for installing snappy packages and configuring.
 0.7.6:
  - open 0.7.6
  - Enable vendordata on CloudSigma datasource (LP: #1303986)
diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py
index 133336d4..131ee7ea 100644
--- a/cloudinit/config/cc_snappy.py
+++ b/cloudinit/config/cc_snappy.py
@@ -1,5 +1,46 @@
 # vi: ts=4 expandtab
 #
+"""
+snappy modules allows configuration of snappy.
+Example config:
+  #cloud-config
+  snappy:
+    system_snappy: auto
+    ssh_enabled: False
+    packages: [etcd, pkg2.smoser]
+    config:
+      pkgname:
+        key2: value2
+      pkg2:
+        key1: value1
+    packages_dir: '/writable/user-data/cloud-init/snaps'
+
+ - ssh_enabled:
+   This defaults to 'False'.  Set to a non-false value to enable ssh service
+ - snap installation and config
+   The above would install 'etcd', and then install 'pkg2.smoser' with a
+   '--config=<file>' argument where 'file' as 'config-blob' inside it.
+   If 'pkgname' is installed already, then 'snappy config pkgname <file>'
+   will be called where 'file' has 'pkgname-config-blob' as its content.
+
+   Entries in 'config' can be namespaced or non-namespaced for a package.
+   In either case, the config provided to snappy command is non-namespaced.
+   The package name is provided as it appears.
+
+   If 'packages_dir' has files in it that end in '.snap', then they are
+   installed.  Given 3 files:
+     <packages_dir>/foo.snap
+     <packages_dir>/foo.config
+     <packages_dir>/bar.snap
+   cloud-init will invoke:
+     snappy install "--config=<packages_dir>/foo.config" \
+         <packages_dir>/foo.snap
+     snappy install <packages_dir>/bar.snap
+
+   Note, that if provided a 'config' entry for 'ubuntu-core', then
+   cloud-init will invoke: snappy config ubuntu-core <config>
+   Allowing you to configure ubuntu-core in this way.
+"""
 
 from cloudinit import log as logging
 from cloudinit import templater
@@ -7,63 +48,168 @@ from cloudinit import util
 from cloudinit.settings import PER_INSTANCE
 
 import glob
+import six
+import tempfile
 import os
 
 LOG = logging.getLogger(__name__)
 
 frequency = PER_INSTANCE
-SNAPPY_ENV_PATH = "/writable/system-data/etc/snappy.env"
+SNAPPY_CMD = "snappy"
+NAMESPACE_DELIM = '.'
 
 BUILTIN_CFG = {
     'packages': [],
-    'packages_dir': '/writable/user-data/cloud-init/click_packages',
+    'packages_dir': '/writable/user-data/cloud-init/snaps',
     'ssh_enabled': False,
-    'system_snappy': "auto"
+    'system_snappy': "auto",
+    'config': {},
 }
 
-"""
-snappy:
-  system_snappy: auto
-  ssh_enabled: True
-  packages:
-    - etcd
-    - {'name': 'pkg1', 'config': "wark"}
-"""
 
+def parse_filename(fname):
+    fname = os.path.basename(fname)
+    fname_noext = fname.rpartition(".")[0]
+    name = fname_noext.partition("_")[0]
+    shortname = name.partition(".")[0]
+    return(name, shortname, fname_noext)
+    
+
+def get_fs_package_ops(fspath):
+    if not fspath:
+        return []
+    ops = []
+    for snapfile in sorted(glob.glob(os.path.sep.join([fspath, '*.snap']))):
+        (name, shortname, fname_noext) = parse_filename(snapfile)
+        cfg = None
+        for cand in (fname_noext, name, shortname):
+            fpcand = os.path.sep.join([fspath, cand]) + ".config"
+            if os.path.isfile(fpcand):
+                cfg = fpcand
+                break
+        ops.append(makeop('install', name, config=None,
+                   path=snapfile, cfgfile=cfg))
+    return ops
+
+
+def makeop(op, name, config=None, path=None, cfgfile=None):
+    return({'op': op, 'name': name, 'config': config, 'path': path,
+            'cfgfile': cfgfile})
+
+
+def get_package_config(configs, name):
+    # load the package's config from the configs dict.
+    # prefer full-name entry (config-example.canonical) 
+    # over short name entry (config-example)
+    if name in configs:
+        return configs[name]
+    return configs.get(name.partition(NAMESPACE_DELIM)[0])
+
+
+def get_package_ops(packages, configs, installed=None, fspath=None):
+    # get the install an config operations that should be done
+    if installed is None:
+        installed = read_installed_packages()
+    short_installed = [p.partition(NAMESPACE_DELIM)[0] for p in installed]
+
+    if not packages:
+        packages = []
+    if not configs:
+        configs = {}
+
+    ops = []
+    ops += get_fs_package_ops(fspath)
+
+    for name in packages:
+        ops.append(makeop('install', name, get_package_config(configs, name)))
+
+    to_install = [f['name'] for f in ops]
+    short_to_install = [f['name'].partition(NAMESPACE_DELIM)[0] for f in ops]
 
-def install_package(pkg_name, config=None):
-    cmd = ["snappy", "install"]
-    if config:
-        if os.path.isfile(config):
-            cmd.append("--config-file=" + config)
+    for name in configs:
+        if name in to_install:
+            continue
+        shortname = name.partition(NAMESPACE_DELIM)[0]
+        if shortname in short_to_install:
+            continue
+        if name in installed or shortname in short_installed:
+            ops.append(makeop('config', name,
+                              config=get_package_config(configs, name)))
+
+    # prefer config entries to filepath entries
+    for op in ops:
+        if op['op'] != 'install' or not op['cfgfile']:
+            continue
+        name = op['name']
+        fromcfg = get_package_config(configs, op['name'])
+        if fromcfg:
+            LOG.debug("preferring configs[%(name)s] over '%(cfgfile)s'", op)
+            op['cfgfile'] = None
+            op['config'] = fromcfg
+
+    return ops
+
+
+def render_snap_op(op, name, path=None, cfgfile=None, config=None):
+    if op not in ('install', 'config'):
+        raise ValueError("cannot render op '%s'" % op)
+
+    shortname = name.partition(NAMESPACE_DELIM)[0]
+    try:
+        cfg_tmpf = None
+        if config is not None:
+            # input to 'snappy config packagename' must have nested data. odd.
+            # config:
+            #   packagename:
+            #      config
+            # Note, however, we do not touch config files on disk.
+            nested_cfg = {'config': {shortname: config}}
+            (fd, cfg_tmpf) = tempfile.mkstemp()
+            os.write(fd, util.yaml_dumps(nested_cfg).encode())
+            os.close(fd)
+            cfgfile = cfg_tmpf
+
+        cmd = [SNAPPY_CMD, op]
+        if op == 'install':
+            if cfgfile:
+                cmd.append('--config=' + cfgfile)
+            if path:
+                cmd.append("--allow-unauthenticated")
+                cmd.append(path)
+            else:
+                cmd.append(name)
+        elif op == 'config':
+            cmd += [name, cfgfile]
+
+        util.subp(cmd)
+
+    finally:
+        if cfg_tmpf:
+            os.unlink(cfg_tmpf)
+
+
+def read_installed_packages():
+    ret = []
+    for (name, date, version, dev) in read_pkg_data():
+        if dev:
+            ret.append(NAMESPACE_DELIM.join([name, dev]))
         else:
-            cmd.append("--config=" + config)
-    cmd.append(pkg_name)
-    util.subp(cmd)
-
-
-def install_packages(package_dir, packages):
-    local_pkgs = glob.glob(os.path.sep.join([package_dir, '*.click']))
-    LOG.debug("installing local packages %s" % local_pkgs)
-    if local_pkgs:
-        for pkg in local_pkgs:
-            cfg = pkg.replace(".click", ".config")
-            if not os.path.isfile(cfg):
-                cfg = None
-            install_package(pkg, config=cfg)
-
-    LOG.debug("installing click packages")
-    if packages:
-        for pkg in packages:
-            if not pkg:
-                continue
-            if isinstance(pkg, str):
-                name = pkg
-                config = None
-            elif pkg:
-                name = pkg.get('name', pkg)
-                config = pkg.get('config')
-            install_package(pkg_name=name, config=config)
+            ret.append(name)
+    return ret
+
+
+def read_pkg_data():
+    out, err = util.subp([SNAPPY_CMD, "list"])
+    pkg_data = []
+    for line in out.splitlines()[1:]:
+        toks = line.split(sep=None, maxsplit=3)
+        if len(toks) == 3:
+            (name, date, version) = toks
+            dev = None
+        else:
+            (name, date, version, dev) = toks
+        pkg_data.append((name, date, version, dev,))
+    return pkg_data
 
 
 def disable_enable_ssh(enabled):
@@ -92,6 +238,15 @@ def system_is_snappy():
     return False
 
 
+def set_snappy_command():
+    global SNAPPY_CMD
+    if util.which("snappy-go"):
+        SNAPPY_CMD = "snappy-go"
+    else:
+        SNAPPY_CMD = "snappy"
+    LOG.debug("snappy command is '%s'", SNAPPY_CMD)
+
+
 def handle(name, cfg, cloud, log, args):
     cfgin = cfg.get('snappy')
     if not cfgin:
@@ -107,7 +262,22 @@ def handle(name, cfg, cloud, log, args):
         LOG.debug("%s: 'auto' mode, and system not snappy", name)
         return
 
-    install_packages(mycfg['packages_dir'],
-                     mycfg['packages'])
+    set_snappy_command()
+
+    pkg_ops = get_package_ops(packages=mycfg['packages'],
+                              configs=mycfg['config'],
+                              fspath=mycfg['packages_dir'])
+
+    fails = []
+    for pkg_op in pkg_ops:
+        try:
+            render_snap_op(**pkg_op)
+        except Exception as e:
+            fails.append((pkg_op, e,))
+            LOG.warn("'%s' failed for '%s': %s",
+                     pkg_op['op'], pkg_op['name'], e)
 
     disable_enable_ssh(mycfg.get('ssh_enabled', False))
+
+    if fails:
+        raise Exception("failed to install/configure snaps")
diff --git a/tests/unittests/test_handler/test_handler_snappy.py b/tests/unittests/test_handler/test_handler_snappy.py
new file mode 100644
index 00000000..8effd99d
--- /dev/null
+++ b/tests/unittests/test_handler/test_handler_snappy.py
@@ -0,0 +1,309 @@
+from cloudinit.config.cc_snappy import (
+    makeop, get_package_ops, render_snap_op)
+from cloudinit import util
+from .. import helpers as t_help
+
+import os
+import shutil
+import tempfile
+import yaml
+
+ALLOWED = (dict, list, int, str)
+
+
+class TestInstallPackages(t_help.TestCase):
+    def setUp(self):
+        super(TestInstallPackages, self).setUp()
+        self.unapply = []
+
+        # by default 'which' has nothing in its path
+        self.apply_patches([(util, 'subp', self._subp)])
+        self.subp_called = []
+        self.snapcmds = []
+        self.tmp = tempfile.mkdtemp(prefix="TestInstallPackages")
+
+    def tearDown(self):
+        apply_patches([i for i in reversed(self.unapply)])
+        shutil.rmtree(self.tmp)
+
+    def apply_patches(self, patches):
+        ret = apply_patches(patches)
+        self.unapply += ret
+
+    def populate_tmp(self, files):
+        return t_help.populate_dir(self.tmp, files)
+
+    def _subp(self, *args, **kwargs):
+        # supports subp calling with cmd as args or kwargs
+        if 'args' not in kwargs:
+            kwargs['args'] = args[0]
+        self.subp_called.append(kwargs)
+        snap_cmds = []
+        args = kwargs['args']
+        # here we basically parse the snappy command invoked
+        # and append to snapcmds a list of (mode, pkg, config)
+        if args[0:2] == ['snappy', 'config']:
+            if args[3] == "-":
+                config = kwargs.get('data', '')
+            else:
+                with open(args[3], "rb") as fp:
+                    config = yaml.safe_load(fp.read())
+            self.snapcmds.append(['config', args[2], config])
+        elif args[0:2] == ['snappy', 'install']:
+            config = None
+            pkg = None
+            for arg in args[2:]:
+                if arg.startswith("--config="):
+                    cfgfile = arg.partition("=")[2]
+                    if cfgfile == "-":
+                        config = kwargs.get('data', '')
+                    elif cfgfile:
+                        with open(cfgfile, "rb") as fp:
+                            config = yaml.safe_load(fp.read())
+                elif not pkg and not arg.startswith("-"):
+                    pkg = arg
+            self.snapcmds.append(['install', pkg, config])
+
+    def test_package_ops_1(self):
+        ret = get_package_ops(
+            packages=['pkg1', 'pkg2', 'pkg3'],
+            configs={'pkg2': b'mycfg2'}, installed=[])
+        self.assertEqual(
+            ret, [makeop('install', 'pkg1', None, None),
+                  makeop('install', 'pkg2', b'mycfg2', None),
+                  makeop('install', 'pkg3', None, None)])
+
+    def test_package_ops_config_only(self):
+        ret = get_package_ops(
+            packages=None,
+            configs={'pkg2': b'mycfg2'}, installed=['pkg1', 'pkg2'])
+        self.assertEqual(
+            ret, [makeop('config', 'pkg2', b'mycfg2')])
+
+    def test_package_ops_install_and_config(self):
+        ret = get_package_ops(
+            packages=['pkg3', 'pkg2'],
+            configs={'pkg2': b'mycfg2', 'xinstalled': b'xcfg'},
+            installed=['xinstalled'])
+        self.assertEqual(
+            ret, [makeop('install', 'pkg3'),
+                  makeop('install', 'pkg2', b'mycfg2'),
+                  makeop('config', 'xinstalled', b'xcfg')])
+
+    def test_package_ops_install_long_config_short(self):
+        # a package can be installed by full name, but have config by short
+        cfg = {'k1': 'k2'}
+        ret = get_package_ops(
+            packages=['config-example.canonical'],
+            configs={'config-example': cfg}, installed=[])
+        self.assertEqual(
+            ret, [makeop('install', 'config-example.canonical', cfg)])
+
+    def test_package_ops_with_file(self):
+        self.populate_tmp(
+            {"snapf1.snap": b"foo1", "snapf1.config": b"snapf1cfg",
+             "snapf2.snap": b"foo2", "foo.bar": "ignored"})
+        ret = get_package_ops(
+            packages=['pkg1'], configs={}, installed=[], fspath=self.tmp)
+        self.assertEqual(
+            ret,
+            [makeop_tmpd(self.tmp, 'install', 'snapf1', path="snapf1.snap",
+                         cfgfile="snapf1.config"),
+             makeop_tmpd(self.tmp, 'install', 'snapf2', path="snapf2.snap"),
+             makeop('install', 'pkg1')])
+
+    def test_package_ops_common_filename(self):
+        # fish package name from filename
+        # package names likely look like: pkgname.namespace_version_arch.snap
+        fname = "xkcd-webserver.canonical_0.3.4_all.snap"
+        name = "xkcd-webserver.canonical"
+        shortname = "xkcd-webserver"
+
+        # find filenames
+        self.populate_tmp(
+            {"pkg-ws.smoser_0.3.4_all.snap": "pkg-ws-snapdata",
+             "pkg-ws.config": "pkg-ws-config",
+             "pkg1.smoser_1.2.3_all.snap": "pkg1.snapdata",
+             "pkg1.smoser.config": "pkg1.smoser.config-data",
+             "pkg1.config": "pkg1.config-data",
+             "pkg2.smoser_0.0_amd64.snap": "pkg2-snapdata",
+             "pkg2.smoser_0.0_amd64.config": "pkg2.config",
+            })
+
+        ret = get_package_ops(
+            packages=[], configs={}, installed=[], fspath=self.tmp)
+        self.assertEqual(
+            ret,
+            [makeop_tmpd(self.tmp, 'install', 'pkg-ws.smoser',
+                         path="pkg-ws.smoser_0.3.4_all.snap",
+                         cfgfile="pkg-ws.config"),
+             makeop_tmpd(self.tmp, 'install', 'pkg1.smoser',
+                         path="pkg1.smoser_1.2.3_all.snap",
+                         cfgfile="pkg1.smoser.config"),
+             makeop_tmpd(self.tmp, 'install', 'pkg2.smoser',
+                         path="pkg2.smoser_0.0_amd64.snap",
+                         cfgfile="pkg2.smoser_0.0_amd64.config"),
+             ])
+
+    def test_package_ops_config_overrides_file(self):
+        # config data overrides local file .config
+        self.populate_tmp(
+            {"snapf1.snap": b"foo1", "snapf1.config": b"snapf1cfg"})
+        ret = get_package_ops(
+            packages=[], configs={'snapf1': 'snapf1cfg-config'},
+            installed=[], fspath=self.tmp)
+        self.assertEqual(
+            ret, [makeop_tmpd(self.tmp, 'install', 'snapf1',
+                              path="snapf1.snap", config="snapf1cfg-config")])
+
+    def test_package_ops_namespacing(self):
+        cfgs = {
+            'config-example': {'k1': 'v1'},
+            'pkg1': {'p1': 'p2'},
+            'ubuntu-core': {'c1': 'c2'},
+            'notinstalled.smoser': {'s1': 's2'},
+        }
+        cfg = {'config-example-k1': 'config-example-k2'}
+        ret = get_package_ops(
+            packages=['config-example.canonical'], configs=cfgs,
+            installed=['config-example.smoser', 'pkg1.canonical',
+                       'ubuntu-core'])
+
+        expected_configs = [
+            makeop('config', 'pkg1', config=cfgs['pkg1']),
+            makeop('config', 'ubuntu-core', config=cfgs['ubuntu-core'])]
+        expected_installs = [
+            makeop('install', 'config-example.canonical',
+                   config=cfgs['config-example'])]
+
+        installs = [i for i in ret if i['op'] == 'install']
+        configs = [c for c in ret if c['op'] == 'config']
+
+        self.assertEqual(installs, expected_installs)
+        # configs are not ordered
+        self.assertEqual(len(configs), len(expected_configs))
+        self.assertTrue(all(found in expected_configs for found in configs))
+
+    def test_render_op_localsnap(self):
+        self.populate_tmp({"snapf1.snap": b"foo1"})
+        op = makeop_tmpd(self.tmp, 'install', 'snapf1',
+                         path='snapf1.snap')
+        render_snap_op(**op)
+        self.assertEqual(
+            self.snapcmds, [['install', op['path'], None]])
+
+    def test_render_op_localsnap_localconfig(self):
+        self.populate_tmp(
+            {"snapf1.snap": b"foo1", 'snapf1.config': b'snapf1cfg'})
+        op = makeop_tmpd(self.tmp, 'install', 'snapf1',
+                         path='snapf1.snap', cfgfile='snapf1.config')
+        render_snap_op(**op)
+        self.assertEqual(
+            self.snapcmds, [['install', op['path'], 'snapf1cfg']])
+
+    def test_render_op_snap(self):
+        op = makeop('install', 'snapf1')
+        render_snap_op(**op)
+        self.assertEqual(
+            self.snapcmds, [['install', 'snapf1', None]])
+
+    def test_render_op_snap_config(self):
+        mycfg = {'key1': 'value1'}
+        name = "snapf1"
+        op = makeop('install', name, config=mycfg)
+        render_snap_op(**op)
+        self.assertEqual(
+            self.snapcmds, [['install', name, {'config': {name: mycfg}}]])
+
+    def test_render_op_config_bytes(self):
+        name = "snapf1"
+        mycfg = b'myconfig'
+        op = makeop('config', name, config=mycfg)
+        render_snap_op(**op)
+        self.assertEqual(
+            self.snapcmds, [['config', 'snapf1', {'config': {name: mycfg}}]])
+
+    def test_render_op_config_string(self):
+        name = 'snapf1'
+        mycfg = 'myconfig: foo\nhisconfig: bar\n'
+        op = makeop('config', name, config=mycfg)
+        render_snap_op(**op)
+        self.assertEqual(
+            self.snapcmds, [['config', 'snapf1', {'config': {name: mycfg}}]])
+
+    def test_render_op_config_dict(self):
+        # config entry for package can be a dict, not a string blob
+        mycfg = {'foo': 'bar'}
+        name = 'snapf1'
+        op = makeop('config', name, config=mycfg)
+        render_snap_op(**op)
+        # snapcmds is a list of 3-entry lists. data_found will be the
+        # blob of data in the file in 'snappy install --config=<file>'
+        data_found = self.snapcmds[0][2]
+        self.assertEqual(mycfg, data_found['config'][name])
+
+    def test_render_op_config_list(self):
+        # config entry for package can be a list, not a string blob
+        mycfg = ['foo', 'bar', 'wark', {'f1': 'b1'}]
+        name = "snapf1"
+        op = makeop('config', name, config=mycfg)
+        render_snap_op(**op)
+        data_found = self.snapcmds[0][2]
+        self.assertEqual(mycfg, data_found['config'][name])
+
+    def test_render_op_config_int(self):
+        # config entry for package can be a list, not a string blob
+        mycfg = 1
+        name = 'snapf1'
+        op = makeop('config', name, config=mycfg)
+        render_snap_op(**op)
+        data_found = self.snapcmds[0][2]
+        self.assertEqual(mycfg, data_found['config'][name])
+
+    def test_render_long_configs_short(self):
+        # install a namespaced package should have un-namespaced config
+        mycfg = {'k1': 'k2'}
+        name = 'snapf1'
+        op = makeop('install', name + ".smoser", config=mycfg)
+        render_snap_op(**op)
+        data_found = self.snapcmds[0][2]
+        self.assertEqual(mycfg, data_found['config'][name])
+
+    def test_render_does_not_pad_cfgfile(self):
+        # package_ops with cfgfile should not modify --file= content.
+        mydata = "foo1: bar1\nk: [l1, l2, l3]\n"
+        self.populate_tmp(
+            {"snapf1.snap": b"foo1", "snapf1.config": mydata.encode()})
+        ret = get_package_ops(
+            packages=[], configs={}, installed=[], fspath=self.tmp)
+        self.assertEqual(
+            ret,
+            [makeop_tmpd(self.tmp, 'install', 'snapf1', path="snapf1.snap",
+                         cfgfile="snapf1.config")])
+
+        # now the op was ok, but test that render didn't mess it up.
+        render_snap_op(**ret[0])
+        data_found = self.snapcmds[0][2]
+        # the data found gets loaded in the snapcmd interpretation
+        # so this comparison is a bit lossy, but input to snappy config
+        # is expected to be yaml loadable, so it should be OK.
+        self.assertEqual(yaml.safe_load(mydata), data_found)
+
+
+def makeop_tmpd(tmpd, op, name, config=None, path=None, cfgfile=None):
+    if cfgfile:
+        cfgfile = os.path.sep.join([tmpd, cfgfile])
+    if path:
+        path = os.path.sep.join([tmpd, path])
+    return(makeop(op=op, name=name, config=config, path=path, cfgfile=cfgfile))
+
+
+def apply_patches(patches):
+    ret = []
+    for (ref, name, replace) in patches:
+        if replace is None:
+            continue
+        orig = getattr(ref, name)
+        setattr(ref, name, replace)
+        ret.append((ref, name, orig))
+    return ret