cc_snap: Add new module to install and configure snapd and snap packages.

Support installing and configuring snaps on ubuntu systems. Now,
cloud-config files can provide a list or dictionary of snap:assertions
which will be allow configuration of snapd on a system via 'snap ack'
calls. The snap:commands configuration option supports arbitrary system
commands intended to interact with snappy's cli. This allows users to run
arbitrary snappy commands to create users, download, install and
configure snap packages and snapd.

This branch also deprecates old snappy and snap_config modules leaving
warnings in documentation and runtime for consumers of these modules.
Deprecated snap* modules will be dropped in cloud-init v.18.2 release.

9 files changed, 222 insertions, 24 deletions

diff --git a/tests/cloud_tests/releases.yaml b/tests/cloud_tests/releases.yaml
index d8bc170f..c7dcbe83 100644
--- a/tests/cloud_tests/releases.yaml
+++ b/tests/cloud_tests/releases.yaml
@@ -30,6 +30,9 @@ default_release_config:
         mirror_url: https://cloud-images.ubuntu.com/daily
         mirror_dir: '/srv/citest/images'
         keyring: /usr/share/keyrings/ubuntu-cloudimage-keyring.gpg
+        # The OS version formatted as Major.Minor is used to compare releases
+        version: null   # Each release needs to define this, for example 16.04
+
     ec2:
         # Choose from: [ebs, instance-store]
         root-store: ebs
diff --git a/tests/cloud_tests/testcases.yaml b/tests/cloud_tests/testcases.yaml
index 8e0fb62f..a3e29900 100644
--- a/tests/cloud_tests/testcases.yaml
+++ b/tests/cloud_tests/testcases.yaml
@@ -15,6 +15,9 @@ base_test_data:
         instance-id: |
             #!/bin/sh
             cat /run/cloud-init/.instance-id
+        instance-data.json: |
+            #!/bin/sh
+            cat /run/cloud-init/instance-data.json
         result.json: |
             #!/bin/sh
             cat /run/cloud-init/result.json
diff --git a/tests/cloud_tests/testcases/__init__.py b/tests/cloud_tests/testcases/__init__.py
index a29a0928..bd548f5a 100644
--- a/tests/cloud_tests/testcases/__init__.py
+++ b/tests/cloud_tests/testcases/__init__.py
@@ -7,6 +7,8 @@ import inspect
 import unittest
 from unittest.util import strclass
 
+from cloudinit.util import read_conf
+
 from tests.cloud_tests import config
 from tests.cloud_tests.testcases.base import CloudTestCase as base_test
 
@@ -48,6 +50,7 @@ def get_suite(test_name, data, conf):
             def setUpClass(cls):
                 cls.data = data
                 cls.conf = conf
+                cls.release_conf = read_conf(config.RELEASES_CONF)['releases']
 
         suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(tmp))
 
diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py
index 20e95955..324c7c91 100644
--- a/tests/cloud_tests/testcases/base.py
+++ b/tests/cloud_tests/testcases/base.py
@@ -4,10 +4,14 @@
 
 import crypt
 import json
+import re
 import unittest
 
+
 from cloudinit import util as c_util
 
+SkipTest = unittest.SkipTest
+
 
 class CloudTestCase(unittest.TestCase):
     """Base test class for verifiers."""
@@ -16,6 +20,43 @@ class CloudTestCase(unittest.TestCase):
     data = {}
     conf = None
     _cloud_config = None
+    release_conf = {}    # The platform's os release configuration
+
+    expected_warnings = ()  # Subclasses set to ignore expected WARN logs
+
+    @property
+    def os_cfg(self):
+        return self.release_conf[self.os_name]['default']
+
+    def is_distro(self, distro_name):
+        return self.os_cfg['os'] == distro_name
+
+    def os_version_cmp(self, cmp_version):
+        """Compare the version of the test to comparison_version.
+
+        @param: cmp_version: Either a float or a string representing
+           a release os from releases.yaml (e.g. centos66)
+
+        @return: -1 when version < cmp_version, 0 when version=cmp_version and
+            1 when version > cmp_version.
+        """
+        version = self.release_conf[self.os_name]['default']['version']
+        if isinstance(cmp_version, str):
+            cmp_version = self.release_conf[cmp_version]['default']['version']
+        if version < cmp_version:
+            return -1
+        elif version == cmp_version:
+            return 0
+        else:
+            return 1
+
+    @property
+    def os_name(self):
+        return self.data.get('os_name', 'UNKNOWN')
+
+    @property
+    def platform(self):
+        return self.data.get('platform', 'UNKNOWN')
 
     @property
     def cloud_config(self):
@@ -72,12 +113,134 @@ class CloudTestCase(unittest.TestCase):
         self.assertEqual(len(result['errors']), 0)
 
     def test_no_warnings_in_log(self):
-        """Warnings should not be found in the log."""
+        """Unexpected warnings should not be found in the log."""
+        warnings = [
+            l for l in self.get_data_file('cloud-init.log').splitlines()
+            if 'WARN' in l]
+        joined_warnings = '\n'.join(warnings)
+        for expected_warning in self.expected_warnings:
+            self.assertIn(
+                expected_warning, joined_warnings,
+                msg="Did not find %s in cloud-init.log" % expected_warning)
+            # Prune expected from discovered warnings
+            warnings = [w for w in warnings if expected_warning not in w]
+        self.assertEqual(
+            [], warnings, msg="'WARN' found inside cloud-init.log")
+
+    def test_instance_data_json_ec2(self):
+        """Validate instance-data.json content by ec2 platform.
+
+        This content is sourced by snapd when determining snapstore endpoints.
+        We validate expected values per cloud type to ensure we don't break
+        snapd.
+        """
+        if self.platform != 'ec2':
+            raise SkipTest(
+                'Skipping ec2 instance-data.json on %s' % self.platform)
+        out = self.get_data_file('instance-data.json')
+        if not out:
+            if self.is_distro('ubuntu') and self.os_version_cmp('bionic') >= 0:
+                raise AssertionError(
+                    'No instance-data.json found on %s' % self.os_name)
+            raise SkipTest(
+                'Skipping instance-data.json test.'
+                ' OS: %s not bionic or newer' % self.os_name)
+        instance_data = json.loads(out)
+        self.assertEqual(
+            ['ds/user-data'], instance_data['base64-encoded-keys'])
+        ds = instance_data.get('ds', {})
+        macs = ds.get('network', {}).get('interfaces', {}).get('macs', {})
+        if not macs:
+            raise AssertionError('No network data from EC2 meta-data')
+        # Check meta-data items we depend on
+        expected_net_keys = [
+            'public-ipv4s', 'ipv4-associations', 'local-hostname',
+            'public-hostname']
+        for mac, mac_data in macs.items():
+            for key in expected_net_keys:
+                self.assertIn(key, mac_data)
+        self.assertIsNotNone(
+            ds.get('placement', {}).get('availability-zone'),
+            'Could not determine EC2 Availability zone placement')
+        ds = instance_data.get('ds', {})
+        v1_data = instance_data.get('v1', {})
+        self.assertIsNotNone(
+            v1_data['availability-zone'], 'expected ec2 availability-zone')
+        self.assertEqual('aws', v1_data['cloud-name'])
+        self.assertIn('i-', v1_data['instance-id'])
+        self.assertIn('ip-', v1_data['local-hostname'])
+        self.assertIsNotNone(v1_data['region'], 'expected ec2 region')
+
+    def test_instance_data_json_lxd(self):
+        """Validate instance-data.json content by lxd platform.
+
+        This content is sourced by snapd when determining snapstore endpoints.
+        We validate expected values per cloud type to ensure we don't break
+        snapd.
+        """
+        if self.platform != 'lxd':
+            raise SkipTest(
+                'Skipping lxd instance-data.json on %s' % self.platform)
+        out = self.get_data_file('instance-data.json')
+        if not out:
+            if self.is_distro('ubuntu') and self.os_version_cmp('bionic') >= 0:
+                raise AssertionError(
+                    'No instance-data.json found on %s' % self.os_name)
+            raise SkipTest(
+                'Skipping instance-data.json test.'
+                ' OS: %s not bionic or newer' % self.os_name)
+        instance_data = json.loads(out)
+        v1_data = instance_data.get('v1', {})
+        self.assertEqual(
+            ['ds/user-data', 'ds/vendor-data'],
+            sorted(instance_data['base64-encoded-keys']))
+        self.assertEqual('nocloud', v1_data['cloud-name'])
+        self.assertIsNone(
+            v1_data['availability-zone'],
+            'found unexpected lxd availability-zone %s' %
+            v1_data['availability-zone'])
+        self.assertIn('cloud-test', v1_data['instance-id'])
+        self.assertIn('cloud-test', v1_data['local-hostname'])
+        self.assertIsNone(
+            v1_data['region'],
+            'found unexpected lxd region %s' % v1_data['region'])
+
+    def test_instance_data_json_kvm(self):
+        """Validate instance-data.json content by nocloud-kvm platform.
+
+        This content is sourced by snapd when determining snapstore endpoints.
+        We validate expected values per cloud type to ensure we don't break
+        snapd.
+        """
+        if self.platform != 'nocloud-kvm':
+            raise SkipTest(
+                'Skipping nocloud-kvm instance-data.json on %s' %
+                self.platform)
+        out = self.get_data_file('instance-data.json')
+        if not out:
+            if self.is_distro('ubuntu') and self.os_version_cmp('bionic') >= 0:
+                raise AssertionError(
+                    'No instance-data.json found on %s' % self.os_name)
+            raise SkipTest(
+                'Skipping instance-data.json test.'
+                ' OS: %s not bionic or newer' % self.os_name)
+        instance_data = json.loads(out)
+        v1_data = instance_data.get('v1', {})
         self.assertEqual(
-            [],
-            [l for l in self.get_data_file('cloud-init.log').splitlines()
-             if 'WARN' in l],
-            msg="'WARN' found inside cloud-init.log")
+            ['ds/user-data'], instance_data['base64-encoded-keys'])
+        self.assertEqual('nocloud', v1_data['cloud-name'])
+        self.assertIsNone(
+            v1_data['availability-zone'],
+            'found unexpected kvm availability-zone %s' %
+            v1_data['availability-zone'])
+        self.assertIsNotNone(
+            re.match('[\da-f]{8}(-[\da-f]{4}){3}-[\da-f]{12}',
+                     v1_data['instance-id']),
+            'kvm instance-id is not a UUID: %s' % v1_data['instance-id'])
+        self.assertIn('ubuntu', v1_data['local-hostname'])
+        self.assertIsNone(
+            v1_data['region'],
+            'found unexpected lxd region %s' % v1_data['region'])
 
 
 class PasswordListTest(CloudTestCase):
diff --git a/tests/cloud_tests/testcases/main/command_output_simple.py b/tests/cloud_tests/testcases/main/command_output_simple.py
index 857881cb..80a2c8d7 100644
--- a/tests/cloud_tests/testcases/main/command_output_simple.py
+++ b/tests/cloud_tests/testcases/main/command_output_simple.py
@@ -7,6 +7,8 @@ from tests.cloud_tests.testcases import base
 class TestCommandOutputSimple(base.CloudTestCase):
     """Test functionality of simple output redirection."""
 
+    expected_warnings = ('Stdout, stderr changing to',)
+
     def test_output_file(self):
         """Ensure that the output file is not empty and has all stages."""
         data = self.get_data_file('cloud-init-test-output')
@@ -15,20 +17,5 @@ class TestCommandOutputSimple(base.CloudTestCase):
                          data.splitlines()[-1].strip())
         # TODO: need to test that all stages redirected here
 
-    def test_no_warnings_in_log(self):
-        """Warnings should not be found in the log.
-
-        This class redirected stderr and stdout, so it expects to find
-        a warning in cloud-init.log to that effect."""
-        redirect_msg = 'Stdout, stderr changing to'
-        warnings = [
-            l for l in self.get_data_file('cloud-init.log').splitlines()
-            if 'WARN' in l]
-        self.assertEqual(
-            [], [w for w in warnings if redirect_msg not in w],
-            msg="'WARN' found inside cloud-init.log")
-        self.assertEqual(
-            1, len(warnings),
-            msg="Did not find %s in cloud-init.log" % redirect_msg)
 
 # vi: ts=4 expandtab
diff --git a/tests/cloud_tests/testcases/modules/snap.py b/tests/cloud_tests/testcases/modules/snap.py
new file mode 100644
index 00000000..ff68abbe
--- /dev/null
+++ b/tests/cloud_tests/testcases/modules/snap.py
@@ -0,0 +1,16 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""cloud-init Integration Test Verify Script"""
+from tests.cloud_tests.testcases import base
+
+
+class TestSnap(base.CloudTestCase):
+    """Test snap module"""
+
+    def test_snappy_version(self):
+        """Expect hello-world and core snaps are installed."""
+        out = self.get_data_file('snaplist')
+        self.assertIn('core', out)
+        self.assertIn('hello-world', out)
+
+# vi: ts=4 expandtab
diff --git a/tests/cloud_tests/testcases/modules/snap.yaml b/tests/cloud_tests/testcases/modules/snap.yaml
new file mode 100644
index 00000000..44043f31
--- /dev/null
+++ b/tests/cloud_tests/testcases/modules/snap.yaml
@@ -0,0 +1,18 @@
+#
+# Install snappy
+#
+required_features:
+  - snap
+cloud_config: |
+  #cloud-config
+  package_update: true
+  snap:
+    squashfuse_in_container: true
+    commands:
+      - snap install hello-world
+collect_scripts:
+  snaplist: |
+    #!/bin/bash
+    snap list
+
+# vi: ts=4 expandtab
diff --git a/tests/cloud_tests/testcases/modules/snappy.py b/tests/cloud_tests/testcases/modules/snappy.py
index b92271c1..7d17fc5b 100644
--- a/tests/cloud_tests/testcases/modules/snappy.py
+++ b/tests/cloud_tests/testcases/modules/snappy.py
@@ -7,6 +7,8 @@ from tests.cloud_tests.testcases import base
 class TestSnappy(base.CloudTestCase):
     """Test snappy module"""
 
+    expected_warnings = ('DEPRECATION',)
+
     def test_snappy_version(self):
         """Test snappy version output"""
         out = self.get_data_file('snapd')
diff --git a/tests/cloud_tests/verify.py b/tests/cloud_tests/verify.py
index 2a9fd520..5a68a484 100644
--- a/tests/cloud_tests/verify.py
+++ b/tests/cloud_tests/verify.py
@@ -8,13 +8,16 @@ import unittest
 from tests.cloud_tests import (config, LOG, util, testcases)
 
 
-def verify_data(base_dir, tests):
+def verify_data(data_dir, platform, os_name, tests):
     """Verify test data is correct.
 
-    @param base_dir: base directory for data
+    @param data_dir: top level directory for all tests
+    @param platform: The platform name we for this test data (e.g. lxd)
+    @param os_name: The operating system under test (xenial, artful, etc.).
     @param tests: list of test names
     @return_value: {<test_name>: {passed: True/False, failures: []}}
     """
+    base_dir = os.sep.join((data_dir, platform, os_name))
     runner = unittest.TextTestRunner(verbosity=util.current_verbosity())
     res = {}
     for test_name in tests:
@@ -26,7 +29,7 @@ def verify_data(base_dir, tests):
         cloud_conf = test_conf['cloud_config']
 
         # load script outputs
-        data = {}
+        data = {'platform': platform, 'os_name': os_name}
         test_dir = os.path.join(base_dir, test_name)
         for script_name in os.listdir(test_dir):
             with open(os.path.join(test_dir, script_name), 'rb') as fp:
@@ -73,7 +76,7 @@ def verify(args):
 
             # run test
             res[platform][os_name] = verify_data(
-                os.sep.join((args.data_dir, platform, os_name)),
+                args.data_dir, platform, os_name,
                 tests[platform][os_name])
 
             # handle results