mcollective: add tests, cleanups and bug fix when no config in /etc.

Things here:
 - restart rather than 'start' the service, to pick up a config change
   that we would have written.
 - update the config and write cert files whether or not the file
   existed on the system.  Previously it would only write the cert
   files if /etc/mcollective/server.cfg already existed.
 - improve test coverage

3 files changed, 200 insertions, 53 deletions

diff --git a/cloudinit/config/cc_mcollective.py b/cloudinit/config/cc_mcollective.py
index 0c84d600..ada535f8 100644
--- a/cloudinit/config/cc_mcollective.py
+++ b/cloudinit/config/cc_mcollective.py
@@ -36,49 +36,53 @@ SERVER_CFG = '/etc/mcollective/server.cfg'
 LOG = logging.getLogger(__name__)
 
 
-def configure(config):
+def configure(config, server_cfg=SERVER_CFG,
+              pubcert_file=PUBCERT_FILE, pricert_file=PRICERT_FILE):
     # Read server.cfg values from the
     # original file in order to be able to mix the rest up
     try:
-        mcollective_config = ConfigObj(SERVER_CFG, file_error=True)
+        mcollective_config = ConfigObj(server_cfg, file_error=True)
+        existed = True
     except IOError:
-        LOG.warn("Did not find file %s", SERVER_CFG)
-        mcollective_config = ConfigObj(config)
-    else:
-        for (cfg_name, cfg) in config.items():
-            if cfg_name == 'public-cert':
-                util.write_file(PUBCERT_FILE, cfg, mode=0o644)
-                mcollective_config[
-                    'plugin.ssl_server_public'] = PUBCERT_FILE
-                mcollective_config['securityprovider'] = 'ssl'
-            elif cfg_name == 'private-cert':
-                util.write_file(PRICERT_FILE, cfg, mode=0o600)
-                mcollective_config[
-                    'plugin.ssl_server_private'] = PRICERT_FILE
-                mcollective_config['securityprovider'] = 'ssl'
+        LOG.debug("Did not find file %s", server_cfg)
+        mcollective_config = ConfigObj()
+        existed = False
+
+    for (cfg_name, cfg) in config.items():
+        if cfg_name == 'public-cert':
+            util.write_file(pubcert_file, cfg, mode=0o644)
+            mcollective_config[
+                'plugin.ssl_server_public'] = pubcert_file
+            mcollective_config['securityprovider'] = 'ssl'
+        elif cfg_name == 'private-cert':
+            util.write_file(pricert_file, cfg, mode=0o600)
+            mcollective_config[
+                'plugin.ssl_server_private'] = pricert_file
+            mcollective_config['securityprovider'] = 'ssl'
+        else:
+            if isinstance(cfg, six.string_types):
+                # Just set it in the 'main' section
+                mcollective_config[cfg_name] = cfg
+            elif isinstance(cfg, (dict)):
+                # Iterate through the config items, create a section if
+                # it is needed and then add/or create items as needed
+                if cfg_name not in mcollective_config.sections:
+                    mcollective_config[cfg_name] = {}
+                for (o, v) in cfg.items():
+                    mcollective_config[cfg_name][o] = v
             else:
-                if isinstance(cfg, six.string_types):
-                    # Just set it in the 'main' section
-                    mcollective_config[cfg_name] = cfg
-                elif isinstance(cfg, (dict)):
-                    # Iterate through the config items, create a section if
-                    # it is needed and then add/or create items as needed
-                    if cfg_name not in mcollective_config.sections:
-                        mcollective_config[cfg_name] = {}
-                    for (o, v) in cfg.items():
-                        mcollective_config[cfg_name][o] = v
-                else:
-                    # Otherwise just try to convert it to a string
-                    mcollective_config[cfg_name] = str(cfg)
+                # Otherwise just try to convert it to a string
+                mcollective_config[cfg_name] = str(cfg)
+
+    if existed:
         # We got all our config as wanted we'll rename
         # the previous server.cfg and create our new one
-        util.rename(SERVER_CFG, "%s.old" % (SERVER_CFG))
+        util.rename(server_cfg, "%s.old" % (server_cfg))
 
     # Now we got the whole file, write to disk...
     contents = BytesIO()
     mcollective_config.write(contents)
-    contents = contents.getvalue()
-    util.write_file(SERVER_CFG, contents, mode=0o644)
+    util.write_file(server_cfg, contents.getvalue(), mode=0o644)
 
 
 def handle(name, cfg, cloud, log, _args):
@@ -98,5 +102,5 @@ def handle(name, cfg, cloud, log, _args):
     if 'conf' in mcollective_cfg:
         configure(config=mcollective_cfg['conf'])
 
-    # Start mcollective
-    util.subp(['service', 'mcollective', 'start'], capture=False)
+    # restart mcollective to handle updated config
+    util.subp(['service', 'mcollective', 'restart'], capture=False)
diff --git a/templates/sources.list.ubuntu.tmpl b/templates/sources.list.ubuntu.tmpl
index d8799726..a0b350df 100644
--- a/templates/sources.list.ubuntu.tmpl
+++ b/templates/sources.list.ubuntu.tmpl
@@ -43,6 +43,61 @@ deb-src {{mirror}} {{codename}}-updates multiverse
 deb {{mirror}} {{codename}}-backports main restricted universe multiverse
 deb-src {{mirror}} {{codename}}-backports main restricted universe multiverse
 
+##
+## "entries" are mega things that can get rendered into sources.list format.
+## pocket_aliases allow for 'release' or 'updates' rather than
+##  {{codename}} and {{codename}}-released
+##
+##
+## one thing not done here is that for 'entry' rendering
+## the components cannot vary per pocket.
+##  ie, each pocket (release or updates) will have the same
+##  list of components
+apt:
+  pocket_aliases:
+    release: {{codename}}
+    updates: {{codename}}-updates
+    proposed: {{codename}}-proposed
+    backports: {{codename}}-backports
+    security: {{codename}}-security
+  sources:
+    main:
+      path: "/etc/apt/sources.list"
+      # if entries is present, then we render each of the entries defined.
+      entries: [primary, security]
+      key: yourkey
+    my-repo2.list:
+      # use source as it would before (supporting templates there)
+      # and entries can be in their rendering namespace
+      source: deb $primary.mirror $RELEASE multiverse
+
+  entries:
+    primary:
+      mirrors:
+       - arches: [i386, amd64]
+         mirror: http://archive.ubuntu.com/ubuntu
+         search:
+           - http://%(ec2_region)s.ec2.archive.ubuntu.com/ubuntu
+           - http://%(availability_zone)s.clouds.archive.ubuntu.com/ubuntu
+           - http://%(region)s.clouds.archive.ubuntu.com/ubuntu
+       - arches: [default]
+         mirror: http://ports.ubuntu.com/ubuntu-ports
+         search: []
+      pockets: [release, updates, proposed, backports]
+      components: [main, restricted, universe, multiverse]
+      types: [deb, deb-src]
+
+    security:
+      mirrors:
+       - arches: [default]
+         mirror: http://archive.ubuntu.com/ubuntu
+      pockets: [securityrelease, updates, proposed, backports]
+      components: [main, restricted, universe]
+      types: [deb, deb-src]
+
+ primary:
+   mirror
+
 deb {{security}} {{codename}}-security main restricted
 deb-src {{security}} {{codename}}-security main restricted
 deb {{security}} {{codename}}-security universe
diff --git a/tests/unittests/test_handler/test_handler_mcollective.py b/tests/unittests/test_handler/test_handler_mcollective.py
index f9448d80..6aefb93d 100644
--- a/tests/unittests/test_handler/test_handler_mcollective.py
+++ b/tests/unittests/test_handler/test_handler_mcollective.py
@@ -1,10 +1,12 @@
+from cloudinit import (cloud, distros, helpers, util)
 from cloudinit.config import cc_mcollective
-from cloudinit import util
+from cloudinit.sources import DataSourceNoCloud
 
-from .. import helpers
+from .. import helpers as t_help
 
 import configobj
 import logging
+import os
 import shutil
 from six import BytesIO
 import tempfile
@@ -12,11 +14,43 @@ import tempfile
 LOG = logging.getLogger(__name__)
 
 
-class TestConfig(helpers.FilesystemMockingTestCase):
+STOCK_CONFIG = """\
+main_collective = mcollective
+collectives = mcollective
+libdir = /usr/share/mcollective/plugins
+logfile = /var/log/mcollective.log
+loglevel = info
+daemonize = 1
+
+# Plugins
+securityprovider = psk
+plugin.psk = unset
+
+connector = activemq
+plugin.activemq.pool.size = 1
+plugin.activemq.pool.1.host = stomp1
+plugin.activemq.pool.1.port = 61613
+plugin.activemq.pool.1.user = mcollective
+plugin.activemq.pool.1.password = marionette
+
+# Facts
+factsource = yaml
+plugin.yaml = /etc/mcollective/facts.yaml
+"""
+
+
+class TestConfig(t_help.FilesystemMockingTestCase):
     def setUp(self):
         super(TestConfig, self).setUp()
         self.tmp = tempfile.mkdtemp()
         self.addCleanup(shutil.rmtree, self.tmp)
+        # "./": make os.path.join behave correctly with abs path as second arg
+        self.server_cfg = os.path.join(
+            self.tmp, "./" + cc_mcollective.SERVER_CFG)
+        self.pubcert_file = os.path.join(
+            self.tmp, "./" + cc_mcollective.PUBCERT_FILE)
+        self.pricert_file= os.path.join(
+            self.tmp, self.tmp, "./" + cc_mcollective.PRICERT_FILE)
 
     def test_basic_config(self):
         cfg = {
@@ -38,23 +72,77 @@ class TestConfig(helpers.FilesystemMockingTestCase):
                 },
             },
         }
+        expected = cfg['mcollective']['conf']
+
         self.patchUtils(self.tmp)
         cc_mcollective.configure(cfg['mcollective']['conf'])
-        contents = util.load_file("/etc/mcollective/server.cfg", decode=False)
+        contents = util.load_file(cc_mcollective.SERVER_CFG, decode=False)
         contents = configobj.ConfigObj(BytesIO(contents))
-        expected = {
-            'loglevel': 'debug',
-            'connector': 'rabbitmq',
-            'logfile': '/var/log/mcollective.log',
-            'ttl': '4294957',
-            'collectives': 'mcollective',
-            'main_collective': 'mcollective',
-            'securityprovider': 'psk',
-            'daemonize': '1',
-            'factsource': 'yaml',
-            'direct_addressing': '1',
-            'plugin.psk': 'unset',
-            'libdir': '/usr/share/mcollective/plugins',
-            'identity': '1',
-        }
         self.assertEqual(expected, dict(contents))
+
+    def test_existing_config_is_saved(self):
+        cfg = {'loglevel': 'warn'}
+        util.write_file(self.server_cfg, STOCK_CONFIG)
+        cc_mcollective.configure(config=cfg, server_cfg=self.server_cfg)
+        self.assertTrue(os.path.exists(self.server_cfg))
+        self.assertTrue(os.path.exists(self.server_cfg + ".old"))
+        self.assertEqual(util.load_file(self.server_cfg + ".old"), STOCK_CONFIG)
+
+    def test_existing_updated(self):
+        cfg = {'loglevel': 'warn'}
+        util.write_file(self.server_cfg, STOCK_CONFIG)
+        cc_mcollective.configure(config=cfg, server_cfg=self.server_cfg)
+        cfgobj = configobj.ConfigObj(self.server_cfg)
+        self.assertEqual(cfg['loglevel'], cfgobj['loglevel'])
+
+    def test_certificats_written(self):
+        # check public-cert and private-cert keys in config get written
+        cfg = {'loglevel': 'debug',
+               'public-cert': "this is my public-certificate",
+               'private-cert': "secret private certificate"}
+
+        cc_mcollective.configure(config=cfg,
+            server_cfg=self.server_cfg, pricert_file=self.pricert_file,
+            pubcert_file=self.pubcert_file)
+
+        found = configobj.ConfigObj(self.server_cfg)
+
+        # make sure these didnt get written in
+        self.assertFalse('public-cert' in found)
+        self.assertFalse('private-cert' in found)
+
+        # these need updating to the specified paths
+        self.assertEqual(found['plugin.ssl_server_public'], self.pubcert_file)
+        self.assertEqual(found['plugin.ssl_server_private'], self.pricert_file)
+
+        # and the security provider should be ssl
+        self.assertEqual(found['securityprovider'], 'ssl')
+
+        self.assertEqual(
+            util.load_file(self.pricert_file), cfg['private-cert'])
+        self.assertEqual(
+            util.load_file(self.pubcert_file), cfg['public-cert'])
+
+
+class TestHandler(t_help.TestCase):
+    def _get_cloud(self, distro):
+        cls = distros.fetch(distro)
+        paths = helpers.Paths({})
+        d = cls(distro, {}, paths)
+        ds = DataSourceNoCloud.DataSourceNoCloud({}, d, paths)
+        cc = cloud.Cloud(ds, paths, {}, d, None)
+        return cc
+
+    @t_help.mock.patch("cloudinit.config.cc_mcollective.util")
+    def test_mcollective_install(self, mock_util):
+        cc = self._get_cloud('ubuntu')
+        cc.distro = t_help.mock.MagicMock()
+        mycfg = {'mcollective': {'conf': {'loglevel': 'debug'}}}
+        cc_mcollective.handle('cc_mcollective', mycfg, cc, LOG, [])
+        self.assertTrue(cc.distro.install_packages.called)
+        install_pkg = cc.distro.install_packages.call_args_list[0][0][0]
+        self.assertEqual(install_pkg, ('mcollective',))
+
+        self.assertTrue(mock_util.subp.called)
+        self.assertEqual(mock_util.subp.call_args_list[0][0][0],
+                         ['service', 'mcollective', 'restart'])