add support for 'include-once' and public and private keys to mcollective

the new 'include-once' type will include a URL only once, and cache its
results.  This way you can use expiring URLs or one-time use urls to
pass sensitive data to the instance.  The instance will cache the result
in a local root only file for subsequent boots.

Additionally, add support for specifying public and private keys for
mcollective via cloud-config.

Both these features come via Marc Cluet.

2 files changed, 57 insertions, 9 deletions

diff --git a/cloudinit/CloudConfig/cc_mcollective.py b/cloudinit/CloudConfig/cc_mcollective.py
index 9aae2d64..c7912aa4 100644
--- a/cloudinit/CloudConfig/cc_mcollective.py
+++ b/cloudinit/CloudConfig/cc_mcollective.py
@@ -24,6 +24,10 @@ import fileinput
 import StringIO
 import ConfigParser
 import cloudinit.CloudConfig as cc
+import cloudinit.util as util
+
+pubcert_file = "/etc/mcollective/ssl/server-public.pem"
+pricert_file = "/etc/mcollective/ssl/server-private.pem"
 
 # Our fake header section
 class FakeSecHead(object):
@@ -50,24 +54,35 @@ def handle(name,cfg,cloud,log,args):
         # Read server.cfg values from original file in order to be able to mix the rest up
         mcollective_config.readfp(FakeSecHead(open('/etc/mcollective/server.cfg')))
         for cfg_name, cfg in mcollective_cfg['conf'].iteritems():
-            # Iterate throug the config items, we'll use ConfigParser.set
-            # to overwrite or create new items as needed
-            for o, v in cfg.iteritems():
-                mcollective_config.set(cfg_name,o,v)
+            if cfg_name == 'public-cert':
+                util.write_file(pubcert_file, cfg, mode=0644)
+                mcollective_config.set(cfg_name,
+                    'plugin.ssl_server_public', pubcert_file)
+                mcollective_config.set(cfg_name,'securityprovider','ssl')
+            elif cfg_name == 'private-cert':
+                util.write_file(pricert_file, cfg, mode=0600)
+                mcollective_config.set(cfg_name,
+                    'plugin.ssl_server_private', pricert_file)
+                mcollective_config.set(cfg_name,'securityprovider','ssl')
+            else:
+                # Iterate throug the config items, we'll use ConfigParser.set
+                # to overwrite or create new items as needed
+                for o, v in cfg.iteritems():
+                    mcollective_config.set(cfg_name,o,v)
         # We got all our config as wanted we'll rename
         # the previous server.cfg and create our new one
         os.rename('/etc/mcollective/server.cfg','/etc/mcollective/server.cfg.old')
         outputfile = StringIO.StringIO()
         mcollective_config.write(outputfile)
         # Now we got the whole file, write to disk except first line
-        final_configfile = open('/etc/mcollective/server.cfg', 'wb')
         # Note below, that we've just used ConfigParser because it generally
         # works.  Below, we remove the initial 'nullsection' header
         # and then change 'key = value' to 'key: value'.  The global
         # search and replace of '=' with ':' could be problematic though.
         # this most likely needs fixing.
-        final_configfile.write(outputfile.getvalue().replace('[nullsection]\n','').replace(' =',':'))
-        final_configfile.close()
+        util.write_file('/etc/mcollective/server.cfg',
+            outputfile.getvalue().replace('[nullsection]\n','').replace(' =',':'),
+            mode=0644)
 
     # Start mcollective
     subprocess.check_call(['service', 'mcollective', 'start'])
diff --git a/cloudinit/UserDataHandler.py b/cloudinit/UserDataHandler.py
index 83377dab..9670c0cb 100644
--- a/cloudinit/UserDataHandler.py
+++ b/cloudinit/UserDataHandler.py
@@ -22,9 +22,13 @@ from email.mime.text import MIMEText
 from email.mime.base import MIMEBase
 from email import encoders
 import yaml
+import cloudinit
+import cloudinit.util as util
+import md5
 
 starts_with_mappings={
     '#include' : 'text/x-include-url',
+    '#include-once' : 'text/x-include-once-url',
     '#!' : 'text/x-shellscript',
     '#cloud-config' : 'text/cloud-config',
     '#upstart-job'  : 'text/upstart-job',
@@ -45,16 +49,41 @@ def decomp_str(str):
 
 def do_include(str,parts):
     import urllib
+    import os
     # is just a list of urls, one per line
     # also support '#include <url here>'
+    includeonce = False
     for line in str.splitlines():
         if line == "#include": continue
-        if line.startswith("#include"):
+        if line == "#include-once":
+            includeonce = True
+            continue
+        if line.startswith("#include-once"):
+            line = line[len("#include-once"):].lstrip()
+            includeonce = True
+        elif line.startswith("#include"):
             line = line[len("#include"):].lstrip()
         if line.startswith("#"): continue
-        content = urllib.urlopen(line).read()
+
+        # urls cannot not have leading or trailing white space
+        msum = md5.new()
+        msum.update(line.strip())
+        includeonce_filename = "%s/urlcache/%s" % (
+            cloudinit.get_ipath_cur("data"), msum.hexdigest())
+        try:
+            if includeonce and os.path.isfile(includeonce_filename):
+                with open(includeonce_filename, "r") as fp:
+                    content = fp.read()
+            else:
+                content = urllib.urlopen(line).read()
+                if includeonce:
+                    util.write_file(includeonce_filename, content, mode=0600)
+        except Exception as e:
+            raise
+
         process_includes(email.message_from_string(decomp_str(content)),parts)
 
+
 def explode_cc_archive(archive,parts):
     for ent in yaml.load(archive):
         # ent can be one of:
@@ -113,6 +142,10 @@ def process_includes(msg,parts):
             do_include(payload,parts)
             continue
 
+        if ctype == 'text/x-include-once-url':
+            do_include(payload,parts)
+            continue
+
         if ctype == "text/cloud-config-archive":
             explode_cc_archive(payload,parts)
             continue