From 57ea45b2bc86895582de65928c555e6f0430b287 Mon Sep 17 00:00:00 2001 From: Marc Cluet Date: Mon, 25 Jul 2011 13:27:27 +0100 Subject: Added new feature include-once --- cloudinit/UserDataHandler.py | 23 +++++++++++++++++++++-- doc/examples/include-once.txt | 7 +++++++ doc/userdata.txt | 8 ++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 doc/examples/include-once.txt diff --git a/cloudinit/UserDataHandler.py b/cloudinit/UserDataHandler.py index 83377dab..4fd6ef28 100644 --- a/cloudinit/UserDataHandler.py +++ b/cloudinit/UserDataHandler.py @@ -25,6 +25,7 @@ import yaml starts_with_mappings={ '#include' : 'text/x-include-url', + '#include-once' : 'text/x-include-once-url', '#!' : 'text/x-shellscript', '#cloud-config' : 'text/cloud-config', '#upstart-job' : 'text/upstart-job', @@ -45,16 +46,34 @@ def decomp_str(str): def do_include(str,parts): import urllib + import os + import base64 # is just a list of urls, one per line # also support '#include ' + includeonce = False for line in str.splitlines(): if line == "#include": continue - if line.startswith("#include"): + if line == "#include-once": + includeonce == True + if line.startswith("#include-once"): + line = line[len("#include-once"):].lstrip() + includeonce = True + elif line.startswith("#include"): line = line[len("#include"):].lstrip() if line.startswith("#"): continue - content = urllib.urlopen(line).read() + if includeonce == True: + uniquestring = base64.encodestring(line).strip('\n') + includeonce_filename = "/var/lib/cloud/instance/.includeonce.%s" % uniquestring + if os.path.isfile(includeonce_filename): continue + includeonce_file = open(includeonce_filename,'w') + includeonce_file.close() + try: + content = urllib.urlopen(line).read() + except Exception as e: + log.debug(traceback.format_exc(e)) process_includes(email.message_from_string(decomp_str(content)),parts) + def explode_cc_archive(archive,parts): for ent in yaml.load(archive): # ent can be one of: diff --git a/doc/examples/include-once.txt b/doc/examples/include-once.txt new file mode 100644 index 00000000..0cf74e5e --- /dev/null +++ b/doc/examples/include-once.txt @@ -0,0 +1,7 @@ +#include-once +# entries are one url per line. comment lines beginning with '#' are allowed +# urls are passed to urllib.urlopen, so the format must be supported there +# This entries will just be processed ONE TIME by cloud-init, any further +# iterations won't process this file +http://www.ubuntu.com/robots.txt +http://www.w3schools.com/html/lastpage.htm diff --git a/doc/userdata.txt b/doc/userdata.txt index 00c16b25..3af1e632 100644 --- a/doc/userdata.txt +++ b/doc/userdata.txt @@ -36,6 +36,14 @@ finds. However, certain types of user-data are handled specially. will be passed through this same set of rules. Ie, the content read from the URL can be gzipped, mime-multi-part, or plain text +* Include File Once + begins with #include-once or Content-Type: text/x-include-once-url + This content is a "include" file. The file contains a list of + urls, one per line. Each of the URLs will be read, and their content + will be passed through this same set of rules. Ie, the content + read from the URL can be gzipped, mime-multi-part, or plain text + This file will just be processed once by cloud-init + * Cloud Config Data begins with #cloud-config or Content-Type: text/cloud-config -- cgit v1.2.3 From f70bc5ddf301517863b48943cd3d8d6df5548c68 Mon Sep 17 00:00:00 2001 From: Marc Cluet Date: Mon, 25 Jul 2011 13:27:48 +0100 Subject: Added ssl cert support to mcollective --- cloudinit/CloudConfig/cc_mcollective.py | 21 ++++++++++++++++---- doc/examples/cloud-config-mcollective.txt | 33 +++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/cloudinit/CloudConfig/cc_mcollective.py b/cloudinit/CloudConfig/cc_mcollective.py index 9aae2d64..3b358302 100644 --- a/cloudinit/CloudConfig/cc_mcollective.py +++ b/cloudinit/CloudConfig/cc_mcollective.py @@ -50,10 +50,23 @@ def handle(name,cfg,cloud,log,args): # Read server.cfg values from original file in order to be able to mix the rest up mcollective_config.readfp(FakeSecHead(open('/etc/mcollective/server.cfg'))) for cfg_name, cfg in mcollective_cfg['conf'].iteritems(): - # Iterate throug the config items, we'll use ConfigParser.set - # to overwrite or create new items as needed - for o, v in cfg.iteritems(): - mcollective_config.set(cfg_name,o,v) + if cfg_name == 'public-cert': + publicrt_fh = open('/etc/mcollective/ssl/server-public.pem', 'w') + publicrt_fh.write(cfg) + publicrt_fh.close() + mcollective_config.set(cfg_name,'plugin.ssl_server_public','/etc/mcollective/ssl/server-public.pem') + mcollective_config.set(cfg_name,'securityprovider','ssl') + elif cfg_name == 'private-cert': + privcrt_fh = open('/etc/mcollective/ssl/server-private.pem', 'w') + privcrt_fh.write(cfg) + privcrt_fh.close() + mcollective_config.set(cfg_name,'plugin.ssl_server_private','/etc/mcollective/ssl/server-private.pem') + mcollective_config.set(cfg_name,'securityprovider','ssl') + else: + # Iterate throug the config items, we'll use ConfigParser.set + # to overwrite or create new items as needed + for o, v in cfg.iteritems(): + mcollective_config.set(cfg_name,o,v) # We got all our config as wanted we'll rename # the previous server.cfg and create our new one os.rename('/etc/mcollective/server.cfg','/etc/mcollective/server.cfg.old') diff --git a/doc/examples/cloud-config-mcollective.txt b/doc/examples/cloud-config-mcollective.txt index ca7ba03e..ddeaf0c6 100644 --- a/doc/examples/cloud-config-mcollective.txt +++ b/doc/examples/cloud-config-mcollective.txt @@ -13,3 +13,36 @@ mcollective: # plugin.stomp.host: dbhost conf: plugin.stomp.host: dbhost + # This will add ssl certs to mcollective + # WARNING WARNING WARNING + # Please remember cloud-init data is transmitted without encryption + # If you want security for this, please use include-once + SSL urls + public-cert: | + -----BEGIN CERTIFICATE----- + MIICCTCCAXKgAwIBAgIBATANBgkqhkiG9w0BAQUFADANMQswCQYDVQQDDAJjYTAe + Fw0xMDAyMTUxNzI5MjFaFw0xNTAyMTQxNzI5MjFaMA0xCzAJBgNVBAMMAmNhMIGf + MA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCu7Q40sm47/E1Pf+r8AYb/V/FWGPgc + b014OmNoX7dgCxTDvps/h8Vw555PdAFsW5+QhsGr31IJNI3kSYprFQcYf7A8tNWu + 1MASW2CfaEiOEi9F1R3R4Qlz4ix+iNoHiUDTjazw/tZwEdxaQXQVLwgTGRwVa+aA + qbutJKi93MILLwIDAQABo3kwdzA4BglghkgBhvhCAQ0EKxYpUHVwcGV0IFJ1Ynkv + T3BlblNTTCBHZW5lcmF0ZWQgQ2VydGlmaWNhdGUwDwYDVR0TAQH/BAUwAwEB/zAd + BgNVHQ4EFgQUu4+jHB+GYE5Vxo+ol1OAhevspjAwCwYDVR0PBAQDAgEGMA0GCSqG + SIb3DQEBBQUAA4GBAH/rxlUIjwNb3n7TXJcDJ6MMHUlwjr03BDJXKb34Ulndkpaf + +GAlzPXWa7bO908M9I8RnPfvtKnteLbvgTK+h+zX1XCty+S2EQWk29i2AdoqOTxb + hppiGMp0tT5Havu4aceCXiy2crVcudj3NFciy8X66SoECemW9UYDCb9T5D0d + -----END CERTIFICATE----- + private-cert: | + -----BEGIN CERTIFICATE----- + MIICCTCCAXKgAwIBAgIBATANBgkqhkiG9w0BAQUFADANMQswCQYDVQQDDAJjYTAe + Fw0xMDAyMTUxNzI5MjFaFw0xNTAyMTQxNzI5MjFaMA0xCzAJBgNVBAMMAmNhMIGf + MA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCu7Q40sm47/E1Pf+r8AYb/V/FWGPgc + b014OmNoX7dgCxTDvps/h8Vw555PdAFsW5+QhsGr31IJNI3kSYprFQcYf7A8tNWu + 1MASW2CfaEiOEi9F1R3R4Qlz4ix+iNoHiUDTjazw/tZwEdxaQXQVLwgTGRwVa+aA + qbutJKi93MILLwIDAQABo3kwdzA4BglghkgBhvhCAQ0EKxYpUHVwcGV0IFJ1Ynkv + T3BlblNTTCBHZW5lcmF0ZWQgQ2VydGlmaWNhdGUwDwYDVR0TAQH/BAUwAwEB/zAd + BgNVHQ4EFgQUu4+jHB+GYE5Vxo+ol1OAhevspjAwCwYDVR0PBAQDAgEGMA0GCSqG + SIb3DQEBBQUAA4GBAH/rxlUIjwNb3n7TXJcDJ6MMHUlwjr03BDJXKb34Ulndkpaf + +GAlzPXWa7bO908M9I8RnPfvtKnteLbvgTK+h+zX1XCty+S2EQWk29i2AdoqOTxb + hppiGMp0tT5Havu4aceCXiy2crVcudj3NFciy8X66SoECemW9UYDCb9T5D0d + -----END CERTIFICATE----- + -- cgit v1.2.3 From 149bfa0e4a2705bbe980a2335c5e7951e9b70925 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 26 Jul 2011 09:40:19 -0400 Subject: clarify warning text in mcollective config example --- doc/examples/cloud-config-mcollective.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/examples/cloud-config-mcollective.txt b/doc/examples/cloud-config-mcollective.txt index ddeaf0c6..67735682 100644 --- a/doc/examples/cloud-config-mcollective.txt +++ b/doc/examples/cloud-config-mcollective.txt @@ -15,7 +15,8 @@ mcollective: plugin.stomp.host: dbhost # This will add ssl certs to mcollective # WARNING WARNING WARNING - # Please remember cloud-init data is transmitted without encryption + # The ec2 metadata service is a network service, and thus is readable + # by non-root users on the system (ie: 'ec2metadata --user-data') # If you want security for this, please use include-once + SSL urls public-cert: | -----BEGIN CERTIFICATE----- -- cgit v1.2.3 From 690086473dccc7489dcb21ccade9c057762e35a3 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 26 Jul 2011 09:50:49 -0400 Subject: cc_mcollective: use util.write_file, change perms of private key file to 0600 --- cloudinit/CloudConfig/cc_mcollective.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/cloudinit/CloudConfig/cc_mcollective.py b/cloudinit/CloudConfig/cc_mcollective.py index 3b358302..b894a7bb 100644 --- a/cloudinit/CloudConfig/cc_mcollective.py +++ b/cloudinit/CloudConfig/cc_mcollective.py @@ -24,6 +24,10 @@ import fileinput import StringIO import ConfigParser import cloudinit.CloudConfig as cc +import cloudinit.util as util + +pubcert_file = "/etc/mcollective/ssl/server-public.pem" +pricert_file = "/etc/mcollective/ssl/server-private.pem" # Our fake header section class FakeSecHead(object): @@ -51,16 +55,14 @@ def handle(name,cfg,cloud,log,args): mcollective_config.readfp(FakeSecHead(open('/etc/mcollective/server.cfg'))) for cfg_name, cfg in mcollective_cfg['conf'].iteritems(): if cfg_name == 'public-cert': - publicrt_fh = open('/etc/mcollective/ssl/server-public.pem', 'w') - publicrt_fh.write(cfg) - publicrt_fh.close() - mcollective_config.set(cfg_name,'plugin.ssl_server_public','/etc/mcollective/ssl/server-public.pem') + util.write_file(pubcert_file, cfg, mode=0644) + mcollective_config.set(cfg_name, + 'plugin.ssl_server_public', pubcert_file) mcollective_config.set(cfg_name,'securityprovider','ssl') elif cfg_name == 'private-cert': - privcrt_fh = open('/etc/mcollective/ssl/server-private.pem', 'w') - privcrt_fh.write(cfg) - privcrt_fh.close() - mcollective_config.set(cfg_name,'plugin.ssl_server_private','/etc/mcollective/ssl/server-private.pem') + util.write_file(pricert_file, cfg, mode=0600) + mcollective_config.set(cfg_name, + 'plugin.ssl_server_private', pricert_file) mcollective_config.set(cfg_name,'securityprovider','ssl') else: # Iterate throug the config items, we'll use ConfigParser.set -- cgit v1.2.3 From a6dce5ac548de073918d679503f447d265847066 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 26 Jul 2011 10:22:23 -0400 Subject: make 'include-once' really "download source once per-instance" Marc's implementation would only ever process the include-once urls a single time. This changes that to process them every time, with the second time coming from a file on disk rather than the url. You can then do expiring or one time use URLs in the include-once and have all function of if the content was there every time. The cached file is readable by root-only. --- cloudinit/UserDataHandler.py | 20 +++++++++++++------- doc/userdata.txt | 4 +++- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/cloudinit/UserDataHandler.py b/cloudinit/UserDataHandler.py index 4fd6ef28..4ac0e2cd 100644 --- a/cloudinit/UserDataHandler.py +++ b/cloudinit/UserDataHandler.py @@ -22,6 +22,7 @@ from email.mime.text import MIMEText from email.mime.base import MIMEBase from email import encoders import yaml +from cloudinit import util, get_ipath_cur starts_with_mappings={ '#include' : 'text/x-include-url', @@ -61,16 +62,21 @@ def do_include(str,parts): elif line.startswith("#include"): line = line[len("#include"):].lstrip() if line.startswith("#"): continue - if includeonce == True: - uniquestring = base64.encodestring(line).strip('\n') - includeonce_filename = "/var/lib/cloud/instance/.includeonce.%s" % uniquestring - if os.path.isfile(includeonce_filename): continue - includeonce_file = open(includeonce_filename,'w') - includeonce_file.close() + + # urls cannot not have leading or trailing white space + uniquestring = base64.encodestring(line).strip() + includeonce_filename = "%/urlcache/%s" % (get_ipath_cur("data"), uniquestring) try: - content = urllib.urlopen(line).read() + if includeonce and os.path.isfile(includeonce_filename): + with open(includeonce_filename, "r") as fp: + content = fp.read() + else: + content = urllib.urlopen(line).read() + if includeonce: + util.write_file(includeonce_filename, content, mode=0600) except Exception as e: log.debug(traceback.format_exc(e)) + process_includes(email.message_from_string(decomp_str(content)),parts) diff --git a/doc/userdata.txt b/doc/userdata.txt index 3af1e632..cc691ae6 100644 --- a/doc/userdata.txt +++ b/doc/userdata.txt @@ -42,7 +42,9 @@ finds. However, certain types of user-data are handled specially. urls, one per line. Each of the URLs will be read, and their content will be passed through this same set of rules. Ie, the content read from the URL can be gzipped, mime-multi-part, or plain text - This file will just be processed once by cloud-init + This file will just be downloaded only once per instance, and its + contents cached for subsequent boots. This allows you to pass in + one-time-use or expiring URLs. * Cloud Config Data begins with #cloud-config or Content-Type: text/cloud-config -- cgit v1.2.3 From 155e519a63703183823ef9368e2fdb3a6b7b0c0a Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 26 Jul 2011 11:18:10 -0400 Subject: use util.write_file in cc_mcollective.py --- cloudinit/CloudConfig/cc_mcollective.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cloudinit/CloudConfig/cc_mcollective.py b/cloudinit/CloudConfig/cc_mcollective.py index b894a7bb..c7912aa4 100644 --- a/cloudinit/CloudConfig/cc_mcollective.py +++ b/cloudinit/CloudConfig/cc_mcollective.py @@ -75,14 +75,14 @@ def handle(name,cfg,cloud,log,args): outputfile = StringIO.StringIO() mcollective_config.write(outputfile) # Now we got the whole file, write to disk except first line - final_configfile = open('/etc/mcollective/server.cfg', 'wb') # Note below, that we've just used ConfigParser because it generally # works. Below, we remove the initial 'nullsection' header # and then change 'key = value' to 'key: value'. The global # search and replace of '=' with ':' could be problematic though. # this most likely needs fixing. - final_configfile.write(outputfile.getvalue().replace('[nullsection]\n','').replace(' =',':')) - final_configfile.close() + util.write_file('/etc/mcollective/server.cfg', + outputfile.getvalue().replace('[nullsection]\n','').replace(' =',':'), + mode=0644) # Start mcollective subprocess.check_call(['service', 'mcollective', 'start']) -- cgit v1.2.3 From cf65c6c8e0da58698139223888f23adf7093d12f Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 26 Jul 2011 13:54:56 -0400 Subject: include-once: fixups found in testing at this point, the following user-data file in /var/lib/cloud/seed/nocloud-net/user-data will do what you would expect: $ cat > /var/lib/cloud/seed/nocloud-net/user-data < Date: Tue, 26 Jul 2011 14:04:02 -0400 Subject: use md5sum as the unique identifier rather than base64 base64 encode will grow with the size of the url, possibly resulting in silly-long filenames. md5sum will keep it to a constant length. --- cloudinit/UserDataHandler.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cloudinit/UserDataHandler.py b/cloudinit/UserDataHandler.py index fa8ce716..9670c0cb 100644 --- a/cloudinit/UserDataHandler.py +++ b/cloudinit/UserDataHandler.py @@ -24,6 +24,7 @@ from email import encoders import yaml import cloudinit import cloudinit.util as util +import md5 starts_with_mappings={ '#include' : 'text/x-include-url', @@ -49,7 +50,6 @@ def decomp_str(str): def do_include(str,parts): import urllib import os - import base64 # is just a list of urls, one per line # also support '#include ' includeonce = False @@ -66,8 +66,10 @@ def do_include(str,parts): if line.startswith("#"): continue # urls cannot not have leading or trailing white space - uniquestring = base64.encodestring(line).strip() - includeonce_filename = "%s/urlcache/%s" % (cloudinit.get_ipath_cur("data"), uniquestring) + msum = md5.new() + msum.update(line.strip()) + includeonce_filename = "%s/urlcache/%s" % ( + cloudinit.get_ipath_cur("data"), msum.hexdigest()) try: if includeonce and os.path.isfile(includeonce_filename): with open(includeonce_filename, "r") as fp: -- cgit v1.2.3