From aed1c1e9fda1e78d19305e90f554d0dcb5388cd7 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 3 Feb 2010 18:09:48 -0500 Subject: globally remove ec2init and rename to cloudinit --- cloud-init-cfg.py | 10 +- cloud-init-run-module.py | 10 +- cloudinit/CloudConfig.py | 407 +++++++++++++++++++++++++++++++++++++ cloudinit/DataSource.py | 30 +++ cloudinit/DataSourceEc2.py | 132 ++++++++++++ cloudinit/UserDataHandler.py | 130 ++++++++++++ cloudinit/__init__.py | 313 ++++++++++++++++++++++++++++ cloudinit/boto_utils.py | 106 ++++++++++ cloudinit/execute.py | 13 ++ cloudinit/util.py | 79 +++++++ ec2-init.py | 6 +- ec2init/CloudConfig.py | 407 ------------------------------------- ec2init/DataSource.py | 30 --- ec2init/DataSourceEc2.py | 132 ------------ ec2init/UserDataHandler.py | 130 ------------ ec2init/__init__.py | 313 ---------------------------- ec2init/boto_utils.py | 106 ---------- ec2init/execute.py | 13 -- ec2init/util.py | 79 ------- setup.py | 2 +- upstart/cloud-run-user-script.conf | 4 +- 21 files changed, 1226 insertions(+), 1226 deletions(-) create mode 100644 cloudinit/CloudConfig.py create mode 100644 cloudinit/DataSource.py create mode 100644 cloudinit/DataSourceEc2.py create mode 100644 cloudinit/UserDataHandler.py create mode 100644 cloudinit/__init__.py create mode 100644 cloudinit/boto_utils.py create mode 100644 cloudinit/execute.py create mode 100644 cloudinit/util.py delete mode 100644 ec2init/CloudConfig.py delete mode 100644 ec2init/DataSource.py delete mode 100644 ec2init/DataSourceEc2.py delete mode 100644 ec2init/UserDataHandler.py delete mode 100644 ec2init/__init__.py delete mode 100644 ec2init/boto_utils.py delete mode 100644 ec2init/execute.py delete mode 100644 ec2init/util.py diff --git a/cloud-init-cfg.py b/cloud-init-cfg.py index 88216e10..9260a767 100755 --- a/cloud-init-cfg.py +++ b/cloud-init-cfg.py @@ -1,7 +1,7 @@ #!/usr/bin/python import sys -import ec2init +import cloudinit def Usage(out = sys.stdout): out.write("Usage: %s name\n" % sys.argv[0]) @@ -16,15 +16,15 @@ def main(): name=sys.argv[1] run_args=sys.argv[2:] - import ec2init.CloudConfig + import cloudinit.CloudConfig import os - cfg_path = ec2init.cloud_config - cfg_env_name = ec2init.cfg_env_name + cfg_path = cloudinit.cloud_config + cfg_env_name = cloudinit.cfg_env_name if os.environ.has_key(cfg_env_name): cfg_path = os.environ[cfg_env_name] - cc = ec2init.CloudConfig.CloudConfig(cfg_path) + cc = cloudinit.CloudConfig.CloudConfig(cfg_path) try: cc.handle(name,run_args) diff --git a/cloud-init-run-module.py b/cloud-init-run-module.py index d2d4feee..de52f116 100755 --- a/cloud-init-run-module.py +++ b/cloud-init-run-module.py @@ -1,7 +1,7 @@ #!/usr/bin/python import sys -import ec2init +import cloudinit def Usage(out = sys.stdout): out.write("Usage: cloud-init-run-module freq sem-name mod-name [args]\n") @@ -16,7 +16,7 @@ def main(): (freq,semname,modname)=sys.argv[1:4] run_args=sys.argv[4:] - cloud = ec2init.EC2Init() + cloud = cloudinit.EC2Init() try: cloud.get_data_source() except Exception as e: @@ -29,16 +29,16 @@ def main(): sys.exit(0) try: - mod = __import__('ec2init.' + modname) + mod = __import__('cloudinit.' + modname) inst = getattr(mod,modname) except: - sys.stderr.write("Failed to load module ec2init.%s\n" % modname) + sys.stderr.write("Failed to load module cloudinit.%s\n" % modname) sys.exit(1) import os cfg_path = None - cfg_env_name = ec2init.cfg_env_name + cfg_env_name = cloudinit.cfg_env_name if os.environ.has_key(cfg_env_name): cfg_path = os.environ[cfg_env_name] diff --git a/cloudinit/CloudConfig.py b/cloudinit/CloudConfig.py new file mode 100644 index 00000000..4a498866 --- /dev/null +++ b/cloudinit/CloudConfig.py @@ -0,0 +1,407 @@ +# +# Common code for the EC2 configuration files in Ubuntu +# Copyright (C) 2008-2010 Canonical Ltd. +# +# Author: Chuck Short +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +import yaml +import re +import cloudinit +import cloudinit.util as util +import subprocess +import os +import glob +import sys + +per_instance="once-per-instance" + +class CloudConfig(): + cfgfile = None + handlers = { } + cfg = None + + def __init__(self,cfgfile): + self.cloud = cloudinit.EC2Init() + self.cfg = self.get_config_obj(cfgfile) + self.cloud.get_data_source() + self.add_handler('apt-update-upgrade', self.h_apt_update_upgrade) + self.add_handler('config-ssh') + self.add_handler('disable-ec2-metadata', + self.h_disable_ec2_metadata, "always") + self.add_handler('config-mounts') + + def get_config_obj(self,cfgfile): + f=file(cfgfile) + cfg=yaml.load(f.read()) + f.close() + if cfg is None: cfg = { } + return(util.mergedict(cfg,self.cloud.cfg)) + + def convert_old_config(self): + # support reading the old ConfigObj format file and turning it + # into a yaml string + try: + f = file(self.conffile) + str=file.read().replace('=',': ') + f.close() + return str + except: + return("") + + def add_handler(self, name, handler=None, freq=None): + if handler is None: + try: + handler=getattr(self,'h_%s' % name.replace('-','_')) + except: + raise Exception("Unknown hander for name %s" %name) + if freq is None: + freq = per_instance + + self.handlers[name]= { 'handler': handler, 'freq': freq } + + def get_handler_info(self, name): + return(self.handlers[name]['handler'], self.handlers[name]['freq']) + + def parse_ssh_keys(self): + disableRoot = self.cfg['disable_root'] + if disableRoot == 'true': + value = 'disabled_root' + return value + else: + ec2Key = self.cfg['ec2_fetch_key'] + if ec2Key != 'none': + value = 'default_key' + return value + else: + return ec2Key + + def handle(self, name, args): + handler = None + freq = None + try: + (handler, freq) = self.get_handler_info(name) + except: + raise Exception("Unknown config key %s\n" % name) + + self.cloud.sem_and_run(name, freq, handler, [ name, args ]) + + def h_apt_update_upgrade(self,name,args): + update = util.get_cfg_option_bool(self.cfg, 'apt_update', False) + upgrade = util.get_cfg_option_bool(self.cfg, 'apt_upgrade', False) + + if not util.get_cfg_option_bool(self.cfg, \ + 'apt_preserve_sources_list', False): + if self.cfg.has_key("apt_mirror"): + mirror = self.cfg["apt_mirror"] + else: + mirror = self.cloud.get_mirror() + generate_sources_list(mirror) + + # process 'apt_sources' + if self.cfg.has_key('apt_sources'): + errors = add_sources(self.cfg['apt_sources']) + for e in errors: + warn("Source Error: %s\n" % ':'.join(e)) + + pkglist = [] + if 'packages' in self.cfg: + if isinstance(self.cfg['packages'],list): + pkglist = self.cfg['packages'] + else: pkglist.append(self.cfg['packages']) + + if update or upgrade or pkglist: + #retcode = subprocess.call(list) + subprocess.Popen(['apt-get', 'update']).communicate() + + e=os.environ.copy() + e['DEBIAN_FRONTEND']='noninteractive' + + if upgrade: + subprocess.Popen(['apt-get', 'upgrade', '--assume-yes'], env=e).communicate() + + if pkglist: + cmd=['apt-get', 'install', '--assume-yes'] + cmd.extend(pkglist) + subprocess.Popen(cmd, env=e).communicate() + + return(True) + + def h_disable_ec2_metadata(self,name,args): + if util.get_cfg_option_bool(self.cfg, "disable_ec2_metadata", False): + fwall="route add -host 169.254.169.254 reject" + subprocess.call(fwall.split(' ')) + + def h_config_ssh(self,name,args): + # remove the static keys from the pristine image + for f in glob.glob("/etc/ssh/ssh_host_*_key*"): + try: os.unlink(f) + except: pass + + if self.cfg.has_key("ssh_keys"): + # if there are keys in cloud-config, use them + key2file = { + "rsa_private" : ("/etc/ssh/ssh_host_rsa_key", 0600), + "rsa_public" : ("/etc/ssh/ssh_host_rsa_key.pub", 0644), + "dsa_private" : ("/etc/ssh/ssh_host_dsa_key", 0600), + "dsa_public" : ("/etc/ssh/ssh_host_dsa_key.pub", 0644) + } + + for key,val in self.cfg["ssh_keys"].items(): + if key2file.has_key(key): + util.write_file(key2file[key][0],val,key2file[key][1]) + else: + # if not, generate them + genkeys ='ssh-keygen -f /etc/ssh/ssh_host_rsa_key -t rsa -N ""; ' + genkeys+='ssh-keygen -f /etc/ssh/ssh_host_dsa_key -t dsa -N ""; ' + subprocess.call(('sh', '-c', "{ %s } 0, toks) + if len(r) == 4: + return("ip-%s" % '-'.join(r)) + except: pass + return toks[0] + + def get_mirror_from_availability_zone(self, availability_zone = None): + # availability is like 'us-west-1b' or 'eu-west-1a' + if availability_zone == None: + availability_zone = self.get_availability_zone() + + try: + host="%s.ec2.archive.ubuntu.com" % availability_zone[:-1] + socket.getaddrinfo(host, None, 0, socket.SOCK_STREAM) + return 'http://%s/ubuntu/' % host + except: + return 'http://archive.ubuntu.com/ubuntu/' + + def wait_for_metadata_service(self, sleeps = 10): + sleeptime = 1 + for x in range(sleeps): + s = socket.socket() + try: + address = '169.254.169.254' + port = 80 + s.connect((address,port)) + s.close() + return True + except socket.error, e: + print "sleeping %s" % sleeptime + time.sleep(sleeptime) + #timeout = timeout * 2 + return False + + def get_public_ssh_keys(self): + keys = [] + if not self.metadata.has_key('public-keys'): return([]) + for keyname, klist in self.metadata['public-keys'].items(): + # lp:506332 uec metadata service responds with + # data that makes boto populate a string for 'klist' rather + # than a list. + if isinstance(klist,str): + klist = [ klist ] + for pkey in klist: + # there is an empty string at the end of the keylist, trim it + if pkey: + keys.append(pkey) + + return(keys) + + def device_name_to_device(self, name): + # consult metadata service, that has + # ephemeral0: sdb + # and return 'sdb' for input 'ephemeral0' + if not self.metadata.has_key('block-device-mapping'): + return(None) + + for entname, device in self.metadata['block-device-mapping'].items(): + if entname == name: + return(device) + # LP: #513842 mapping in Euca has 'ephemeral' not 'ephemeral0' + if entname == "ephemeral" and name == "ephemeral0": + return(device) + return None diff --git a/cloudinit/UserDataHandler.py b/cloudinit/UserDataHandler.py new file mode 100644 index 00000000..56feb0ff --- /dev/null +++ b/cloudinit/UserDataHandler.py @@ -0,0 +1,130 @@ +import email + +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText + + +starts_with_mappings={ + '#include' : 'text/x-include-url', + '#!' : 'text/x-shellscript', + '#cloud-config' : 'text/cloud-config', + '#upstart-job' : 'text/upstart-job', + '#part-handler' : 'text/part-handler' +} + +# if 'str' is compressed return decompressed otherwise return it +def decomp_str(str): + import StringIO + import gzip + try: + uncomp = gzip.GzipFile(None,"rb",1,StringIO.StringIO(str)).read() + return(uncomp) + except: + return(str) + +def do_include(str,parts): + import urllib + # is just a list of urls, one per line + for line in str.splitlines(): + if line == "#include": continue + if line.startswith("#"): continue + content = urllib.urlopen(line).read() + process_includes(email.message_from_string(decomp_str(content)),parts) + +def process_includes(msg,parts): + # parts is a dictionary of arrays + # parts['content'] + # parts['names'] + # parts['types'] + for t in ( 'content', 'names', 'types' ): + if not parts.has_key(t): + parts[t]=[ ] + for part in msg.walk(): + # multipart/* are just containers + if part.get_content_maintype() == 'multipart': + continue + + payload = part.get_payload() + + ctype = None + for str, gtype in starts_with_mappings.items(): + if payload.startswith(str): + ctype = gtype + break + + if ctype is None: + ctype = part.get_content_type() + + if ctype == 'text/x-include-url': + do_include(payload,parts) + continue + + filename = part.get_filename() + if not filename: + filename = 'part-%03d' % len(parts['content']) + + parts['content'].append(payload) + parts['types'].append(ctype) + parts['names'].append(filename) + +def parts2mime(parts): + outer = MIMEMultipart() + + i = 0 + while i < len(parts['content']): + if parts['types'][i] is None: + # No guess could be made, or the file is encoded (compressed), so + # use a generic bag-of-bits type. + ctype = 'application/octet-stream' + else: ctype = parts['types'][i] + maintype, subtype = ctype.split('/', 1) + if maintype == 'text': + msg = MIMEText(parts['content'][i], _subtype=subtype) + else: + msg = MIMEBase(maintype, subtype) + msg.set_payload(parts['content'][i]) + # Encode the payload using Base64 + encoders.encode_base64(msg) + # Set the filename parameter + msg.add_header('Content-Disposition', 'attachment', + filename=parts['names'][i]) + outer.attach(msg) + + i=i+1 + return(outer.as_string()) + +# this is heavily wasteful, reads through userdata string input +def preprocess_userdata(data): + parts = { } + process_includes(email.message_from_string(decomp_str(data)),parts) + return(parts2mime(parts)) + +# callbacks is a dictionary with: +# { 'content-type': handler(data,content_type,filename,payload) } +def walk_userdata(str, callbacks, data = None): + partnum = 0 + for part in email.message_from_string(str).walk(): + # multipart/* are just containers + if part.get_content_maintype() == 'multipart': + continue + + ctype = part.get_content_type() + if ctype is None: + ctype = 'application/octet-stream' + + filename = part.get_filename() + if not filename: + filename = 'part-%03d' % partnum + + if callbacks.has_key(ctype): + callbacks[ctype](data,ctype,filename,part.get_payload()) + + partnum = partnum+1 + +if __name__ == "__main__": + import sys + data = decomp_str(file(sys.argv[1]).read()) + parts = { } + process_includes(email.message_from_string(data),parts) + print "#found %s parts" % len(parts['content']) + print parts2mime(parts) diff --git a/cloudinit/__init__.py b/cloudinit/__init__.py new file mode 100644 index 00000000..76aa34f0 --- /dev/null +++ b/cloudinit/__init__.py @@ -0,0 +1,313 @@ +# +# Common code for the EC2 initialisation scripts in Ubuntu +# Copyright (C) 2008-2009 Canonical Ltd +# +# Author: Soren Hansen +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +import os +from configobj import ConfigObj + +import cPickle +import sys +import os.path +import errno +import pwd +import subprocess +import yaml + +datadir = '/var/lib/cloud/data' +semdir = '/var/lib/cloud/sem' +pluginsdir = datadir + '/plugins' +cachedir = datadir + '/cache' +userdata_raw = datadir + '/user-data.txt' +userdata = datadir + '/user-data.txt.i' +user_scripts_dir = datadir + "/scripts" +cloud_config = datadir + '/cloud-config.txt' +data_source_cache = cachedir + '/obj.pkl' +system_config = '/etc/cloud/cloud.cfg' +cfg_env_name = "CLOUD_CFG" + +import DataSourceEc2 +import UserDataHandler +import util + +class EC2Init: + datasource_map = { + "ec2" : DataSourceEc2.DataSourceEc2, + } + datasource = None + auto_order = [ 'ec2' ] + + cfg = None + part_handlers = { } + old_conffile = '/etc/ec2-init/ec2-config.cfg' + + def __init__(self): + self.part_handlers = { + 'text/x-shellscript' : self.handle_user_script, + 'text/cloud-config' : self.handle_cloud_config, + 'text/upstart-job' : self.handle_upstart_job, + 'text/part-handler' : self.handle_handler + } + self.cfg=self.read_cfg() + + def read_cfg(self): + if self.cfg: + return(self.cfg) + + conf = { } + try: + stream = file(system_config) + conf = yaml.load(stream) + stream.close() + except: + pass + + if conf is None: conf = { } + + # support reading the old ConfigObj format file and merging + # it into the yaml dictionary + try: + from configobj import ConfigObj + oldcfg = ConfigObj(self.old_conffile) + if oldcfg is None: oldcfg = { } + conf = util.mergedict(conf,oldcfg) + except: + pass + + if not conf.has_key("cloud_type"): + conf["cloud_type"]=None + + return(conf) + + def restore_from_cache(self): + try: + f=open(data_source_cache, "rb") + data = cPickle.load(f) + self.datasource = data + return True + except: + return False + + def write_to_cache(self): + try: + f=open(data_source_cache, "wb") + data = cPickle.dump(self.datasource,f) + return True + except: + return False + + def get_cloud_type(self): + pass + + def get_data_source(self): + if self.datasource is not None: return True + + if self.restore_from_cache(): + return True + + dslist=[ ] + cfglist=self.cfg['cloud_type'] + if cfglist == "auto": + dslist = self.auto_order + elif cfglist: + for ds in cfglist.split(','): + dslist.append(strip(ds).tolower()) + + for ds in dslist: + if ds not in self.datasource_map: continue + try: + s = self.datasource_map[ds]() + if s.get_data(): + self.datasource = s + self.datasource_name = ds + return True + except Exception as e: + pass + raise Exception("Could not find data source") + + def get_userdata(self): + return(self.datasource.get_userdata()) + + def update_cache(self): + self.write_to_cache() + self.store_userdata() + + def store_userdata(self): + util.write_file(userdata_raw, self.datasource.get_userdata_raw(), 0600) + util.write_file(userdata, self.datasource.get_userdata(), 0600) + + def initctl_emit(self): + subprocess.Popen(['initctl', 'emit', 'cloud-config', + '%s=%s' % (cfg_env_name,cloud_config)]).communicate() + + def sem_getpath(self,name,freq): + freqtok = freq + if freq == 'once-per-instance': + freqtok = self.datasource.get_instance_id() + + return("%s/%s.%s" % (semdir,name,freqtok)) + + def sem_has_run(self,name,freq): + if freq is "always": return False + semfile = self.sem_getpath(name,freq) + if os.path.exists(semfile): + return True + return False + + def sem_acquire(self,name,freq): + from time import time + semfile = self.sem_getpath(name,freq) + + try: + os.makedirs(os.path.dirname(semfile)) + except OSError as e: + if e.errno != errno.EEXIST: + raise e + + if os.path.exists(semfile) and freq is not "always": + return False + + # race condition + try: + f = open(semfile,"w") + f.write("%s\n" % str(time())) + f.close() + except: + return(False) + return(True) + + def sem_clear(self,name,freq): + semfile = self.sem_getpath(name,freq) + try: + os.unlink(semfile) + except OSError as e: + if e.errno != errno.ENOENT: + return False + + return True + + # acquire lock on 'name' for given 'freq' + # if that does not exist, then call 'func' with given 'args' + # if 'clear_on_fail' is True and func throws an exception + # then remove the lock (so it would run again) + def sem_and_run(self,semname,freq,func,args=[],clear_on_fail=False): + if self.sem_has_run(semname,freq): return + try: + if not self.sem_acquire(semname,freq): + raise Exception("Failed to acquire lock on %s\n" % semname) + + func(*args) + except: + if clear_on_fail: + self.sem_clear(semname,freq) + raise + + def consume_userdata(self): + self.get_userdata() + data = self + # give callbacks opportunity to initialize + for ctype, func in self.part_handlers.items(): + func(data, "__begin__",None,None) + UserDataHandler.walk_userdata(self.get_userdata(), + self.part_handlers, data) + + # give callbacks opportunity to finalize + for ctype, func in self.part_handlers.items(): + func(data,"__end__",None,None) + + def handle_handler(self,data,ctype,filename,payload): + if ctype == "__end__": return + if ctype == "__begin__" : + self.handlercount = 0 + return + + # add the path to the plugins dir to the top of our list for import + if self.handlercount == 0: + sys.path.insert(0,pluginsdir) + + self.handlercount=self.handlercount+1 + + # write content to pluginsdir + modname = 'part-handler-%03d' % self.handlercount + modfname = modname + ".py" + util.write_file("%s/%s" % (pluginsdir,modfname), payload, 0600) + + try: + mod = __import__(modname) + lister = getattr(mod, "list_types") + handler = getattr(mod, "handle_part") + except: + import traceback + traceback.print_exc(file=sys.stderr) + return + + # - call it with '__begin__' + handler(data, "__begin__", None, None) + + # - add it self.part_handlers + for mtype in lister(): + self.part_handlers[mtype]=handler + + def handle_user_script(self,data,ctype,filename,payload): + if ctype == "__end__": return + if ctype == "__begin__": + # maybe delete existing things here + return + + filename=filename.replace(os.sep,'_') + util.write_file("%s/%s" % (user_scripts_dir,filename), payload, 0700) + + def handle_upstart_job(self,data,ctype,filename,payload): + if ctype == "__end__" or ctype == "__begin__": return + if not filename.endswith(".conf"): + filename=filename+".conf" + + util.write_file("%s/%s" % ("/etc/init",filename), payload, 0644) + + def handle_cloud_config(self,data,ctype,filename,payload): + if ctype == "__begin__": + self.cloud_config_str="" + return + if ctype == "__end__": + util.write_file(cloud_config, self.cloud_config_str, 0600) + + ## this could merge the cloud config with the system config + ## for now, not doing this as it seems somewhat circular + ## as CloudConfig does that also, merging it with this cfg + ## + # ccfg = yaml.load(self.cloud_config_str) + # if ccfg is None: ccfg = { } + # self.cfg = util.mergedict(ccfg, self.cfg) + + return + + self.cloud_config_str+="\n#%s\n%s" % (filename,payload) + + def get_public_ssh_keys(self): + return(self.datasource.get_public_ssh_keys()) + + def get_locale(self): + return(self.datasource.get_locale()) + + def get_mirror(self): + return(self.datasource.get_local_mirror()) + + def get_hostname(self): + return(self.datasource.get_hostname()) + + def device_name_to_device(self,name): + return(self.datasource.device_name_to_device(name)) diff --git a/cloudinit/boto_utils.py b/cloudinit/boto_utils.py new file mode 100644 index 00000000..2f9de95f --- /dev/null +++ b/cloudinit/boto_utils.py @@ -0,0 +1,106 @@ +# The contents of this file are taken from boto 1.9b's boto/utils.py +# +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +# +# Parts of this code were copied or derived from sample code supplied by AWS. +# The following notice applies to that code. +# +# This software code is made available "AS IS" without warranties of any +# kind. You may copy, display, modify and redistribute the software +# code either by itself or as incorporated into your code; provided that +# you do not remove any proprietary notices. Your use of this software +# code is at your own risk and you waive any claim against Amazon +# Digital Services, Inc. or its affiliates with respect to your use of +# this software code. (c) 2006 Amazon Digital Services, Inc. or its +# affiliates. +import urllib2 +import sys + +def retry_url(url, retry_on_404=True): + for i in range(0, 10): + try: + req = urllib2.Request(url) + resp = urllib2.urlopen(req) + return resp.read() + except urllib2.HTTPError, e: + # in 2.6 you use getcode(), in 2.5 and earlier you use code + if hasattr(e, 'getcode'): + code = e.getcode() + else: + code = e.code + if code == 404 and not retry_on_404: + return '' + except: + pass + #boto.log.exception('Caught exception reading instance data') + sys.stderr.write('Caught exception reading instance data') + time.sleep(2**i) + #boto.log.error('Unable to read instance data, giving up') + sys.stderr.write('Caught exception reading instance data') + return '' + +def get_instance_metadata(version='latest'): + """ + Returns the instance metadata as a nested Python dictionary. + Simple values (e.g. local_hostname, hostname, etc.) will be + stored as string values. Values such as ancestor-ami-ids will + be stored in the dict as a list of string values. More complex + fields such as public-keys and will be stored as nested dicts. + """ + url = 'http://169.254.169.254/%s/meta-data/' % version + return _get_instance_metadata(url) + +def get_instance_userdata(version='latest', sep=None): + url = 'http://169.254.169.254/%s/user-data' % version + user_data = retry_url(url, retry_on_404=False) + if user_data: + if sep: + l = user_data.split(sep) + user_data = {} + for nvpair in l: + t = nvpair.split('=') + user_data[t[0].strip()] = t[1].strip() + return user_data + + +def _get_instance_metadata(url): + d = {} + data = retry_url(url) + if data: + fields = data.split('\n') + for field in fields: + if field.endswith('/'): + d[field[0:-1]] = _get_instance_metadata(url + field) + else: + p = field.find('=') + if p > 0: + key = field[p+1:] + resource = field[0:p] + '/openssh-key' + else: + key = resource = field + val = retry_url(url + resource) + p = val.find('\n') + if p > 0: + val = val.split('\n') + d[key] = val + return d diff --git a/cloudinit/execute.py b/cloudinit/execute.py new file mode 100644 index 00000000..033da8f7 --- /dev/null +++ b/cloudinit/execute.py @@ -0,0 +1,13 @@ +def run(list,cfg): + import subprocess + retcode = subprocess.call(list) + + if retcode == 0: + return + + if retcode < 0: + str="Cmd terminated by signal %s\n" % -retcode + else: + str="Cmd returned %s\n" % retcode + str+=' '.join(list) + raise Exception(str) diff --git a/cloudinit/util.py b/cloudinit/util.py new file mode 100644 index 00000000..9b9f6f4d --- /dev/null +++ b/cloudinit/util.py @@ -0,0 +1,79 @@ +import yaml +import os +import errno +import subprocess +from Cheetah.Template import Template + +def read_conf(fname): + stream = file(fname) + conf = yaml.load(stream) + stream.close() + return conf + +def get_cfg_option_bool(yobj, key, default=False): + if not yobj.has_key(key): return default + val = yobj[key] + if yobj[key] in [ True, '1', 'on', 'yes', 'true']: + return True + return False + +def get_cfg_option_str(yobj, key, default=None): + if not yobj.has_key(key): return default + return yobj[key] + +# merge values from src into cand. +# if src has a key, cand will not override +def mergedict(src,cand): + if isinstance(src,dict) and isinstance(cand,dict): + for k,v in cand.iteritems(): + if k not in src: + src[k] = v + else: + src[k] = mergedict(src[k],v) + return src + +def write_file(file,content,mode=0644): + try: + os.makedirs(os.path.dirname(file)) + except OSError as e: + if e.errno != errno.EEXIST: + raise e + + f=open(file,"wb") + os.chmod(file,mode) + f.write(content) + f.close() + +# get keyid from keyserver +def getkeybyid(keyid,keyserver): + shcmd=""" + k=${1} ks=${2}; + exec 2>/dev/null + [ -n "$k" ] || exit 1; + armour=$(gpg --list-keys --armour "${k}") + if [ -z "${armour}" ]; then + gpg --keyserver ${ks} --recv $k >/dev/null && + armour=$(gpg --export --armour "${k}") && + gpg --batch --yes --delete-keys "${k}" + fi + [ -n "${armour}" ] && echo "${armour}" + """ + args=['sh', '-c', shcmd, "export-gpg-keyid", keyid, keyserver] + return(subp(args)[0]) + +def subp(args, input=None): + s_in = None + if input is not None: + s_in = subprocess.PIPE + sp = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=s_in) + out,err = sp.communicate(input) + if sp.returncode is not 0: + raise subprocess.CalledProcessError(sp.returncode,args) + return(out,err) + +def render_to_file(template, outfile, searchList): + t = Template(file='/etc/cloud/templates/%s.tmpl' % template, searchList=[searchList]) + f = open(outfile, 'w') + f.write(t.respond()) + f.close() + diff --git a/ec2-init.py b/ec2-init.py index 06662f7b..723ae3df 100755 --- a/ec2-init.py +++ b/ec2-init.py @@ -3,14 +3,14 @@ import subprocess import sys -import ec2init -import ec2init.util as util +import cloudinit +import cloudinit.util as util def warn(str): sys.stderr.write(str) def main(): - cloud = ec2init.EC2Init() + cloud = cloudinit.EC2Init() try: cloud.get_data_source() diff --git a/ec2init/CloudConfig.py b/ec2init/CloudConfig.py deleted file mode 100644 index d1699f09..00000000 --- a/ec2init/CloudConfig.py +++ /dev/null @@ -1,407 +0,0 @@ -# -# Common code for the EC2 configuration files in Ubuntu -# Copyright (C) 2008-2010 Canonical Ltd. -# -# Author: Chuck Short -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -import yaml -import re -import ec2init -import ec2init.util as util -import subprocess -import os -import glob -import sys - -per_instance="once-per-instance" - -class CloudConfig(): - cfgfile = None - handlers = { } - cfg = None - - def __init__(self,cfgfile): - self.cloud = ec2init.EC2Init() - self.cfg = self.get_config_obj(cfgfile) - self.cloud.get_data_source() - self.add_handler('apt-update-upgrade', self.h_apt_update_upgrade) - self.add_handler('config-ssh') - self.add_handler('disable-ec2-metadata', - self.h_disable_ec2_metadata, "always") - self.add_handler('config-mounts') - - def get_config_obj(self,cfgfile): - f=file(cfgfile) - cfg=yaml.load(f.read()) - f.close() - if cfg is None: cfg = { } - return(util.mergedict(cfg,self.cloud.cfg)) - - def convert_old_config(self): - # support reading the old ConfigObj format file and turning it - # into a yaml string - try: - f = file(self.conffile) - str=file.read().replace('=',': ') - f.close() - return str - except: - return("") - - def add_handler(self, name, handler=None, freq=None): - if handler is None: - try: - handler=getattr(self,'h_%s' % name.replace('-','_')) - except: - raise Exception("Unknown hander for name %s" %name) - if freq is None: - freq = per_instance - - self.handlers[name]= { 'handler': handler, 'freq': freq } - - def get_handler_info(self, name): - return(self.handlers[name]['handler'], self.handlers[name]['freq']) - - def parse_ssh_keys(self): - disableRoot = self.cfg['disable_root'] - if disableRoot == 'true': - value = 'disabled_root' - return value - else: - ec2Key = self.cfg['ec2_fetch_key'] - if ec2Key != 'none': - value = 'default_key' - return value - else: - return ec2Key - - def handle(self, name, args): - handler = None - freq = None - try: - (handler, freq) = self.get_handler_info(name) - except: - raise Exception("Unknown config key %s\n" % name) - - self.cloud.sem_and_run(name, freq, handler, [ name, args ]) - - def h_apt_update_upgrade(self,name,args): - update = util.get_cfg_option_bool(self.cfg, 'apt_update', False) - upgrade = util.get_cfg_option_bool(self.cfg, 'apt_upgrade', False) - - if not util.get_cfg_option_bool(self.cfg, \ - 'apt_preserve_sources_list', False): - if self.cfg.has_key("apt_mirror"): - mirror = self.cfg["apt_mirror"] - else: - mirror = self.cloud.get_mirror() - generate_sources_list(mirror) - - # process 'apt_sources' - if self.cfg.has_key('apt_sources'): - errors = add_sources(self.cfg['apt_sources']) - for e in errors: - warn("Source Error: %s\n" % ':'.join(e)) - - pkglist = [] - if 'packages' in self.cfg: - if isinstance(self.cfg['packages'],list): - pkglist = self.cfg['packages'] - else: pkglist.append(self.cfg['packages']) - - if update or upgrade or pkglist: - #retcode = subprocess.call(list) - subprocess.Popen(['apt-get', 'update']).communicate() - - e=os.environ.copy() - e['DEBIAN_FRONTEND']='noninteractive' - - if upgrade: - subprocess.Popen(['apt-get', 'upgrade', '--assume-yes'], env=e).communicate() - - if pkglist: - cmd=['apt-get', 'install', '--assume-yes'] - cmd.extend(pkglist) - subprocess.Popen(cmd, env=e).communicate() - - return(True) - - def h_disable_ec2_metadata(self,name,args): - if util.get_cfg_option_bool(self.cfg, "disable_ec2_metadata", False): - fwall="route add -host 169.254.169.254 reject" - subprocess.call(fwall.split(' ')) - - def h_config_ssh(self,name,args): - # remove the static keys from the pristine image - for f in glob.glob("/etc/ssh/ssh_host_*_key*"): - try: os.unlink(f) - except: pass - - if self.cfg.has_key("ssh_keys"): - # if there are keys in cloud-config, use them - key2file = { - "rsa_private" : ("/etc/ssh/ssh_host_rsa_key", 0600), - "rsa_public" : ("/etc/ssh/ssh_host_rsa_key.pub", 0644), - "dsa_private" : ("/etc/ssh/ssh_host_dsa_key", 0600), - "dsa_public" : ("/etc/ssh/ssh_host_dsa_key.pub", 0644) - } - - for key,val in self.cfg["ssh_keys"].items(): - if key2file.has_key(key): - util.write_file(key2file[key][0],val,key2file[key][1]) - else: - # if not, generate them - genkeys ='ssh-keygen -f /etc/ssh/ssh_host_rsa_key -t rsa -N ""; ' - genkeys+='ssh-keygen -f /etc/ssh/ssh_host_dsa_key -t dsa -N ""; ' - subprocess.call(('sh', '-c', "{ %s } 0, toks) - if len(r) == 4: - return("ip-%s" % '-'.join(r)) - except: pass - return toks[0] - - def get_mirror_from_availability_zone(self, availability_zone = None): - # availability is like 'us-west-1b' or 'eu-west-1a' - if availability_zone == None: - availability_zone = self.get_availability_zone() - - try: - host="%s.ec2.archive.ubuntu.com" % availability_zone[:-1] - socket.getaddrinfo(host, None, 0, socket.SOCK_STREAM) - return 'http://%s/ubuntu/' % host - except: - return 'http://archive.ubuntu.com/ubuntu/' - - def wait_for_metadata_service(self, sleeps = 10): - sleeptime = 1 - for x in range(sleeps): - s = socket.socket() - try: - address = '169.254.169.254' - port = 80 - s.connect((address,port)) - s.close() - return True - except socket.error, e: - print "sleeping %s" % sleeptime - time.sleep(sleeptime) - #timeout = timeout * 2 - return False - - def get_public_ssh_keys(self): - keys = [] - if not self.metadata.has_key('public-keys'): return([]) - for keyname, klist in self.metadata['public-keys'].items(): - # lp:506332 uec metadata service responds with - # data that makes boto populate a string for 'klist' rather - # than a list. - if isinstance(klist,str): - klist = [ klist ] - for pkey in klist: - # there is an empty string at the end of the keylist, trim it - if pkey: - keys.append(pkey) - - return(keys) - - def device_name_to_device(self, name): - # consult metadata service, that has - # ephemeral0: sdb - # and return 'sdb' for input 'ephemeral0' - if not self.metadata.has_key('block-device-mapping'): - return(None) - - for entname, device in self.metadata['block-device-mapping'].items(): - if entname == name: - return(device) - # LP: #513842 mapping in Euca has 'ephemeral' not 'ephemeral0' - if entname == "ephemeral" and name == "ephemeral0": - return(device) - return None diff --git a/ec2init/UserDataHandler.py b/ec2init/UserDataHandler.py deleted file mode 100644 index 56feb0ff..00000000 --- a/ec2init/UserDataHandler.py +++ /dev/null @@ -1,130 +0,0 @@ -import email - -from email.mime.multipart import MIMEMultipart -from email.mime.text import MIMEText - - -starts_with_mappings={ - '#include' : 'text/x-include-url', - '#!' : 'text/x-shellscript', - '#cloud-config' : 'text/cloud-config', - '#upstart-job' : 'text/upstart-job', - '#part-handler' : 'text/part-handler' -} - -# if 'str' is compressed return decompressed otherwise return it -def decomp_str(str): - import StringIO - import gzip - try: - uncomp = gzip.GzipFile(None,"rb",1,StringIO.StringIO(str)).read() - return(uncomp) - except: - return(str) - -def do_include(str,parts): - import urllib - # is just a list of urls, one per line - for line in str.splitlines(): - if line == "#include": continue - if line.startswith("#"): continue - content = urllib.urlopen(line).read() - process_includes(email.message_from_string(decomp_str(content)),parts) - -def process_includes(msg,parts): - # parts is a dictionary of arrays - # parts['content'] - # parts['names'] - # parts['types'] - for t in ( 'content', 'names', 'types' ): - if not parts.has_key(t): - parts[t]=[ ] - for part in msg.walk(): - # multipart/* are just containers - if part.get_content_maintype() == 'multipart': - continue - - payload = part.get_payload() - - ctype = None - for str, gtype in starts_with_mappings.items(): - if payload.startswith(str): - ctype = gtype - break - - if ctype is None: - ctype = part.get_content_type() - - if ctype == 'text/x-include-url': - do_include(payload,parts) - continue - - filename = part.get_filename() - if not filename: - filename = 'part-%03d' % len(parts['content']) - - parts['content'].append(payload) - parts['types'].append(ctype) - parts['names'].append(filename) - -def parts2mime(parts): - outer = MIMEMultipart() - - i = 0 - while i < len(parts['content']): - if parts['types'][i] is None: - # No guess could be made, or the file is encoded (compressed), so - # use a generic bag-of-bits type. - ctype = 'application/octet-stream' - else: ctype = parts['types'][i] - maintype, subtype = ctype.split('/', 1) - if maintype == 'text': - msg = MIMEText(parts['content'][i], _subtype=subtype) - else: - msg = MIMEBase(maintype, subtype) - msg.set_payload(parts['content'][i]) - # Encode the payload using Base64 - encoders.encode_base64(msg) - # Set the filename parameter - msg.add_header('Content-Disposition', 'attachment', - filename=parts['names'][i]) - outer.attach(msg) - - i=i+1 - return(outer.as_string()) - -# this is heavily wasteful, reads through userdata string input -def preprocess_userdata(data): - parts = { } - process_includes(email.message_from_string(decomp_str(data)),parts) - return(parts2mime(parts)) - -# callbacks is a dictionary with: -# { 'content-type': handler(data,content_type,filename,payload) } -def walk_userdata(str, callbacks, data = None): - partnum = 0 - for part in email.message_from_string(str).walk(): - # multipart/* are just containers - if part.get_content_maintype() == 'multipart': - continue - - ctype = part.get_content_type() - if ctype is None: - ctype = 'application/octet-stream' - - filename = part.get_filename() - if not filename: - filename = 'part-%03d' % partnum - - if callbacks.has_key(ctype): - callbacks[ctype](data,ctype,filename,part.get_payload()) - - partnum = partnum+1 - -if __name__ == "__main__": - import sys - data = decomp_str(file(sys.argv[1]).read()) - parts = { } - process_includes(email.message_from_string(data),parts) - print "#found %s parts" % len(parts['content']) - print parts2mime(parts) diff --git a/ec2init/__init__.py b/ec2init/__init__.py deleted file mode 100644 index 76aa34f0..00000000 --- a/ec2init/__init__.py +++ /dev/null @@ -1,313 +0,0 @@ -# -# Common code for the EC2 initialisation scripts in Ubuntu -# Copyright (C) 2008-2009 Canonical Ltd -# -# Author: Soren Hansen -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# - -import os -from configobj import ConfigObj - -import cPickle -import sys -import os.path -import errno -import pwd -import subprocess -import yaml - -datadir = '/var/lib/cloud/data' -semdir = '/var/lib/cloud/sem' -pluginsdir = datadir + '/plugins' -cachedir = datadir + '/cache' -userdata_raw = datadir + '/user-data.txt' -userdata = datadir + '/user-data.txt.i' -user_scripts_dir = datadir + "/scripts" -cloud_config = datadir + '/cloud-config.txt' -data_source_cache = cachedir + '/obj.pkl' -system_config = '/etc/cloud/cloud.cfg' -cfg_env_name = "CLOUD_CFG" - -import DataSourceEc2 -import UserDataHandler -import util - -class EC2Init: - datasource_map = { - "ec2" : DataSourceEc2.DataSourceEc2, - } - datasource = None - auto_order = [ 'ec2' ] - - cfg = None - part_handlers = { } - old_conffile = '/etc/ec2-init/ec2-config.cfg' - - def __init__(self): - self.part_handlers = { - 'text/x-shellscript' : self.handle_user_script, - 'text/cloud-config' : self.handle_cloud_config, - 'text/upstart-job' : self.handle_upstart_job, - 'text/part-handler' : self.handle_handler - } - self.cfg=self.read_cfg() - - def read_cfg(self): - if self.cfg: - return(self.cfg) - - conf = { } - try: - stream = file(system_config) - conf = yaml.load(stream) - stream.close() - except: - pass - - if conf is None: conf = { } - - # support reading the old ConfigObj format file and merging - # it into the yaml dictionary - try: - from configobj import ConfigObj - oldcfg = ConfigObj(self.old_conffile) - if oldcfg is None: oldcfg = { } - conf = util.mergedict(conf,oldcfg) - except: - pass - - if not conf.has_key("cloud_type"): - conf["cloud_type"]=None - - return(conf) - - def restore_from_cache(self): - try: - f=open(data_source_cache, "rb") - data = cPickle.load(f) - self.datasource = data - return True - except: - return False - - def write_to_cache(self): - try: - f=open(data_source_cache, "wb") - data = cPickle.dump(self.datasource,f) - return True - except: - return False - - def get_cloud_type(self): - pass - - def get_data_source(self): - if self.datasource is not None: return True - - if self.restore_from_cache(): - return True - - dslist=[ ] - cfglist=self.cfg['cloud_type'] - if cfglist == "auto": - dslist = self.auto_order - elif cfglist: - for ds in cfglist.split(','): - dslist.append(strip(ds).tolower()) - - for ds in dslist: - if ds not in self.datasource_map: continue - try: - s = self.datasource_map[ds]() - if s.get_data(): - self.datasource = s - self.datasource_name = ds - return True - except Exception as e: - pass - raise Exception("Could not find data source") - - def get_userdata(self): - return(self.datasource.get_userdata()) - - def update_cache(self): - self.write_to_cache() - self.store_userdata() - - def store_userdata(self): - util.write_file(userdata_raw, self.datasource.get_userdata_raw(), 0600) - util.write_file(userdata, self.datasource.get_userdata(), 0600) - - def initctl_emit(self): - subprocess.Popen(['initctl', 'emit', 'cloud-config', - '%s=%s' % (cfg_env_name,cloud_config)]).communicate() - - def sem_getpath(self,name,freq): - freqtok = freq - if freq == 'once-per-instance': - freqtok = self.datasource.get_instance_id() - - return("%s/%s.%s" % (semdir,name,freqtok)) - - def sem_has_run(self,name,freq): - if freq is "always": return False - semfile = self.sem_getpath(name,freq) - if os.path.exists(semfile): - return True - return False - - def sem_acquire(self,name,freq): - from time import time - semfile = self.sem_getpath(name,freq) - - try: - os.makedirs(os.path.dirname(semfile)) - except OSError as e: - if e.errno != errno.EEXIST: - raise e - - if os.path.exists(semfile) and freq is not "always": - return False - - # race condition - try: - f = open(semfile,"w") - f.write("%s\n" % str(time())) - f.close() - except: - return(False) - return(True) - - def sem_clear(self,name,freq): - semfile = self.sem_getpath(name,freq) - try: - os.unlink(semfile) - except OSError as e: - if e.errno != errno.ENOENT: - return False - - return True - - # acquire lock on 'name' for given 'freq' - # if that does not exist, then call 'func' with given 'args' - # if 'clear_on_fail' is True and func throws an exception - # then remove the lock (so it would run again) - def sem_and_run(self,semname,freq,func,args=[],clear_on_fail=False): - if self.sem_has_run(semname,freq): return - try: - if not self.sem_acquire(semname,freq): - raise Exception("Failed to acquire lock on %s\n" % semname) - - func(*args) - except: - if clear_on_fail: - self.sem_clear(semname,freq) - raise - - def consume_userdata(self): - self.get_userdata() - data = self - # give callbacks opportunity to initialize - for ctype, func in self.part_handlers.items(): - func(data, "__begin__",None,None) - UserDataHandler.walk_userdata(self.get_userdata(), - self.part_handlers, data) - - # give callbacks opportunity to finalize - for ctype, func in self.part_handlers.items(): - func(data,"__end__",None,None) - - def handle_handler(self,data,ctype,filename,payload): - if ctype == "__end__": return - if ctype == "__begin__" : - self.handlercount = 0 - return - - # add the path to the plugins dir to the top of our list for import - if self.handlercount == 0: - sys.path.insert(0,pluginsdir) - - self.handlercount=self.handlercount+1 - - # write content to pluginsdir - modname = 'part-handler-%03d' % self.handlercount - modfname = modname + ".py" - util.write_file("%s/%s" % (pluginsdir,modfname), payload, 0600) - - try: - mod = __import__(modname) - lister = getattr(mod, "list_types") - handler = getattr(mod, "handle_part") - except: - import traceback - traceback.print_exc(file=sys.stderr) - return - - # - call it with '__begin__' - handler(data, "__begin__", None, None) - - # - add it self.part_handlers - for mtype in lister(): - self.part_handlers[mtype]=handler - - def handle_user_script(self,data,ctype,filename,payload): - if ctype == "__end__": return - if ctype == "__begin__": - # maybe delete existing things here - return - - filename=filename.replace(os.sep,'_') - util.write_file("%s/%s" % (user_scripts_dir,filename), payload, 0700) - - def handle_upstart_job(self,data,ctype,filename,payload): - if ctype == "__end__" or ctype == "__begin__": return - if not filename.endswith(".conf"): - filename=filename+".conf" - - util.write_file("%s/%s" % ("/etc/init",filename), payload, 0644) - - def handle_cloud_config(self,data,ctype,filename,payload): - if ctype == "__begin__": - self.cloud_config_str="" - return - if ctype == "__end__": - util.write_file(cloud_config, self.cloud_config_str, 0600) - - ## this could merge the cloud config with the system config - ## for now, not doing this as it seems somewhat circular - ## as CloudConfig does that also, merging it with this cfg - ## - # ccfg = yaml.load(self.cloud_config_str) - # if ccfg is None: ccfg = { } - # self.cfg = util.mergedict(ccfg, self.cfg) - - return - - self.cloud_config_str+="\n#%s\n%s" % (filename,payload) - - def get_public_ssh_keys(self): - return(self.datasource.get_public_ssh_keys()) - - def get_locale(self): - return(self.datasource.get_locale()) - - def get_mirror(self): - return(self.datasource.get_local_mirror()) - - def get_hostname(self): - return(self.datasource.get_hostname()) - - def device_name_to_device(self,name): - return(self.datasource.device_name_to_device(name)) diff --git a/ec2init/boto_utils.py b/ec2init/boto_utils.py deleted file mode 100644 index 2f9de95f..00000000 --- a/ec2init/boto_utils.py +++ /dev/null @@ -1,106 +0,0 @@ -# The contents of this file are taken from boto 1.9b's boto/utils.py -# -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -# -# Parts of this code were copied or derived from sample code supplied by AWS. -# The following notice applies to that code. -# -# This software code is made available "AS IS" without warranties of any -# kind. You may copy, display, modify and redistribute the software -# code either by itself or as incorporated into your code; provided that -# you do not remove any proprietary notices. Your use of this software -# code is at your own risk and you waive any claim against Amazon -# Digital Services, Inc. or its affiliates with respect to your use of -# this software code. (c) 2006 Amazon Digital Services, Inc. or its -# affiliates. -import urllib2 -import sys - -def retry_url(url, retry_on_404=True): - for i in range(0, 10): - try: - req = urllib2.Request(url) - resp = urllib2.urlopen(req) - return resp.read() - except urllib2.HTTPError, e: - # in 2.6 you use getcode(), in 2.5 and earlier you use code - if hasattr(e, 'getcode'): - code = e.getcode() - else: - code = e.code - if code == 404 and not retry_on_404: - return '' - except: - pass - #boto.log.exception('Caught exception reading instance data') - sys.stderr.write('Caught exception reading instance data') - time.sleep(2**i) - #boto.log.error('Unable to read instance data, giving up') - sys.stderr.write('Caught exception reading instance data') - return '' - -def get_instance_metadata(version='latest'): - """ - Returns the instance metadata as a nested Python dictionary. - Simple values (e.g. local_hostname, hostname, etc.) will be - stored as string values. Values such as ancestor-ami-ids will - be stored in the dict as a list of string values. More complex - fields such as public-keys and will be stored as nested dicts. - """ - url = 'http://169.254.169.254/%s/meta-data/' % version - return _get_instance_metadata(url) - -def get_instance_userdata(version='latest', sep=None): - url = 'http://169.254.169.254/%s/user-data' % version - user_data = retry_url(url, retry_on_404=False) - if user_data: - if sep: - l = user_data.split(sep) - user_data = {} - for nvpair in l: - t = nvpair.split('=') - user_data[t[0].strip()] = t[1].strip() - return user_data - - -def _get_instance_metadata(url): - d = {} - data = retry_url(url) - if data: - fields = data.split('\n') - for field in fields: - if field.endswith('/'): - d[field[0:-1]] = _get_instance_metadata(url + field) - else: - p = field.find('=') - if p > 0: - key = field[p+1:] - resource = field[0:p] + '/openssh-key' - else: - key = resource = field - val = retry_url(url + resource) - p = val.find('\n') - if p > 0: - val = val.split('\n') - d[key] = val - return d diff --git a/ec2init/execute.py b/ec2init/execute.py deleted file mode 100644 index 033da8f7..00000000 --- a/ec2init/execute.py +++ /dev/null @@ -1,13 +0,0 @@ -def run(list,cfg): - import subprocess - retcode = subprocess.call(list) - - if retcode == 0: - return - - if retcode < 0: - str="Cmd terminated by signal %s\n" % -retcode - else: - str="Cmd returned %s\n" % retcode - str+=' '.join(list) - raise Exception(str) diff --git a/ec2init/util.py b/ec2init/util.py deleted file mode 100644 index 9b9f6f4d..00000000 --- a/ec2init/util.py +++ /dev/null @@ -1,79 +0,0 @@ -import yaml -import os -import errno -import subprocess -from Cheetah.Template import Template - -def read_conf(fname): - stream = file(fname) - conf = yaml.load(stream) - stream.close() - return conf - -def get_cfg_option_bool(yobj, key, default=False): - if not yobj.has_key(key): return default - val = yobj[key] - if yobj[key] in [ True, '1', 'on', 'yes', 'true']: - return True - return False - -def get_cfg_option_str(yobj, key, default=None): - if not yobj.has_key(key): return default - return yobj[key] - -# merge values from src into cand. -# if src has a key, cand will not override -def mergedict(src,cand): - if isinstance(src,dict) and isinstance(cand,dict): - for k,v in cand.iteritems(): - if k not in src: - src[k] = v - else: - src[k] = mergedict(src[k],v) - return src - -def write_file(file,content,mode=0644): - try: - os.makedirs(os.path.dirname(file)) - except OSError as e: - if e.errno != errno.EEXIST: - raise e - - f=open(file,"wb") - os.chmod(file,mode) - f.write(content) - f.close() - -# get keyid from keyserver -def getkeybyid(keyid,keyserver): - shcmd=""" - k=${1} ks=${2}; - exec 2>/dev/null - [ -n "$k" ] || exit 1; - armour=$(gpg --list-keys --armour "${k}") - if [ -z "${armour}" ]; then - gpg --keyserver ${ks} --recv $k >/dev/null && - armour=$(gpg --export --armour "${k}") && - gpg --batch --yes --delete-keys "${k}" - fi - [ -n "${armour}" ] && echo "${armour}" - """ - args=['sh', '-c', shcmd, "export-gpg-keyid", keyid, keyserver] - return(subp(args)[0]) - -def subp(args, input=None): - s_in = None - if input is not None: - s_in = subprocess.PIPE - sp = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=s_in) - out,err = sp.communicate(input) - if sp.returncode is not 0: - raise subprocess.CalledProcessError(sp.returncode,args) - return(out,err) - -def render_to_file(template, outfile, searchList): - t = Template(file='/etc/cloud/templates/%s.tmpl' % template, searchList=[searchList]) - f = open(outfile, 'w') - f.write(t.respond()) - f.close() - diff --git a/setup.py b/setup.py index 92c2c79f..2a9bdce8 100755 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ setup(name='EC2-init', author='Scott Moser', author_email='scott.moser@canonical.com', url='http://launchpad.net/ec2-init/', - packages=['ec2init'], + packages=['cloudinit'], scripts=['ec2-init.py', 'cloud-init-run-module.py', 'cloud-init-cfg.py' diff --git a/upstart/cloud-run-user-script.conf b/upstart/cloud-run-user-script.conf index 272bc409..3e5089d5 100644 --- a/upstart/cloud-run-user-script.conf +++ b/upstart/cloud-run-user-script.conf @@ -1,9 +1,9 @@ # this runs user scripts found in user-data, that are -# stored in /var/lib/cloud/scripts by the initial ec2init upstart job +# stored in /var/lib/cloud/scripts by the initial cloudinit upstart job description "execute cloud user scripts" -start on (stopped ec2init and stopped rc RUNLEVEL=[2345]) +start on (stopped cloudinit and stopped rc RUNLEVEL=[2345]) console output task -- cgit v1.2.3