diff options
26 files changed, 605 insertions, 680 deletions
diff --git a/cloud-init-cfg.py b/cloud-init-cfg.py index 2ef9bb04..76f34ae0 100755 --- a/cloud-init-cfg.py +++ b/cloud-init-cfg.py @@ -28,7 +28,12 @@ def Usage(out = sys.stdout): def main(): # expect to be called with - # name freq [ args ] + # name [ args ] + # run the cloud-config job 'name' at with given args + # or + # read cloud config jobs from config (builtin -> system) + # and run all in order + if len(sys.argv) < 2: Usage(sys.stderr) sys.exit(1) @@ -40,8 +45,6 @@ def main(): log = logging.getLogger() log.info("cloud-init-cfg %s" % sys.argv[1:]) - cloud = cloudinit.CloudInit() - cfg_path = cloudinit.cloud_config cfg_env_name = cloudinit.cfg_env_name if os.environ.has_key(cfg_env_name): @@ -49,15 +52,55 @@ def main(): cc = cloudinit.CloudConfig.CloudConfig(cfg_path) - try: - cc.handle(name,run_args) - except: - import traceback - traceback.print_exc(file=sys.stderr) - sys.stderr.write("config handling of %s failed\n" % name) - sys.exit(1) + module_list = [ ] + if name == "all": + # create 'module_list', an array of arrays + # where array[0] = config + # array[1] = freq + # array[2:] = arguemnts + if "cloud_config_modules" in cc.cfg: + for item in cc.cfg["cloud_config_modules"]: + if isinstance(item,str): + module_list.append((item,)) + elif isinstance(item,list): + module_list.append(item) + else: + fail("Failed to parse cloud_config_modules",log) + else: + fail("No cloud_config_modules found in config",log) + else: + args = [ name, None ] + run_args + module_list.append = ( args ) + + failures = [] + for cfg_mod in module_list: + name = cfg_mod[0] + freq = None + run_args = [ ] + if len(cfg_mod) > 1: + freq = cfg_mod[1] + if len(cfg_mod) > 2: + run_args = cfg_mod[2:] + + try: + cc.handle(name, run_args, freq=freq) + except: + import traceback + traceback.print_exc(file=sys.stderr) + err("config handling of %s failed\n" % name,log) + failures.append(name) + sys.exit(len(failures)) + + sys.exit(len(failures)) + +def err(msg,log=None): + if log: + log.error(msg) + sys.stderr.write(msg + "\n") - sys.exit(0) +def fail(msg,log=None): + err(msg,log) + sys.exit(1) if __name__ == '__main__': main() diff --git a/cloudinit/CloudConfig.py b/cloudinit/CloudConfig.py deleted file mode 100644 index 9c050abc..00000000 --- a/cloudinit/CloudConfig.py +++ /dev/null @@ -1,564 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2008-2010 Canonical Ltd. -# -# Author: Chuck Short <chuck.short@canonical.com> -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -# -import yaml -import re -import cloudinit -import cloudinit.util as util -import pwd -import socket -import subprocess -import os -import glob -import sys -import time -import re -import string - -per_instance="once-per-instance" -cronpre = "/etc/cron.d/cloudinit" - -class CloudConfig(): - cfgfile = None - handlers = { } - cfg = None - - def __init__(self,cfgfile): - self.cloud = cloudinit.CloudInit() - self.cfg = self.get_config_obj(cfgfile) - self.cloud.get_data_source() - self.add_handler('apt-update-upgrade', self.h_apt_update_upgrade) - self.add_handler('config-ssh') - self.add_handler('disable-ec2-metadata', - self.h_disable_ec2_metadata, "always") - self.add_handler('config-mounts') - self.add_handler('config-puppet') - self.add_handler('config-misc') - - def get_config_obj(self,cfgfile): - f=file(cfgfile) - cfg=yaml.load(f.read()) - f.close() - if cfg is None: cfg = { } - return(util.mergedict(cfg,self.cloud.cfg)) - - def convert_old_config(self): - # support reading the old ConfigObj format file and turning it - # into a yaml string - try: - f = file(self.conffile) - str=file.read().replace('=',': ') - f.close() - return str - except: - return("") - - def add_handler(self, name, handler=None, freq=None): - if handler is None: - try: - handler=getattr(self,'h_%s' % name.replace('-','_')) - except: - raise Exception("Unknown hander for name %s" %name) - if freq is None: - freq = per_instance - - self.handlers[name]= { 'handler': handler, 'freq': freq } - - def get_handler_info(self, name): - return(self.handlers[name]['handler'], self.handlers[name]['freq']) - - def parse_ssh_keys(self): - disableRoot = self.cfg['disable_root'] - if disableRoot == 'true': - value = 'disabled_root' - return value - else: - ec2Key = self.cfg['ec2_fetch_key'] - if ec2Key != 'none': - value = 'default_key' - return value - else: - return ec2Key - - def handle(self, name, args): - handler = None - freq = None - try: - (handler, freq) = self.get_handler_info(name) - except: - raise Exception("Unknown config key %s\n" % name) - - self.cloud.sem_and_run(name, freq, handler, [ name, args ]) - - def h_apt_update_upgrade(self,name,args): - update = util.get_cfg_option_bool(self.cfg, 'apt_update', False) - upgrade = util.get_cfg_option_bool(self.cfg, 'apt_upgrade', False) - - if not util.get_cfg_option_bool(self.cfg, \ - 'apt_preserve_sources_list', False): - if self.cfg.has_key("apt_mirror"): - mirror = self.cfg["apt_mirror"] - else: - mirror = self.cloud.get_mirror() - generate_sources_list(mirror) - old_mir = util.get_cfg_option_str(self.cfg,'apt_old_mirror', \ - "archive.ubuntu.com/ubuntu") - rename_apt_lists(old_mir, mirror) - - # process 'apt_sources' - if self.cfg.has_key('apt_sources'): - errors = add_sources(self.cfg['apt_sources']) - for e in errors: - warn("Source Error: %s\n" % ':'.join(e)) - - pkglist = [] - if 'packages' in self.cfg: - if isinstance(self.cfg['packages'],list): - pkglist = self.cfg['packages'] - else: pkglist.append(self.cfg['packages']) - - if update or upgrade or pkglist: - #retcode = subprocess.call(list) - subprocess.Popen(['apt-get', 'update']).communicate() - - e=os.environ.copy() - e['DEBIAN_FRONTEND']='noninteractive' - - if upgrade: - subprocess.Popen(['apt-get', 'upgrade', '--assume-yes'], env=e).communicate() - - if pkglist: - cmd=['apt-get', 'install', '--assume-yes'] - cmd.extend(pkglist) - subprocess.Popen(cmd, env=e).communicate() - - return(True) - - def h_disable_ec2_metadata(self,name,args): - if util.get_cfg_option_bool(self.cfg, "disable_ec2_metadata", False): - fwall="route add -host 169.254.169.254 reject" - subprocess.call(fwall.split(' ')) - - def h_config_ssh(self,name,args): - # remove the static keys from the pristine image - for f in glob.glob("/etc/ssh/ssh_host_*_key*"): - try: os.unlink(f) - except: pass - - if self.cfg.has_key("ssh_keys"): - # if there are keys in cloud-config, use them - key2file = { - "rsa_private" : ("/etc/ssh/ssh_host_rsa_key", 0600), - "rsa_public" : ("/etc/ssh/ssh_host_rsa_key.pub", 0644), - "dsa_private" : ("/etc/ssh/ssh_host_dsa_key", 0600), - "dsa_public" : ("/etc/ssh/ssh_host_dsa_key.pub", 0644) - } - - for key,val in self.cfg["ssh_keys"].items(): - if key2file.has_key(key): - util.write_file(key2file[key][0],val,key2file[key][1]) - else: - # if not, generate them - genkeys ='ssh-keygen -f /etc/ssh/ssh_host_rsa_key -t rsa -N ""; ' - genkeys+='ssh-keygen -f /etc/ssh/ssh_host_dsa_key -t dsa -N ""; ' - subprocess.call(('sh', '-c', "{ %s } </dev/null" % (genkeys))) - - try: - user = util.get_cfg_option_str(self.cfg,'user') - disable_root = util.get_cfg_option_bool(self.cfg, "disable_root", True) - keys = self.cloud.get_public_ssh_keys() - - if self.cfg.has_key("ssh_authorized_keys"): - cfgkeys = self.cfg["ssh_authorized_keys"] - keys.extend(cfgkeys) - - apply_credentials(keys,user,disable_root) - except: - warn("applying credentials failed!\n") - - send_ssh_keys_to_console() - - def h_config_misc(self,name,args): - handle_updates_check(self.cfg) - handle_runcmd(self.cfg) - - def h_config_puppet(self,name,args): - # If there isn't a puppet key in the configuration don't do anything - if not self.cfg.has_key('puppet'): return - puppet_cfg = self.cfg['puppet'] - # Start by installing the puppet package ... - e=os.environ.copy() - e['DEBIAN_FRONTEND']='noninteractive' - # Make sure that the apt database is updated since it's not run by - # default - # Note: we should have a helper to check if apt-get update - # has already been run on this instance to speed the boot time. - subprocess.check_call(['apt-get', 'update'], env=e) - subprocess.check_call(['apt-get', 'install', '--assume-yes', - 'puppet'], env=e) - # ... and then update the puppet configuration - if puppet_cfg.has_key('conf'): - # Add all sections from the conf object to puppet.conf - puppet_conf_fh = open('/etc/puppet/puppet.conf', 'a') - for cfg_name, cfg in puppet_cfg['conf'].iteritems(): - # ca_cert configuration is a special case - # Dump the puppetmaster ca certificate in the correct place - if cfg_name == 'ca_cert': - # Puppet ssl sub-directory isn't created yet - # Create it with the proper permissions and ownership - os.makedirs('/var/lib/puppet/ssl') - os.chmod('/var/lib/puppet/ssl', 0771) - os.chown('/var/lib/puppet/ssl', - pwd.getpwnam('puppet').pw_uid, 0) - os.makedirs('/var/lib/puppet/ssl/certs/') - os.chown('/var/lib/puppet/ssl/certs/', - pwd.getpwnam('puppet').pw_uid, 0) - ca_fh = open('/var/lib/puppet/ssl/certs/ca.pem', 'w') - ca_fh.write(cfg) - ca_fh.close() - os.chown('/var/lib/puppet/ssl/certs/ca.pem', - pwd.getpwnam('puppet').pw_uid, 0) - else: - puppet_conf_fh.write("\n[%s]\n" % (cfg_name)) - for o, v in cfg.iteritems(): - if o == 'certname': - # Expand %f as the fqdn - v = v.replace("%f", socket.getfqdn()) - # Expand %i as the instance id - v = v.replace("%i", - self.cloud.datasource.get_instance_id()) - # certname needs to be downcase - v = v.lower() - puppet_conf_fh.write("%s=\"%s\"\n" % (o, v)) - puppet_conf_fh.close() - # Set puppet default file to automatically start - subprocess.check_call(['sed', '-i', - '-e', 's/^START=.*/START=yes/', - '/etc/default/puppet']) - # Start puppetd - subprocess.check_call(['service', 'puppet', 'start']) - - def h_ec2_ebs_mounts(self,name,args): - print "Warning, not doing anything for config %s" % name - - def h_config_setup_raid(self,name,args): - print "Warning, not doing anything for config %s" % name - - def h_config_runurl(self,name,args): - print "Warning, not doing anything for config %s" % name - - def h_config_mounts(self,name,args): - # handle 'mounts' - - # these are our default set of mounts - defmnts = [ [ "ephemeral0", "/mnt", "auto", "defaults", "0", "0" ], - [ "swap", "none", "swap", "sw", "0", "0" ] ] - - # fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno - defvals = [ None, None, "auto", "defaults", "0", "0" ] - - cfgmnt = [ ] - if self.cfg.has_key("mounts"): - cfgmnt = self.cfg["mounts"] - - for i in range(len(cfgmnt)): - # skip something that wasn't a list - if not isinstance(cfgmnt[i],list): continue - - # workaround, allow user to specify 'ephemeral' - # rather than more ec2 correct 'ephemeral0' - if cfgmnt[i][0] == "ephemeral": - cfgmnt[i][0] = "ephemeral0" - - newname = cfgmnt[i][0] - if not newname.startswith("/"): - newname = self.cloud.device_name_to_device(cfgmnt[i][0]) - if newname is not None: - cfgmnt[i][0] = newname - else: - # there is no good way of differenciating between - # a name that *couldn't* exist in the md service and - # one that merely didnt - # in order to allow user to specify 'sda3' rather - # than '/dev/sda3', go through some hoops - ok = False - for f in [ "/", "sd", "hd", "vd", "xvd" ]: - if cfgmnt[i][0].startswith(f): - ok = True - break - if not ok: - cfgmnt[i][1] = None - - for i in range(len(cfgmnt)): - # fill in values with - for j in range(len(defvals)): - if len(cfgmnt[i]) <= j: - cfgmnt[i].append(defvals[j]) - elif cfgmnt[i][j] is None: - cfgmnt[i][j] = defvals[j] - - if not cfgmnt[i][0].startswith("/"): - cfgmnt[i][0]="/dev/%s" % cfgmnt[i][0] - - # if the second entry in the list is 'None' this - # clears all previous entries of that same 'fs_spec' - # (fs_spec is the first field in /etc/fstab, ie, that device) - if cfgmnt[i][1] is None: - for j in range(i): - if cfgmnt[j][0] == cfgmnt[i][0]: - cfgmnt[j][1] = None - - - # for each of the "default" mounts, add them only if no other - # entry has the same device name - for defmnt in defmnts: - devname = self.cloud.device_name_to_device(defmnt[0]) - if devname is None: continue - if devname.startswith("/"): - defmnt[0] = devname - else: - defmnt[0] = "/dev/%s" % devname - - cfgmnt_has = False - for cfgm in cfgmnt: - if cfgm[0] == defmnt[0]: - cfgmnt_has = True - break - - if cfgmnt_has: continue - cfgmnt.append(defmnt) - - - # now, each entry in the cfgmnt list has all fstab values - # if the second field is None (not the string, the value) we skip it - actlist = filter(lambda x: x[1] is not None, cfgmnt) - - if len(actlist) == 0: return - - comment="comment=cloudconfig" - cc_lines = [ ] - needswap = False - dirs = [ ] - for line in actlist: - # write 'comment' in the fs_mntops, entry, claiming this - line[3]="%s,comment=cloudconfig" % line[3] - if line[2] == "swap": needswap = True - if line[1].startswith("/"): dirs.append(line[1]) - cc_lines.append('\t'.join(line)) - - fstab_lines = [ ] - fstab=open("/etc/fstab","r+") - ws = re.compile("[%s]+" % string.whitespace) - for line in fstab.read().splitlines(): - try: - toks = ws.split(line) - if toks[3].find(comment) != -1: continue - except: - pass - fstab_lines.append(line) - - fstab_lines.extend(cc_lines) - - fstab.seek(0) - fstab.write("%s\n" % '\n'.join(fstab_lines)) - fstab.truncate() - fstab.close() - - if needswap: - try: util.subp(("swapon", "-a")) - except: warn("Failed to enable swap") - - for d in dirs: - if os.path.exists(d): continue - try: os.makedirs(d) - except: warn("Failed to make '%s' config-mount\n",d) - - try: util.subp(("mount","-a")) - except: pass - - - - -def apply_credentials(keys, user, disable_root): - keys = set(keys) - if user: - setup_user_keys(keys, user, '') - - if disable_root: - key_prefix = 'command="echo \'Please login as the %s user rather than root user.\';echo;sleep 10" ' % user - else: - key_prefix = '' - - setup_user_keys(keys, 'root', key_prefix) - -def setup_user_keys(keys, user, key_prefix): - import pwd - saved_umask = os.umask(077) - - pwent = pwd.getpwnam(user) - - ssh_dir = '%s/.ssh' % pwent.pw_dir - if not os.path.exists(ssh_dir): - os.mkdir(ssh_dir) - os.chown(ssh_dir, pwent.pw_uid, pwent.pw_gid) - - authorized_keys = '%s/.ssh/authorized_keys' % pwent.pw_dir - fp = open(authorized_keys, 'a') - fp.write(''.join(['%s%s\n' % (key_prefix, key) for key in keys])) - fp.close() - - os.chown(authorized_keys, pwent.pw_uid, pwent.pw_gid) - - os.umask(saved_umask) - -def send_ssh_keys_to_console(): - send_keys_sh = """ - { - echo - echo "#############################################################" - echo "-----BEGIN SSH HOST KEY FINGERPRINTS-----" - ssh-keygen -l -f /etc/ssh/ssh_host_rsa_key.pub - ssh-keygen -l -f /etc/ssh/ssh_host_dsa_key.pub - echo "-----END SSH HOST KEY FINGERPRINTS-----" - echo "#############################################################" - } | logger -p user.info -s -t "ec2" - """ - subprocess.call(('sh', '-c', send_keys_sh)) - - -def warn(str): - sys.stderr.write("Warning:%s\n" % str) - -# srclist is a list of dictionaries, -# each entry must have: 'source' -# may have: key, ( keyid and keyserver) -def add_sources(srclist): - elst = [] - - for ent in srclist: - if not ent.has_key('source'): - elst.append([ "", "missing source" ]) - continue - - source=ent['source'] - if source.startswith("ppa:"): - try: util.subp(["add-apt-repository",source]) - except: - elst.append([source, "add-apt-repository failed"]) - continue - - if not ent.has_key('filename'): - ent['filename']='cloud_config_sources.list' - - if not ent['filename'].startswith("/"): - ent['filename'] = "%s/%s" % \ - ("/etc/apt/sources.list.d/", ent['filename']) - - if ( ent.has_key('keyid') and not ent.has_key('key') ): - ks = "keyserver.ubuntu.com" - if ent.has_key('keyserver'): ks = ent['keyserver'] - try: - ent['key'] = util.getkeybyid(ent['keyid'], ks) - except: - elst.append([source,"failed to get key from %s" % ks]) - continue - - if ent.has_key('key'): - try: util.subp(('apt-key', 'add', '-'), ent['key']) - except: - elst.append([source, "failed add key"]) - - try: util.write_file(ent['filename'], source + "\n") - except: - elst.append([source, "failed write to file %s" % ent['filename']]) - - return(elst) - - -def generate_sources_list(mirror): - stdout, stderr = subprocess.Popen(['lsb_release', '-cs'], stdout=subprocess.PIPE).communicate() - codename = stdout.strip() - - util.render_to_file('sources.list', '/etc/apt/sources.list', \ - { 'mirror' : mirror, 'codename' : codename }) - -def handle_updates_check(cfg): - if not util.get_cfg_option_bool(cfg, 'updates-check', True): - return - build_info = "/etc/cloud/build.info" - if not os.path.isfile(build_info): - warn("no %s file" % build_info) - - avail="%s/%s" % ( cloudinit.datadir, "available.build" ) - cmd=( "uec-query-builds", "--system-suite", "--config", "%s" % build_info, - "--output", "%s" % avail, "is-update-available" ) - try: - util.subp(cmd) - except: - warn("failed to execute uec-query-build for updates check") - - # add a cron entry for this hour and this minute every day - try: - cron=open("%s-%s" % (cronpre, "updates") ,"w") - cron.write("%s root %s\n" % \ - (time.strftime("%M %H * * *"),' '.join(cmd))) - cron.close() - except: - warn("failed to enable cron update system check") - -def handle_runcmd(cfg): - if not cfg.has_key("runcmd"): - return - outfile="%s/runcmd" % cloudinit.user_scripts_dir - - content="#!/bin/sh\n" - escaped="%s%s%s%s" % ( "'", '\\', "'", "'" ) - try: - for args in cfg["runcmd"]: - # if the item is a list, wrap all items in single tick - # if its not, then just write it directly - if isinstance(args,list): - fixed = [ ] - for f in args: - fixed.append("'%s'" % str(f).replace("'",escaped)) - content="%s%s\n" % ( content, ' '.join(fixed) ) - else: - content="%s%s\n" % ( content, str(args) ) - - util.write_file(outfile,content,0700) - except: - warn("failed to open %s for runcmd" % outfile) - -def mirror2lists_fileprefix(mirror): - file=mirror - # take of http:// or ftp:// - if file.endswith("/"): file=file[0:-1] - pos=file.find("://") - if pos >= 0: - file=file[pos+3:] - file=file.replace("/","_") - return file - -def rename_apt_lists(omirror,new_mirror,lists_d="/var/lib/apt/lists"): - - oprefix="%s/%s" % (lists_d,mirror2lists_fileprefix(omirror)) - nprefix="%s/%s" % (lists_d,mirror2lists_fileprefix(new_mirror)) - if(oprefix==nprefix): return - olen=len(oprefix) - for file in glob.glob("%s_*" % oprefix): - os.rename(file,"%s%s" % (nprefix, file[olen:])) diff --git a/cloudinit/CloudConfig/__init__.py b/cloudinit/CloudConfig/__init__.py new file mode 100644 index 00000000..600b21ab --- /dev/null +++ b/cloudinit/CloudConfig/__init__.py @@ -0,0 +1,57 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2008-2010 Canonical Ltd. +# +# Author: Chuck Short <chuck.short@canonical.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +import yaml +import cloudinit +import cloudinit.util as util +import sys + +per_instance="once-per-instance" +per_always="always" + +class CloudConfig(): + cfgfile = None + cfg = None + + def __init__(self,cfgfile): + self.cloud = cloudinit.CloudInit() + self.cfg = self.get_config_obj(cfgfile) + self.cloud.get_data_source() + + def get_config_obj(self,cfgfile): + f=file(cfgfile) + cfg=yaml.load(f.read()) + f.close() + if cfg is None: cfg = { } + return(util.mergedict(cfg,self.cloud.cfg)) + + def handle(self, name, args, freq=None): + try: + mod = __import__("cc_" + name.replace("-","_"),globals()) + def_freq = getattr(mod, "frequency",per_instance) + handler = getattr(mod, "handle") + + if not freq: + freq = def_freq + + self.cloud.sem_and_run(name, freq, handler, + [ name, self.cfg, self.cloud, cloudinit.log, args ]) + except: + cloudinit.log.error(traceback.format_exc()) + raise + diff --git a/cloudinit/CloudConfig/cc_apt_update_upgrade.py b/cloudinit/CloudConfig/cc_apt_update_upgrade.py new file mode 100644 index 00000000..ab2ece93 --- /dev/null +++ b/cloudinit/CloudConfig/cc_apt_update_upgrade.py @@ -0,0 +1,120 @@ +import cloudinit.util as util +import subprocess +import os + +def handle(name,cfg,cloud,log,args): + update = util.get_cfg_option_bool(cfg, 'apt_update', False) + upgrade = util.get_cfg_option_bool(cfg, 'apt_upgrade', False) + + if not util.get_cfg_option_bool(cfg, \ + 'apt_preserve_sources_list', False): + if cfg.has_key("apt_mirror"): + mirror = cfg["apt_mirror"] + else: + mirror = cloud.get_mirror() + generate_sources_list(mirror) + old_mir = util.get_cfg_option_str(cfg,'apt_old_mirror', \ + "archive.ubuntu.com/ubuntu") + rename_apt_lists(old_mir, mirror) + + # process 'apt_sources' + if cfg.has_key('apt_sources'): + errors = add_sources(cfg['apt_sources']) + for e in errors: + log.warn("Source Error: %s\n" % ':'.join(e)) + + pkglist = [] + if 'packages' in cfg: + if isinstance(cfg['packages'],list): + pkglist = cfg['packages'] + else: pkglist.append(cfg['packages']) + + if update or upgrade or pkglist: + #retcode = subprocess.call(list) + subprocess.Popen(['apt-get', 'update']).communicate() + + e=os.environ.copy() + e['DEBIAN_FRONTEND']='noninteractive' + + if upgrade: + subprocess.Popen(['apt-get', 'upgrade', '--assume-yes'], env=e).communicate() + + if pkglist: + cmd=['apt-get', 'install', '--assume-yes'] + cmd.extend(pkglist) + subprocess.Popen(cmd, env=e).communicate() + + return(True) + +def mirror2lists_fileprefix(mirror): + file=mirror + # take of http:// or ftp:// + if file.endswith("/"): file=file[0:-1] + pos=file.find("://") + if pos >= 0: + file=file[pos+3:] + file=file.replace("/","_") + return file + +def rename_apt_lists(omirror,new_mirror,lists_d="/var/lib/apt/lists"): + + oprefix="%s/%s" % (lists_d,mirror2lists_fileprefix(omirror)) + nprefix="%s/%s" % (lists_d,mirror2lists_fileprefix(new_mirror)) + if(oprefix==nprefix): return + olen=len(oprefix) + for file in glob.glob("%s_*" % oprefix): + os.rename(file,"%s%s" % (nprefix, file[olen:])) + +def generate_sources_list(mirror): + stdout, stderr = subprocess.Popen(['lsb_release', '-cs'], stdout=subprocess.PIPE).communicate() + codename = stdout.strip() + + util.render_to_file('sources.list', '/etc/apt/sources.list', \ + { 'mirror' : mirror, 'codename' : codename }) + +# srclist is a list of dictionaries, +# each entry must have: 'source' +# may have: key, ( keyid and keyserver) +def add_sources(srclist): + elst = [] + + for ent in srclist: + if not ent.has_key('source'): + elst.append([ "", "missing source" ]) + continue + + source=ent['source'] + if source.startswith("ppa:"): + try: util.subp(["add-apt-repository",source]) + except: + elst.append([source, "add-apt-repository failed"]) + continue + + if not ent.has_key('filename'): + ent['filename']='cloud_config_sources.list' + + if not ent['filename'].startswith("/"): + ent['filename'] = "%s/%s" % \ + ("/etc/apt/sources.list.d/", ent['filename']) + + if ( ent.has_key('keyid') and not ent.has_key('key') ): + ks = "keyserver.ubuntu.com" + if ent.has_key('keyserver'): ks = ent['keyserver'] + try: + ent['key'] = util.getkeybyid(ent['keyid'], ks) + except: + elst.append([source,"failed to get key from %s" % ks]) + continue + + if ent.has_key('key'): + try: util.subp(('apt-key', 'add', '-'), ent['key']) + except: + elst.append([source, "failed add key"]) + + try: util.write_file(ent['filename'], source + "\n") + except: + elst.append([source, "failed write to file %s" % ent['filename']]) + + return(elst) + + diff --git a/cloudinit/CloudConfig/cc_disable_ec2_metadata.py b/cloudinit/CloudConfig/cc_disable_ec2_metadata.py new file mode 100644 index 00000000..a4b4280f --- /dev/null +++ b/cloudinit/CloudConfig/cc_disable_ec2_metadata.py @@ -0,0 +1,9 @@ +import cloudinit.util as util +from cloudinit.CloudConfig import per_always + +frequency = per_always + +def handle(name,cfg,cloud,log,args): + if util.get_cfg_option_bool(cfg, "disable_ec2_metadata", False): + fwall="route add -host 169.254.169.254 reject" + subprocess.call(fwall.split(' ')) diff --git a/cloudinit/CloudConfig/cc_foo.py b/cloudinit/CloudConfig/cc_foo.py new file mode 100644 index 00000000..dc5d453e --- /dev/null +++ b/cloudinit/CloudConfig/cc_foo.py @@ -0,0 +1,7 @@ +import cloudinit +import cloudinit.util as util +from cloudinit.CloudConfig import per_instance + +frequency = per_instance +def handle(name,cfg,cloud,log,args): + print "hi" diff --git a/cloudinit/CloudConfig/cc_mounts.py b/cloudinit/CloudConfig/cc_mounts.py new file mode 100644 index 00000000..8bd16240 --- /dev/null +++ b/cloudinit/CloudConfig/cc_mounts.py @@ -0,0 +1,131 @@ +import cloudinit.util as util +import os +import re +import string + +def handle(name,cfg,cloud,log,args): + # these are our default set of mounts + defmnts = [ [ "ephemeral0", "/mnt", "auto", "defaults", "0", "0" ], + [ "swap", "none", "swap", "sw", "0", "0" ] ] + + # fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno + defvals = [ None, None, "auto", "defaults", "0", "0" ] + + cfgmnt = [ ] + if cfg.has_key("mounts"): + cfgmnt = cfg["mounts"] + + for i in range(len(cfgmnt)): + # skip something that wasn't a list + if not isinstance(cfgmnt[i],list): continue + + # workaround, allow user to specify 'ephemeral' + # rather than more ec2 correct 'ephemeral0' + if cfgmnt[i][0] == "ephemeral": + cfgmnt[i][0] = "ephemeral0" + + newname = cfgmnt[i][0] + if not newname.startswith("/"): + newname = cloud.device_name_to_device(cfgmnt[i][0]) + if newname is not None: + cfgmnt[i][0] = newname + else: + # there is no good way of differenciating between + # a name that *couldn't* exist in the md service and + # one that merely didnt + # in order to allow user to specify 'sda3' rather + # than '/dev/sda3', go through some hoops + ok = False + for f in [ "/", "sd", "hd", "vd", "xvd" ]: + if cfgmnt[i][0].startswith(f): + ok = True + break + if not ok: + cfgmnt[i][1] = None + + for i in range(len(cfgmnt)): + # fill in values with + for j in range(len(defvals)): + if len(cfgmnt[i]) <= j: + cfgmnt[i].append(defvals[j]) + elif cfgmnt[i][j] is None: + cfgmnt[i][j] = defvals[j] + + if not cfgmnt[i][0].startswith("/"): + cfgmnt[i][0]="/dev/%s" % cfgmnt[i][0] + + # if the second entry in the list is 'None' this + # clears all previous entries of that same 'fs_spec' + # (fs_spec is the first field in /etc/fstab, ie, that device) + if cfgmnt[i][1] is None: + for j in range(i): + if cfgmnt[j][0] == cfgmnt[i][0]: + cfgmnt[j][1] = None + + + # for each of the "default" mounts, add them only if no other + # entry has the same device name + for defmnt in defmnts: + devname = cloud.device_name_to_device(defmnt[0]) + if devname is None: continue + if devname.startswith("/"): + defmnt[0] = devname + else: + defmnt[0] = "/dev/%s" % devname + + cfgmnt_has = False + for cfgm in cfgmnt: + if cfgm[0] == defmnt[0]: + cfgmnt_has = True + break + + if cfgmnt_has: continue + cfgmnt.append(defmnt) + + + # now, each entry in the cfgmnt list has all fstab values + # if the second field is None (not the string, the value) we skip it + actlist = filter(lambda x: x[1] is not None, cfgmnt) + + if len(actlist) == 0: return + + comment="comment=cloudconfig" + cc_lines = [ ] + needswap = False + dirs = [ ] + for line in actlist: + # write 'comment' in the fs_mntops, entry, claiming this + line[3]="%s,comment=cloudconfig" % line[3] + if line[2] == "swap": needswap = True + if line[1].startswith("/"): dirs.append(line[1]) + cc_lines.append('\t'.join(line)) + + fstab_lines = [ ] + fstab=open("/etc/fstab","r+") + ws = re.compile("[%s]+" % string.whitespace) + for line in fstab.read().splitlines(): + try: + toks = ws.split(line) + if toks[3].find(comment) != -1: continue + except: + pass + fstab_lines.append(line) + + fstab_lines.extend(cc_lines) + + fstab.seek(0) + fstab.write("%s\n" % '\n'.join(fstab_lines)) + fstab.truncate() + fstab.close() + + if needswap: + try: util.subp(("swapon", "-a")) + except: log.warn("Failed to enable swap") + + for d in dirs: + if os.path.exists(d): continue + try: os.makedirs(d) + except: log.warn("Failed to make '%s' config-mount\n",d) + + try: util.subp(("mount","-a")) + except: pass diff --git a/cloudinit/CloudConfig/cc_puppet.py b/cloudinit/CloudConfig/cc_puppet.py new file mode 100644 index 00000000..542bced0 --- /dev/null +++ b/cloudinit/CloudConfig/cc_puppet.py @@ -0,0 +1,59 @@ +import os +import subprocess + +def handle(name,cfg,cloud,log,args): + # If there isn't a puppet key in the configuration don't do anything + if not cfg.has_key('puppet'): return + puppet_cfg = cfg['puppet'] + # Start by installing the puppet package ... + e=os.environ.copy() + e['DEBIAN_FRONTEND']='noninteractive' + # Make sure that the apt database is updated since it's not run by + # default + # Note: we should have a helper to check if apt-get update + # has already been run on this instance to speed the boot time. + subprocess.check_call(['apt-get', 'update'], env=e) + subprocess.check_call(['apt-get', 'install', '--assume-yes', + 'puppet'], env=e) + # ... and then update the puppet configuration + if puppet_cfg.has_key('conf'): + # Add all sections from the conf object to puppet.conf + puppet_conf_fh = open('/etc/puppet/puppet.conf', 'a') + for cfg_name, cfg in puppet_cfg['conf'].iteritems(): + # ca_cert configuration is a special case + # Dump the puppetmaster ca certificate in the correct place + if cfg_name == 'ca_cert': + # Puppet ssl sub-directory isn't created yet + # Create it with the proper permissions and ownership + os.makedirs('/var/lib/puppet/ssl') + os.chmod('/var/lib/puppet/ssl', 0771) + os.chown('/var/lib/puppet/ssl', + pwd.getpwnam('puppet').pw_uid, 0) + os.makedirs('/var/lib/puppet/ssl/certs/') + os.chown('/var/lib/puppet/ssl/certs/', + pwd.getpwnam('puppet').pw_uid, 0) + ca_fh = open('/var/lib/puppet/ssl/certs/ca.pem', 'w') + ca_fh.write(cfg) + ca_fh.close() + os.chown('/var/lib/puppet/ssl/certs/ca.pem', + pwd.getpwnam('puppet').pw_uid, 0) + else: + puppet_conf_fh.write("\n[%s]\n" % (cfg_name)) + for o, v in cfg.iteritems(): + if o == 'certname': + # Expand %f as the fqdn + v = v.replace("%f", socket.getfqdn()) + # Expand %i as the instance id + v = v.replace("%i", + cloud.datasource.get_instance_id()) + # certname needs to be downcase + v = v.lower() + puppet_conf_fh.write("%s=\"%s\"\n" % (o, v)) + puppet_conf_fh.close() + # Set puppet default file to automatically start + subprocess.check_call(['sed', '-i', + '-e', 's/^START=.*/START=yes/', + '/etc/default/puppet']) + # Start puppetd + subprocess.check_call(['service', 'puppet', 'start']) + diff --git a/cloudinit/CloudConfig/cc_runcmd.py b/cloudinit/CloudConfig/cc_runcmd.py new file mode 100644 index 00000000..fb5739da --- /dev/null +++ b/cloudinit/CloudConfig/cc_runcmd.py @@ -0,0 +1,25 @@ +import cloudinit +import cloudinit.util as util + +def handle(name,cfg,cloud,log,args): + if not cfg.has_key("runcmd"): + return + outfile="%s/runcmd" % cloudinit.user_scripts_dir + + content="#!/bin/sh\n" + escaped="%s%s%s%s" % ( "'", '\\', "'", "'" ) + try: + for args in cfg["runcmd"]: + # if the item is a list, wrap all items in single tick + # if its not, then just write it directly + if isinstance(args,list): + fixed = [ ] + for f in args: + fixed.append("'%s'" % str(f).replace("'",escaped)) + content="%s%s\n" % ( content, ' '.join(fixed) ) + else: + content="%s%s\n" % ( content, str(args) ) + + util.write_file(outfile,content,0700) + except: + log.warn("failed to open %s for runcmd" % outfile) diff --git a/cloudinit/CloudConfig/cc_ssh.py b/cloudinit/CloudConfig/cc_ssh.py new file mode 100644 index 00000000..08d1e243 --- /dev/null +++ b/cloudinit/CloudConfig/cc_ssh.py @@ -0,0 +1,91 @@ +import cloudinit.util as util +import os +import glob +import subprocess + +def handle(name,cfg,cloud,log,args): + # remove the static keys from the pristine image + for f in glob.glob("/etc/ssh/ssh_host_*_key*"): + try: os.unlink(f) + except: pass + + if cfg.has_key("ssh_keys"): + # if there are keys in cloud-config, use them + key2file = { + "rsa_private" : ("/etc/ssh/ssh_host_rsa_key", 0600), + "rsa_public" : ("/etc/ssh/ssh_host_rsa_key.pub", 0644), + "dsa_private" : ("/etc/ssh/ssh_host_dsa_key", 0600), + "dsa_public" : ("/etc/ssh/ssh_host_dsa_key.pub", 0644) + } + + for key,val in cfg["ssh_keys"].items(): + if key2file.has_key(key): + util.write_file(key2file[key][0],val,key2file[key][1]) + else: + # if not, generate them + genkeys ='ssh-keygen -f /etc/ssh/ssh_host_rsa_key -t rsa -N ""; ' + genkeys+='ssh-keygen -f /etc/ssh/ssh_host_dsa_key -t dsa -N ""; ' + subprocess.call(('sh', '-c', "{ %s } </dev/null" % (genkeys))) + + try: + user = util.get_cfg_option_str(cfg,'user') + disable_root = util.get_cfg_option_bool(cfg, "disable_root", True) + keys = cloud.get_public_ssh_keys() + + if cfg.has_key("ssh_authorized_keys"): + cfgkeys = cfg["ssh_authorized_keys"] + keys.extend(cfgkeys) + + apply_credentials(keys,user,disable_root) + except: + log.warn("applying credentials failed!\n") + + send_ssh_keys_to_console() + +def send_ssh_keys_to_console(): + send_keys_sh = """ + { + echo + echo "#############################################################" + echo "-----BEGIN SSH HOST KEY FINGERPRINTS-----" + ssh-keygen -l -f /etc/ssh/ssh_host_rsa_key.pub + ssh-keygen -l -f /etc/ssh/ssh_host_dsa_key.pub + echo "-----END SSH HOST KEY FINGERPRINTS-----" + echo "#############################################################" + } | logger -p user.info -s -t "ec2" + """ + subprocess.call(('sh', '-c', send_keys_sh)) + +def apply_credentials(keys, user, disable_root): + keys = set(keys) + if user: + setup_user_keys(keys, user, '') + + if disable_root: + key_prefix = 'command="echo \'Please login as the %s user rather than root user.\';echo;sleep 10" ' % user + else: + key_prefix = '' + + setup_user_keys(keys, 'root', key_prefix) + +def setup_user_keys(keys, user, key_prefix): + import pwd + saved_umask = os.umask(077) + + pwent = pwd.getpwnam(user) + + ssh_dir = '%s/.ssh' % pwent.pw_dir + if not os.path.exists(ssh_dir): + os.mkdir(ssh_dir) + os.chown(ssh_dir, pwent.pw_uid, pwent.pw_gid) + + authorized_keys = '%s/.ssh/authorized_keys' % pwent.pw_dir + fp = open(authorized_keys, 'a') + fp.write(''.join(['%s%s\n' % (key_prefix, key) for key in keys])) + fp.close() + + os.chown(authorized_keys, pwent.pw_uid, pwent.pw_gid) + + os.umask(saved_umask) + + diff --git a/cloudinit/CloudConfig/cc_updates_check.py b/cloudinit/CloudConfig/cc_updates_check.py new file mode 100644 index 00000000..b24b8c61 --- /dev/null +++ b/cloudinit/CloudConfig/cc_updates_check.py @@ -0,0 +1,32 @@ +import cloudinit.util as util +import cloudinit +import os +import time + +cronpre = "/etc/cron.d/cloudinit" + +def handle(name,cfg,cloud,log,args): + if not util.get_cfg_option_bool(cfg, 'updates-check', True): + return + build_info = "/etc/cloud/build.info" + if not os.path.isfile(build_info): + log.warn("no %s file" % build_info) + + avail="%s/%s" % ( cloudinit.datadir, "available.build" ) + cmd=( "uec-query-builds", "--system-suite", "--config", "%s" % build_info, + "--output", "%s" % avail, "is-update-available" ) + try: + util.subp(cmd) + except: + log.warn("failed to execute uec-query-build for updates check") + + # add a cron entry for this hour and this minute every day + try: + cron=open("%s-%s" % (cronpre, "updates") ,"w") + cron.write("%s root %s\n" % \ + (time.strftime("%M %H * * *"),' '.join(cmd))) + cron.close() + except: + log.warn("failed to enable cron update system check") + raise + diff --git a/cloudinit/__init__.py b/cloudinit/__init__.py index d8f2c8c9..2b76a73f 100644 --- a/cloudinit/__init__.py +++ b/cloudinit/__init__.py @@ -26,6 +26,7 @@ userdata_raw = datadir + '/user-data.txt' userdata = datadir + '/user-data.txt.i' user_scripts_dir = datadir + "/scripts" cloud_config = datadir + '/cloud-config.txt' +#cloud_config = '/tmp/cloud-config.txt' data_source_cache = cachedir + '/obj.pkl' system_config = '/etc/cloud/cloud.cfg' cfg_env_name = "CLOUD_CFG" @@ -36,12 +37,13 @@ user: ubuntu disable_root: 1 cloud_config_modules: + - mounts + - ssh - apt-update-upgrade - - config-misc - - config-mounts - - config-puppet - - config-ssh + - puppet + - updates-check - disable-ec2-metadata + - runcmd log_cfg: built_in """ @@ -316,7 +318,9 @@ class CloudInit: # if 'clear_on_fail' is True and func throws an exception # then remove the lock (so it would run again) def sem_and_run(self,semname,freq,func,args=[],clear_on_fail=False): - if self.sem_has_run(semname,freq): return + if self.sem_has_run(semname,freq): + log.debug("%s already ran %s", semname, freq) + return try: if not self.sem_acquire(semname,freq): raise Exception("Failed to acquire lock on %s\n" % semname) diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index 1bde5b10..144820a9 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -196,7 +196,7 @@ runcmd: # once-per-instance # always # a python file in the CloudConfig/ module directory named -# cloud_config_name.py +# cc_<name>.py # example: cloud_config_modules: - [apt-update-upgrade, always] @@ -24,12 +24,12 @@ import os.path import subprocess setup(name='cloud-init', - version='0.5.10', + version='0.5.11', description='EC2 initialisation magic', author='Scott Moser', author_email='scott.moser@canonical.com', url='http://launchpad.net/cloud-init/', - packages=['cloudinit'], + packages=['cloudinit', 'cloudinit.CloudConfig' ], scripts=['cloud-init.py', 'cloud-init-run-module.py', 'cloud-init-cfg.py' diff --git a/upstart/cloud-apt-update-upgrade.conf b/upstart/cloud-apt-update-upgrade.conf deleted file mode 100644 index ef907360..00000000 --- a/upstart/cloud-apt-update-upgrade.conf +++ /dev/null @@ -1,7 +0,0 @@ -# cloud-apt-update-upgrade - Update software at boot -description "Update software at boot" - -start on filesystem -console output - -exec cloud-init-cfg apt-update-upgrade diff --git a/upstart/cloud-config-cat.conf.debug b/upstart/cloud-config-cat.conf.debug deleted file mode 100644 index 9cc3b2ce..00000000 --- a/upstart/cloud-config-cat.conf.debug +++ /dev/null @@ -1,7 +0,0 @@ -description "Cat the Config" - -start on cloud-config -console output -task - -exec cloud-init-run-module once-per-instance catconfig execute cat $CLOUD_CFG diff --git a/upstart/cloud-config-misc.conf b/upstart/cloud-config-misc.conf deleted file mode 100644 index 9133794e..00000000 --- a/upstart/cloud-config-misc.conf +++ /dev/null @@ -1,9 +0,0 @@ -# cloud-config-misc - Miscellaneous cloud-config items -# includes enabling 'updates check' -description "Run miscellaneous cloud-config items" - -start on filesystem -console output -task - -exec cloud-init-cfg config-misc diff --git a/upstart/cloud-config-mounts.conf b/upstart/cloud-config-mounts.conf deleted file mode 100644 index cbe15256..00000000 --- a/upstart/cloud-config-mounts.conf +++ /dev/null @@ -1,9 +0,0 @@ -# cloud-config-mounts - setup mount points from cloud-config -# includes enabling swap -description "Setup mount points in fstab per config" - -start on filesystem -console output -task - -exec cloud-init-cfg config-mounts diff --git a/upstart/cloud-config-puppet.conf b/upstart/cloud-config-puppet.conf deleted file mode 100644 index a42b0dc6..00000000 --- a/upstart/cloud-config-puppet.conf +++ /dev/null @@ -1,11 +0,0 @@ -# cloud-config-puppet - Setup puppetd -description "Setup puppetd" - -# Make sure puppet is started after repositories have been setup. -# This can be useful if the puppet package should be pulled from -# a different repository. -start on stopped cloud-apt-update-upgrade -console output -task - -exec cloud-init-cfg config-puppet diff --git a/upstart/cloud-config-ssh.conf b/upstart/cloud-config-ssh.conf deleted file mode 100644 index 98b3d6cc..00000000 --- a/upstart/cloud-config-ssh.conf +++ /dev/null @@ -1,8 +0,0 @@ -# cloud-config-ssh - obtain ssh keys from metadata service -description "Download preconfigured ssh keys" - -start on filesystem -console output -task - -exec cloud-init-cfg config-ssh diff --git a/upstart/cloud-config.conf b/upstart/cloud-config.conf new file mode 100644 index 00000000..12caaaa7 --- /dev/null +++ b/upstart/cloud-config.conf @@ -0,0 +1,7 @@ +# cloud-config - Handle applying the settings specified in cloud-config +description "Handle applying cloud-config" + +start on filesystem and started syslogd +console output + +exec cloud-init-cfg all diff --git a/upstart/cloud-disable-ec2-metadata.conf b/upstart/cloud-disable-ec2-metadata.conf deleted file mode 100644 index 7cb044bc..00000000 --- a/upstart/cloud-disable-ec2-metadata.conf +++ /dev/null @@ -1,8 +0,0 @@ -# cloud-disable-ec2-metadata - Disable the ec2 metadata service -description "Disable the ec2 metadata service" - -start on filesystem -console output -task - -exec cloud-init-cfg disable-ec2-metadata diff --git a/upstart/cloud-ebs-mounts.conf.disabled b/upstart/cloud-ebs-mounts.conf.disabled deleted file mode 100644 index 03ddfa40..00000000 --- a/upstart/cloud-ebs-mounts.conf.disabled +++ /dev/null @@ -1,12 +0,0 @@ -# ec2-ebs-mounts -# -# Mount EC2 EBS mount points - -description "Populate EBS mountpoints" - -start on cloud-config - -console output -task - -exec cloud-init-cfg ec2-ebs-mounts diff --git a/upstart/cloud-raid.conf.disabled b/upstart/cloud-raid.conf.disabled deleted file mode 100644 index d18dd551..00000000 --- a/upstart/cloud-raid.conf.disabled +++ /dev/null @@ -1,12 +0,0 @@ -# ec2-raid - Setup ephemeral storage RAID and mount points -# -# Setup ephemeral storage RAID and mount points - -description "Setup RAID storage and moint points" - -start on (cloud-config - and local-filesystems) -console output -task - -exec cloud-init-cfg setup-raid diff --git a/upstart/cloud-run-user-script.conf b/upstart/cloud-run-user-script.conf index 886781b3..e50006d4 100644 --- a/upstart/cloud-run-user-script.conf +++ b/upstart/cloud-run-user-script.conf @@ -2,7 +2,7 @@ # stored in /var/lib/cloud/scripts by the initial cloudinit upstart job description "execute cloud user scripts" -start on (stopped rc RUNLEVEL=[2345] and stopped cloud-config-misc) +start on (stopped rc RUNLEVEL=[2345] and stopped cloud-config) console output task diff --git a/upstart/cloud-runurl.conf.disabled b/upstart/cloud-runurl.conf.disabled deleted file mode 100644 index 3e2c46e0..00000000 --- a/upstart/cloud-runurl.conf.disabled +++ /dev/null @@ -1,13 +0,0 @@ -# ec2-runurl - Run runurl at boot -# -# Runurl at boot - -description "Run runurl" - -start on (cloud-config - and local-filesystems - and net-device-ifup IFACE=eth0) -console output -task - -exec cloud-init-cfg runurl |