From c59e06c6d20ce585927f336630e8ae3cca12c110 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Wed, 12 Sep 2012 10:59:19 +0200 Subject: Initial support for OpenNebula's contextualization disk. --- cloudinit/sources/DataSourceOpenNebula.py | 227 ++++++++++++++++++++++++++++++ 1 file changed, 227 insertions(+) create mode 100644 cloudinit/sources/DataSourceOpenNebula.py (limited to 'cloudinit/sources/DataSourceOpenNebula.py') diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py new file mode 100644 index 00000000..03c0103e --- /dev/null +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -0,0 +1,227 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Yahoo! Inc. +# Copyright (C) 2012 CERIT-Scientific Cloud +# +# Author: Scott Moser +# Author: Joshua Harlow +# Author: Vlastimil Holer +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import re +import subprocess + +from cloudinit import log as logging +from cloudinit import sources +from cloudinit import util + +LOG = logging.getLogger(__name__) + +DEFAULT_IID = "iid-dsopennebula" +CONTEXT_DISK_FILES = ["context.sh"] + +class DataSourceOpenNebula(sources.DataSource): + def __init__(self, sys_cfg, distro, paths): + sources.DataSource.__init__(self, sys_cfg, distro, paths) + self.dsmode = 'local' + self.seed = None + self.seed_dir = os.path.join(paths.seed_dir, 'opennebula') + + def __str__(self): + mstr = "%s [seed=%s][dsmode=%s]" % (util.obj_name(self), + self.seed, self.dsmode) + return mstr + + def get_data(self): + defaults = { + "instance-id": DEFAULT_IID, + "dsmode": self.dsmode, + } + + found = None + md = {} + ud = "" + + results = {} + if os.path.isdir(self.seed_dir): + try: + results=read_on_context_device_dir(self.seed_dir) + found = self.seed_dir + except NonContextDeviceDir: + util.logexc(LOG, "Failed reading context device from %s", + self.seed_dir) + if not found: + devlist = find_candidate_devs() + for dev in devlist: + try: + results = util.mount_cb(dev, read_context_disk_dir) + found = dev + break + except (NonConfigDriveDir, util.MountFailedError): + pass + except BrokenConfigDriveDir: + util.logexc(LOG, "broken config drive: %s", dev) + + if not found: + return False + + md = results['metadata'] + md = util.mergedict(md, defaults) + + # update interfaces and ifup only on the local datasource + # this way the DataSourceConfigDriveNet doesn't do it also. +# if 'network-interfaces' in md and self.dsmode == "local": +# if md['dsmode'] == "pass": +# log.info("updating network interfaces from configdrive") +# else: +# log.debug("updating network interfaces from configdrive") +# +# util.write_file("/etc/network/interfaces", +# md['network-interfaces']) +# try: +# (out, err) = util.subp(['ifup', '--all']) +# if len(out) or len(err): +# log.warn("ifup --all had stderr: %s" % err) +# +# except subprocess.CalledProcessError as exc: +# log.warn("ifup --all failed: %s" % (exc.output[1])) +# + + if md['dsmode'] == self.dsmode: + self.seed = found + self.metadata = md + self.userdata_raw = ud + return True + + LOG.debug("%s: not claiming datasource, dsmode=%s", self, md['dsmode']) + return False + + +class DataSourceOpenNebulaNet(DataSourceOpenNebula): + dsmode = "net" + + +class NonContextDeviceDir(Exception): + pass + + +def find_candidate_devs(): + """ + Return a list of devices that may contain the context disk. + """ + by_fstype = util.find_devs_with("TYPE=iso9660") + by_label = util.find_devs_with("LABEL=CDROM") + + by_fstype.sort() + by_label.sort() + + # combine list of items by putting by-label items first + # followed by fstype items, but with dupes removed + combined = (by_label + [d for d in by_fstype if d not in by_label]) + + # We are looking for block device (sda, not sda1), ignore partitions + combined = [d for d in combined if d[-1] not in "0123456789"] + + return combined + + +def read_context_disk_dir(source_dir): + """ + read_context_disk_dir(source_dir): + read source_dir and return a tuple with metadata dict and user-data + string populated. If not a valid dir, raise a NonContextDeviceDir + """ + + found = {} + for af in CONTEXT_DISK_FILES: + fn = os.path.join(source_dir, af) + if os.path.isfile(fn): + found[af] = fn + + if len(found) == 0: + raise NonContextDeviceDir("%s: %s" % (source_dir, "no files found")) + + context_sh = {} + results = { + 'userdata':None, + 'metadata':{}, + } + + if "context.sh" in found: + # let bash process the contextualization script; + # write out data in normalized output NAME=\$?'?VALUE'? + # TODO: don't trust context.sh! parse manually !!! + try: + BASH_CMD='VARS=`set | sort -u `;' \ + '. %s/context.sh;' \ + 'comm -23 <(set | sort -u) <(echo "$VARS") | egrep -v "^(VARS|PIPESTATUS|_)="' + + (out,err) = util.subp(['bash', + '--noprofile', + '--norc', + '-c', + BASH_CMD % (source_dir) ]) + + for (key,value) in [ l.split('=',1) for l in out.rstrip().split("\n") ]: + # with backslash escapes + r=re.match("^\$'(.*)'$",value) + if r: + context_sh[key.lower()]=r.group(1).\ + replace('\\\\','\\').\ + replace('\\t','\t').\ + replace('\\n','\n').\ + replace("\\'","'") + else: + # multiword values + r=re.match("^'(.*)'$",value) + if r: + context_sh[key.lower()]=r.group(1) + else: + # simple values + context_sh[key.lower()]=value + except subprocess.CalledProcessError as exc: + LOG.warn("context script faled to read" % (exc.output[1])) + results['metadata']=context_sh + + # process single or multiple SSH keys + if "ssh_key" in context_sh: + lines = context_sh.get('ssh_key').splitlines() + results['metadata']['public-keys'] = [l for l in lines + if len(l) and not l.startswith("#")] + + # custom hostname + if 'hostname' in context_sh: + results['metadata']['local-hostname'] = context_sh['hostname'] + + # raw user data + if "user_data" in context_sh: + results['userdata'] = context_sh["user_data"] + if "userdata" in context_sh: + results['userdata'] = context_sh["userdata"] + + return results + + +# Used to match classes to dependencies +datasources = [ + (DataSourceOpenNebula, (sources.DEP_FILESYSTEM, )), + (DataSourceOpenNebulaNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), +] + + +# Return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return sources.list_from_depends(depends, datasources) -- cgit v1.2.3 From 0659e75ee2a2c25c113e8e652a3366433e76eff0 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Wed, 19 Sep 2012 15:32:10 +0200 Subject: Use and resolve PUBLIC_IP from context disk if no HOSTNAME is specified. --- cloudinit/sources/DataSourceOpenNebula.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'cloudinit/sources/DataSourceOpenNebula.py') diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 03c0103e..e8e79c7e 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -109,6 +109,8 @@ class DataSourceOpenNebula(sources.DataSource): LOG.debug("%s: not claiming datasource, dsmode=%s", self, md['dsmode']) return False + def get_hostname(self, fqdn=False, resolve_ip=True): + return sources.DataSource.get_hostname(self, fqdn, resolve_ip) class DataSourceOpenNebulaNet(DataSourceOpenNebula): dsmode = "net" @@ -205,6 +207,8 @@ def read_context_disk_dir(source_dir): # custom hostname if 'hostname' in context_sh: results['metadata']['local-hostname'] = context_sh['hostname'] + elif 'public_ip'in context_sh: + results['metadata']['local-hostname'] = context_sh['public_ip'] # raw user data if "user_data" in context_sh: -- cgit v1.2.3 From ea42bf67999c1ef61fa46ab805b0093b95dc575f Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Thu, 20 Sep 2012 18:42:18 +0200 Subject: Configurable dsmode. Resolve IPv4 hostname only with dsmode=net. --- cloudinit/sources/DataSourceOpenNebula.py | 42 ++++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 9 deletions(-) (limited to 'cloudinit/sources/DataSourceOpenNebula.py') diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index e8e79c7e..0b498a54 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -32,6 +32,7 @@ LOG = logging.getLogger(__name__) DEFAULT_IID = "iid-dsopennebula" CONTEXT_DISK_FILES = ["context.sh"] +VALID_DSMODES = ("local", "net", "disabled") class DataSourceOpenNebula(sources.DataSource): def __init__(self, sys_cfg, distro, paths): @@ -81,6 +82,20 @@ class DataSourceOpenNebula(sources.DataSource): md = results['metadata'] md = util.mergedict(md, defaults) + dsmode = results.get('dsmode', None) + if dsmode not in VALID_DSMODES + (None,): + LOG.warn("user specified invalid mode: %s" % dsmode) + dsmode = None + + if (dsmode is None) and self.ds_cfg.get('dsmode'): + dsmode = self.ds_cfg.get('dsmode') + else: + dsmode = self.dsmode + + if dsmode == "disabled": + # most likely user specified + return False + # update interfaces and ifup only on the local datasource # this way the DataSourceConfigDriveNet doesn't do it also. # if 'network-interfaces' in md and self.dsmode == "local": @@ -100,20 +115,29 @@ class DataSourceOpenNebula(sources.DataSource): # log.warn("ifup --all failed: %s" % (exc.output[1])) # - if md['dsmode'] == self.dsmode: - self.seed = found - self.metadata = md - self.userdata_raw = ud - return True + if dsmode != self.dsmode: + LOG.debug("%s: not claiming datasource, dsmode=%s", self, dsmode) + return False + + self.seed = found + self.metadata = md + self.userdata_raw = ud - LOG.debug("%s: not claiming datasource, dsmode=%s", self, md['dsmode']) - return False + return True - def get_hostname(self, fqdn=False, resolve_ip=True): + def get_hostname(self, fqdn=False, resolve_ip=None): + if resolve_ip is None: + if self.dsmode == 'net': + resolve_ip = True + else: + resolve_ip = False return sources.DataSource.get_hostname(self, fqdn, resolve_ip) + class DataSourceOpenNebulaNet(DataSourceOpenNebula): - dsmode = "net" + def __init__(self, sys_cfg, distro, paths): + DataSourceOpenNebula.__init__(self, sys_cfg, distro, paths) + self.dsmode = 'net' class NonContextDeviceDir(Exception): -- cgit v1.2.3 From 9eaafc7379e0c6ded70a851a703a5bc5c7e56e42 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Mon, 24 Sep 2012 13:28:51 +0200 Subject: Process userdata, ignored by mistake. --- cloudinit/sources/DataSourceOpenNebula.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'cloudinit/sources/DataSourceOpenNebula.py') diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 0b498a54..ad65a36e 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -54,7 +54,6 @@ class DataSourceOpenNebula(sources.DataSource): found = None md = {} - ud = "" results = {} if os.path.isdir(self.seed_dir): @@ -121,7 +120,7 @@ class DataSourceOpenNebula(sources.DataSource): self.seed = found self.metadata = md - self.userdata_raw = ud + self.userdata_raw = results.get('userdata') return True @@ -237,7 +236,7 @@ def read_context_disk_dir(source_dir): # raw user data if "user_data" in context_sh: results['userdata'] = context_sh["user_data"] - if "userdata" in context_sh: + elif "userdata" in context_sh: results['userdata'] = context_sh["userdata"] return results -- cgit v1.2.3 From 1567b1bc3b2948aa80e0b150d34542e2ff02428f Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Wed, 3 Oct 2012 13:26:36 +0200 Subject: Delete old ConfigDrive's code. --- cloudinit/sources/DataSourceOpenNebula.py | 19 ------------------- 1 file changed, 19 deletions(-) (limited to 'cloudinit/sources/DataSourceOpenNebula.py') diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index ad65a36e..325269f6 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -95,25 +95,6 @@ class DataSourceOpenNebula(sources.DataSource): # most likely user specified return False - # update interfaces and ifup only on the local datasource - # this way the DataSourceConfigDriveNet doesn't do it also. -# if 'network-interfaces' in md and self.dsmode == "local": -# if md['dsmode'] == "pass": -# log.info("updating network interfaces from configdrive") -# else: -# log.debug("updating network interfaces from configdrive") -# -# util.write_file("/etc/network/interfaces", -# md['network-interfaces']) -# try: -# (out, err) = util.subp(['ifup', '--all']) -# if len(out) or len(err): -# log.warn("ifup --all had stderr: %s" % err) -# -# except subprocess.CalledProcessError as exc: -# log.warn("ifup --all failed: %s" % (exc.output[1])) -# - if dsmode != self.dsmode: LOG.debug("%s: not claiming datasource, dsmode=%s", self, dsmode) return False -- cgit v1.2.3 From 0e7ce376fdb84345c97f936f5ccb1fee964a4c03 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Wed, 3 Oct 2012 13:29:24 +0200 Subject: Delete hyphen. --- cloudinit/sources/DataSourceOpenNebula.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/sources/DataSourceOpenNebula.py') diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 325269f6..00c076e6 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -2,7 +2,7 @@ # # Copyright (C) 2012 Canonical Ltd. # Copyright (C) 2012 Yahoo! Inc. -# Copyright (C) 2012 CERIT-Scientific Cloud +# Copyright (C) 2012 CERIT Scientific Cloud # # Author: Scott Moser # Author: Joshua Harlow -- cgit v1.2.3 From 5746598d3cffe07dbae155347eaa44aae48ff572 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Thu, 20 Dec 2012 13:20:57 +0100 Subject: Change context variables .replace with .decode --- cloudinit/sources/DataSourceOpenNebula.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'cloudinit/sources/DataSourceOpenNebula.py') diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 00c076e6..1622a66e 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -185,11 +185,7 @@ def read_context_disk_dir(source_dir): # with backslash escapes r=re.match("^\$'(.*)'$",value) if r: - context_sh[key.lower()]=r.group(1).\ - replace('\\\\','\\').\ - replace('\\t','\t').\ - replace('\\n','\n').\ - replace("\\'","'") + context_sh[key.lower()]=r.group(1).decode('string_escape') else: # multiword values r=re.match("^'(.*)'$",value) -- cgit v1.2.3 From 21d3a5af7d2e1884009cfd1ad650a937438f5991 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Thu, 20 Dec 2012 13:57:57 +0100 Subject: Add explanation on how context variables parsing works. --- cloudinit/sources/DataSourceOpenNebula.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) (limited to 'cloudinit/sources/DataSourceOpenNebula.py') diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 1622a66e..a50b0c10 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -167,10 +167,24 @@ def read_context_disk_dir(source_dir): } if "context.sh" in found: - # let bash process the contextualization script; - # write out data in normalized output NAME=\$?'?VALUE'? - # TODO: don't trust context.sh! parse manually !!! try: + # Note: context.sh is a "shell" script with defined context + # variables, like: X="Y" . It's ready to use as a shell source + # e.g.: ". context.sh" and as a shell script it can also reference + # to already defined shell variables. So to have same context var. + # values as we can have in custom shell script, we use bash itself + # to read context.sh and dump variables in easily parsable way. + # + # normalized variables dump format (get by cmd "set"): + # 1. simple single word assignment ........ X=Y + # 2. multiword assignment ................. X='Y Z' + # 3. assignments with backslash escapes ... X=$'Y\nZ' + # + # how context variables are read: + # 1. list existing ("old") shell variables and store into $VARS + # 2. read context variables + # 3. use comm to filter "old" variables from all current + # variables and excl. few other vars with grep BASH_CMD='VARS=`set | sort -u `;' \ '. %s/context.sh;' \ 'comm -23 <(set | sort -u) <(echo "$VARS") | egrep -v "^(VARS|PIPESTATUS|_)="' -- cgit v1.2.3 From 46e646fabf3a0f2593dc9d9f231fd075134de36f Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Thu, 20 Dec 2012 15:17:32 +0100 Subject: Change subp exception handling to util.ProcessExecutionError --- cloudinit/sources/DataSourceOpenNebula.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cloudinit/sources/DataSourceOpenNebula.py') diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index a50b0c10..967d7170 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -208,8 +208,8 @@ def read_context_disk_dir(source_dir): else: # simple values context_sh[key.lower()]=value - except subprocess.CalledProcessError as exc: - LOG.warn("context script faled to read" % (exc.output[1])) + except util.ProcessExecutionError, _err: + LOG.warn("Failed to read context variables: %s" % (_err.message)) results['metadata']=context_sh # process single or multiple SSH keys -- cgit v1.2.3 From 2efdb4b8d1c17eea352ae0dc022d2ddca80833da Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Thu, 20 Dec 2012 17:16:47 +0100 Subject: Fix exception handlers for data read. Fix name read_context_disk_dir. --- cloudinit/sources/DataSourceOpenNebula.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'cloudinit/sources/DataSourceOpenNebula.py') diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 967d7170..2f1ce459 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -58,7 +58,7 @@ class DataSourceOpenNebula(sources.DataSource): results = {} if os.path.isdir(self.seed_dir): try: - results=read_on_context_device_dir(self.seed_dir) + results=read_context_disk_dir(self.seed_dir) found = self.seed_dir except NonContextDeviceDir: util.logexc(LOG, "Failed reading context device from %s", @@ -70,10 +70,8 @@ class DataSourceOpenNebula(sources.DataSource): results = util.mount_cb(dev, read_context_disk_dir) found = dev break - except (NonConfigDriveDir, util.MountFailedError): + except (NonContextDeviceDir, util.MountFailedError): pass - except BrokenConfigDriveDir: - util.logexc(LOG, "broken config drive: %s", dev) if not found: return False @@ -211,6 +209,8 @@ def read_context_disk_dir(source_dir): except util.ProcessExecutionError, _err: LOG.warn("Failed to read context variables: %s" % (_err.message)) results['metadata']=context_sh + else: + raise NonContextDeviceDir("Missing context.sh") # process single or multiple SSH keys if "ssh_key" in context_sh: -- cgit v1.2.3 From 45641d2e31ffa1b7235a2f36d234e804484ff4a3 Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Fri, 21 Dec 2012 12:47:59 +0100 Subject: Add OpenNebula contextualization options to cloud-init --- cloudinit/sources/DataSourceOpenNebula.py | 115 ++++++++++++++++++++++++++++-- 1 file changed, 111 insertions(+), 4 deletions(-) (limited to 'cloudinit/sources/DataSourceOpenNebula.py') diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 2f1ce459..8af6ad3d 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -101,6 +101,12 @@ class DataSourceOpenNebula(sources.DataSource): self.metadata = md self.userdata_raw = results.get('userdata') + if 'network-interfaces' in results: + self.distro.apply_network(results['network-interfaces']) + + if 'dns' in results: + self.distro.apply_resolv_conf(results['dns']) + return True def get_hostname(self, fqdn=False, resolve_ip=None): @@ -122,6 +128,93 @@ class NonContextDeviceDir(Exception): pass +class OpenNebulaNetwork(object): + REG_ETH=re.compile('^eth') + REG_DEV_MAC=re.compile('^(eth\d+).*HWaddr (..:..:..:..:..:..)') + + def __init__(self, ifconfig, context_sh): + self.ifconfig=ifconfig + self.context_sh=context_sh + self.ifaces=self.get_ifaces() + + def get_ifaces(self): + return [self.REG_DEV_MAC.search(f).groups() for f in self.ifconfig.split("\n") if self.REG_ETH.match(f)] + + def mac2ip(self, mac): + components=mac.split(':')[2:] + + return [str(int(c, 16)) for c in components] + + def get_ip(self, dev, components): + var_name=dev+'_ip' + if var_name in self.context_sh: + return self.context_sh[var_name] + else: + return '.'.join(components) + + def get_mask(self, dev, components): + var_name=dev+'_mask' + if var_name in self.context_sh: + return self.context_sh[var_name] + else: + return '255.255.255.0' + + def get_network(self, dev, components): + var_name=dev+'_network' + if var_name in self.context_sh: + return self.context_sh[var_name] + else: + return '.'.join(components[:-1])+'.0' + + def get_gateway(self, dev, components): + var_name=dev+'_gateway' + if var_name in self.context_sh: + return self.context_sh[var_name] + else: + None + + def gen_conf(self): + conf=[] + conf.append('auto lo') + conf.append('iface lo inet loopback') + conf.append('') + + for i in self.ifaces: + dev=i[0] + mac=i[1] + ip_components=self.mac2ip(mac) + + conf.append('auto '+dev) + conf.append('iface '+dev+' inet static') + conf.append(' address '+self.get_ip(dev, ip_components)) + conf.append(' network '+self.get_network(dev, ip_components)) + conf.append(' netmask '+self.get_mask(dev, ip_components)) + + gateway=self.get_gateway(dev, ip_components) + if gateway: + conf.append(' gateway '+gateway) + + conf.append('') + + return "\n".join(conf) + + def gen_dns(self): + dnss=[] + + if 'dns' in self.context_sh: + dnss.append('nameserver '+self.context_sh['dns']) + + keys=[d for d in self.context_sh.keys() if re.match('^eth\d+_dns$', d)] + + for k in sorted(keys): + dnss.append('nameserver '+self.context_sh[k]) + + if not dnss: + return None + else: + return "\n".join(dnss)+"\n" + + def find_candidate_devs(): """ Return a list of devices that may contain the context disk. @@ -136,9 +229,6 @@ def find_candidate_devs(): # followed by fstype items, but with dupes removed combined = (by_label + [d for d in by_fstype if d not in by_label]) - # We are looking for block device (sda, not sda1), ignore partitions - combined = [d for d in combined if d[-1] not in "0123456789"] - return combined @@ -213,8 +303,15 @@ def read_context_disk_dir(source_dir): raise NonContextDeviceDir("Missing context.sh") # process single or multiple SSH keys + ssh_key_var=None + if "ssh_key" in context_sh: - lines = context_sh.get('ssh_key').splitlines() + ssh_key_var="ssh_key" + elif "ssh_public_key" in context_sh: + ssh_key_var="ssh_public_key" + + if ssh_key_var: + lines = context_sh.get(ssh_key_var).splitlines() results['metadata']['public-keys'] = [l for l in lines if len(l) and not l.startswith("#")] @@ -223,6 +320,8 @@ def read_context_disk_dir(source_dir): results['metadata']['local-hostname'] = context_sh['hostname'] elif 'public_ip'in context_sh: results['metadata']['local-hostname'] = context_sh['public_ip'] + elif 'eth0_ip' in context_sh: + results['metadata']['local-hostname'] = context_sh['eth0_ip'] # raw user data if "user_data" in context_sh: @@ -230,6 +329,14 @@ def read_context_disk_dir(source_dir): elif "userdata" in context_sh: results['userdata'] = context_sh["userdata"] + (out, err) = util.subp(['/sbin/ifconfig', '-a']) + net=OpenNebulaNetwork(out, context_sh) + results['network-interfaces']=net.gen_conf() + + dns=net.gen_dns() + if dns: + results['dns']=dns + return results -- cgit v1.2.3 From 5aab64f71c1f07670b59dd7be18d704611dc0ab5 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Wed, 2 Jan 2013 15:20:33 +0100 Subject: Add OpenNebula.org copyright. --- cloudinit/sources/DataSourceOpenNebula.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'cloudinit/sources/DataSourceOpenNebula.py') diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 8af6ad3d..57b0c62c 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -2,11 +2,13 @@ # # Copyright (C) 2012 Canonical Ltd. # Copyright (C) 2012 Yahoo! Inc. -# Copyright (C) 2012 CERIT Scientific Cloud +# Copyright (C) 2012-2013 CERIT Scientific Cloud +# Copyright (C) 2012 OpenNebula.org # # Author: Scott Moser # Author: Joshua Harlow # Author: Vlastimil Holer +# Author: Javier Fontan # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as -- cgit v1.2.3 From 5bfbf7a81872afd426271b0ed3ed65e76b9a584e Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Fri, 4 Jan 2013 15:26:11 +0100 Subject: Append DNS related stuff to network interfaces configuration. Minor cleanups. --- cloudinit/sources/DataSourceOpenNebula.py | 76 +++++++++++++++++-------------- 1 file changed, 41 insertions(+), 35 deletions(-) (limited to 'cloudinit/sources/DataSourceOpenNebula.py') diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 57b0c62c..52474675 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -103,12 +103,11 @@ class DataSourceOpenNebula(sources.DataSource): self.metadata = md self.userdata_raw = results.get('userdata') - if 'network-interfaces' in results: + # apply static network configuration only in 'local' dsmode + if ('network-interfaces' in results and self.dsmode == "local"): + LOG.debug("Updating network interfaces from %s", self) self.distro.apply_network(results['network-interfaces']) - if 'dns' in results: - self.distro.apply_resolv_conf(results['dns']) - return True def get_hostname(self, fqdn=False, resolve_ip=None): @@ -175,7 +174,25 @@ class OpenNebulaNetwork(object): else: None + def get_dns(self, dev, components): + var_name=dev+'_dns' + if var_name in self.context_sh: + return self.context_sh[var_name] + else: + None + + def get_domain(self, dev, components): + var_name=dev+'_domain' + if var_name in self.context_sh: + return self.context_sh[var_name] + else: + None + def gen_conf(self): + global_dns=[] + if 'dns' in self.context_sh: + global_dns.append(self.context_sh['dns']) + conf=[] conf.append('auto lo') conf.append('iface lo inet loopback') @@ -196,25 +213,21 @@ class OpenNebulaNetwork(object): if gateway: conf.append(' gateway '+gateway) - conf.append('') - - return "\n".join(conf) - - def gen_dns(self): - dnss=[] + domain=self.get_domain(dev, ip_components) + if domain: + conf.append(' dns-search '+domain) - if 'dns' in self.context_sh: - dnss.append('nameserver '+self.context_sh['dns']) + # add global DNS servers to all interfaces + dns=self.get_dns(dev, ip_components) + if global_dns or dns: + all_dns=global_dns + if dns: + all_dns.append(dns) + conf.append(' dns-nameservers '+' '.join(all_dns)) - keys=[d for d in self.context_sh.keys() if re.match('^eth\d+_dns$', d)] - - for k in sorted(keys): - dnss.append('nameserver '+self.context_sh[k]) + conf.append('') - if not dnss: - return None - else: - return "\n".join(dnss)+"\n" + return "\n".join(conf) def find_candidate_devs(): @@ -280,10 +293,8 @@ def read_context_disk_dir(source_dir): 'comm -23 <(set | sort -u) <(echo "$VARS") | egrep -v "^(VARS|PIPESTATUS|_)="' (out,err) = util.subp(['bash', - '--noprofile', - '--norc', - '-c', - BASH_CMD % (source_dir) ]) + '--noprofile', '--norc', + '-c', BASH_CMD % (source_dir) ]) for (key,value) in [ l.split('=',1) for l in out.rstrip().split("\n") ]: # with backslash escapes @@ -317,13 +328,12 @@ def read_context_disk_dir(source_dir): results['metadata']['public-keys'] = [l for l in lines if len(l) and not l.startswith("#")] - # custom hostname - if 'hostname' in context_sh: - results['metadata']['local-hostname'] = context_sh['hostname'] - elif 'public_ip'in context_sh: - results['metadata']['local-hostname'] = context_sh['public_ip'] - elif 'eth0_ip' in context_sh: - results['metadata']['local-hostname'] = context_sh['eth0_ip'] + # custom hostname -- try hostname or leave cloud-init + # itself create hostname from IP address later + for k in ('hostname','public_ip','ip_public','eth0_ip'): + if k in context_sh: + results['metadata']['local-hostname'] = context_sh[k] + break # raw user data if "user_data" in context_sh: @@ -335,10 +345,6 @@ def read_context_disk_dir(source_dir): net=OpenNebulaNetwork(out, context_sh) results['network-interfaces']=net.gen_conf() - dns=net.gen_dns() - if dns: - results['dns']=dns - return results -- cgit v1.2.3 From 54f6ccccfb4f75dc6877b04b42987e834b7a0015 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Tue, 19 Feb 2013 12:35:12 +0100 Subject: Change network ifaces detection from ifconfig to ip command. --- cloudinit/sources/DataSourceOpenNebula.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'cloudinit/sources/DataSourceOpenNebula.py') diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 52474675..9b1b5f9c 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -130,16 +130,15 @@ class NonContextDeviceDir(Exception): class OpenNebulaNetwork(object): - REG_ETH=re.compile('^eth') - REG_DEV_MAC=re.compile('^(eth\d+).*HWaddr (..:..:..:..:..:..)') + REG_DEV_MAC=re.compile('^\d+: (eth\d+):.*link\/ether (..:..:..:..:..:..) ') - def __init__(self, ifconfig, context_sh): - self.ifconfig=ifconfig + def __init__(self, ip, context_sh): + self.ip=ip self.context_sh=context_sh self.ifaces=self.get_ifaces() def get_ifaces(self): - return [self.REG_DEV_MAC.search(f).groups() for f in self.ifconfig.split("\n") if self.REG_ETH.match(f)] + return [self.REG_DEV_MAC.search(f).groups() for f in self.ip.split("\n") if self.REG_DEV_MAC.match(f)] def mac2ip(self, mac): components=mac.split(':')[2:] @@ -341,7 +340,7 @@ def read_context_disk_dir(source_dir): elif "userdata" in context_sh: results['userdata'] = context_sh["userdata"] - (out, err) = util.subp(['/sbin/ifconfig', '-a']) + (out, err) = util.subp(['/sbin/ip', '-o', 'link']) net=OpenNebulaNetwork(out, context_sh) results['network-interfaces']=net.gen_conf() -- cgit v1.2.3 From eda36b6116a79433cc29b778b0008cf35d6a2afa Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Tue, 19 Feb 2013 14:21:42 +0100 Subject: Name unification Context.*Device -> Context.*Disk --- cloudinit/sources/DataSourceOpenNebula.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'cloudinit/sources/DataSourceOpenNebula.py') diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 9b1b5f9c..bc199461 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -62,8 +62,8 @@ class DataSourceOpenNebula(sources.DataSource): try: results=read_context_disk_dir(self.seed_dir) found = self.seed_dir - except NonContextDeviceDir: - util.logexc(LOG, "Failed reading context device from %s", + except NonContextDiskDir: + util.logexc(LOG, "Failed reading context disk from %s", self.seed_dir) if not found: devlist = find_candidate_devs() @@ -72,7 +72,7 @@ class DataSourceOpenNebula(sources.DataSource): results = util.mount_cb(dev, read_context_disk_dir) found = dev break - except (NonContextDeviceDir, util.MountFailedError): + except (NonContextDiskDir, util.MountFailedError): pass if not found: @@ -125,7 +125,7 @@ class DataSourceOpenNebulaNet(DataSourceOpenNebula): self.dsmode = 'net' -class NonContextDeviceDir(Exception): +class NonContextDiskDir(Exception): pass @@ -250,7 +250,7 @@ def read_context_disk_dir(source_dir): """ read_context_disk_dir(source_dir): read source_dir and return a tuple with metadata dict and user-data - string populated. If not a valid dir, raise a NonContextDeviceDir + string populated. If not a valid dir, raise a NonContextDiskDir """ found = {} @@ -260,7 +260,7 @@ def read_context_disk_dir(source_dir): found[af] = fn if len(found) == 0: - raise NonContextDeviceDir("%s: %s" % (source_dir, "no files found")) + raise NonContextDiskDir("%s: %s" % (source_dir, "no files found")) context_sh = {} results = { @@ -312,7 +312,7 @@ def read_context_disk_dir(source_dir): LOG.warn("Failed to read context variables: %s" % (_err.message)) results['metadata']=context_sh else: - raise NonContextDeviceDir("Missing context.sh") + raise NonContextDiskDir("Missing context.sh") # process single or multiple SSH keys ssh_key_var=None -- cgit v1.2.3 From a8e39f502cffdf8639263fb1dcb0ad36158c2d83 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Tue, 19 Feb 2013 16:00:16 +0100 Subject: Context.sh parsing cleanups, fix single quotes handling in multiword variables. --- cloudinit/sources/DataSourceOpenNebula.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) (limited to 'cloudinit/sources/DataSourceOpenNebula.py') diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index bc199461..dad64bd4 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -288,28 +288,35 @@ def read_context_disk_dir(source_dir): # 3. use comm to filter "old" variables from all current # variables and excl. few other vars with grep BASH_CMD='VARS=`set | sort -u `;' \ - '. %s/context.sh;' \ + 'source %s/context.sh;' \ 'comm -23 <(set | sort -u) <(echo "$VARS") | egrep -v "^(VARS|PIPESTATUS|_)="' - (out,err) = util.subp(['bash', - '--noprofile', '--norc', + (out,err) = util.subp(['bash','--noprofile', '--norc', '-c', BASH_CMD % (source_dir) ]) for (key,value) in [ l.split('=',1) for l in out.rstrip().split("\n") ]: - # with backslash escapes + k=key.lower() + + # with backslash escapes, e.g. + # X=$'Y\nZ' r=re.match("^\$'(.*)'$",value) if r: - context_sh[key.lower()]=r.group(1).decode('string_escape') + context_sh[k]=r.group(1).decode('string_escape') else: - # multiword values + # multiword values, e.g.: + # X='Y Z' + # X='Y'\''Z' for "Y'Z" r=re.match("^'(.*)'$",value) if r: - context_sh[key.lower()]=r.group(1) + context_sh[k]=r.group(1).replace("'\\''","'") else: - # simple values - context_sh[key.lower()]=value - except util.ProcessExecutionError, _err: - LOG.warn("Failed to read context variables: %s" % (_err.message)) + # simple values, e.g.: + # X=Y + context_sh[k]=value + + except util.ProcessExecutionError as e: + raise NonContextDiskDir("Error reading context.sh: %s" % (e)) + results['metadata']=context_sh else: raise NonContextDiskDir("Missing context.sh") -- cgit v1.2.3 From 5e42d558c2589701c8ad22cd77fd7060ab3f1c02 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Wed, 20 Feb 2013 14:19:21 +0100 Subject: Minor OpenNebula DS cleanups (style, dsmode, static network). --- cloudinit/sources/DataSourceOpenNebula.py | 70 +++++++++++++++++-------------- 1 file changed, 39 insertions(+), 31 deletions(-) (limited to 'cloudinit/sources/DataSourceOpenNebula.py') diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index dad64bd4..bfa7eeaf 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -33,6 +33,7 @@ from cloudinit import util LOG = logging.getLogger(__name__) DEFAULT_IID = "iid-dsopennebula" +DEFAULT_MODE = 'net' CONTEXT_DISK_FILES = ["context.sh"] VALID_DSMODES = ("local", "net", "disabled") @@ -44,30 +45,29 @@ class DataSourceOpenNebula(sources.DataSource): self.seed_dir = os.path.join(paths.seed_dir, 'opennebula') def __str__(self): - mstr = "%s [seed=%s][dsmode=%s]" % (util.obj_name(self), - self.seed, self.dsmode) - return mstr + return "%s [seed=%s][dsmode=%s]" % \ + (util.obj_name(self), self.seed, self.dsmode) def get_data(self): defaults = { "instance-id": DEFAULT_IID, - "dsmode": self.dsmode, - } + "dsmode": self.dsmode } found = None md = {} - results = {} + if os.path.isdir(self.seed_dir): try: - results=read_context_disk_dir(self.seed_dir) + results = read_context_disk_dir(self.seed_dir) found = self.seed_dir except NonContextDiskDir: - util.logexc(LOG, "Failed reading context disk from %s", - self.seed_dir) + util.logexc(LOG, "Failed reading context disk from %s", self.seed_dir) + + # find candidate devices, try to mount them and + # read context script if present if not found: - devlist = find_candidate_devs() - for dev in devlist: + for dev in find_candidate_devs(): try: results = util.mount_cb(dev, read_context_disk_dir) found = dev @@ -81,20 +81,30 @@ class DataSourceOpenNebula(sources.DataSource): md = results['metadata'] md = util.mergedict(md, defaults) - dsmode = results.get('dsmode', None) - if dsmode not in VALID_DSMODES + (None,): - LOG.warn("user specified invalid mode: %s" % dsmode) - dsmode = None + # check for valid user specified dsmode + user_dsmode = results.get('dsmode', None) + if user_dsmode not in VALID_DSMODES + (None,): + LOG.warn("user specified invalid mode: %s" % user_dsmode) + user_dsmode = None - if (dsmode is None) and self.ds_cfg.get('dsmode'): + # decide dsmode + if user_dsmode: + dsmode = user_dsmode + elif self.ds_cfg.get('dsmode'): dsmode = self.ds_cfg.get('dsmode') else: - dsmode = self.dsmode + dsmode = DEFAULT_MODE if dsmode == "disabled": # most likely user specified return False + # apply static network configuration only in 'local' dsmode + # TODO: first boot? + if ('network-interfaces' in results and self.dsmode == "local"): + LOG.debug("Updating network interfaces from %s", self) + self.distro.apply_network(results['network-interfaces']) + if dsmode != self.dsmode: LOG.debug("%s: not claiming datasource, dsmode=%s", self, dsmode) return False @@ -103,11 +113,6 @@ class DataSourceOpenNebula(sources.DataSource): self.metadata = md self.userdata_raw = results.get('userdata') - # apply static network configuration only in 'local' dsmode - if ('network-interfaces' in results and self.dsmode == "local"): - LOG.debug("Updating network interfaces from %s", self) - self.distro.apply_network(results['network-interfaces']) - return True def get_hostname(self, fqdn=False, resolve_ip=None): @@ -234,9 +239,9 @@ def find_candidate_devs(): Return a list of devices that may contain the context disk. """ by_fstype = util.find_devs_with("TYPE=iso9660") - by_label = util.find_devs_with("LABEL=CDROM") - by_fstype.sort() + + by_label = util.find_devs_with("LABEL=CDROM") by_label.sort() # combine list of items by putting by-label items first @@ -262,11 +267,8 @@ def read_context_disk_dir(source_dir): if len(found) == 0: raise NonContextDiskDir("%s: %s" % (source_dir, "no files found")) + results = {'userdata':None, 'metadata':{}} context_sh = {} - results = { - 'userdata':None, - 'metadata':{}, - } if "context.sh" in found: try: @@ -347,9 +349,15 @@ def read_context_disk_dir(source_dir): elif "userdata" in context_sh: results['userdata'] = context_sh["userdata"] - (out, err) = util.subp(['/sbin/ip', '-o', 'link']) - net=OpenNebulaNetwork(out, context_sh) - results['network-interfaces']=net.gen_conf() + # generate static /etc/network/interfaces + # only if there are any required context variables + # http://opennebula.org/documentation:rel3.8:cong#network_configuration + for k in context_sh.keys(): + if re.match('^eth\d+_ip$',k): + (out, err) = util.subp(['/sbin/ip', '-o', 'link']) + net=OpenNebulaNetwork(out, context_sh) + results['network-interfaces']=net.gen_conf() + break return results -- cgit v1.2.3 From b83c0fb7a35ea5c4f61ef4ce94d037a4f10f3c1e Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Thu, 21 Feb 2013 14:45:34 +0100 Subject: Remove TODO --- cloudinit/sources/DataSourceOpenNebula.py | 1 - 1 file changed, 1 deletion(-) (limited to 'cloudinit/sources/DataSourceOpenNebula.py') diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index bfa7eeaf..b22c8aed 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -100,7 +100,6 @@ class DataSourceOpenNebula(sources.DataSource): return False # apply static network configuration only in 'local' dsmode - # TODO: first boot? if ('network-interfaces' in results and self.dsmode == "local"): LOG.debug("Updating network interfaces from %s", self) self.distro.apply_network(results['network-interfaces']) -- cgit v1.2.3 From 1d35141960f2c0183d3f306d4604011ca9d5d2e8 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Tue, 2 Apr 2013 19:55:15 +0200 Subject: PEP8 fixes. --- cloudinit/sources/DataSourceOpenNebula.py | 109 +++++++++++++++--------------- 1 file changed, 55 insertions(+), 54 deletions(-) (limited to 'cloudinit/sources/DataSourceOpenNebula.py') diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index b22c8aed..ff8e6b6b 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -37,6 +37,7 @@ DEFAULT_MODE = 'net' CONTEXT_DISK_FILES = ["context.sh"] VALID_DSMODES = ("local", "net", "disabled") + class DataSourceOpenNebula(sources.DataSource): def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) @@ -51,7 +52,7 @@ class DataSourceOpenNebula(sources.DataSource): def get_data(self): defaults = { "instance-id": DEFAULT_IID, - "dsmode": self.dsmode } + "dsmode": self.dsmode} found = None md = {} @@ -62,7 +63,8 @@ class DataSourceOpenNebula(sources.DataSource): results = read_context_disk_dir(self.seed_dir) found = self.seed_dir except NonContextDiskDir: - util.logexc(LOG, "Failed reading context disk from %s", self.seed_dir) + util.logexc(LOG, "Failed reading context disk from %s", + self.seed_dir) # find candidate devices, try to mount them and # read context script if present @@ -134,99 +136,98 @@ class NonContextDiskDir(Exception): class OpenNebulaNetwork(object): - REG_DEV_MAC=re.compile('^\d+: (eth\d+):.*link\/ether (..:..:..:..:..:..) ') + REG_DEV_MAC = re.compile('^\d+: (eth\d+):.*link\/ether (..:..:..:..:..:..) ') def __init__(self, ip, context_sh): - self.ip=ip - self.context_sh=context_sh - self.ifaces=self.get_ifaces() + self.ip = ip + self.context_sh = context_sh + self.ifaces = self.get_ifaces() def get_ifaces(self): return [self.REG_DEV_MAC.search(f).groups() for f in self.ip.split("\n") if self.REG_DEV_MAC.match(f)] def mac2ip(self, mac): - components=mac.split(':')[2:] - + components = mac.split(':')[2:] return [str(int(c, 16)) for c in components] - + def get_ip(self, dev, components): - var_name=dev+'_ip' + var_name = dev + '_ip' if var_name in self.context_sh: return self.context_sh[var_name] else: return '.'.join(components) def get_mask(self, dev, components): - var_name=dev+'_mask' + var_name = dev + '_mask' if var_name in self.context_sh: return self.context_sh[var_name] else: return '255.255.255.0' def get_network(self, dev, components): - var_name=dev+'_network' + var_name = dev + '_network' if var_name in self.context_sh: return self.context_sh[var_name] else: - return '.'.join(components[:-1])+'.0' + return '.'.join(components[:-1]) + '.0' def get_gateway(self, dev, components): - var_name=dev+'_gateway' + var_name = dev + '_gateway' if var_name in self.context_sh: return self.context_sh[var_name] else: None def get_dns(self, dev, components): - var_name=dev+'_dns' + var_name = dev + '_dns' if var_name in self.context_sh: return self.context_sh[var_name] else: None def get_domain(self, dev, components): - var_name=dev+'_domain' + var_name = dev + '_domain' if var_name in self.context_sh: return self.context_sh[var_name] else: None def gen_conf(self): - global_dns=[] + global_dns = [] if 'dns' in self.context_sh: global_dns.append(self.context_sh['dns']) - conf=[] + conf = [] conf.append('auto lo') conf.append('iface lo inet loopback') conf.append('') for i in self.ifaces: - dev=i[0] - mac=i[1] - ip_components=self.mac2ip(mac) + dev = i[0] + mac = i[1] + ip_components = self.mac2ip(mac) - conf.append('auto '+dev) - conf.append('iface '+dev+' inet static') - conf.append(' address '+self.get_ip(dev, ip_components)) - conf.append(' network '+self.get_network(dev, ip_components)) - conf.append(' netmask '+self.get_mask(dev, ip_components)) + conf.append('auto ' + dev) + conf.append('iface ' + dev + ' inet static') + conf.append(' address ' + self.get_ip(dev, ip_components)) + conf.append(' network ' + self.get_network(dev, ip_components)) + conf.append(' netmask ' + self.get_mask(dev, ip_components)) - gateway=self.get_gateway(dev, ip_components) + gateway = self.get_gateway(dev, ip_components) if gateway: - conf.append(' gateway '+gateway) + conf.append(' gateway ' + gateway) - domain=self.get_domain(dev, ip_components) + domain = self.get_domain(dev, ip_components) if domain: - conf.append(' dns-search '+domain) + conf.append(' dns-search ' + domain) # add global DNS servers to all interfaces - dns=self.get_dns(dev, ip_components) + dns = self.get_dns(dev, ip_components) if global_dns or dns: - all_dns=global_dns + all_dns = global_dns if dns: all_dns.append(dns) - conf.append(' dns-nameservers '+' '.join(all_dns)) + conf.append(' dns-nameservers ' + ' '.join(all_dns)) conf.append('') @@ -266,7 +267,7 @@ def read_context_disk_dir(source_dir): if len(found) == 0: raise NonContextDiskDir("%s: %s" % (source_dir, "no files found")) - results = {'userdata':None, 'metadata':{}} + results = {'userdata': None, 'metadata': {}} context_sh = {} if "context.sh" in found: @@ -288,47 +289,47 @@ def read_context_disk_dir(source_dir): # 2. read context variables # 3. use comm to filter "old" variables from all current # variables and excl. few other vars with grep - BASH_CMD='VARS=`set | sort -u `;' \ + BASH_CMD = 'VARS=`set | sort -u `;' \ 'source %s/context.sh;' \ 'comm -23 <(set | sort -u) <(echo "$VARS") | egrep -v "^(VARS|PIPESTATUS|_)="' - (out,err) = util.subp(['bash','--noprofile', '--norc', - '-c', BASH_CMD % (source_dir) ]) + (out, err) = util.subp(['bash', '--noprofile', '--norc', + '-c', BASH_CMD % (source_dir)]) - for (key,value) in [ l.split('=',1) for l in out.rstrip().split("\n") ]: - k=key.lower() + for (key, value) in [l.split('=', 1) for l in out.rstrip().split("\n")]: + k = key.lower() # with backslash escapes, e.g. # X=$'Y\nZ' - r=re.match("^\$'(.*)'$",value) + r = re.match("^\$'(.*)'$", value) if r: - context_sh[k]=r.group(1).decode('string_escape') + context_sh[k] = r.group(1).decode('string_escape') else: # multiword values, e.g.: - # X='Y Z' + # X='Y Z' # X='Y'\''Z' for "Y'Z" - r=re.match("^'(.*)'$",value) + r = re.match("^'(.*)'$", value) if r: - context_sh[k]=r.group(1).replace("'\\''","'") + context_sh[k] = r.group(1).replace("'\\''", "'") else: # simple values, e.g.: - # X=Y - context_sh[k]=value + # X=Y + context_sh[k] = value except util.ProcessExecutionError as e: raise NonContextDiskDir("Error reading context.sh: %s" % (e)) - results['metadata']=context_sh + results['metadata'] = context_sh else: raise NonContextDiskDir("Missing context.sh") # process single or multiple SSH keys - ssh_key_var=None + ssh_key_var = None if "ssh_key" in context_sh: - ssh_key_var="ssh_key" + ssh_key_var = "ssh_key" elif "ssh_public_key" in context_sh: - ssh_key_var="ssh_public_key" + ssh_key_var = "ssh_public_key" if ssh_key_var: lines = context_sh.get(ssh_key_var).splitlines() @@ -337,7 +338,7 @@ def read_context_disk_dir(source_dir): # custom hostname -- try hostname or leave cloud-init # itself create hostname from IP address later - for k in ('hostname','public_ip','ip_public','eth0_ip'): + for k in ('hostname', 'public_ip', 'ip_public', 'eth0_ip'): if k in context_sh: results['metadata']['local-hostname'] = context_sh[k] break @@ -352,10 +353,10 @@ def read_context_disk_dir(source_dir): # only if there are any required context variables # http://opennebula.org/documentation:rel3.8:cong#network_configuration for k in context_sh.keys(): - if re.match('^eth\d+_ip$',k): + if re.match('^eth\d+_ip$', k): (out, err) = util.subp(['/sbin/ip', '-o', 'link']) - net=OpenNebulaNetwork(out, context_sh) - results['network-interfaces']=net.gen_conf() + net = OpenNebulaNetwork(out, context_sh) + results['network-interfaces'] = net.gen_conf() break return results -- cgit v1.2.3 From 77c8798388637fcb2f7dbc057cad81e8fd5fbe58 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Wed, 4 Sep 2013 14:32:24 +0200 Subject: Apply parse.diff by Javier Fontan --- cloudinit/sources/DataSourceOpenNebula.py | 96 ++++++++-------------- tests/unittests/helpers.py | 1 + tests/unittests/test_datasource/test_opennebula.py | 16 ++-- 3 files changed, 45 insertions(+), 68 deletions(-) (limited to 'cloudinit/sources/DataSourceOpenNebula.py') diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index ff8e6b6b..d2ab348f 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -24,7 +24,6 @@ import os import re -import subprocess from cloudinit import log as logging from cloudinit import sources @@ -86,7 +85,7 @@ class DataSourceOpenNebula(sources.DataSource): # check for valid user specified dsmode user_dsmode = results.get('dsmode', None) if user_dsmode not in VALID_DSMODES + (None,): - LOG.warn("user specified invalid mode: %s" % user_dsmode) + LOG.warn("user specified invalid mode: %s", user_dsmode) user_dsmode = None # decide dsmode @@ -136,7 +135,9 @@ class NonContextDiskDir(Exception): class OpenNebulaNetwork(object): - REG_DEV_MAC = re.compile('^\d+: (eth\d+):.*link\/ether (..:..:..:..:..:..) ') + REG_DEV_MAC = re.compile( + r'^\d+: (eth\d+):.*?link\/ether (..:..:..:..:..:..) ?', + re.MULTILINE | re.DOTALL) def __init__(self, ip, context_sh): self.ip = ip @@ -144,7 +145,7 @@ class OpenNebulaNetwork(object): self.ifaces = self.get_ifaces() def get_ifaces(self): - return [self.REG_DEV_MAC.search(f).groups() for f in self.ip.split("\n") if self.REG_DEV_MAC.match(f)] + return self.REG_DEV_MAC.findall(self.ip) def mac2ip(self, mac): components = mac.split(':')[2:] @@ -157,7 +158,7 @@ class OpenNebulaNetwork(object): else: return '.'.join(components) - def get_mask(self, dev, components): + def get_mask(self, dev): var_name = dev + '_mask' if var_name in self.context_sh: return self.context_sh[var_name] @@ -171,26 +172,26 @@ class OpenNebulaNetwork(object): else: return '.'.join(components[:-1]) + '.0' - def get_gateway(self, dev, components): + def get_gateway(self, dev): var_name = dev + '_gateway' if var_name in self.context_sh: return self.context_sh[var_name] else: - None + return None - def get_dns(self, dev, components): + def get_dns(self, dev): var_name = dev + '_dns' if var_name in self.context_sh: return self.context_sh[var_name] else: - None + return None - def get_domain(self, dev, components): + def get_domain(self, dev): var_name = dev + '_domain' if var_name in self.context_sh: return self.context_sh[var_name] else: - None + return None def gen_conf(self): global_dns = [] @@ -211,18 +212,18 @@ class OpenNebulaNetwork(object): conf.append('iface ' + dev + ' inet static') conf.append(' address ' + self.get_ip(dev, ip_components)) conf.append(' network ' + self.get_network(dev, ip_components)) - conf.append(' netmask ' + self.get_mask(dev, ip_components)) + conf.append(' netmask ' + self.get_mask(dev)) - gateway = self.get_gateway(dev, ip_components) + gateway = self.get_gateway(dev) if gateway: conf.append(' gateway ' + gateway) - domain = self.get_domain(dev, ip_components) + domain = self.get_domain(dev) if domain: conf.append(' dns-search ' + domain) # add global DNS servers to all interfaces - dns = self.get_dns(dev, ip_components) + dns = self.get_dns(dev) if global_dns or dns: all_dns = global_dns if dns: @@ -272,51 +273,22 @@ def read_context_disk_dir(source_dir): if "context.sh" in found: try: - # Note: context.sh is a "shell" script with defined context - # variables, like: X="Y" . It's ready to use as a shell source - # e.g.: ". context.sh" and as a shell script it can also reference - # to already defined shell variables. So to have same context var. - # values as we can have in custom shell script, we use bash itself - # to read context.sh and dump variables in easily parsable way. - # - # normalized variables dump format (get by cmd "set"): - # 1. simple single word assignment ........ X=Y - # 2. multiword assignment ................. X='Y Z' - # 3. assignments with backslash escapes ... X=$'Y\nZ' - # - # how context variables are read: - # 1. list existing ("old") shell variables and store into $VARS - # 2. read context variables - # 3. use comm to filter "old" variables from all current - # variables and excl. few other vars with grep - BASH_CMD = 'VARS=`set | sort -u `;' \ - 'source %s/context.sh;' \ - 'comm -23 <(set | sort -u) <(echo "$VARS") | egrep -v "^(VARS|PIPESTATUS|_)="' - - (out, err) = util.subp(['bash', '--noprofile', '--norc', - '-c', BASH_CMD % (source_dir)]) - - for (key, value) in [l.split('=', 1) for l in out.rstrip().split("\n")]: - k = key.lower() - - # with backslash escapes, e.g. - # X=$'Y\nZ' - r = re.match("^\$'(.*)'$", value) - if r: - context_sh[k] = r.group(1).decode('string_escape') - else: - # multiword values, e.g.: - # X='Y Z' - # X='Y'\''Z' for "Y'Z" - r = re.match("^'(.*)'$", value) - if r: - context_sh[k] = r.group(1).replace("'\\''", "'") - else: - # simple values, e.g.: - # X=Y - context_sh[k] = value - - except util.ProcessExecutionError as e: + f = open('%s/context.sh' % (source_dir), 'r') + text = f.read() + f.close() + + context_reg = re.compile(r"^([\w_]+)=['\"](.*?[^\\])['\"]$", + re.MULTILINE | re.DOTALL) + variables = context_reg.findall(text) + + if not variables: + raise NonContextDiskDir("No variables in context") + + context_sh = {} + for k, v in variables: + context_sh[k.lower()] = v + + except (IOError, NonContextDiskDir) as e: raise NonContextDiskDir("Error reading context.sh: %s" % (e)) results['metadata'] = context_sh @@ -353,8 +325,8 @@ def read_context_disk_dir(source_dir): # only if there are any required context variables # http://opennebula.org/documentation:rel3.8:cong#network_configuration for k in context_sh.keys(): - if re.match('^eth\d+_ip$', k): - (out, err) = util.subp(['/sbin/ip', '-o', 'link']) + if re.match(r'^eth\d+_ip$', k): + (out, _) = util.subp(['/sbin/ip', '-o', 'link']) net = OpenNebulaNetwork(out, context_sh) results['network-interfaces'] = net.gen_conf() break diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 91a50e18..904677f1 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -183,6 +183,7 @@ class FilesystemMockingTestCase(ResourceUsingTestCase): setattr(mod, f, trap_func) self.patched_funcs.append((mod, f, func)) + def populate_dir(path, files): os.makedirs(path) for (name, content) in files.iteritems(): diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index bc6c4b73..27725930 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -19,9 +19,11 @@ SSH_KEY = 'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460-%i' HOSTNAME = 'foo.example.com' PUBLIC_IP = '10.0.0.3' -CMD_IP_OUT = '''\ -1: lo: mtu 16436 qdisc noqueue state UNKNOWN \ link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 -2: eth0: mtu 1500 qdisc mq state UP qlen 1000\ link/ether 02:00:0a:12:01:01 brd ff:ff:ff:ff:ff:ff +CMD_IP_OUT = ''' +1: lo: mtu 16436 qdisc noqueue state UNKNOWN + link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 +2: eth0: mtu 1500 qdisc mq state UP qlen 1000 + link/ether 02:00:0a:12:01:01 brd ff:ff:ff:ff:ff:ff ''' @@ -52,7 +54,7 @@ class TestOpenNebulaDataSource(MockerTestCase): self.assertEqual(TEST_VARS, results['metadata']) def test_ssh_key(self): - public_keys = [] + public_keys = ['first key', 'second key'] for c in range(4): for k in ('SSH_KEY', 'SSH_PUBLIC_KEY'): my_d = os.path.join(self.tmp, "%s-%i" % (k, c)) @@ -61,7 +63,8 @@ class TestOpenNebulaDataSource(MockerTestCase): self.assertTrue('metadata' in results) self.assertTrue('public-keys' in results['metadata']) - self.assertEqual(public_keys, results['metadata']['public-keys']) + self.assertEqual(public_keys, + results['metadata']['public-keys']) public_keys.append(SSH_KEY % (c + 1,)) @@ -96,7 +99,8 @@ class TestOpenNebulaDataSource(MockerTestCase): try: orig_find_devs_with = util.find_devs_with util.find_devs_with = my_devs_with - self.assertEqual(["/dev/sr0", "/dev/vdb"], ds.find_candidate_devs()) + self.assertEqual(["/dev/sr0", "/dev/vdb"], + ds.find_candidate_devs()) finally: util.find_devs_with = orig_find_devs_with -- cgit v1.2.3 From 1adc68b643b2f73a2a08ca7d19c3fc8e759f06c2 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Wed, 4 Sep 2013 16:19:54 +0200 Subject: Fix RE matching context variables. Test cleanups. --- cloudinit/sources/DataSourceOpenNebula.py | 26 ++++++++++++++++------ tests/unittests/test_datasource/test_opennebula.py | 12 +++++----- 2 files changed, 26 insertions(+), 12 deletions(-) (limited to 'cloudinit/sources/DataSourceOpenNebula.py') diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index d2ab348f..0ab23b25 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -277,16 +277,28 @@ def read_context_disk_dir(source_dir): text = f.read() f.close() - context_reg = re.compile(r"^([\w_]+)=['\"](.*?[^\\])['\"]$", - re.MULTILINE | re.DOTALL) - variables = context_reg.findall(text) + # lame matching: + # 1. group = key + # 2. group = single quoted value, respect '\'' + # 3. group = old double quoted value, but doesn't end with \" + context_reg = re.compile( + r"^([\w_]+)=(?:'((?:[^']|'\\'')*?)'|\"(.*?[^\\])\")$", + re.MULTILINE | re.DOTALL) + variables = context_reg.findall(text) if not variables: raise NonContextDiskDir("No variables in context") - context_sh = {} - for k, v in variables: - context_sh[k.lower()] = v + for k, v1, v2 in variables: + k = k.lower() + if v1: + # take single quoted variable 'xyz' + # (ON>=4) and unquote '\'' -> ' + context_sh[k] = v1.replace(r"'\''", r"'") + elif v2: + # take double quoted variable "xyz" + # (old ON<4) and unquote \" -> " + context_sh[k] = v2.replace(r'\"', r'"') except (IOError, NonContextDiskDir) as e: raise NonContextDiskDir("Error reading context.sh: %s" % (e)) @@ -326,7 +338,7 @@ def read_context_disk_dir(source_dir): # http://opennebula.org/documentation:rel3.8:cong#network_configuration for k in context_sh.keys(): if re.match(r'^eth\d+_ip$', k): - (out, _) = util.subp(['/sbin/ip', '-o', 'link']) + (out, _) = util.subp(['/sbin/ip', 'link']) net = OpenNebulaNetwork(out, context_sh) results['network-interfaces'] = net.gen_conf() break diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index 27725930..66a38e31 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -1,7 +1,8 @@ import os -from mocker import MockerTestCase -from cloudinit import util + from cloudinit.sources import DataSourceOpenNebula as ds +from cloudinit import util +from mocker import MockerTestCase TEST_VARS = { 'var1': 'single', @@ -19,7 +20,7 @@ SSH_KEY = 'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460-%i' HOSTNAME = 'foo.example.com' PUBLIC_IP = '10.0.0.3' -CMD_IP_OUT = ''' +CMD_IP_OUT = '''\ 1: lo: mtu 16436 qdisc noqueue state UNKNOWN link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2: eth0: mtu 1500 qdisc mq state UP qlen 1000 @@ -91,6 +92,7 @@ class TestOpenNebulaDataSource(MockerTestCase): devs_with_answers = { "TYPE=iso9660": ["/dev/vdb"], "LABEL=CDROM": ["/dev/sr0"], + "LABEL=CONTEXT": ["/dev/sdb"], } def my_devs_with(criteria): @@ -99,7 +101,7 @@ class TestOpenNebulaDataSource(MockerTestCase): try: orig_find_devs_with = util.find_devs_with util.find_devs_with = my_devs_with - self.assertEqual(["/dev/sr0", "/dev/vdb"], + self.assertEqual(["/dev/sdb", "/dev/sr0", "/dev/vdb"], ds.find_candidate_devs()) finally: util.find_devs_with = orig_find_devs_with @@ -161,7 +163,7 @@ def populate_dir(seed_dir, files): with open(os.path.join(seed_dir, "context.sh"), "w") as fp: fp.write("# Context variables generated by OpenNebula\n") for (name, content) in files.iteritems(): - fp.write('%s="%s"\n' % (name.upper(), content)) + fp.write("%s='%s'\n" % (name.upper(), content.replace(r"'", r"'\''"))) fp.close() # vi: ts=4 expandtab -- cgit v1.2.3 From 8a2a88e0bb4520eabe99b6686413a548f3d59652 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Wed, 4 Sep 2013 16:36:00 +0200 Subject: Search for contextualization CDROM by LABEL=CONTEXT --- cloudinit/sources/DataSourceOpenNebula.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) (limited to 'cloudinit/sources/DataSourceOpenNebula.py') diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 0ab23b25..fef7beac 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -239,15 +239,11 @@ def find_candidate_devs(): """ Return a list of devices that may contain the context disk. """ - by_fstype = util.find_devs_with("TYPE=iso9660") - by_fstype.sort() - - by_label = util.find_devs_with("LABEL=CDROM") - by_label.sort() - - # combine list of items by putting by-label items first - # followed by fstype items, but with dupes removed - combined = (by_label + [d for d in by_fstype if d not in by_label]) + combined = [] + for f in ('LABEL=CONTEXT', 'LABEL=CDROM', 'TYPE=iso9660'): + for d in util.find_devs_with(f).sort(): + if d not in combined: + combined.append(d) return combined -- cgit v1.2.3 From d0e7898d4d0aef452fbfdd9b51fa2da6437397ff Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Thu, 5 Sep 2013 17:55:53 +0200 Subject: PEP8 and Pylint fixes. Move context.sh "parser" into separate function. Fix fetching user specified dsmode (from context). Rename context_sh->context. Reuse unittests.helpers.populate_dir. --- cloudinit/sources/DataSourceOpenNebula.py | 163 +++++++++++---------- tests/unittests/test_datasource/test_opennebula.py | 62 ++++---- 2 files changed, 119 insertions(+), 106 deletions(-) (limited to 'cloudinit/sources/DataSourceOpenNebula.py') diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index fef7beac..87ec51bd 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -3,7 +3,7 @@ # Copyright (C) 2012 Canonical Ltd. # Copyright (C) 2012 Yahoo! Inc. # Copyright (C) 2012-2013 CERIT Scientific Cloud -# Copyright (C) 2012 OpenNebula.org +# Copyright (C) 2012-2013 OpenNebula.org # # Author: Scott Moser # Author: Joshua Harlow @@ -45,45 +45,47 @@ class DataSourceOpenNebula(sources.DataSource): self.seed_dir = os.path.join(paths.seed_dir, 'opennebula') def __str__(self): - return "%s [seed=%s][dsmode=%s]" % \ - (util.obj_name(self), self.seed, self.dsmode) + root = sources.DataSource.__str__(self) + return "%s [seed=%s][dsmode=%s]" % (root, self.seed, self.dsmode) def get_data(self): defaults = { "instance-id": DEFAULT_IID, - "dsmode": self.dsmode} + "dsmode": self.dsmode + } - found = None - md = {} + seed = None results = {} + # first try to read local seed_dir if os.path.isdir(self.seed_dir): try: results = read_context_disk_dir(self.seed_dir) - found = self.seed_dir + seed = self.seed_dir except NonContextDiskDir: - util.logexc(LOG, "Failed reading context disk from %s", + util.logexc(LOG, "Failed reading context from %s", self.seed_dir) - # find candidate devices, try to mount them and - # read context script if present - if not found: + if not seed: + # then try to detect and mount candidate devices and + # read contextualization if present for dev in find_candidate_devs(): try: results = util.mount_cb(dev, read_context_disk_dir) - found = dev + seed = dev break except (NonContextDiskDir, util.MountFailedError): pass - if not found: + if not seed: return False + # merge fetched metadata with datasource defaults md = results['metadata'] - md = util.mergedict(md, defaults) + md = util.mergemanydict([md, defaults]) # check for valid user specified dsmode - user_dsmode = results.get('dsmode', None) + user_dsmode = results['metadata'].get('dsmode', None) if user_dsmode not in VALID_DSMODES + (None,): LOG.warn("user specified invalid mode: %s", user_dsmode) user_dsmode = None @@ -109,10 +111,9 @@ class DataSourceOpenNebula(sources.DataSource): LOG.debug("%s: not claiming datasource, dsmode=%s", self, dsmode) return False - self.seed = found + self.seed = seed self.metadata = md self.userdata_raw = results.get('userdata') - return True def get_hostname(self, fqdn=False, resolve_ip=None): @@ -139,9 +140,9 @@ class OpenNebulaNetwork(object): r'^\d+: (eth\d+):.*?link\/ether (..:..:..:..:..:..) ?', re.MULTILINE | re.DOTALL) - def __init__(self, ip, context_sh): + def __init__(self, ip, context): self.ip = ip - self.context_sh = context_sh + self.context = context self.ifaces = self.get_ifaces() def get_ifaces(self): @@ -153,50 +154,50 @@ class OpenNebulaNetwork(object): def get_ip(self, dev, components): var_name = dev + '_ip' - if var_name in self.context_sh: - return self.context_sh[var_name] + if var_name in self.context: + return self.context[var_name] else: return '.'.join(components) def get_mask(self, dev): var_name = dev + '_mask' - if var_name in self.context_sh: - return self.context_sh[var_name] + if var_name in self.context: + return self.context[var_name] else: return '255.255.255.0' def get_network(self, dev, components): var_name = dev + '_network' - if var_name in self.context_sh: - return self.context_sh[var_name] + if var_name in self.context: + return self.context[var_name] else: return '.'.join(components[:-1]) + '.0' def get_gateway(self, dev): var_name = dev + '_gateway' - if var_name in self.context_sh: - return self.context_sh[var_name] + if var_name in self.context: + return self.context[var_name] else: return None def get_dns(self, dev): var_name = dev + '_dns' - if var_name in self.context_sh: - return self.context_sh[var_name] + if var_name in self.context: + return self.context[var_name] else: return None def get_domain(self, dev): var_name = dev + '_domain' - if var_name in self.context_sh: - return self.context_sh[var_name] + if var_name in self.context: + return self.context[var_name] else: return None def gen_conf(self): global_dns = [] - if 'dns' in self.context_sh: - global_dns.append(self.context_sh['dns']) + if 'dns' in self.context: + global_dns.append(self.context['dns']) conf = [] conf.append('auto lo') @@ -241,101 +242,113 @@ def find_candidate_devs(): """ combined = [] for f in ('LABEL=CONTEXT', 'LABEL=CDROM', 'TYPE=iso9660'): - for d in util.find_devs_with(f).sort(): + devs = util.find_devs_with(f) + devs.sort() + for d in devs: if d not in combined: combined.append(d) return combined +def parse_context_data(data): + """ + parse_context_data(data) + parse context.sh variables provided as a single string. Uses + very simple matching RE. Returns None if nothing is matched. + """ + # RE groups: + # 1: key + # 2: single quoted value, respect '\'' + # 3: old double quoted value, but doesn't end with \" + context_reg = re.compile( + r"^([\w_]+)=(?:'((?:[^']|'\\'')*?)'|\"(.*?[^\\])\")$", + re.MULTILINE | re.DOTALL) + + found = context_reg.findall(data) + if not found: + return None + + variables = {} + for k, v1, v2 in found: + k = k.lower() + if v1: + # take single quoted variable 'xyz' + # (ON>=4) and unquote '\'' -> ' + variables[k] = v1.replace(r"'\''", r"'") + elif v2: + # take double quoted variable "xyz" + # (old ON<4) and unquote \" -> " + variables[k] = v2.replace(r'\"', r'"') + + return variables + + def read_context_disk_dir(source_dir): """ read_context_disk_dir(source_dir): read source_dir and return a tuple with metadata dict and user-data string populated. If not a valid dir, raise a NonContextDiskDir """ - found = {} for af in CONTEXT_DISK_FILES: fn = os.path.join(source_dir, af) if os.path.isfile(fn): found[af] = fn - if len(found) == 0: + if not found: raise NonContextDiskDir("%s: %s" % (source_dir, "no files found")) results = {'userdata': None, 'metadata': {}} - context_sh = {} + context = {} if "context.sh" in found: try: - f = open('%s/context.sh' % (source_dir), 'r') - text = f.read() - f.close() - - # lame matching: - # 1. group = key - # 2. group = single quoted value, respect '\'' - # 3. group = old double quoted value, but doesn't end with \" - context_reg = re.compile( - r"^([\w_]+)=(?:'((?:[^']|'\\'')*?)'|\"(.*?[^\\])\")$", - re.MULTILINE | re.DOTALL) - - variables = context_reg.findall(text) - if not variables: + with open(os.path.join(source_dir, 'context.sh'), 'r') as f: + context = parse_context_data(f.read()) + f.close() + if not context: raise NonContextDiskDir("No variables in context") - for k, v1, v2 in variables: - k = k.lower() - if v1: - # take single quoted variable 'xyz' - # (ON>=4) and unquote '\'' -> ' - context_sh[k] = v1.replace(r"'\''", r"'") - elif v2: - # take double quoted variable "xyz" - # (old ON<4) and unquote \" -> " - context_sh[k] = v2.replace(r'\"', r'"') - except (IOError, NonContextDiskDir) as e: raise NonContextDiskDir("Error reading context.sh: %s" % (e)) - results['metadata'] = context_sh + results['metadata'] = context else: raise NonContextDiskDir("Missing context.sh") # process single or multiple SSH keys ssh_key_var = None - - if "ssh_key" in context_sh: + if "ssh_key" in context: ssh_key_var = "ssh_key" - elif "ssh_public_key" in context_sh: + elif "ssh_public_key" in context: ssh_key_var = "ssh_public_key" if ssh_key_var: - lines = context_sh.get(ssh_key_var).splitlines() + lines = context.get(ssh_key_var).splitlines() results['metadata']['public-keys'] = [l for l in lines if len(l) and not l.startswith("#")] # custom hostname -- try hostname or leave cloud-init # itself create hostname from IP address later for k in ('hostname', 'public_ip', 'ip_public', 'eth0_ip'): - if k in context_sh: - results['metadata']['local-hostname'] = context_sh[k] + if k in context: + results['metadata']['local-hostname'] = context[k] break # raw user data - if "user_data" in context_sh: - results['userdata'] = context_sh["user_data"] - elif "userdata" in context_sh: - results['userdata'] = context_sh["userdata"] + if "user_data" in context: + results['userdata'] = context["user_data"] + elif "userdata" in context: + results['userdata'] = context["userdata"] # generate static /etc/network/interfaces # only if there are any required context variables # http://opennebula.org/documentation:rel3.8:cong#network_configuration - for k in context_sh.keys(): + for k in context.keys(): if re.match(r'^eth\d+_ip$', k): (out, _) = util.subp(['/sbin/ip', 'link']) - net = OpenNebulaNetwork(out, context_sh) + net = OpenNebulaNetwork(out, context) results['network-interfaces'] = net.gen_conf() break diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index 66a38e31..4b82a49c 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -1,8 +1,9 @@ -import os - from cloudinit.sources import DataSourceOpenNebula as ds from cloudinit import util from mocker import MockerTestCase +from tests.unittests.helpers import populate_dir + +import os TEST_VARS = { 'var1': 'single', @@ -13,7 +14,11 @@ TEST_VARS = { 'var6': "'multi\nline\n'", 'var7': 'single\\t', 'var8': 'double\\tword', - 'var9': 'multi\\t\nline\n'} + 'var9': 'multi\\t\nline\n', + 'var10': '\\', # expect \ + 'var11': '\'', # expect ' + 'var12': '$', # expect $ +} USER_DATA = '#cloud-config\napt_upgrade: true' SSH_KEY = 'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460-%i' @@ -38,17 +43,15 @@ class TestOpenNebulaDataSource(MockerTestCase): my_d = os.path.join(self.tmp, 'non-contextdisk') self.assertRaises(ds.NonContextDiskDir, ds.read_context_disk_dir, my_d) - def test_seed_dir_bad_context_sh(self): - my_d = os.path.join(self.tmp, 'bad-context-sh') + def test_seed_dir_bad_context(self): + my_d = os.path.join(self.tmp, 'bad-context') os.mkdir(my_d) - with open(os.path.join(my_d, "context.sh"), "w") as fp: - fp.write('/bin/false\n') - fp.close() + open(os.path.join(my_d, "context.sh"), "w").close() self.assertRaises(ds.NonContextDiskDir, ds.read_context_disk_dir, my_d) - def test_context_sh_parser(self): - my_d = os.path.join(self.tmp, 'context-sh-parser') - populate_dir(my_d, TEST_VARS) + def test_context_parser(self): + my_d = os.path.join(self.tmp, 'context-parser') + populate_context_dir(my_d, TEST_VARS) results = ds.read_context_disk_dir(my_d) self.assertTrue('metadata' in results) @@ -59,7 +62,7 @@ class TestOpenNebulaDataSource(MockerTestCase): for c in range(4): for k in ('SSH_KEY', 'SSH_PUBLIC_KEY'): my_d = os.path.join(self.tmp, "%s-%i" % (k, c)) - populate_dir(my_d, {k: '\n'.join(public_keys)}) + populate_context_dir(my_d, {k: '\n'.join(public_keys)}) results = ds.read_context_disk_dir(my_d) self.assertTrue('metadata' in results) @@ -72,7 +75,7 @@ class TestOpenNebulaDataSource(MockerTestCase): def test_user_data(self): for k in ('USER_DATA', 'USERDATA'): my_d = os.path.join(self.tmp, k) - populate_dir(my_d, {k: USER_DATA}) + populate_context_dir(my_d, {k: USER_DATA}) results = ds.read_context_disk_dir(my_d) self.assertTrue('userdata' in results) @@ -81,7 +84,7 @@ class TestOpenNebulaDataSource(MockerTestCase): def test_hostname(self): for k in ('HOSTNAME', 'PUBLIC_IP', 'IP_PUBLIC', 'ETH0_IP'): my_d = os.path.join(self.tmp, k) - populate_dir(my_d, {k: PUBLIC_IP}) + populate_context_dir(my_d, {k: PUBLIC_IP}) results = ds.read_context_disk_dir(my_d) self.assertTrue('metadata' in results) @@ -89,14 +92,12 @@ class TestOpenNebulaDataSource(MockerTestCase): self.assertEqual(PUBLIC_IP, results['metadata']['local-hostname']) def test_find_candidates(self): - devs_with_answers = { - "TYPE=iso9660": ["/dev/vdb"], - "LABEL=CDROM": ["/dev/sr0"], - "LABEL=CONTEXT": ["/dev/sdb"], - } - def my_devs_with(criteria): - return devs_with_answers[criteria] + return { + "LABEL=CONTEXT": ["/dev/sdb"], + "LABEL=CDROM": ["/dev/sr0"], + "TYPE=iso9660": ["/dev/vdb"], + }.get(criteria, []) try: orig_find_devs_with = util.find_devs_with @@ -133,16 +134,17 @@ iface eth0 inet static ''') def test_eth0_override(self): - context_sh = { + context = { 'dns': '1.2.3.8', 'eth0_ip': '1.2.3.4', 'eth0_network': '1.2.3.0', 'eth0_mask': '255.255.0.0', 'eth0_gateway': '1.2.3.5', 'eth0_domain': 'example.com', - 'eth0_dns': '1.2.3.6 1.2.3.7'} + 'eth0_dns': '1.2.3.6 1.2.3.7' + } - net = ds.OpenNebulaNetwork(CMD_IP_OUT, context_sh) + net = ds.OpenNebulaNetwork(CMD_IP_OUT, context) self.assertEqual(net.gen_conf(), u'''\ auto lo iface lo inet loopback @@ -158,12 +160,10 @@ iface eth0 inet static ''') -def populate_dir(seed_dir, files): - os.mkdir(seed_dir) - with open(os.path.join(seed_dir, "context.sh"), "w") as fp: - fp.write("# Context variables generated by OpenNebula\n") - for (name, content) in files.iteritems(): - fp.write("%s='%s'\n" % (name.upper(), content.replace(r"'", r"'\''"))) - fp.close() +def populate_context_dir(path, variables): + data = "# Context variables generated by OpenNebula\n" + for (k, v) in variables.iteritems(): + data += ("%s='%s'\n" % (k.upper(), v.replace(r"'", r"'\''"))) + populate_dir(path, {'context.sh': data}) # vi: ts=4 expandtab -- cgit v1.2.3 From 781d105a6ca9222e59909341dbd86a13bab5a67e Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Tue, 10 Sep 2013 00:40:16 +0200 Subject: Replace RE context.sh parser with Scott's rewrite of bash dumper. Upper case context variable names. --- cloudinit/sources/DataSourceOpenNebula.py | 178 ++++++++++++++------- tests/unittests/test_datasource/test_opennebula.py | 38 ++--- 2 files changed, 137 insertions(+), 79 deletions(-) (limited to 'cloudinit/sources/DataSourceOpenNebula.py') diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 87ec51bd..a1995aca 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -24,6 +24,8 @@ import os import re +import string +import subprocess from cloudinit import log as logging from cloudinit import sources @@ -50,8 +52,8 @@ class DataSourceOpenNebula(sources.DataSource): def get_data(self): defaults = { - "instance-id": DEFAULT_IID, - "dsmode": self.dsmode + "instance-id": DEFAULT_IID, #TODO:???? + "DSMODE": self.dsmode } seed = None @@ -85,7 +87,7 @@ class DataSourceOpenNebula(sources.DataSource): md = util.mergemanydict([md, defaults]) # check for valid user specified dsmode - user_dsmode = results['metadata'].get('dsmode', None) + user_dsmode = results['metadata'].get('DSMODE', None) if user_dsmode not in VALID_DSMODES + (None,): LOG.warn("user specified invalid mode: %s", user_dsmode) user_dsmode = None @@ -93,7 +95,7 @@ class DataSourceOpenNebula(sources.DataSource): # decide dsmode if user_dsmode: dsmode = user_dsmode - elif self.ds_cfg.get('dsmode'): + elif self.ds_cfg.get('dsmode'): #TODO: fakt se k tomu nekdy dostane? dsmode = self.ds_cfg.get('dsmode') else: dsmode = DEFAULT_MODE @@ -153,42 +155,42 @@ class OpenNebulaNetwork(object): return [str(int(c, 16)) for c in components] def get_ip(self, dev, components): - var_name = dev + '_ip' + var_name = dev.upper() + '_IP' if var_name in self.context: return self.context[var_name] else: return '.'.join(components) def get_mask(self, dev): - var_name = dev + '_mask' + var_name = dev.upper() + '_MASK' if var_name in self.context: return self.context[var_name] else: return '255.255.255.0' def get_network(self, dev, components): - var_name = dev + '_network' + var_name = dev.upper() + '_NETWORK' if var_name in self.context: return self.context[var_name] else: return '.'.join(components[:-1]) + '.0' def get_gateway(self, dev): - var_name = dev + '_gateway' + var_name = dev.upper() + '_GATEWAY' if var_name in self.context: return self.context[var_name] else: return None def get_dns(self, dev): - var_name = dev + '_dns' + var_name = dev.upper() + '_DNS' if var_name in self.context: return self.context[var_name] else: return None def get_domain(self, dev): - var_name = dev + '_domain' + var_name = dev.upper() + '_DOMAIN' if var_name in self.context: return self.context[var_name] else: @@ -196,8 +198,8 @@ class OpenNebulaNetwork(object): def gen_conf(self): global_dns = [] - if 'dns' in self.context: - global_dns.append(self.context['dns']) + if 'DNS' in self.context: + global_dns.append(self.context['DNS']) conf = [] conf.append('auto lo') @@ -251,37 +253,91 @@ def find_candidate_devs(): return combined -def parse_context_data(data): - """ - parse_context_data(data) - parse context.sh variables provided as a single string. Uses - very simple matching RE. Returns None if nothing is matched. - """ - # RE groups: - # 1: key - # 2: single quoted value, respect '\'' - # 3: old double quoted value, but doesn't end with \" - context_reg = re.compile( - r"^([\w_]+)=(?:'((?:[^']|'\\'')*?)'|\"(.*?[^\\])\")$", - re.MULTILINE | re.DOTALL) - - found = context_reg.findall(data) - if not found: - return None - - variables = {} - for k, v1, v2 in found: - k = k.lower() - if v1: - # take single quoted variable 'xyz' - # (ON>=4) and unquote '\'' -> ' - variables[k] = v1.replace(r"'\''", r"'") - elif v2: - # take double quoted variable "xyz" - # (old ON<4) and unquote \" -> " - variables[k] = v2.replace(r'\"', r'"') - - return variables +def parse_shell_config(content, keylist=None, bash=None, asuser=None): + + if isinstance(bash, str): + bash = [bash] + elif bash is None: + bash = ['bash', '-e'] + + # allvars expands to all existing variables by using '${!x*}' notation + # where x is lower or upper case letters or '_' + allvars = ["${!%s*}" % x for x in string.letters + "_"] + + keylist_in = keylist + if keylist is None: + keylist = allvars + keylist_in = [] + + setup = '\n'.join(('__v="";', '',)) + + def varprinter(vlist): + # output '\0'.join(['_start_', key=value NULL for vars in vlist] + return '\n'.join(( + 'printf "%s\\0" _start_', + 'for __v in %s; do' % ' '.join(vlist), + ' printf "%s=%s\\0" "$__v" "${!__v}";', + 'done', + '' + )) + + # the rendered 'bcmd' is bash syntax that does + # setup: declare variables we use (so they show up in 'all') + # varprinter(allvars): print all variables known at beginning + # content: execute the provided content + # varprinter(keylist): print all variables known after content + # + # output is then a null terminated array of: + # literal '_start_' + # key=value (for each preset variable) + # literal '_start_' + # key=value (for each post set variable) + bcmd = ('unset IFS\n' + + setup + + varprinter(allvars) + + '{\n%s\n\n} > /dev/null\n' % content + + 'unset IFS\n' + + varprinter(keylist) + "\n") + + cmd = [] + if asuser is not None: + cmd = ['sudo', '-u', asuser] + + cmd.extend(bash) + + sp = subprocess.Popen(cmd, stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + (output, error) = sp.communicate(input=bcmd) + + if sp.returncode != 0: + raise Exception("Process returned %d" % sp.returncode) + + # exclude vars in bash that change on their own or that we used + excluded = ("RANDOM", "LINENO", "_", "__v") + preset = {} + ret = {} + target = None + output = output[0:-1] # remove trailing null + + # go through output. First _start_ is for 'preset', second for 'target'. + # Add to target only things were changed and not in volitile + for line in output.split("\0"): + try: + (key, val) = line.split("=", 1) + if target is preset: + target[key] = val + elif (key not in excluded and + (key in keylist_in or preset.get(key) != val)): + ret[key] = val + except ValueError: + if line != "_start_": + raise + if target is None: + target = preset + elif target is preset: + target = ret + + return ret def read_context_disk_dir(source_dir): @@ -305,24 +361,26 @@ def read_context_disk_dir(source_dir): if "context.sh" in found: try: with open(os.path.join(source_dir, 'context.sh'), 'r') as f: - context = parse_context_data(f.read()) + content = f.read().strip() + if content: + context = parse_shell_config(content) f.close() - if not context: - raise NonContextDiskDir("No variables in context") - - except (IOError, NonContextDiskDir) as e: + except (IOError, Exception) as e: raise NonContextDiskDir("Error reading context.sh: %s" % (e)) - - results['metadata'] = context else: raise NonContextDiskDir("Missing context.sh") + if not context: + raise NonContextDiskDir("No context variables found") + + results['metadata'] = context + # process single or multiple SSH keys ssh_key_var = None - if "ssh_key" in context: - ssh_key_var = "ssh_key" - elif "ssh_public_key" in context: - ssh_key_var = "ssh_public_key" + if "SSH_KEY" in context: + ssh_key_var = "SSH_KEY" + elif "SSH_PUBLIC_KEY" in context: + ssh_key_var = "SSH_PUBLIC_KEY" if ssh_key_var: lines = context.get(ssh_key_var).splitlines() @@ -331,22 +389,22 @@ def read_context_disk_dir(source_dir): # custom hostname -- try hostname or leave cloud-init # itself create hostname from IP address later - for k in ('hostname', 'public_ip', 'ip_public', 'eth0_ip'): + for k in ('HOSTNAME', 'PUBLIC_IP', 'IP_PUBLIC', 'ETH0_IP'): if k in context: results['metadata']['local-hostname'] = context[k] break # raw user data - if "user_data" in context: - results['userdata'] = context["user_data"] - elif "userdata" in context: - results['userdata'] = context["userdata"] + if "USER_DATA" in context: + results['userdata'] = context["USER_DATA"] + elif "USERDATA" in context: + results['userdata'] = context["USERDATA"] # generate static /etc/network/interfaces # only if there are any required context variables # http://opennebula.org/documentation:rel3.8:cong#network_configuration for k in context.keys(): - if re.match(r'^eth\d+_ip$', k): + if re.match(r'^ETH\d+_ip$', k): (out, _) = util.subp(['/sbin/ip', 'link']) net = OpenNebulaNetwork(out, context) results['network-interfaces'] = net.gen_conf() diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index 4b82a49c..f544e145 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -6,18 +6,18 @@ from tests.unittests.helpers import populate_dir import os TEST_VARS = { - 'var1': 'single', - 'var2': 'double word', - 'var3': 'multi\nline\n', - 'var4': "'single'", - 'var5': "'double word'", - 'var6': "'multi\nline\n'", - 'var7': 'single\\t', - 'var8': 'double\\tword', - 'var9': 'multi\\t\nline\n', - 'var10': '\\', # expect \ - 'var11': '\'', # expect ' - 'var12': '$', # expect $ + 'VAR1': 'single', + 'VAR2': 'double word', + 'VAR3': 'multi\nline\n', + 'VAR4': "'single'", + 'VAR5': "'double word'", + 'VAR6': "'multi\nline\n'", + 'VAR7': 'single\\t', + 'VAR8': 'double\\tword', + 'VAR9': 'multi\\t\nline\n', + 'VAR10': '\\', # expect \ + 'VAR11': '\'', # expect ' + 'VAR12': '$', # expect $ } USER_DATA = '#cloud-config\napt_upgrade: true' @@ -135,13 +135,13 @@ iface eth0 inet static def test_eth0_override(self): context = { - 'dns': '1.2.3.8', - 'eth0_ip': '1.2.3.4', - 'eth0_network': '1.2.3.0', - 'eth0_mask': '255.255.0.0', - 'eth0_gateway': '1.2.3.5', - 'eth0_domain': 'example.com', - 'eth0_dns': '1.2.3.6 1.2.3.7' + 'DNS': '1.2.3.8', + 'ETH0_IP': '1.2.3.4', + 'ETH0_NETWORK': '1.2.3.0', + 'ETH0_MASK': '255.255.0.0', + 'ETH0_GATEWAY': '1.2.3.5', + 'ETH0_DOMAIN': 'example.com', + 'ETH0_DNS': '1.2.3.6 1.2.3.7' } net = ds.OpenNebulaNetwork(CMD_IP_OUT, context) -- cgit v1.2.3 From 64a2f3c7d4eae1354134c021db232c117eb1c772 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Tue, 10 Sep 2013 15:37:15 +0200 Subject: Configurable OpenNebula::parseuser. Seed search dir+dev merge. Eat shell parser error output. Few tests for tests for get_data. --- cloudinit/sources/DataSourceOpenNebula.py | 78 +++++++++++------- tests/unittests/test_datasource/test_opennebula.py | 95 +++++++++++++++++++--- 2 files changed, 132 insertions(+), 41 deletions(-) (limited to 'cloudinit/sources/DataSourceOpenNebula.py') diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index a1995aca..1b419cfd 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -35,6 +35,7 @@ LOG = logging.getLogger(__name__) DEFAULT_IID = "iid-dsopennebula" DEFAULT_MODE = 'net' +DEFAULT_PARSEUSER = 'nobody' CONTEXT_DISK_FILES = ["context.sh"] VALID_DSMODES = ("local", "net", "disabled") @@ -51,33 +52,35 @@ class DataSourceOpenNebula(sources.DataSource): return "%s [seed=%s][dsmode=%s]" % (root, self.seed, self.dsmode) def get_data(self): - defaults = { - "instance-id": DEFAULT_IID, #TODO:???? - "DSMODE": self.dsmode - } - + defaults = {"instance-id": DEFAULT_IID} + results = None seed = None - results = {} - # first try to read local seed_dir - if os.path.isdir(self.seed_dir): + # decide parseuser for context.sh shell reader + parseuser = DEFAULT_PARSEUSER + if self.ds_cfg.get('parseuser'): + parseuser = self.ds_cfg.get('parseuser') + + candidates = [self.seed_dir] + candidates.extend(find_candidate_devs()) + for cdev in candidates: try: - results = read_context_disk_dir(self.seed_dir) - seed = self.seed_dir + if os.path.isdir(self.seed_dir): + results = read_context_disk_dir(cdev, asuser=parseuser) + elif cdev.startswith("/dev"): + results = util.mount_cb(cdev, read_context_disk_dir, + data=parseuser) except NonContextDiskDir: - util.logexc(LOG, "Failed reading context from %s", - self.seed_dir) + continue + except BrokenContextDiskDir as exc: + raise exc + except util.MountFailedError: + LOG.warn("%s was not mountable" % cdev) - if not seed: - # then try to detect and mount candidate devices and - # read contextualization if present - for dev in find_candidate_devs(): - try: - results = util.mount_cb(dev, read_context_disk_dir) - seed = dev - break - except (NonContextDiskDir, util.MountFailedError): - pass + if results: + seed = cdev + LOG.debug("found datasource in %s", cdev) + break if not seed: return False @@ -95,7 +98,7 @@ class DataSourceOpenNebula(sources.DataSource): # decide dsmode if user_dsmode: dsmode = user_dsmode - elif self.ds_cfg.get('dsmode'): #TODO: fakt se k tomu nekdy dostane? + elif self.ds_cfg.get('dsmode'): dsmode = self.ds_cfg.get('dsmode') else: dsmode = DEFAULT_MODE @@ -137,6 +140,10 @@ class NonContextDiskDir(Exception): pass +class BrokenContextDiskDir(Exception): + pass + + class OpenNebulaNetwork(object): REG_DEV_MAC = re.compile( r'^\d+: (eth\d+):.*?link\/ether (..:..:..:..:..:..) ?', @@ -306,7 +313,8 @@ def parse_shell_config(content, keylist=None, bash=None, asuser=None): cmd.extend(bash) sp = subprocess.Popen(cmd, stdin=subprocess.PIPE, - stdout=subprocess.PIPE) + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) (output, error) = sp.communicate(input=bcmd) if sp.returncode != 0: @@ -340,7 +348,7 @@ def parse_shell_config(content, keylist=None, bash=None, asuser=None): return ret -def read_context_disk_dir(source_dir): +def read_context_disk_dir(source_dir, asuser=None): """ read_context_disk_dir(source_dir): read source_dir and return a tuple with metadata dict and user-data @@ -355,23 +363,31 @@ def read_context_disk_dir(source_dir): if not found: raise NonContextDiskDir("%s: %s" % (source_dir, "no files found")) - results = {'userdata': None, 'metadata': {}} context = {} + results = {'userdata': None, 'metadata': {}} if "context.sh" in found: try: with open(os.path.join(source_dir, 'context.sh'), 'r') as f: - content = f.read().strip() - if content: - context = parse_shell_config(content) + content = f.read().strip() f.close() - except (IOError, Exception) as e: + + # don't pass empty context script + # to shell parser + non_empty = re.match(r'.*?^\s*([^# ]+)', content, + re.MULTILINE | re.DOTALL) + + if non_empty: + context = parse_shell_config(content, asuser=asuser) + except IOError as e: raise NonContextDiskDir("Error reading context.sh: %s" % (e)) + except Exception as e: + raise BrokenContextDiskDir("Error processing context.sh: %s" % (e)) else: raise NonContextDiskDir("Missing context.sh") if not context: - raise NonContextDiskDir("No context variables found") + return results results['metadata'] = context diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index f544e145..752638b6 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -1,4 +1,5 @@ from cloudinit.sources import DataSourceOpenNebula as ds +from cloudinit import helpers from cloudinit import util from mocker import MockerTestCase from tests.unittests.helpers import populate_dir @@ -20,6 +21,8 @@ TEST_VARS = { 'VAR12': '$', # expect $ } +INVALID_PARSEUSER = 'cloud-init-mocker-opennebula-invalid' +INVALID_CONTEXT = ';' USER_DATA = '#cloud-config\napt_upgrade: true' SSH_KEY = 'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460-%i' HOSTNAME = 'foo.example.com' @@ -38,21 +41,93 @@ class TestOpenNebulaDataSource(MockerTestCase): def setUp(self): super(TestOpenNebulaDataSource, self).setUp() self.tmp = self.makeDir() + self.paths = helpers.Paths({'cloud_dir': self.tmp}) + + # defaults for few tests + self.ds = ds.DataSourceOpenNebula + self.seed_dir = os.path.join(self.paths.seed_dir, "opennebula") + self.sys_cfg = {'datasource': {'OpenNebula': {'dsmode': 'local'}}} + + def test_get_data_non_contextdisk(self): + try: + # dont' try to lookup for CDs + orig_find_devs_with = util.find_devs_with + util.find_devs_with = lambda n: [] + + dsrc = self.ds(sys_cfg=self.sys_cfg, distro=None, paths=self.paths) + ret = dsrc.get_data() + self.assertFalse(ret) + finally: + util.find_devs_with = orig_find_devs_with + + def test_get_data_broken_contextdisk(self): + try: + # dont' try to lookup for CDs + orig_find_devs_with = util.find_devs_with + util.find_devs_with = lambda n: [] + + populate_dir(self.seed_dir, {'context.sh': INVALID_CONTEXT}) + dsrc = self.ds(sys_cfg=self.sys_cfg, distro=None, paths=self.paths) + self.assertRaises(ds.BrokenContextDiskDir, dsrc.get_data) + finally: + util.find_devs_with = orig_find_devs_with + + def test_get_data_invalid_identity(self): + try: + # dont' try to lookup for CDs + orig_find_devs_with = util.find_devs_with + util.find_devs_with = lambda n: [] + + sys_cfg = self.sys_cfg + sys_cfg['datasource']['OpenNebula']['parseuser'] = \ + INVALID_PARSEUSER + + populate_context_dir(self.seed_dir, {'KEY1': 'val1'}) + dsrc = self.ds(sys_cfg=sys_cfg, distro=None, paths=self.paths) + self.assertRaises(ds.BrokenContextDiskDir, dsrc.get_data) + finally: + util.find_devs_with = orig_find_devs_with + + def test_get_data(self): + try: + # dont' try to lookup for CDs + orig_find_devs_with = util.find_devs_with + util.find_devs_with = lambda n: [] + populate_context_dir(self.seed_dir, {'KEY1': 'val1'}) + dsrc = self.ds(sys_cfg=self.sys_cfg, distro=None, paths=self.paths) + ret = dsrc.get_data() + self.assertTrue(ret) + finally: + util.find_devs_with = orig_find_devs_with def test_seed_dir_non_contextdisk(self): - my_d = os.path.join(self.tmp, 'non-contextdisk') - self.assertRaises(ds.NonContextDiskDir, ds.read_context_disk_dir, my_d) + self.assertRaises(ds.NonContextDiskDir, ds.read_context_disk_dir, + self.seed_dir) + + def test_seed_dir_empty1_context(self): + populate_dir(self.seed_dir, {'context.sh': ''}) + results = ds.read_context_disk_dir(self.seed_dir) + + self.assertEqual(results['userdata'], None) + self.assertEqual(results['metadata'], {}) + + def test_seed_dir_empty2_context(self): + populate_context_dir(self.seed_dir, {}) + results = ds.read_context_disk_dir(self.seed_dir) + + self.assertEqual(results['userdata'], None) + self.assertEqual(results['metadata'], {}) + + def test_seed_dir_broken_context(self): + populate_dir(self.seed_dir, {'context.sh': INVALID_CONTEXT}) - def test_seed_dir_bad_context(self): - my_d = os.path.join(self.tmp, 'bad-context') - os.mkdir(my_d) - open(os.path.join(my_d, "context.sh"), "w").close() - self.assertRaises(ds.NonContextDiskDir, ds.read_context_disk_dir, my_d) + self.assertRaises(ds.BrokenContextDiskDir, + ds.read_context_disk_dir, + self.seed_dir) def test_context_parser(self): - my_d = os.path.join(self.tmp, 'context-parser') - populate_context_dir(my_d, TEST_VARS) - results = ds.read_context_disk_dir(my_d) + populate_context_dir(self.seed_dir, TEST_VARS) + results = ds.read_context_disk_dir(self.seed_dir) self.assertTrue('metadata' in results) self.assertEqual(TEST_VARS, results['metadata']) -- cgit v1.2.3 From c0d1a59d96a16c080ad8b8278251294cccc21894 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Tue, 10 Sep 2013 18:35:19 +0200 Subject: Fix detection of ETHx_IP context variable, add test. --- cloudinit/sources/DataSourceOpenNebula.py | 3 +-- tests/unittests/test_datasource/test_opennebula.py | 6 ++++++ 2 files changed, 7 insertions(+), 2 deletions(-) (limited to 'cloudinit/sources/DataSourceOpenNebula.py') diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 1b419cfd..141bd454 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -376,7 +376,6 @@ def read_context_disk_dir(source_dir, asuser=None): # to shell parser non_empty = re.match(r'.*?^\s*([^# ]+)', content, re.MULTILINE | re.DOTALL) - if non_empty: context = parse_shell_config(content, asuser=asuser) except IOError as e: @@ -420,7 +419,7 @@ def read_context_disk_dir(source_dir, asuser=None): # only if there are any required context variables # http://opennebula.org/documentation:rel3.8:cong#network_configuration for k in context.keys(): - if re.match(r'^ETH\d+_ip$', k): + if re.match(r'^ETH\d+_IP$', k): (out, _) = util.subp(['/sbin/ip', 'link']) net = OpenNebulaNetwork(out, context) results['network-interfaces'] = net.gen_conf() diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index f5103dc0..f2457657 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -173,6 +173,12 @@ class TestOpenNebulaDataSource(MockerTestCase): self.assertTrue('local-hostname' in results['metadata']) self.assertEqual(PUBLIC_IP, results['metadata']['local-hostname']) + def test_network_interfaces(self): + populate_context_dir(self.seed_dir, {'ETH0_IP': '1.2.3.4'}) + results = ds.read_context_disk_dir(self.seed_dir) + + self.assertTrue('network-interfaces' in results) + def test_find_candidates(self): def my_devs_with(criteria): return { -- cgit v1.2.3 From 7dff14c285d6562b9cd0b47876628607feac4a18 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 10 Sep 2013 14:18:58 -0400 Subject: some cleanups and changes * use util.subp from inside parse_shell_config, and adjust exception handling accordingly. * add 'switch_user_cmd' as a callback function to pass to parse_shell_config, which allows us to mock this to avoid 'sudo' when running test cases. Basically the test cases just return '[]' here. * fix some pylint * handle empty 'content' in parse_shell_config and remove the protection that was present. --- cloudinit/sources/DataSourceOpenNebula.py | 50 +++++++++++----------- tests/unittests/test_datasource/test_opennebula.py | 15 ++++++- 2 files changed, 40 insertions(+), 25 deletions(-) (limited to 'cloudinit/sources/DataSourceOpenNebula.py') diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 141bd454..07dc25ff 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -23,9 +23,9 @@ # along with this program. If not, see . import os +import pwd import re -import string -import subprocess +import string # pylint: disable=W0402 from cloudinit import log as logging from cloudinit import sources @@ -58,7 +58,7 @@ class DataSourceOpenNebula(sources.DataSource): # decide parseuser for context.sh shell reader parseuser = DEFAULT_PARSEUSER - if self.ds_cfg.get('parseuser'): + if 'parseuser' in self.ds_cfg: parseuser = self.ds_cfg.get('parseuser') candidates = [self.seed_dir] @@ -260,13 +260,21 @@ def find_candidate_devs(): return combined -def parse_shell_config(content, keylist=None, bash=None, asuser=None): +def switch_user_cmd(user): + return ['sudo', '-u', user] + + +def parse_shell_config(content, keylist=None, bash=None, asuser=None, + switch_user_cb=None): if isinstance(bash, str): bash = [bash] elif bash is None: bash = ['bash', '-e'] + if switch_user_cb is None: + switch_user_cb = switch_user_cmd + # allvars expands to all existing variables by using '${!x*}' notation # where x is lower or upper case letters or '_' allvars = ["${!%s*}" % x for x in string.letters + "_"] @@ -302,23 +310,17 @@ def parse_shell_config(content, keylist=None, bash=None, asuser=None): bcmd = ('unset IFS\n' + setup + varprinter(allvars) + - '{\n%s\n\n} > /dev/null\n' % content + + '{\n%s\n\n:\n} > /dev/null\n' % content + 'unset IFS\n' + varprinter(keylist) + "\n") cmd = [] if asuser is not None: - cmd = ['sudo', '-u', asuser] + cmd = switch_user_cb(asuser) cmd.extend(bash) - sp = subprocess.Popen(cmd, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - (output, error) = sp.communicate(input=bcmd) - - if sp.returncode != 0: - raise Exception("Process returned %d" % sp.returncode) + (output, _error) = util.subp(cmd, data=bcmd) # exclude vars in bash that change on their own or that we used excluded = ("RANDOM", "LINENO", "_", "__v") @@ -329,7 +331,7 @@ def parse_shell_config(content, keylist=None, bash=None, asuser=None): # go through output. First _start_ is for 'preset', second for 'target'. # Add to target only things were changed and not in volitile - for line in output.split("\0"): + for line in output.split("\x00"): try: (key, val) = line.split("=", 1) if target is preset: @@ -367,21 +369,21 @@ def read_context_disk_dir(source_dir, asuser=None): results = {'userdata': None, 'metadata': {}} if "context.sh" in found: + if asuser is not None: + try: + pwd.getpwnam(asuser) + except KeyError as e: + raise BrokenContextDiskDir("configured user '%s' " + "does not exist", asuser) try: with open(os.path.join(source_dir, 'context.sh'), 'r') as f: content = f.read().strip() - f.close() - - # don't pass empty context script - # to shell parser - non_empty = re.match(r'.*?^\s*([^# ]+)', content, - re.MULTILINE | re.DOTALL) - if non_empty: - context = parse_shell_config(content, asuser=asuser) + + context = parse_shell_config(content, asuser=asuser) + except util.ProcessExecutionError as e: + raise BrokenContextDiskDir("Error processing context.sh: %s" % (e)) except IOError as e: raise NonContextDiskDir("Error reading context.sh: %s" % (e)) - except Exception as e: - raise BrokenContextDiskDir("Error processing context.sh: %s" % (e)) else: raise NonContextDiskDir("Missing context.sh") diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index f2457657..9c7a644a 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -37,6 +37,7 @@ CMD_IP_OUT = '''\ class TestOpenNebulaDataSource(MockerTestCase): + parsed_user = None def setUp(self): super(TestOpenNebulaDataSource, self).setUp() @@ -48,6 +49,18 @@ class TestOpenNebulaDataSource(MockerTestCase): self.seed_dir = os.path.join(self.paths.seed_dir, "opennebula") self.sys_cfg = {'datasource': {'OpenNebula': {'dsmode': 'local'}}} + # we don't want 'sudo' called in tests. so we patch switch_user_cmd + def my_switch_user_cmd(user): + self.parsed_user = user + return [] + + self.switch_user_cmd_real = ds.switch_user_cmd + ds.switch_user_cmd = my_switch_user_cmd + + def tearDown(self): + ds.switch_user_cmd = self.switch_user_cmd_real + super(TestOpenNebulaDataSource, self).tearDown() + def test_get_data_non_contextdisk(self): try: # dont' try to lookup for CDs @@ -96,9 +109,9 @@ class TestOpenNebulaDataSource(MockerTestCase): util.find_devs_with = orig_find_devs_with def test_get_data(self): + orig_find_devs_with = util.find_devs_with try: # dont' try to lookup for CDs - orig_find_devs_with = util.find_devs_with util.find_devs_with = lambda n: [] populate_context_dir(self.seed_dir, {'KEY1': 'val1'}) dsrc = self.ds(sys_cfg=self.sys_cfg, distro=None, paths=self.paths) -- cgit v1.2.3