From c59e06c6d20ce585927f336630e8ae3cca12c110 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Wed, 12 Sep 2012 10:59:19 +0200 Subject: Initial support for OpenNebula's contextualization disk. --- cloudinit/settings.py | 1 + cloudinit/sources/DataSourceOpenNebula.py | 227 ++++++++++++++++++++++++++++++ 2 files changed, 228 insertions(+) create mode 100644 cloudinit/sources/DataSourceOpenNebula.py diff --git a/cloudinit/settings.py b/cloudinit/settings.py index 8cc9e3b4..4b95b5b7 100644 --- a/cloudinit/settings.py +++ b/cloudinit/settings.py @@ -31,6 +31,7 @@ CFG_BUILTIN = { 'datasource_list': [ 'NoCloud', 'ConfigDrive', + 'OpenNebula', 'AltCloud', 'OVF', 'MAAS', diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py new file mode 100644 index 00000000..03c0103e --- /dev/null +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -0,0 +1,227 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Yahoo! Inc. +# Copyright (C) 2012 CERIT-Scientific Cloud +# +# Author: Scott Moser +# Author: Joshua Harlow +# Author: Vlastimil Holer +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import re +import subprocess + +from cloudinit import log as logging +from cloudinit import sources +from cloudinit import util + +LOG = logging.getLogger(__name__) + +DEFAULT_IID = "iid-dsopennebula" +CONTEXT_DISK_FILES = ["context.sh"] + +class DataSourceOpenNebula(sources.DataSource): + def __init__(self, sys_cfg, distro, paths): + sources.DataSource.__init__(self, sys_cfg, distro, paths) + self.dsmode = 'local' + self.seed = None + self.seed_dir = os.path.join(paths.seed_dir, 'opennebula') + + def __str__(self): + mstr = "%s [seed=%s][dsmode=%s]" % (util.obj_name(self), + self.seed, self.dsmode) + return mstr + + def get_data(self): + defaults = { + "instance-id": DEFAULT_IID, + "dsmode": self.dsmode, + } + + found = None + md = {} + ud = "" + + results = {} + if os.path.isdir(self.seed_dir): + try: + results=read_on_context_device_dir(self.seed_dir) + found = self.seed_dir + except NonContextDeviceDir: + util.logexc(LOG, "Failed reading context device from %s", + self.seed_dir) + if not found: + devlist = find_candidate_devs() + for dev in devlist: + try: + results = util.mount_cb(dev, read_context_disk_dir) + found = dev + break + except (NonConfigDriveDir, util.MountFailedError): + pass + except BrokenConfigDriveDir: + util.logexc(LOG, "broken config drive: %s", dev) + + if not found: + return False + + md = results['metadata'] + md = util.mergedict(md, defaults) + + # update interfaces and ifup only on the local datasource + # this way the DataSourceConfigDriveNet doesn't do it also. +# if 'network-interfaces' in md and self.dsmode == "local": +# if md['dsmode'] == "pass": +# log.info("updating network interfaces from configdrive") +# else: +# log.debug("updating network interfaces from configdrive") +# +# util.write_file("/etc/network/interfaces", +# md['network-interfaces']) +# try: +# (out, err) = util.subp(['ifup', '--all']) +# if len(out) or len(err): +# log.warn("ifup --all had stderr: %s" % err) +# +# except subprocess.CalledProcessError as exc: +# log.warn("ifup --all failed: %s" % (exc.output[1])) +# + + if md['dsmode'] == self.dsmode: + self.seed = found + self.metadata = md + self.userdata_raw = ud + return True + + LOG.debug("%s: not claiming datasource, dsmode=%s", self, md['dsmode']) + return False + + +class DataSourceOpenNebulaNet(DataSourceOpenNebula): + dsmode = "net" + + +class NonContextDeviceDir(Exception): + pass + + +def find_candidate_devs(): + """ + Return a list of devices that may contain the context disk. + """ + by_fstype = util.find_devs_with("TYPE=iso9660") + by_label = util.find_devs_with("LABEL=CDROM") + + by_fstype.sort() + by_label.sort() + + # combine list of items by putting by-label items first + # followed by fstype items, but with dupes removed + combined = (by_label + [d for d in by_fstype if d not in by_label]) + + # We are looking for block device (sda, not sda1), ignore partitions + combined = [d for d in combined if d[-1] not in "0123456789"] + + return combined + + +def read_context_disk_dir(source_dir): + """ + read_context_disk_dir(source_dir): + read source_dir and return a tuple with metadata dict and user-data + string populated. If not a valid dir, raise a NonContextDeviceDir + """ + + found = {} + for af in CONTEXT_DISK_FILES: + fn = os.path.join(source_dir, af) + if os.path.isfile(fn): + found[af] = fn + + if len(found) == 0: + raise NonContextDeviceDir("%s: %s" % (source_dir, "no files found")) + + context_sh = {} + results = { + 'userdata':None, + 'metadata':{}, + } + + if "context.sh" in found: + # let bash process the contextualization script; + # write out data in normalized output NAME=\$?'?VALUE'? + # TODO: don't trust context.sh! parse manually !!! + try: + BASH_CMD='VARS=`set | sort -u `;' \ + '. %s/context.sh;' \ + 'comm -23 <(set | sort -u) <(echo "$VARS") | egrep -v "^(VARS|PIPESTATUS|_)="' + + (out,err) = util.subp(['bash', + '--noprofile', + '--norc', + '-c', + BASH_CMD % (source_dir) ]) + + for (key,value) in [ l.split('=',1) for l in out.rstrip().split("\n") ]: + # with backslash escapes + r=re.match("^\$'(.*)'$",value) + if r: + context_sh[key.lower()]=r.group(1).\ + replace('\\\\','\\').\ + replace('\\t','\t').\ + replace('\\n','\n').\ + replace("\\'","'") + else: + # multiword values + r=re.match("^'(.*)'$",value) + if r: + context_sh[key.lower()]=r.group(1) + else: + # simple values + context_sh[key.lower()]=value + except subprocess.CalledProcessError as exc: + LOG.warn("context script faled to read" % (exc.output[1])) + results['metadata']=context_sh + + # process single or multiple SSH keys + if "ssh_key" in context_sh: + lines = context_sh.get('ssh_key').splitlines() + results['metadata']['public-keys'] = [l for l in lines + if len(l) and not l.startswith("#")] + + # custom hostname + if 'hostname' in context_sh: + results['metadata']['local-hostname'] = context_sh['hostname'] + + # raw user data + if "user_data" in context_sh: + results['userdata'] = context_sh["user_data"] + if "userdata" in context_sh: + results['userdata'] = context_sh["userdata"] + + return results + + +# Used to match classes to dependencies +datasources = [ + (DataSourceOpenNebula, (sources.DEP_FILESYSTEM, )), + (DataSourceOpenNebulaNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), +] + + +# Return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return sources.list_from_depends(depends, datasources) -- cgit v1.2.3 From 34bebd8569e9319b791802f4fd551537967aec69 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Wed, 19 Sep 2012 15:31:10 +0200 Subject: Optionally resolve IPv4 hostname. --- cloudinit/sources/__init__.py | 10 ++++++++-- cloudinit/util.py | 7 +++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 3f611d44..9b68f99e 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -149,7 +149,7 @@ class DataSource(object): return "iid-datasource" return str(self.metadata['instance-id']) - def get_hostname(self, fqdn=False): + def get_hostname(self, fqdn=False, resolve_ip=False): defdomain = "localdomain" defhost = "localhost" domain = defdomain @@ -173,7 +173,13 @@ class DataSource(object): # make up a hostname (LP: #475354) in format ip-xx.xx.xx.xx lhost = self.metadata['local-hostname'] if util.is_ipv4(lhost): - toks = "ip-%s" % lhost.replace(".", "-") + if resolve_ip: + toks = util.gethostbyaddr(lhost) + + if toks: + toks = toks.split('.') + else: + toks = "ip-%s" % lhost.replace(".", "-") else: toks = lhost.split(".") diff --git a/cloudinit/util.py b/cloudinit/util.py index 33da73eb..b25ded0d 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -874,6 +874,13 @@ def get_hostname(): return hostname +def gethostbyaddr(ip): + try: + return socket.gethostbyaddr(ip)[0] + except socket.herror: + return None + + def is_resolvable_url(url): """determine if this url is resolvable (existing or ip).""" return (is_resolvable(urlparse.urlparse(url).hostname)) -- cgit v1.2.3 From 0659e75ee2a2c25c113e8e652a3366433e76eff0 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Wed, 19 Sep 2012 15:32:10 +0200 Subject: Use and resolve PUBLIC_IP from context disk if no HOSTNAME is specified. --- cloudinit/sources/DataSourceOpenNebula.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 03c0103e..e8e79c7e 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -109,6 +109,8 @@ class DataSourceOpenNebula(sources.DataSource): LOG.debug("%s: not claiming datasource, dsmode=%s", self, md['dsmode']) return False + def get_hostname(self, fqdn=False, resolve_ip=True): + return sources.DataSource.get_hostname(self, fqdn, resolve_ip) class DataSourceOpenNebulaNet(DataSourceOpenNebula): dsmode = "net" @@ -205,6 +207,8 @@ def read_context_disk_dir(source_dir): # custom hostname if 'hostname' in context_sh: results['metadata']['local-hostname'] = context_sh['hostname'] + elif 'public_ip'in context_sh: + results['metadata']['local-hostname'] = context_sh['public_ip'] # raw user data if "user_data" in context_sh: -- cgit v1.2.3 From 17ac62b5d4d0cb3f25f431ff85411a7cca860d12 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Thu, 20 Sep 2012 18:39:54 +0200 Subject: Initialize toks variable. Fix EC2-like hostname generation based on IPv4. --- cloudinit/sources/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 9b68f99e..a89f4703 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -173,13 +173,14 @@ class DataSource(object): # make up a hostname (LP: #475354) in format ip-xx.xx.xx.xx lhost = self.metadata['local-hostname'] if util.is_ipv4(lhost): + toks = [] if resolve_ip: toks = util.gethostbyaddr(lhost) if toks: toks = toks.split('.') else: - toks = "ip-%s" % lhost.replace(".", "-") + toks = ["ip-%s" % lhost.replace(".", "-")] else: toks = lhost.split(".") -- cgit v1.2.3 From ea42bf67999c1ef61fa46ab805b0093b95dc575f Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Thu, 20 Sep 2012 18:42:18 +0200 Subject: Configurable dsmode. Resolve IPv4 hostname only with dsmode=net. --- cloudinit/sources/DataSourceOpenNebula.py | 42 ++++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index e8e79c7e..0b498a54 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -32,6 +32,7 @@ LOG = logging.getLogger(__name__) DEFAULT_IID = "iid-dsopennebula" CONTEXT_DISK_FILES = ["context.sh"] +VALID_DSMODES = ("local", "net", "disabled") class DataSourceOpenNebula(sources.DataSource): def __init__(self, sys_cfg, distro, paths): @@ -81,6 +82,20 @@ class DataSourceOpenNebula(sources.DataSource): md = results['metadata'] md = util.mergedict(md, defaults) + dsmode = results.get('dsmode', None) + if dsmode not in VALID_DSMODES + (None,): + LOG.warn("user specified invalid mode: %s" % dsmode) + dsmode = None + + if (dsmode is None) and self.ds_cfg.get('dsmode'): + dsmode = self.ds_cfg.get('dsmode') + else: + dsmode = self.dsmode + + if dsmode == "disabled": + # most likely user specified + return False + # update interfaces and ifup only on the local datasource # this way the DataSourceConfigDriveNet doesn't do it also. # if 'network-interfaces' in md and self.dsmode == "local": @@ -100,20 +115,29 @@ class DataSourceOpenNebula(sources.DataSource): # log.warn("ifup --all failed: %s" % (exc.output[1])) # - if md['dsmode'] == self.dsmode: - self.seed = found - self.metadata = md - self.userdata_raw = ud - return True + if dsmode != self.dsmode: + LOG.debug("%s: not claiming datasource, dsmode=%s", self, dsmode) + return False + + self.seed = found + self.metadata = md + self.userdata_raw = ud - LOG.debug("%s: not claiming datasource, dsmode=%s", self, md['dsmode']) - return False + return True - def get_hostname(self, fqdn=False, resolve_ip=True): + def get_hostname(self, fqdn=False, resolve_ip=None): + if resolve_ip is None: + if self.dsmode == 'net': + resolve_ip = True + else: + resolve_ip = False return sources.DataSource.get_hostname(self, fqdn, resolve_ip) + class DataSourceOpenNebulaNet(DataSourceOpenNebula): - dsmode = "net" + def __init__(self, sys_cfg, distro, paths): + DataSourceOpenNebula.__init__(self, sys_cfg, distro, paths) + self.dsmode = 'net' class NonContextDeviceDir(Exception): -- cgit v1.2.3 From 9eaafc7379e0c6ded70a851a703a5bc5c7e56e42 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Mon, 24 Sep 2012 13:28:51 +0200 Subject: Process userdata, ignored by mistake. --- cloudinit/sources/DataSourceOpenNebula.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 0b498a54..ad65a36e 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -54,7 +54,6 @@ class DataSourceOpenNebula(sources.DataSource): found = None md = {} - ud = "" results = {} if os.path.isdir(self.seed_dir): @@ -121,7 +120,7 @@ class DataSourceOpenNebula(sources.DataSource): self.seed = found self.metadata = md - self.userdata_raw = ud + self.userdata_raw = results.get('userdata') return True @@ -237,7 +236,7 @@ def read_context_disk_dir(source_dir): # raw user data if "user_data" in context_sh: results['userdata'] = context_sh["user_data"] - if "userdata" in context_sh: + elif "userdata" in context_sh: results['userdata'] = context_sh["userdata"] return results -- cgit v1.2.3 From 1567b1bc3b2948aa80e0b150d34542e2ff02428f Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Wed, 3 Oct 2012 13:26:36 +0200 Subject: Delete old ConfigDrive's code. --- cloudinit/sources/DataSourceOpenNebula.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index ad65a36e..325269f6 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -95,25 +95,6 @@ class DataSourceOpenNebula(sources.DataSource): # most likely user specified return False - # update interfaces and ifup only on the local datasource - # this way the DataSourceConfigDriveNet doesn't do it also. -# if 'network-interfaces' in md and self.dsmode == "local": -# if md['dsmode'] == "pass": -# log.info("updating network interfaces from configdrive") -# else: -# log.debug("updating network interfaces from configdrive") -# -# util.write_file("/etc/network/interfaces", -# md['network-interfaces']) -# try: -# (out, err) = util.subp(['ifup', '--all']) -# if len(out) or len(err): -# log.warn("ifup --all had stderr: %s" % err) -# -# except subprocess.CalledProcessError as exc: -# log.warn("ifup --all failed: %s" % (exc.output[1])) -# - if dsmode != self.dsmode: LOG.debug("%s: not claiming datasource, dsmode=%s", self, dsmode) return False -- cgit v1.2.3 From 0e7ce376fdb84345c97f936f5ccb1fee964a4c03 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Wed, 3 Oct 2012 13:29:24 +0200 Subject: Delete hyphen. --- cloudinit/sources/DataSourceOpenNebula.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 325269f6..00c076e6 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -2,7 +2,7 @@ # # Copyright (C) 2012 Canonical Ltd. # Copyright (C) 2012 Yahoo! Inc. -# Copyright (C) 2012 CERIT-Scientific Cloud +# Copyright (C) 2012 CERIT Scientific Cloud # # Author: Scott Moser # Author: Joshua Harlow -- cgit v1.2.3 From 8a8f6095586c6c6d49df09e09f513b2408714282 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Wed, 3 Oct 2012 13:30:23 +0200 Subject: Add documentation for OpenNebula datasource. --- doc/sources/opennebula/README | 66 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 doc/sources/opennebula/README diff --git a/doc/sources/opennebula/README b/doc/sources/opennebula/README new file mode 100644 index 00000000..97d8fb8b --- /dev/null +++ b/doc/sources/opennebula/README @@ -0,0 +1,66 @@ +The 'OpenNebula' DataSource supports the OpenNebula contextualization disk. + +The following criteria are required to be identified by +DataSourceOpenNebula as contextualization disk: + * must be formatted with iso9660 filesystem or labeled as CDROM + * must be un-partitioned block device (/dev/vdb, not /dev/vdb1) + * must contain + * context.sh + +== Content of config-drive == + * context.sh + This is the only mandatory file on context disk, the rest depends + on contextualization parameter FILES and thus are optional. It's + a shell script defining all context parameters. This script is + processed by bash (/bin/bash) to simulate behaviour of common + OpenNebula context scripts. Processed variables are handed over + back to cloud-init for further processing. + +== Configuration == +Cloud-init's behaviour can be modified by context variables found +in the context.sh file in the folowing ways: + * dsmode: + values: local, net, disabled + default: None + + Tells if this datasource will be processed in local (pre-networking) or + net (post-networking) stage or even completely disabled. + + * ssh_key: + default: None + If present, these key(s) will be used as the public key(s) for + the instance. More keys can be specified in this single context + variable, but each key must be on it's own line. I.e. keys must + be separated by newlines. + + * hostname: + default: None + Custom hostname for the instance. + + * public_ip: + default: None + If hostname not specified, public_ip is used to resolve hostname. + + * 'user_data' or 'userdata': + default: None + This provides cloud-init user-data. See other documentation for what + all can be present here. + +== Example OpenNebula's Virtual Machine template == + +CONTEXT=[ + PUBLIC_IP="$NIC[IP]", + SSH_KEY="$USER[SSH_KEY] +$USER[SSH_KEY1] +$USER[SSH_KEY2] ", + USER_DATA="#cloud-config +# see https://help.ubuntu.com/community/CloudInit + +packages: [] + +mounts: +- [vdc,none,swap,sw,0,0] +runcmd: +- echo 'Instance has been configured by cloud-init.' | wall + +" ] -- cgit v1.2.3 From a9939fe768e04d52fe530c7467357d79b78a21f4 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Wed, 3 Oct 2012 13:47:24 +0200 Subject: Minor OpenNebula's documentation tuning. --- doc/sources/opennebula/README | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/sources/opennebula/README b/doc/sources/opennebula/README index 97d8fb8b..772a5b99 100644 --- a/doc/sources/opennebula/README +++ b/doc/sources/opennebula/README @@ -9,8 +9,8 @@ DataSourceOpenNebula as contextualization disk: == Content of config-drive == * context.sh - This is the only mandatory file on context disk, the rest depends - on contextualization parameter FILES and thus are optional. It's + This is the only mandatory file on context disk, the rest content depends + on contextualization parameter FILES and thus is optional. It's a shell script defining all context parameters. This script is processed by bash (/bin/bash) to simulate behaviour of common OpenNebula context scripts. Processed variables are handed over @@ -18,7 +18,8 @@ DataSourceOpenNebula as contextualization disk: == Configuration == Cloud-init's behaviour can be modified by context variables found -in the context.sh file in the folowing ways: +in the context.sh file in the folowing ways (variable names are +case-insensitive): * dsmode: values: local, net, disabled default: None @@ -39,7 +40,7 @@ in the context.sh file in the folowing ways: * public_ip: default: None - If hostname not specified, public_ip is used to resolve hostname. + If hostname not specified, public_ip is used to DNS resolve hostname. * 'user_data' or 'userdata': default: None @@ -62,5 +63,4 @@ mounts: - [vdc,none,swap,sw,0,0] runcmd: - echo 'Instance has been configured by cloud-init.' | wall - " ] -- cgit v1.2.3 From 268cdf3458849eb7be41253c38a86c68bbddde59 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Wed, 19 Dec 2012 18:12:37 +0100 Subject: Remove conflicting merge block --- cloudinit/sources/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index f98493de..0bad4c8b 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -163,7 +163,6 @@ class DataSource(object): # make up a hostname (LP: #475354) in format ip-xx.xx.xx.xx lhost = self.metadata['local-hostname'] if util.is_ipv4(lhost): -<<<<<<< TREE toks = [] if resolve_ip: toks = util.gethostbyaddr(lhost) @@ -172,9 +171,6 @@ class DataSource(object): toks = toks.split('.') else: toks = ["ip-%s" % lhost.replace(".", "-")] -======= - toks = ["ip-%s" % lhost.replace(".", "-")] ->>>>>>> MERGE-SOURCE else: toks = lhost.split(".") -- cgit v1.2.3 From 5746598d3cffe07dbae155347eaa44aae48ff572 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Thu, 20 Dec 2012 13:20:57 +0100 Subject: Change context variables .replace with .decode --- cloudinit/sources/DataSourceOpenNebula.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 00c076e6..1622a66e 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -185,11 +185,7 @@ def read_context_disk_dir(source_dir): # with backslash escapes r=re.match("^\$'(.*)'$",value) if r: - context_sh[key.lower()]=r.group(1).\ - replace('\\\\','\\').\ - replace('\\t','\t').\ - replace('\\n','\n').\ - replace("\\'","'") + context_sh[key.lower()]=r.group(1).decode('string_escape') else: # multiword values r=re.match("^'(.*)'$",value) -- cgit v1.2.3 From 21d3a5af7d2e1884009cfd1ad650a937438f5991 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Thu, 20 Dec 2012 13:57:57 +0100 Subject: Add explanation on how context variables parsing works. --- cloudinit/sources/DataSourceOpenNebula.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 1622a66e..a50b0c10 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -167,10 +167,24 @@ def read_context_disk_dir(source_dir): } if "context.sh" in found: - # let bash process the contextualization script; - # write out data in normalized output NAME=\$?'?VALUE'? - # TODO: don't trust context.sh! parse manually !!! try: + # Note: context.sh is a "shell" script with defined context + # variables, like: X="Y" . It's ready to use as a shell source + # e.g.: ". context.sh" and as a shell script it can also reference + # to already defined shell variables. So to have same context var. + # values as we can have in custom shell script, we use bash itself + # to read context.sh and dump variables in easily parsable way. + # + # normalized variables dump format (get by cmd "set"): + # 1. simple single word assignment ........ X=Y + # 2. multiword assignment ................. X='Y Z' + # 3. assignments with backslash escapes ... X=$'Y\nZ' + # + # how context variables are read: + # 1. list existing ("old") shell variables and store into $VARS + # 2. read context variables + # 3. use comm to filter "old" variables from all current + # variables and excl. few other vars with grep BASH_CMD='VARS=`set | sort -u `;' \ '. %s/context.sh;' \ 'comm -23 <(set | sort -u) <(echo "$VARS") | egrep -v "^(VARS|PIPESTATUS|_)="' -- cgit v1.2.3 From 46e646fabf3a0f2593dc9d9f231fd075134de36f Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Thu, 20 Dec 2012 15:17:32 +0100 Subject: Change subp exception handling to util.ProcessExecutionError --- cloudinit/sources/DataSourceOpenNebula.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index a50b0c10..967d7170 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -208,8 +208,8 @@ def read_context_disk_dir(source_dir): else: # simple values context_sh[key.lower()]=value - except subprocess.CalledProcessError as exc: - LOG.warn("context script faled to read" % (exc.output[1])) + except util.ProcessExecutionError, _err: + LOG.warn("Failed to read context variables: %s" % (_err.message)) results['metadata']=context_sh # process single or multiple SSH keys -- cgit v1.2.3 From 2efdb4b8d1c17eea352ae0dc022d2ddca80833da Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Thu, 20 Dec 2012 17:16:47 +0100 Subject: Fix exception handlers for data read. Fix name read_context_disk_dir. --- cloudinit/sources/DataSourceOpenNebula.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 967d7170..2f1ce459 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -58,7 +58,7 @@ class DataSourceOpenNebula(sources.DataSource): results = {} if os.path.isdir(self.seed_dir): try: - results=read_on_context_device_dir(self.seed_dir) + results=read_context_disk_dir(self.seed_dir) found = self.seed_dir except NonContextDeviceDir: util.logexc(LOG, "Failed reading context device from %s", @@ -70,10 +70,8 @@ class DataSourceOpenNebula(sources.DataSource): results = util.mount_cb(dev, read_context_disk_dir) found = dev break - except (NonConfigDriveDir, util.MountFailedError): + except (NonContextDeviceDir, util.MountFailedError): pass - except BrokenConfigDriveDir: - util.logexc(LOG, "broken config drive: %s", dev) if not found: return False @@ -211,6 +209,8 @@ def read_context_disk_dir(source_dir): except util.ProcessExecutionError, _err: LOG.warn("Failed to read context variables: %s" % (_err.message)) results['metadata']=context_sh + else: + raise NonContextDeviceDir("Missing context.sh") # process single or multiple SSH keys if "ssh_key" in context_sh: -- cgit v1.2.3 From 48016ceed0840305bb57804ecaa1a6c2fbd455a7 Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Fri, 21 Dec 2012 12:46:14 +0100 Subject: Add resolv.conf configuration function --- cloudinit/distros/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 6a684b89..ae81be50 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -59,6 +59,10 @@ class Distro(object): # to write this blob out in a distro format raise NotImplementedError() + def apply_resolv_conf(self, settings): + net_fn = self._paths.join(False, "/etc/resolv.conf") + util.write_file(net_fn, settings) + def get_option(self, opt_name, default=None): return self._cfg.get(opt_name, default) -- cgit v1.2.3 From 45641d2e31ffa1b7235a2f36d234e804484ff4a3 Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Fri, 21 Dec 2012 12:47:59 +0100 Subject: Add OpenNebula contextualization options to cloud-init --- cloudinit/sources/DataSourceOpenNebula.py | 115 ++++++++++++++++++++++++++++-- 1 file changed, 111 insertions(+), 4 deletions(-) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 2f1ce459..8af6ad3d 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -101,6 +101,12 @@ class DataSourceOpenNebula(sources.DataSource): self.metadata = md self.userdata_raw = results.get('userdata') + if 'network-interfaces' in results: + self.distro.apply_network(results['network-interfaces']) + + if 'dns' in results: + self.distro.apply_resolv_conf(results['dns']) + return True def get_hostname(self, fqdn=False, resolve_ip=None): @@ -122,6 +128,93 @@ class NonContextDeviceDir(Exception): pass +class OpenNebulaNetwork(object): + REG_ETH=re.compile('^eth') + REG_DEV_MAC=re.compile('^(eth\d+).*HWaddr (..:..:..:..:..:..)') + + def __init__(self, ifconfig, context_sh): + self.ifconfig=ifconfig + self.context_sh=context_sh + self.ifaces=self.get_ifaces() + + def get_ifaces(self): + return [self.REG_DEV_MAC.search(f).groups() for f in self.ifconfig.split("\n") if self.REG_ETH.match(f)] + + def mac2ip(self, mac): + components=mac.split(':')[2:] + + return [str(int(c, 16)) for c in components] + + def get_ip(self, dev, components): + var_name=dev+'_ip' + if var_name in self.context_sh: + return self.context_sh[var_name] + else: + return '.'.join(components) + + def get_mask(self, dev, components): + var_name=dev+'_mask' + if var_name in self.context_sh: + return self.context_sh[var_name] + else: + return '255.255.255.0' + + def get_network(self, dev, components): + var_name=dev+'_network' + if var_name in self.context_sh: + return self.context_sh[var_name] + else: + return '.'.join(components[:-1])+'.0' + + def get_gateway(self, dev, components): + var_name=dev+'_gateway' + if var_name in self.context_sh: + return self.context_sh[var_name] + else: + None + + def gen_conf(self): + conf=[] + conf.append('auto lo') + conf.append('iface lo inet loopback') + conf.append('') + + for i in self.ifaces: + dev=i[0] + mac=i[1] + ip_components=self.mac2ip(mac) + + conf.append('auto '+dev) + conf.append('iface '+dev+' inet static') + conf.append(' address '+self.get_ip(dev, ip_components)) + conf.append(' network '+self.get_network(dev, ip_components)) + conf.append(' netmask '+self.get_mask(dev, ip_components)) + + gateway=self.get_gateway(dev, ip_components) + if gateway: + conf.append(' gateway '+gateway) + + conf.append('') + + return "\n".join(conf) + + def gen_dns(self): + dnss=[] + + if 'dns' in self.context_sh: + dnss.append('nameserver '+self.context_sh['dns']) + + keys=[d for d in self.context_sh.keys() if re.match('^eth\d+_dns$', d)] + + for k in sorted(keys): + dnss.append('nameserver '+self.context_sh[k]) + + if not dnss: + return None + else: + return "\n".join(dnss)+"\n" + + def find_candidate_devs(): """ Return a list of devices that may contain the context disk. @@ -136,9 +229,6 @@ def find_candidate_devs(): # followed by fstype items, but with dupes removed combined = (by_label + [d for d in by_fstype if d not in by_label]) - # We are looking for block device (sda, not sda1), ignore partitions - combined = [d for d in combined if d[-1] not in "0123456789"] - return combined @@ -213,8 +303,15 @@ def read_context_disk_dir(source_dir): raise NonContextDeviceDir("Missing context.sh") # process single or multiple SSH keys + ssh_key_var=None + if "ssh_key" in context_sh: - lines = context_sh.get('ssh_key').splitlines() + ssh_key_var="ssh_key" + elif "ssh_public_key" in context_sh: + ssh_key_var="ssh_public_key" + + if ssh_key_var: + lines = context_sh.get(ssh_key_var).splitlines() results['metadata']['public-keys'] = [l for l in lines if len(l) and not l.startswith("#")] @@ -223,6 +320,8 @@ def read_context_disk_dir(source_dir): results['metadata']['local-hostname'] = context_sh['hostname'] elif 'public_ip'in context_sh: results['metadata']['local-hostname'] = context_sh['public_ip'] + elif 'eth0_ip' in context_sh: + results['metadata']['local-hostname'] = context_sh['eth0_ip'] # raw user data if "user_data" in context_sh: @@ -230,6 +329,14 @@ def read_context_disk_dir(source_dir): elif "userdata" in context_sh: results['userdata'] = context_sh["userdata"] + (out, err) = util.subp(['/sbin/ifconfig', '-a']) + net=OpenNebulaNetwork(out, context_sh) + results['network-interfaces']=net.gen_conf() + + dns=net.gen_dns() + if dns: + results['dns']=dns + return results -- cgit v1.2.3 From 5aab64f71c1f07670b59dd7be18d704611dc0ab5 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Wed, 2 Jan 2013 15:20:33 +0100 Subject: Add OpenNebula.org copyright. --- cloudinit/sources/DataSourceOpenNebula.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 8af6ad3d..57b0c62c 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -2,11 +2,13 @@ # # Copyright (C) 2012 Canonical Ltd. # Copyright (C) 2012 Yahoo! Inc. -# Copyright (C) 2012 CERIT Scientific Cloud +# Copyright (C) 2012-2013 CERIT Scientific Cloud +# Copyright (C) 2012 OpenNebula.org # # Author: Scott Moser # Author: Joshua Harlow # Author: Vlastimil Holer +# Author: Javier Fontan # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as -- cgit v1.2.3 From 5bfbf7a81872afd426271b0ed3ed65e76b9a584e Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Fri, 4 Jan 2013 15:26:11 +0100 Subject: Append DNS related stuff to network interfaces configuration. Minor cleanups. --- cloudinit/sources/DataSourceOpenNebula.py | 76 +++++++++++++++++-------------- 1 file changed, 41 insertions(+), 35 deletions(-) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 57b0c62c..52474675 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -103,12 +103,11 @@ class DataSourceOpenNebula(sources.DataSource): self.metadata = md self.userdata_raw = results.get('userdata') - if 'network-interfaces' in results: + # apply static network configuration only in 'local' dsmode + if ('network-interfaces' in results and self.dsmode == "local"): + LOG.debug("Updating network interfaces from %s", self) self.distro.apply_network(results['network-interfaces']) - if 'dns' in results: - self.distro.apply_resolv_conf(results['dns']) - return True def get_hostname(self, fqdn=False, resolve_ip=None): @@ -175,7 +174,25 @@ class OpenNebulaNetwork(object): else: None + def get_dns(self, dev, components): + var_name=dev+'_dns' + if var_name in self.context_sh: + return self.context_sh[var_name] + else: + None + + def get_domain(self, dev, components): + var_name=dev+'_domain' + if var_name in self.context_sh: + return self.context_sh[var_name] + else: + None + def gen_conf(self): + global_dns=[] + if 'dns' in self.context_sh: + global_dns.append(self.context_sh['dns']) + conf=[] conf.append('auto lo') conf.append('iface lo inet loopback') @@ -196,25 +213,21 @@ class OpenNebulaNetwork(object): if gateway: conf.append(' gateway '+gateway) - conf.append('') - - return "\n".join(conf) - - def gen_dns(self): - dnss=[] + domain=self.get_domain(dev, ip_components) + if domain: + conf.append(' dns-search '+domain) - if 'dns' in self.context_sh: - dnss.append('nameserver '+self.context_sh['dns']) + # add global DNS servers to all interfaces + dns=self.get_dns(dev, ip_components) + if global_dns or dns: + all_dns=global_dns + if dns: + all_dns.append(dns) + conf.append(' dns-nameservers '+' '.join(all_dns)) - keys=[d for d in self.context_sh.keys() if re.match('^eth\d+_dns$', d)] - - for k in sorted(keys): - dnss.append('nameserver '+self.context_sh[k]) + conf.append('') - if not dnss: - return None - else: - return "\n".join(dnss)+"\n" + return "\n".join(conf) def find_candidate_devs(): @@ -280,10 +293,8 @@ def read_context_disk_dir(source_dir): 'comm -23 <(set | sort -u) <(echo "$VARS") | egrep -v "^(VARS|PIPESTATUS|_)="' (out,err) = util.subp(['bash', - '--noprofile', - '--norc', - '-c', - BASH_CMD % (source_dir) ]) + '--noprofile', '--norc', + '-c', BASH_CMD % (source_dir) ]) for (key,value) in [ l.split('=',1) for l in out.rstrip().split("\n") ]: # with backslash escapes @@ -317,13 +328,12 @@ def read_context_disk_dir(source_dir): results['metadata']['public-keys'] = [l for l in lines if len(l) and not l.startswith("#")] - # custom hostname - if 'hostname' in context_sh: - results['metadata']['local-hostname'] = context_sh['hostname'] - elif 'public_ip'in context_sh: - results['metadata']['local-hostname'] = context_sh['public_ip'] - elif 'eth0_ip' in context_sh: - results['metadata']['local-hostname'] = context_sh['eth0_ip'] + # custom hostname -- try hostname or leave cloud-init + # itself create hostname from IP address later + for k in ('hostname','public_ip','ip_public','eth0_ip'): + if k in context_sh: + results['metadata']['local-hostname'] = context_sh[k] + break # raw user data if "user_data" in context_sh: @@ -335,10 +345,6 @@ def read_context_disk_dir(source_dir): net=OpenNebulaNetwork(out, context_sh) results['network-interfaces']=net.gen_conf() - dns=net.gen_dns() - if dns: - results['dns']=dns - return results -- cgit v1.2.3 From 54f6ccccfb4f75dc6877b04b42987e834b7a0015 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Tue, 19 Feb 2013 12:35:12 +0100 Subject: Change network ifaces detection from ifconfig to ip command. --- cloudinit/sources/DataSourceOpenNebula.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 52474675..9b1b5f9c 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -130,16 +130,15 @@ class NonContextDeviceDir(Exception): class OpenNebulaNetwork(object): - REG_ETH=re.compile('^eth') - REG_DEV_MAC=re.compile('^(eth\d+).*HWaddr (..:..:..:..:..:..)') + REG_DEV_MAC=re.compile('^\d+: (eth\d+):.*link\/ether (..:..:..:..:..:..) ') - def __init__(self, ifconfig, context_sh): - self.ifconfig=ifconfig + def __init__(self, ip, context_sh): + self.ip=ip self.context_sh=context_sh self.ifaces=self.get_ifaces() def get_ifaces(self): - return [self.REG_DEV_MAC.search(f).groups() for f in self.ifconfig.split("\n") if self.REG_ETH.match(f)] + return [self.REG_DEV_MAC.search(f).groups() for f in self.ip.split("\n") if self.REG_DEV_MAC.match(f)] def mac2ip(self, mac): components=mac.split(':')[2:] @@ -341,7 +340,7 @@ def read_context_disk_dir(source_dir): elif "userdata" in context_sh: results['userdata'] = context_sh["userdata"] - (out, err) = util.subp(['/sbin/ifconfig', '-a']) + (out, err) = util.subp(['/sbin/ip', '-o', 'link']) net=OpenNebulaNetwork(out, context_sh) results['network-interfaces']=net.gen_conf() -- cgit v1.2.3 From eda36b6116a79433cc29b778b0008cf35d6a2afa Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Tue, 19 Feb 2013 14:21:42 +0100 Subject: Name unification Context.*Device -> Context.*Disk --- cloudinit/sources/DataSourceOpenNebula.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 9b1b5f9c..bc199461 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -62,8 +62,8 @@ class DataSourceOpenNebula(sources.DataSource): try: results=read_context_disk_dir(self.seed_dir) found = self.seed_dir - except NonContextDeviceDir: - util.logexc(LOG, "Failed reading context device from %s", + except NonContextDiskDir: + util.logexc(LOG, "Failed reading context disk from %s", self.seed_dir) if not found: devlist = find_candidate_devs() @@ -72,7 +72,7 @@ class DataSourceOpenNebula(sources.DataSource): results = util.mount_cb(dev, read_context_disk_dir) found = dev break - except (NonContextDeviceDir, util.MountFailedError): + except (NonContextDiskDir, util.MountFailedError): pass if not found: @@ -125,7 +125,7 @@ class DataSourceOpenNebulaNet(DataSourceOpenNebula): self.dsmode = 'net' -class NonContextDeviceDir(Exception): +class NonContextDiskDir(Exception): pass @@ -250,7 +250,7 @@ def read_context_disk_dir(source_dir): """ read_context_disk_dir(source_dir): read source_dir and return a tuple with metadata dict and user-data - string populated. If not a valid dir, raise a NonContextDeviceDir + string populated. If not a valid dir, raise a NonContextDiskDir """ found = {} @@ -260,7 +260,7 @@ def read_context_disk_dir(source_dir): found[af] = fn if len(found) == 0: - raise NonContextDeviceDir("%s: %s" % (source_dir, "no files found")) + raise NonContextDiskDir("%s: %s" % (source_dir, "no files found")) context_sh = {} results = { @@ -312,7 +312,7 @@ def read_context_disk_dir(source_dir): LOG.warn("Failed to read context variables: %s" % (_err.message)) results['metadata']=context_sh else: - raise NonContextDeviceDir("Missing context.sh") + raise NonContextDiskDir("Missing context.sh") # process single or multiple SSH keys ssh_key_var=None -- cgit v1.2.3 From a8e39f502cffdf8639263fb1dcb0ad36158c2d83 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Tue, 19 Feb 2013 16:00:16 +0100 Subject: Context.sh parsing cleanups, fix single quotes handling in multiword variables. --- cloudinit/sources/DataSourceOpenNebula.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index bc199461..dad64bd4 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -288,28 +288,35 @@ def read_context_disk_dir(source_dir): # 3. use comm to filter "old" variables from all current # variables and excl. few other vars with grep BASH_CMD='VARS=`set | sort -u `;' \ - '. %s/context.sh;' \ + 'source %s/context.sh;' \ 'comm -23 <(set | sort -u) <(echo "$VARS") | egrep -v "^(VARS|PIPESTATUS|_)="' - (out,err) = util.subp(['bash', - '--noprofile', '--norc', + (out,err) = util.subp(['bash','--noprofile', '--norc', '-c', BASH_CMD % (source_dir) ]) for (key,value) in [ l.split('=',1) for l in out.rstrip().split("\n") ]: - # with backslash escapes + k=key.lower() + + # with backslash escapes, e.g. + # X=$'Y\nZ' r=re.match("^\$'(.*)'$",value) if r: - context_sh[key.lower()]=r.group(1).decode('string_escape') + context_sh[k]=r.group(1).decode('string_escape') else: - # multiword values + # multiword values, e.g.: + # X='Y Z' + # X='Y'\''Z' for "Y'Z" r=re.match("^'(.*)'$",value) if r: - context_sh[key.lower()]=r.group(1) + context_sh[k]=r.group(1).replace("'\\''","'") else: - # simple values - context_sh[key.lower()]=value - except util.ProcessExecutionError, _err: - LOG.warn("Failed to read context variables: %s" % (_err.message)) + # simple values, e.g.: + # X=Y + context_sh[k]=value + + except util.ProcessExecutionError as e: + raise NonContextDiskDir("Error reading context.sh: %s" % (e)) + results['metadata']=context_sh else: raise NonContextDiskDir("Missing context.sh") -- cgit v1.2.3 From 73c666ca9b20d3e025cb38621afd704d8c8356d4 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Tue, 19 Feb 2013 16:03:09 +0100 Subject: New unit tests for OpenNebula data source. --- tests/unittests/test_datasource/test_opennebula.py | 160 +++++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 tests/unittests/test_datasource/test_opennebula.py diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py new file mode 100644 index 00000000..d0606227 --- /dev/null +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -0,0 +1,160 @@ +import os +from mocker import MockerTestCase +from cloudinit import util +from cloudinit.sources import DataSourceOpenNebula as ds + +TEST_VARS={ + 'var1': 'single', + 'var2': 'double word', + 'var3': 'multi\nline\n', + 'var4': "'single'", + 'var5': "'double word'", + 'var6': "'multi\nline\n'" } + +USER_DATA='#cloud-config\napt_upgrade: true' +SSH_KEY='ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460-%i' +HOSTNAME='foo.example.com' +PUBLIC_IP='10.0.0.3' + +CMD_IP_OUT='''\ +1: lo: mtu 16436 qdisc noqueue state UNKNOWN \ link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 +2: eth0: mtu 1500 qdisc mq state UP qlen 1000\ link/ether 02:00:0a:12:01:01 brd ff:ff:ff:ff:ff:ff +''' + +class TestOpenNebulaDataSource(MockerTestCase): + + def setUp(self): + super(TestOpenNebulaDataSource, self).setUp() + self.tmp = self.makeDir() + + def test_seed_dir_non_contextdisk(self): + my_d = os.path.join(self.tmp, 'non-contextdisk') + self.assertRaises(ds.NonContextDiskDir,ds.read_context_disk_dir,my_d) + + def test_seed_dir_bad_context_sh(self): + my_d = os.path.join(self.tmp, 'bad-context-sh') + os.mkdir(my_d) + with open(os.path.join(my_d,"context.sh"), "w") as fp: + fp.write('/bin/false\n') + fp.close() + self.assertRaises(ds.NonContextDiskDir,ds.read_context_disk_dir,my_d) + + def test_context_sh_parser(self): + my_d = os.path.join(self.tmp,'context-sh-parser') + populate_dir(my_d, TEST_VARS) + results=ds.read_context_disk_dir(my_d) + + self.assertTrue('metadata' in results) + self.assertEqual(TEST_VARS,results['metadata']) + + def test_ssh_key(self): + public_keys=[] + for c in range(4): + for k in ('SSH_KEY','SSH_PUBLIC_KEY'): + my_d = os.path.join(self.tmp, "%s-%i" % (k,c)) + populate_dir(my_d, {k:'\n'.join(public_keys)}) + results=ds.read_context_disk_dir(my_d) + + self.assertTrue('metadata' in results) + self.assertTrue('public-keys' in results['metadata']) + self.assertEqual(public_keys,results['metadata']['public-keys']) + + public_keys.append(SSH_KEY % (c+1,)) + + def test_user_data(self): + for k in ('USER_DATA','USERDATA'): + my_d = os.path.join(self.tmp, k) + populate_dir(my_d, {k:USER_DATA}) + results=ds.read_context_disk_dir(my_d) + + self.assertTrue('userdata' in results) + self.assertEqual(USER_DATA,results['userdata']) + + def test_hostname(self): + for k in ('HOSTNAME','PUBLIC_IP','IP_PUBLIC','ETH0_IP'): + my_d = os.path.join(self.tmp, k) + populate_dir(my_d, {k:PUBLIC_IP}) + results=ds.read_context_disk_dir(my_d) + + self.assertTrue('metadata' in results) + self.assertTrue('local-hostname' in results['metadata']) + self.assertEqual(PUBLIC_IP,results['metadata']['local-hostname']) + + + def test_find_candidates(self): + devs_with_answers = { + "TYPE=iso9660": ["/dev/vdb"], + "LABEL=CDROM": ["/dev/sr0"], + } + + def my_devs_with(criteria): + return devs_with_answers[criteria] + + try: + orig_find_devs_with = util.find_devs_with + util.find_devs_with = my_devs_with + self.assertEqual(["/dev/sr0","/dev/vdb"], ds.find_candidate_devs()) + finally: + util.find_devs_with = orig_find_devs_with + + +class TestOpenNebulaNetwork(MockerTestCase): + + def setUp(self): + super(TestOpenNebulaNetwork, self).setUp() + + def test_lo(self): + net=ds.OpenNebulaNetwork('',{}) + self.assertEqual(net.gen_conf(),u'''\ +auto lo +iface lo inet loopback +''') + + def test_eth0(self): + net=ds.OpenNebulaNetwork(CMD_IP_OUT,{}) + self.assertEqual(net.gen_conf(),u'''\ +auto lo +iface lo inet loopback + +auto eth0 +iface eth0 inet static + address 10.18.1.1 + network 10.18.1.0 + netmask 255.255.255.0 +''') + + def test_eth0_override(self): + context_sh = { + 'dns': '1.2.3.8', + 'eth0_ip':'1.2.3.4', + 'eth0_network':'1.2.3.0', + 'eth0_mask':'255.255.0.0', + 'eth0_gateway':'1.2.3.5', + 'eth0_domain':'example.com', + 'eth0_dns':'1.2.3.6 1.2.3.7'} + + net=ds.OpenNebulaNetwork(CMD_IP_OUT,context_sh) + self.assertEqual(net.gen_conf(),u'''\ +auto lo +iface lo inet loopback + +auto eth0 +iface eth0 inet static + address 1.2.3.4 + network 1.2.3.0 + netmask 255.255.0.0 + gateway 1.2.3.5 + dns-search example.com + dns-nameservers 1.2.3.8 1.2.3.6 1.2.3.7 +''') + + +def populate_dir(seed_dir, files): + os.mkdir(seed_dir) + with open(os.path.join(seed_dir,"context.sh"), "w") as fp: + fp.write("# Context variables generated by OpenNebula\n") + for (name, content) in files.iteritems(): + fp.write('%s="%s"\n' % (name.upper(),content)) + fp.close() + +# vi: ts=4 expandtab -- cgit v1.2.3 From e18f0f8a382729cc7c9f8df3ad0573af7eeb8f47 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Tue, 19 Feb 2013 16:18:25 +0100 Subject: Test for more variable types in OpenNebula unit test. --- tests/unittests/test_datasource/test_opennebula.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index d0606227..af8bd347 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -9,7 +9,10 @@ TEST_VARS={ 'var3': 'multi\nline\n', 'var4': "'single'", 'var5': "'double word'", - 'var6': "'multi\nline\n'" } + 'var6': "'multi\nline\n'", + 'var7': 'single\\t', + 'var8': 'double\\tword', + 'var9': 'multi\\t\nline\n' } USER_DATA='#cloud-config\napt_upgrade: true' SSH_KEY='ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460-%i' -- cgit v1.2.3 From b6db3ad471dca682f31658ebd2907feb376bc79f Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Wed, 20 Feb 2013 13:23:14 +0100 Subject: OpenNebula datasource documentation update. --- doc/sources/opennebula/README | 66 --------------------- doc/sources/opennebula/README.rst | 117 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+), 66 deletions(-) delete mode 100644 doc/sources/opennebula/README create mode 100644 doc/sources/opennebula/README.rst diff --git a/doc/sources/opennebula/README b/doc/sources/opennebula/README deleted file mode 100644 index 772a5b99..00000000 --- a/doc/sources/opennebula/README +++ /dev/null @@ -1,66 +0,0 @@ -The 'OpenNebula' DataSource supports the OpenNebula contextualization disk. - -The following criteria are required to be identified by -DataSourceOpenNebula as contextualization disk: - * must be formatted with iso9660 filesystem or labeled as CDROM - * must be un-partitioned block device (/dev/vdb, not /dev/vdb1) - * must contain - * context.sh - -== Content of config-drive == - * context.sh - This is the only mandatory file on context disk, the rest content depends - on contextualization parameter FILES and thus is optional. It's - a shell script defining all context parameters. This script is - processed by bash (/bin/bash) to simulate behaviour of common - OpenNebula context scripts. Processed variables are handed over - back to cloud-init for further processing. - -== Configuration == -Cloud-init's behaviour can be modified by context variables found -in the context.sh file in the folowing ways (variable names are -case-insensitive): - * dsmode: - values: local, net, disabled - default: None - - Tells if this datasource will be processed in local (pre-networking) or - net (post-networking) stage or even completely disabled. - - * ssh_key: - default: None - If present, these key(s) will be used as the public key(s) for - the instance. More keys can be specified in this single context - variable, but each key must be on it's own line. I.e. keys must - be separated by newlines. - - * hostname: - default: None - Custom hostname for the instance. - - * public_ip: - default: None - If hostname not specified, public_ip is used to DNS resolve hostname. - - * 'user_data' or 'userdata': - default: None - This provides cloud-init user-data. See other documentation for what - all can be present here. - -== Example OpenNebula's Virtual Machine template == - -CONTEXT=[ - PUBLIC_IP="$NIC[IP]", - SSH_KEY="$USER[SSH_KEY] -$USER[SSH_KEY1] -$USER[SSH_KEY2] ", - USER_DATA="#cloud-config -# see https://help.ubuntu.com/community/CloudInit - -packages: [] - -mounts: -- [vdc,none,swap,sw,0,0] -runcmd: -- echo 'Instance has been configured by cloud-init.' | wall -" ] diff --git a/doc/sources/opennebula/README.rst b/doc/sources/opennebula/README.rst new file mode 100644 index 00000000..d4c3dc39 --- /dev/null +++ b/doc/sources/opennebula/README.rst @@ -0,0 +1,117 @@ +The `OpenNebula`_ DataSource supports the OpenNebula contextualization disk. + + See `contextualization overview`_, `contextualizing VMs`_ and + `network configuration`_ in the public documentation for + more information. + +OpenNebula's virtual machines are contextualized (parametrized) by +CD-ROM image data, which contains a shell script *context.sh* with +custom variables defined on virtual machine start. There are no +fixed contextualization variables, but the datasource accepts +many used and recommended across OpenNebula's documentation. + +Datasource configuration +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Datasource accepts following configuration options. + +:: + + dsmode: + values: local, net, disabled + default: net + +Tells if this datasource will be processed in 'local' (pre-networking) or +'net' (post-networking) stage or even completely 'disabled'. + +Contextualization disk +~~~~~~~~~~~~~~~~~~~~~~ + +The following criteria are required: + +1. Must be formatted with `iso9660`_ fs. or have fs. label of **CDROM** +2. Must contain file *context.sh* with contextualization variables. + File is generated by OpenNebula, it has a KEY="VALUE" format and + can be easily read by shell script. + +Contextualization variables +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +There are no fixed contextualization variables in OpenNebula, no standard. +Following variables were found on various places and revisions of +the OpenNebula documentation. Where multiple similar variables are +specified, only first found is taken. + +:: + + DSMODE + +Datasource mode configuration override. Values: local, net, disabled. + +:: + + DNS + ETH_IP + ETH_NETWORK + ETH_MASK + ETH_GATEWAY + ETH_DOMAIN + ETH_DNS + +Static `network configuration`_. + +:: + + HOSTNAME + +Instance hostname. + +:: + + PUBLIC_IP + IP_PUBLIC + ETH0_IP + +If no hostname has been specified, cloud-init will try to create hostname +from instance's IP address in 'local' dsmode. In 'net' dsmode, cloud-init +try to resolve one of its IP addresses to get hostname. + +:: + + SSH_KEY + SSH_PUBLIC_KEY + +One or multiple SSH keys (separated by newlines) can be specified. + +:: + + USER_DATA + USERDATA + +cloud-init user data. + + +Example OpenNebula's Virtual Machine template +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +CONTEXT=[ + PUBLIC_IP="$NIC[IP]", + SSH_KEY="$USER[SSH_KEY] +$USER[SSH_KEY1] +$USER[SSH_KEY2] ", + USER_DATA="#cloud-config +# see https://help.ubuntu.com/community/CloudInit + +packages: [] + +mounts: +- [vdc,none,swap,sw,0,0] +runcmd: +- echo 'Instance has been configured by cloud-init.' | wall +" ] + +.. _OpenNebula: http://opennebula.org/ +.. _contextualization overview: http://opennebula.org/documentation:documentation:context_overview +.. _contextualizing VMs: http://opennebula.org/documentation:documentation:cong +.. _network configuration: http://opennebula.org/documentation:documentation:cong#network_configuration +.. _iso9660: https://en.wikipedia.org/wiki/ISO_9660 -- cgit v1.2.3 From 0ae1b7588e906afc4f6e4be5ae4e4473e3477e1e Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Wed, 20 Feb 2013 13:24:37 +0100 Subject: Add OpenNebula DS on list of datasources. --- doc/rtd/topics/datasources.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/rtd/topics/datasources.rst b/doc/rtd/topics/datasources.rst index 59c58805..5543ed34 100644 --- a/doc/rtd/topics/datasources.rst +++ b/doc/rtd/topics/datasources.rst @@ -140,6 +140,12 @@ Config Drive .. include:: ../../sources/configdrive/README.rst +--------------------------- +OpenNebula +--------------------------- + +.. include:: ../../sources/opennebula/README.rst + --------------------------- Alt cloud --------------------------- -- cgit v1.2.3 From 444120e896dffd1d7d788da5bfaa98ef762fba35 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Wed, 20 Feb 2013 13:46:18 +0100 Subject: OpenNebula documentation tuning. --- doc/sources/opennebula/README.rst | 46 ++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/doc/sources/opennebula/README.rst b/doc/sources/opennebula/README.rst index d4c3dc39..5cbc4366 100644 --- a/doc/sources/opennebula/README.rst +++ b/doc/sources/opennebula/README.rst @@ -1,14 +1,14 @@ -The `OpenNebula`_ DataSource supports the OpenNebula contextualization disk. +The `OpenNebula`_ (ON) datasource supports the contextualization disk. See `contextualization overview`_, `contextualizing VMs`_ and `network configuration`_ in the public documentation for more information. OpenNebula's virtual machines are contextualized (parametrized) by -CD-ROM image data, which contains a shell script *context.sh* with +CD-ROM image, which contains a shell script *context.sh* with custom variables defined on virtual machine start. There are no fixed contextualization variables, but the datasource accepts -many used and recommended across OpenNebula's documentation. +many used and recommended across the documentation. Datasource configuration ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -32,7 +32,7 @@ The following criteria are required: 1. Must be formatted with `iso9660`_ fs. or have fs. label of **CDROM** 2. Must contain file *context.sh* with contextualization variables. File is generated by OpenNebula, it has a KEY="VALUE" format and - can be easily read by shell script. + can be easily read (via *source*) by shell Contextualization variables ~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -74,7 +74,7 @@ Instance hostname. If no hostname has been specified, cloud-init will try to create hostname from instance's IP address in 'local' dsmode. In 'net' dsmode, cloud-init -try to resolve one of its IP addresses to get hostname. +tries to resolve one of its IP addresses to get hostname. :: @@ -91,24 +91,26 @@ One or multiple SSH keys (separated by newlines) can be specified. cloud-init user data. -Example OpenNebula's Virtual Machine template -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Example VM's context section +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -CONTEXT=[ - PUBLIC_IP="$NIC[IP]", - SSH_KEY="$USER[SSH_KEY] -$USER[SSH_KEY1] -$USER[SSH_KEY2] ", - USER_DATA="#cloud-config -# see https://help.ubuntu.com/community/CloudInit - -packages: [] - -mounts: -- [vdc,none,swap,sw,0,0] -runcmd: -- echo 'Instance has been configured by cloud-init.' | wall -" ] +:: + + CONTEXT=[ + PUBLIC_IP="$NIC[IP]", + SSH_KEY="$USER[SSH_KEY] + $USER[SSH_KEY1] + $USER[SSH_KEY2] ", + USER_DATA="#cloud-config + # see https://help.ubuntu.com/community/CloudInit + + packages: [] + + mounts: + - [vdc,none,swap,sw,0,0] + runcmd: + - echo 'Instance has been configured by cloud-init.' | wall + " ] .. _OpenNebula: http://opennebula.org/ .. _contextualization overview: http://opennebula.org/documentation:documentation:context_overview -- cgit v1.2.3 From 5e42d558c2589701c8ad22cd77fd7060ab3f1c02 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Wed, 20 Feb 2013 14:19:21 +0100 Subject: Minor OpenNebula DS cleanups (style, dsmode, static network). --- cloudinit/sources/DataSourceOpenNebula.py | 70 +++++++++++++++++-------------- 1 file changed, 39 insertions(+), 31 deletions(-) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index dad64bd4..bfa7eeaf 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -33,6 +33,7 @@ from cloudinit import util LOG = logging.getLogger(__name__) DEFAULT_IID = "iid-dsopennebula" +DEFAULT_MODE = 'net' CONTEXT_DISK_FILES = ["context.sh"] VALID_DSMODES = ("local", "net", "disabled") @@ -44,30 +45,29 @@ class DataSourceOpenNebula(sources.DataSource): self.seed_dir = os.path.join(paths.seed_dir, 'opennebula') def __str__(self): - mstr = "%s [seed=%s][dsmode=%s]" % (util.obj_name(self), - self.seed, self.dsmode) - return mstr + return "%s [seed=%s][dsmode=%s]" % \ + (util.obj_name(self), self.seed, self.dsmode) def get_data(self): defaults = { "instance-id": DEFAULT_IID, - "dsmode": self.dsmode, - } + "dsmode": self.dsmode } found = None md = {} - results = {} + if os.path.isdir(self.seed_dir): try: - results=read_context_disk_dir(self.seed_dir) + results = read_context_disk_dir(self.seed_dir) found = self.seed_dir except NonContextDiskDir: - util.logexc(LOG, "Failed reading context disk from %s", - self.seed_dir) + util.logexc(LOG, "Failed reading context disk from %s", self.seed_dir) + + # find candidate devices, try to mount them and + # read context script if present if not found: - devlist = find_candidate_devs() - for dev in devlist: + for dev in find_candidate_devs(): try: results = util.mount_cb(dev, read_context_disk_dir) found = dev @@ -81,20 +81,30 @@ class DataSourceOpenNebula(sources.DataSource): md = results['metadata'] md = util.mergedict(md, defaults) - dsmode = results.get('dsmode', None) - if dsmode not in VALID_DSMODES + (None,): - LOG.warn("user specified invalid mode: %s" % dsmode) - dsmode = None + # check for valid user specified dsmode + user_dsmode = results.get('dsmode', None) + if user_dsmode not in VALID_DSMODES + (None,): + LOG.warn("user specified invalid mode: %s" % user_dsmode) + user_dsmode = None - if (dsmode is None) and self.ds_cfg.get('dsmode'): + # decide dsmode + if user_dsmode: + dsmode = user_dsmode + elif self.ds_cfg.get('dsmode'): dsmode = self.ds_cfg.get('dsmode') else: - dsmode = self.dsmode + dsmode = DEFAULT_MODE if dsmode == "disabled": # most likely user specified return False + # apply static network configuration only in 'local' dsmode + # TODO: first boot? + if ('network-interfaces' in results and self.dsmode == "local"): + LOG.debug("Updating network interfaces from %s", self) + self.distro.apply_network(results['network-interfaces']) + if dsmode != self.dsmode: LOG.debug("%s: not claiming datasource, dsmode=%s", self, dsmode) return False @@ -103,11 +113,6 @@ class DataSourceOpenNebula(sources.DataSource): self.metadata = md self.userdata_raw = results.get('userdata') - # apply static network configuration only in 'local' dsmode - if ('network-interfaces' in results and self.dsmode == "local"): - LOG.debug("Updating network interfaces from %s", self) - self.distro.apply_network(results['network-interfaces']) - return True def get_hostname(self, fqdn=False, resolve_ip=None): @@ -234,9 +239,9 @@ def find_candidate_devs(): Return a list of devices that may contain the context disk. """ by_fstype = util.find_devs_with("TYPE=iso9660") - by_label = util.find_devs_with("LABEL=CDROM") - by_fstype.sort() + + by_label = util.find_devs_with("LABEL=CDROM") by_label.sort() # combine list of items by putting by-label items first @@ -262,11 +267,8 @@ def read_context_disk_dir(source_dir): if len(found) == 0: raise NonContextDiskDir("%s: %s" % (source_dir, "no files found")) + results = {'userdata':None, 'metadata':{}} context_sh = {} - results = { - 'userdata':None, - 'metadata':{}, - } if "context.sh" in found: try: @@ -347,9 +349,15 @@ def read_context_disk_dir(source_dir): elif "userdata" in context_sh: results['userdata'] = context_sh["userdata"] - (out, err) = util.subp(['/sbin/ip', '-o', 'link']) - net=OpenNebulaNetwork(out, context_sh) - results['network-interfaces']=net.gen_conf() + # generate static /etc/network/interfaces + # only if there are any required context variables + # http://opennebula.org/documentation:rel3.8:cong#network_configuration + for k in context_sh.keys(): + if re.match('^eth\d+_ip$',k): + (out, err) = util.subp(['/sbin/ip', '-o', 'link']) + net=OpenNebulaNetwork(out, context_sh) + results['network-interfaces']=net.gen_conf() + break return results -- cgit v1.2.3 From 4c78588baa7ab2d17582b3888fdeb136c3aa73d0 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Thu, 21 Feb 2013 14:45:09 +0100 Subject: Example cloud.cfg --- doc/sources/opennebula/README.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/doc/sources/opennebula/README.rst b/doc/sources/opennebula/README.rst index 5cbc4366..a84aebd4 100644 --- a/doc/sources/opennebula/README.rst +++ b/doc/sources/opennebula/README.rst @@ -90,6 +90,19 @@ One or multiple SSH keys (separated by newlines) can be specified. cloud-init user data. +Example configuration +~~~~~~~~~~~~~~~~~~~~~ + +This example cloud-init configuration (*cloud.cfg*) enables +OpenNebula datasource only in 'net' mode. + +:: + + disable_ec2_metadata: True + datasource_list: ['OpenNebula'] + datasource: + OpenNebula: + dsmode: net Example VM's context section ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- cgit v1.2.3 From b83c0fb7a35ea5c4f61ef4ce94d037a4f10f3c1e Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Thu, 21 Feb 2013 14:45:34 +0100 Subject: Remove TODO --- cloudinit/sources/DataSourceOpenNebula.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index bfa7eeaf..b22c8aed 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -100,7 +100,6 @@ class DataSourceOpenNebula(sources.DataSource): return False # apply static network configuration only in 'local' dsmode - # TODO: first boot? if ('network-interfaces' in results and self.dsmode == "local"): LOG.debug("Updating network interfaces from %s", self) self.distro.apply_network(results['network-interfaces']) -- cgit v1.2.3 From f6b7cfc308a54e05fc38fbcae9d653818459b105 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Sun, 24 Feb 2013 22:40:58 +0100 Subject: Remove commit "Add resolv.conf configuration function" --- cloudinit/distros/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 35577466..0db4aac7 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -64,10 +64,6 @@ class Distro(object): # to write this blob out in a distro format raise NotImplementedError() - def apply_resolv_conf(self, settings): - net_fn = self._paths.join(False, "/etc/resolv.conf") - util.write_file(net_fn, settings) - def get_option(self, opt_name, default=None): return self._cfg.get(opt_name, default) -- cgit v1.2.3 From 648b3f46790c60119571ebd20e1296ec1523b482 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Tue, 2 Apr 2013 19:05:36 +0200 Subject: Apply pep8.patch by Javier Fontan --- tests/unittests/test_datasource/test_opennebula.py | 92 +++++++++++----------- 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index af8bd347..bc6c4b73 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -3,7 +3,7 @@ from mocker import MockerTestCase from cloudinit import util from cloudinit.sources import DataSourceOpenNebula as ds -TEST_VARS={ +TEST_VARS = { 'var1': 'single', 'var2': 'double word', 'var3': 'multi\nline\n', @@ -12,18 +12,19 @@ TEST_VARS={ 'var6': "'multi\nline\n'", 'var7': 'single\\t', 'var8': 'double\\tword', - 'var9': 'multi\\t\nline\n' } + 'var9': 'multi\\t\nline\n'} -USER_DATA='#cloud-config\napt_upgrade: true' -SSH_KEY='ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460-%i' -HOSTNAME='foo.example.com' -PUBLIC_IP='10.0.0.3' +USER_DATA = '#cloud-config\napt_upgrade: true' +SSH_KEY = 'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460-%i' +HOSTNAME = 'foo.example.com' +PUBLIC_IP = '10.0.0.3' -CMD_IP_OUT='''\ +CMD_IP_OUT = '''\ 1: lo: mtu 16436 qdisc noqueue state UNKNOWN \ link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2: eth0: mtu 1500 qdisc mq state UP qlen 1000\ link/ether 02:00:0a:12:01:01 brd ff:ff:ff:ff:ff:ff ''' + class TestOpenNebulaDataSource(MockerTestCase): def setUp(self): @@ -32,57 +33,56 @@ class TestOpenNebulaDataSource(MockerTestCase): def test_seed_dir_non_contextdisk(self): my_d = os.path.join(self.tmp, 'non-contextdisk') - self.assertRaises(ds.NonContextDiskDir,ds.read_context_disk_dir,my_d) + self.assertRaises(ds.NonContextDiskDir, ds.read_context_disk_dir, my_d) def test_seed_dir_bad_context_sh(self): my_d = os.path.join(self.tmp, 'bad-context-sh') os.mkdir(my_d) - with open(os.path.join(my_d,"context.sh"), "w") as fp: + with open(os.path.join(my_d, "context.sh"), "w") as fp: fp.write('/bin/false\n') fp.close() - self.assertRaises(ds.NonContextDiskDir,ds.read_context_disk_dir,my_d) + self.assertRaises(ds.NonContextDiskDir, ds.read_context_disk_dir, my_d) def test_context_sh_parser(self): - my_d = os.path.join(self.tmp,'context-sh-parser') + my_d = os.path.join(self.tmp, 'context-sh-parser') populate_dir(my_d, TEST_VARS) - results=ds.read_context_disk_dir(my_d) + results = ds.read_context_disk_dir(my_d) self.assertTrue('metadata' in results) - self.assertEqual(TEST_VARS,results['metadata']) + self.assertEqual(TEST_VARS, results['metadata']) def test_ssh_key(self): - public_keys=[] + public_keys = [] for c in range(4): - for k in ('SSH_KEY','SSH_PUBLIC_KEY'): - my_d = os.path.join(self.tmp, "%s-%i" % (k,c)) - populate_dir(my_d, {k:'\n'.join(public_keys)}) - results=ds.read_context_disk_dir(my_d) + for k in ('SSH_KEY', 'SSH_PUBLIC_KEY'): + my_d = os.path.join(self.tmp, "%s-%i" % (k, c)) + populate_dir(my_d, {k: '\n'.join(public_keys)}) + results = ds.read_context_disk_dir(my_d) self.assertTrue('metadata' in results) self.assertTrue('public-keys' in results['metadata']) - self.assertEqual(public_keys,results['metadata']['public-keys']) + self.assertEqual(public_keys, results['metadata']['public-keys']) - public_keys.append(SSH_KEY % (c+1,)) + public_keys.append(SSH_KEY % (c + 1,)) def test_user_data(self): - for k in ('USER_DATA','USERDATA'): + for k in ('USER_DATA', 'USERDATA'): my_d = os.path.join(self.tmp, k) - populate_dir(my_d, {k:USER_DATA}) - results=ds.read_context_disk_dir(my_d) + populate_dir(my_d, {k: USER_DATA}) + results = ds.read_context_disk_dir(my_d) self.assertTrue('userdata' in results) - self.assertEqual(USER_DATA,results['userdata']) + self.assertEqual(USER_DATA, results['userdata']) def test_hostname(self): - for k in ('HOSTNAME','PUBLIC_IP','IP_PUBLIC','ETH0_IP'): + for k in ('HOSTNAME', 'PUBLIC_IP', 'IP_PUBLIC', 'ETH0_IP'): my_d = os.path.join(self.tmp, k) - populate_dir(my_d, {k:PUBLIC_IP}) - results=ds.read_context_disk_dir(my_d) + populate_dir(my_d, {k: PUBLIC_IP}) + results = ds.read_context_disk_dir(my_d) self.assertTrue('metadata' in results) self.assertTrue('local-hostname' in results['metadata']) - self.assertEqual(PUBLIC_IP,results['metadata']['local-hostname']) - + self.assertEqual(PUBLIC_IP, results['metadata']['local-hostname']) def test_find_candidates(self): devs_with_answers = { @@ -96,7 +96,7 @@ class TestOpenNebulaDataSource(MockerTestCase): try: orig_find_devs_with = util.find_devs_with util.find_devs_with = my_devs_with - self.assertEqual(["/dev/sr0","/dev/vdb"], ds.find_candidate_devs()) + self.assertEqual(["/dev/sr0", "/dev/vdb"], ds.find_candidate_devs()) finally: util.find_devs_with = orig_find_devs_with @@ -107,15 +107,15 @@ class TestOpenNebulaNetwork(MockerTestCase): super(TestOpenNebulaNetwork, self).setUp() def test_lo(self): - net=ds.OpenNebulaNetwork('',{}) - self.assertEqual(net.gen_conf(),u'''\ + net = ds.OpenNebulaNetwork('', {}) + self.assertEqual(net.gen_conf(), u'''\ auto lo iface lo inet loopback ''') def test_eth0(self): - net=ds.OpenNebulaNetwork(CMD_IP_OUT,{}) - self.assertEqual(net.gen_conf(),u'''\ + net = ds.OpenNebulaNetwork(CMD_IP_OUT, {}) + self.assertEqual(net.gen_conf(), u'''\ auto lo iface lo inet loopback @@ -126,18 +126,18 @@ iface eth0 inet static netmask 255.255.255.0 ''') - def test_eth0_override(self): + def test_eth0_override(self): context_sh = { 'dns': '1.2.3.8', - 'eth0_ip':'1.2.3.4', - 'eth0_network':'1.2.3.0', - 'eth0_mask':'255.255.0.0', - 'eth0_gateway':'1.2.3.5', - 'eth0_domain':'example.com', - 'eth0_dns':'1.2.3.6 1.2.3.7'} - - net=ds.OpenNebulaNetwork(CMD_IP_OUT,context_sh) - self.assertEqual(net.gen_conf(),u'''\ + 'eth0_ip': '1.2.3.4', + 'eth0_network': '1.2.3.0', + 'eth0_mask': '255.255.0.0', + 'eth0_gateway': '1.2.3.5', + 'eth0_domain': 'example.com', + 'eth0_dns': '1.2.3.6 1.2.3.7'} + + net = ds.OpenNebulaNetwork(CMD_IP_OUT, context_sh) + self.assertEqual(net.gen_conf(), u'''\ auto lo iface lo inet loopback @@ -154,10 +154,10 @@ iface eth0 inet static def populate_dir(seed_dir, files): os.mkdir(seed_dir) - with open(os.path.join(seed_dir,"context.sh"), "w") as fp: + with open(os.path.join(seed_dir, "context.sh"), "w") as fp: fp.write("# Context variables generated by OpenNebula\n") for (name, content) in files.iteritems(): - fp.write('%s="%s"\n' % (name.upper(),content)) + fp.write('%s="%s"\n' % (name.upper(), content)) fp.close() # vi: ts=4 expandtab -- cgit v1.2.3 From 1d35141960f2c0183d3f306d4604011ca9d5d2e8 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Tue, 2 Apr 2013 19:55:15 +0200 Subject: PEP8 fixes. --- cloudinit/sources/DataSourceOpenNebula.py | 109 +++++++++++++++--------------- 1 file changed, 55 insertions(+), 54 deletions(-) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index b22c8aed..ff8e6b6b 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -37,6 +37,7 @@ DEFAULT_MODE = 'net' CONTEXT_DISK_FILES = ["context.sh"] VALID_DSMODES = ("local", "net", "disabled") + class DataSourceOpenNebula(sources.DataSource): def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) @@ -51,7 +52,7 @@ class DataSourceOpenNebula(sources.DataSource): def get_data(self): defaults = { "instance-id": DEFAULT_IID, - "dsmode": self.dsmode } + "dsmode": self.dsmode} found = None md = {} @@ -62,7 +63,8 @@ class DataSourceOpenNebula(sources.DataSource): results = read_context_disk_dir(self.seed_dir) found = self.seed_dir except NonContextDiskDir: - util.logexc(LOG, "Failed reading context disk from %s", self.seed_dir) + util.logexc(LOG, "Failed reading context disk from %s", + self.seed_dir) # find candidate devices, try to mount them and # read context script if present @@ -134,99 +136,98 @@ class NonContextDiskDir(Exception): class OpenNebulaNetwork(object): - REG_DEV_MAC=re.compile('^\d+: (eth\d+):.*link\/ether (..:..:..:..:..:..) ') + REG_DEV_MAC = re.compile('^\d+: (eth\d+):.*link\/ether (..:..:..:..:..:..) ') def __init__(self, ip, context_sh): - self.ip=ip - self.context_sh=context_sh - self.ifaces=self.get_ifaces() + self.ip = ip + self.context_sh = context_sh + self.ifaces = self.get_ifaces() def get_ifaces(self): return [self.REG_DEV_MAC.search(f).groups() for f in self.ip.split("\n") if self.REG_DEV_MAC.match(f)] def mac2ip(self, mac): - components=mac.split(':')[2:] - + components = mac.split(':')[2:] return [str(int(c, 16)) for c in components] - + def get_ip(self, dev, components): - var_name=dev+'_ip' + var_name = dev + '_ip' if var_name in self.context_sh: return self.context_sh[var_name] else: return '.'.join(components) def get_mask(self, dev, components): - var_name=dev+'_mask' + var_name = dev + '_mask' if var_name in self.context_sh: return self.context_sh[var_name] else: return '255.255.255.0' def get_network(self, dev, components): - var_name=dev+'_network' + var_name = dev + '_network' if var_name in self.context_sh: return self.context_sh[var_name] else: - return '.'.join(components[:-1])+'.0' + return '.'.join(components[:-1]) + '.0' def get_gateway(self, dev, components): - var_name=dev+'_gateway' + var_name = dev + '_gateway' if var_name in self.context_sh: return self.context_sh[var_name] else: None def get_dns(self, dev, components): - var_name=dev+'_dns' + var_name = dev + '_dns' if var_name in self.context_sh: return self.context_sh[var_name] else: None def get_domain(self, dev, components): - var_name=dev+'_domain' + var_name = dev + '_domain' if var_name in self.context_sh: return self.context_sh[var_name] else: None def gen_conf(self): - global_dns=[] + global_dns = [] if 'dns' in self.context_sh: global_dns.append(self.context_sh['dns']) - conf=[] + conf = [] conf.append('auto lo') conf.append('iface lo inet loopback') conf.append('') for i in self.ifaces: - dev=i[0] - mac=i[1] - ip_components=self.mac2ip(mac) + dev = i[0] + mac = i[1] + ip_components = self.mac2ip(mac) - conf.append('auto '+dev) - conf.append('iface '+dev+' inet static') - conf.append(' address '+self.get_ip(dev, ip_components)) - conf.append(' network '+self.get_network(dev, ip_components)) - conf.append(' netmask '+self.get_mask(dev, ip_components)) + conf.append('auto ' + dev) + conf.append('iface ' + dev + ' inet static') + conf.append(' address ' + self.get_ip(dev, ip_components)) + conf.append(' network ' + self.get_network(dev, ip_components)) + conf.append(' netmask ' + self.get_mask(dev, ip_components)) - gateway=self.get_gateway(dev, ip_components) + gateway = self.get_gateway(dev, ip_components) if gateway: - conf.append(' gateway '+gateway) + conf.append(' gateway ' + gateway) - domain=self.get_domain(dev, ip_components) + domain = self.get_domain(dev, ip_components) if domain: - conf.append(' dns-search '+domain) + conf.append(' dns-search ' + domain) # add global DNS servers to all interfaces - dns=self.get_dns(dev, ip_components) + dns = self.get_dns(dev, ip_components) if global_dns or dns: - all_dns=global_dns + all_dns = global_dns if dns: all_dns.append(dns) - conf.append(' dns-nameservers '+' '.join(all_dns)) + conf.append(' dns-nameservers ' + ' '.join(all_dns)) conf.append('') @@ -266,7 +267,7 @@ def read_context_disk_dir(source_dir): if len(found) == 0: raise NonContextDiskDir("%s: %s" % (source_dir, "no files found")) - results = {'userdata':None, 'metadata':{}} + results = {'userdata': None, 'metadata': {}} context_sh = {} if "context.sh" in found: @@ -288,47 +289,47 @@ def read_context_disk_dir(source_dir): # 2. read context variables # 3. use comm to filter "old" variables from all current # variables and excl. few other vars with grep - BASH_CMD='VARS=`set | sort -u `;' \ + BASH_CMD = 'VARS=`set | sort -u `;' \ 'source %s/context.sh;' \ 'comm -23 <(set | sort -u) <(echo "$VARS") | egrep -v "^(VARS|PIPESTATUS|_)="' - (out,err) = util.subp(['bash','--noprofile', '--norc', - '-c', BASH_CMD % (source_dir) ]) + (out, err) = util.subp(['bash', '--noprofile', '--norc', + '-c', BASH_CMD % (source_dir)]) - for (key,value) in [ l.split('=',1) for l in out.rstrip().split("\n") ]: - k=key.lower() + for (key, value) in [l.split('=', 1) for l in out.rstrip().split("\n")]: + k = key.lower() # with backslash escapes, e.g. # X=$'Y\nZ' - r=re.match("^\$'(.*)'$",value) + r = re.match("^\$'(.*)'$", value) if r: - context_sh[k]=r.group(1).decode('string_escape') + context_sh[k] = r.group(1).decode('string_escape') else: # multiword values, e.g.: - # X='Y Z' + # X='Y Z' # X='Y'\''Z' for "Y'Z" - r=re.match("^'(.*)'$",value) + r = re.match("^'(.*)'$", value) if r: - context_sh[k]=r.group(1).replace("'\\''","'") + context_sh[k] = r.group(1).replace("'\\''", "'") else: # simple values, e.g.: - # X=Y - context_sh[k]=value + # X=Y + context_sh[k] = value except util.ProcessExecutionError as e: raise NonContextDiskDir("Error reading context.sh: %s" % (e)) - results['metadata']=context_sh + results['metadata'] = context_sh else: raise NonContextDiskDir("Missing context.sh") # process single or multiple SSH keys - ssh_key_var=None + ssh_key_var = None if "ssh_key" in context_sh: - ssh_key_var="ssh_key" + ssh_key_var = "ssh_key" elif "ssh_public_key" in context_sh: - ssh_key_var="ssh_public_key" + ssh_key_var = "ssh_public_key" if ssh_key_var: lines = context_sh.get(ssh_key_var).splitlines() @@ -337,7 +338,7 @@ def read_context_disk_dir(source_dir): # custom hostname -- try hostname or leave cloud-init # itself create hostname from IP address later - for k in ('hostname','public_ip','ip_public','eth0_ip'): + for k in ('hostname', 'public_ip', 'ip_public', 'eth0_ip'): if k in context_sh: results['metadata']['local-hostname'] = context_sh[k] break @@ -352,10 +353,10 @@ def read_context_disk_dir(source_dir): # only if there are any required context variables # http://opennebula.org/documentation:rel3.8:cong#network_configuration for k in context_sh.keys(): - if re.match('^eth\d+_ip$',k): + if re.match('^eth\d+_ip$', k): (out, err) = util.subp(['/sbin/ip', '-o', 'link']) - net=OpenNebulaNetwork(out, context_sh) - results['network-interfaces']=net.gen_conf() + net = OpenNebulaNetwork(out, context_sh) + results['network-interfaces'] = net.gen_conf() break return results -- cgit v1.2.3 From 77c8798388637fcb2f7dbc057cad81e8fd5fbe58 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Wed, 4 Sep 2013 14:32:24 +0200 Subject: Apply parse.diff by Javier Fontan --- cloudinit/sources/DataSourceOpenNebula.py | 96 ++++++++-------------- tests/unittests/helpers.py | 1 + tests/unittests/test_datasource/test_opennebula.py | 16 ++-- 3 files changed, 45 insertions(+), 68 deletions(-) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index ff8e6b6b..d2ab348f 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -24,7 +24,6 @@ import os import re -import subprocess from cloudinit import log as logging from cloudinit import sources @@ -86,7 +85,7 @@ class DataSourceOpenNebula(sources.DataSource): # check for valid user specified dsmode user_dsmode = results.get('dsmode', None) if user_dsmode not in VALID_DSMODES + (None,): - LOG.warn("user specified invalid mode: %s" % user_dsmode) + LOG.warn("user specified invalid mode: %s", user_dsmode) user_dsmode = None # decide dsmode @@ -136,7 +135,9 @@ class NonContextDiskDir(Exception): class OpenNebulaNetwork(object): - REG_DEV_MAC = re.compile('^\d+: (eth\d+):.*link\/ether (..:..:..:..:..:..) ') + REG_DEV_MAC = re.compile( + r'^\d+: (eth\d+):.*?link\/ether (..:..:..:..:..:..) ?', + re.MULTILINE | re.DOTALL) def __init__(self, ip, context_sh): self.ip = ip @@ -144,7 +145,7 @@ class OpenNebulaNetwork(object): self.ifaces = self.get_ifaces() def get_ifaces(self): - return [self.REG_DEV_MAC.search(f).groups() for f in self.ip.split("\n") if self.REG_DEV_MAC.match(f)] + return self.REG_DEV_MAC.findall(self.ip) def mac2ip(self, mac): components = mac.split(':')[2:] @@ -157,7 +158,7 @@ class OpenNebulaNetwork(object): else: return '.'.join(components) - def get_mask(self, dev, components): + def get_mask(self, dev): var_name = dev + '_mask' if var_name in self.context_sh: return self.context_sh[var_name] @@ -171,26 +172,26 @@ class OpenNebulaNetwork(object): else: return '.'.join(components[:-1]) + '.0' - def get_gateway(self, dev, components): + def get_gateway(self, dev): var_name = dev + '_gateway' if var_name in self.context_sh: return self.context_sh[var_name] else: - None + return None - def get_dns(self, dev, components): + def get_dns(self, dev): var_name = dev + '_dns' if var_name in self.context_sh: return self.context_sh[var_name] else: - None + return None - def get_domain(self, dev, components): + def get_domain(self, dev): var_name = dev + '_domain' if var_name in self.context_sh: return self.context_sh[var_name] else: - None + return None def gen_conf(self): global_dns = [] @@ -211,18 +212,18 @@ class OpenNebulaNetwork(object): conf.append('iface ' + dev + ' inet static') conf.append(' address ' + self.get_ip(dev, ip_components)) conf.append(' network ' + self.get_network(dev, ip_components)) - conf.append(' netmask ' + self.get_mask(dev, ip_components)) + conf.append(' netmask ' + self.get_mask(dev)) - gateway = self.get_gateway(dev, ip_components) + gateway = self.get_gateway(dev) if gateway: conf.append(' gateway ' + gateway) - domain = self.get_domain(dev, ip_components) + domain = self.get_domain(dev) if domain: conf.append(' dns-search ' + domain) # add global DNS servers to all interfaces - dns = self.get_dns(dev, ip_components) + dns = self.get_dns(dev) if global_dns or dns: all_dns = global_dns if dns: @@ -272,51 +273,22 @@ def read_context_disk_dir(source_dir): if "context.sh" in found: try: - # Note: context.sh is a "shell" script with defined context - # variables, like: X="Y" . It's ready to use as a shell source - # e.g.: ". context.sh" and as a shell script it can also reference - # to already defined shell variables. So to have same context var. - # values as we can have in custom shell script, we use bash itself - # to read context.sh and dump variables in easily parsable way. - # - # normalized variables dump format (get by cmd "set"): - # 1. simple single word assignment ........ X=Y - # 2. multiword assignment ................. X='Y Z' - # 3. assignments with backslash escapes ... X=$'Y\nZ' - # - # how context variables are read: - # 1. list existing ("old") shell variables and store into $VARS - # 2. read context variables - # 3. use comm to filter "old" variables from all current - # variables and excl. few other vars with grep - BASH_CMD = 'VARS=`set | sort -u `;' \ - 'source %s/context.sh;' \ - 'comm -23 <(set | sort -u) <(echo "$VARS") | egrep -v "^(VARS|PIPESTATUS|_)="' - - (out, err) = util.subp(['bash', '--noprofile', '--norc', - '-c', BASH_CMD % (source_dir)]) - - for (key, value) in [l.split('=', 1) for l in out.rstrip().split("\n")]: - k = key.lower() - - # with backslash escapes, e.g. - # X=$'Y\nZ' - r = re.match("^\$'(.*)'$", value) - if r: - context_sh[k] = r.group(1).decode('string_escape') - else: - # multiword values, e.g.: - # X='Y Z' - # X='Y'\''Z' for "Y'Z" - r = re.match("^'(.*)'$", value) - if r: - context_sh[k] = r.group(1).replace("'\\''", "'") - else: - # simple values, e.g.: - # X=Y - context_sh[k] = value - - except util.ProcessExecutionError as e: + f = open('%s/context.sh' % (source_dir), 'r') + text = f.read() + f.close() + + context_reg = re.compile(r"^([\w_]+)=['\"](.*?[^\\])['\"]$", + re.MULTILINE | re.DOTALL) + variables = context_reg.findall(text) + + if not variables: + raise NonContextDiskDir("No variables in context") + + context_sh = {} + for k, v in variables: + context_sh[k.lower()] = v + + except (IOError, NonContextDiskDir) as e: raise NonContextDiskDir("Error reading context.sh: %s" % (e)) results['metadata'] = context_sh @@ -353,8 +325,8 @@ def read_context_disk_dir(source_dir): # only if there are any required context variables # http://opennebula.org/documentation:rel3.8:cong#network_configuration for k in context_sh.keys(): - if re.match('^eth\d+_ip$', k): - (out, err) = util.subp(['/sbin/ip', '-o', 'link']) + if re.match(r'^eth\d+_ip$', k): + (out, _) = util.subp(['/sbin/ip', '-o', 'link']) net = OpenNebulaNetwork(out, context_sh) results['network-interfaces'] = net.gen_conf() break diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 91a50e18..904677f1 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -183,6 +183,7 @@ class FilesystemMockingTestCase(ResourceUsingTestCase): setattr(mod, f, trap_func) self.patched_funcs.append((mod, f, func)) + def populate_dir(path, files): os.makedirs(path) for (name, content) in files.iteritems(): diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index bc6c4b73..27725930 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -19,9 +19,11 @@ SSH_KEY = 'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460-%i' HOSTNAME = 'foo.example.com' PUBLIC_IP = '10.0.0.3' -CMD_IP_OUT = '''\ -1: lo: mtu 16436 qdisc noqueue state UNKNOWN \ link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 -2: eth0: mtu 1500 qdisc mq state UP qlen 1000\ link/ether 02:00:0a:12:01:01 brd ff:ff:ff:ff:ff:ff +CMD_IP_OUT = ''' +1: lo: mtu 16436 qdisc noqueue state UNKNOWN + link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 +2: eth0: mtu 1500 qdisc mq state UP qlen 1000 + link/ether 02:00:0a:12:01:01 brd ff:ff:ff:ff:ff:ff ''' @@ -52,7 +54,7 @@ class TestOpenNebulaDataSource(MockerTestCase): self.assertEqual(TEST_VARS, results['metadata']) def test_ssh_key(self): - public_keys = [] + public_keys = ['first key', 'second key'] for c in range(4): for k in ('SSH_KEY', 'SSH_PUBLIC_KEY'): my_d = os.path.join(self.tmp, "%s-%i" % (k, c)) @@ -61,7 +63,8 @@ class TestOpenNebulaDataSource(MockerTestCase): self.assertTrue('metadata' in results) self.assertTrue('public-keys' in results['metadata']) - self.assertEqual(public_keys, results['metadata']['public-keys']) + self.assertEqual(public_keys, + results['metadata']['public-keys']) public_keys.append(SSH_KEY % (c + 1,)) @@ -96,7 +99,8 @@ class TestOpenNebulaDataSource(MockerTestCase): try: orig_find_devs_with = util.find_devs_with util.find_devs_with = my_devs_with - self.assertEqual(["/dev/sr0", "/dev/vdb"], ds.find_candidate_devs()) + self.assertEqual(["/dev/sr0", "/dev/vdb"], + ds.find_candidate_devs()) finally: util.find_devs_with = orig_find_devs_with -- cgit v1.2.3 From 1adc68b643b2f73a2a08ca7d19c3fc8e759f06c2 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Wed, 4 Sep 2013 16:19:54 +0200 Subject: Fix RE matching context variables. Test cleanups. --- cloudinit/sources/DataSourceOpenNebula.py | 26 ++++++++++++++++------ tests/unittests/test_datasource/test_opennebula.py | 12 +++++----- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index d2ab348f..0ab23b25 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -277,16 +277,28 @@ def read_context_disk_dir(source_dir): text = f.read() f.close() - context_reg = re.compile(r"^([\w_]+)=['\"](.*?[^\\])['\"]$", - re.MULTILINE | re.DOTALL) - variables = context_reg.findall(text) + # lame matching: + # 1. group = key + # 2. group = single quoted value, respect '\'' + # 3. group = old double quoted value, but doesn't end with \" + context_reg = re.compile( + r"^([\w_]+)=(?:'((?:[^']|'\\'')*?)'|\"(.*?[^\\])\")$", + re.MULTILINE | re.DOTALL) + variables = context_reg.findall(text) if not variables: raise NonContextDiskDir("No variables in context") - context_sh = {} - for k, v in variables: - context_sh[k.lower()] = v + for k, v1, v2 in variables: + k = k.lower() + if v1: + # take single quoted variable 'xyz' + # (ON>=4) and unquote '\'' -> ' + context_sh[k] = v1.replace(r"'\''", r"'") + elif v2: + # take double quoted variable "xyz" + # (old ON<4) and unquote \" -> " + context_sh[k] = v2.replace(r'\"', r'"') except (IOError, NonContextDiskDir) as e: raise NonContextDiskDir("Error reading context.sh: %s" % (e)) @@ -326,7 +338,7 @@ def read_context_disk_dir(source_dir): # http://opennebula.org/documentation:rel3.8:cong#network_configuration for k in context_sh.keys(): if re.match(r'^eth\d+_ip$', k): - (out, _) = util.subp(['/sbin/ip', '-o', 'link']) + (out, _) = util.subp(['/sbin/ip', 'link']) net = OpenNebulaNetwork(out, context_sh) results['network-interfaces'] = net.gen_conf() break diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index 27725930..66a38e31 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -1,7 +1,8 @@ import os -from mocker import MockerTestCase -from cloudinit import util + from cloudinit.sources import DataSourceOpenNebula as ds +from cloudinit import util +from mocker import MockerTestCase TEST_VARS = { 'var1': 'single', @@ -19,7 +20,7 @@ SSH_KEY = 'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460-%i' HOSTNAME = 'foo.example.com' PUBLIC_IP = '10.0.0.3' -CMD_IP_OUT = ''' +CMD_IP_OUT = '''\ 1: lo: mtu 16436 qdisc noqueue state UNKNOWN link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2: eth0: mtu 1500 qdisc mq state UP qlen 1000 @@ -91,6 +92,7 @@ class TestOpenNebulaDataSource(MockerTestCase): devs_with_answers = { "TYPE=iso9660": ["/dev/vdb"], "LABEL=CDROM": ["/dev/sr0"], + "LABEL=CONTEXT": ["/dev/sdb"], } def my_devs_with(criteria): @@ -99,7 +101,7 @@ class TestOpenNebulaDataSource(MockerTestCase): try: orig_find_devs_with = util.find_devs_with util.find_devs_with = my_devs_with - self.assertEqual(["/dev/sr0", "/dev/vdb"], + self.assertEqual(["/dev/sdb", "/dev/sr0", "/dev/vdb"], ds.find_candidate_devs()) finally: util.find_devs_with = orig_find_devs_with @@ -161,7 +163,7 @@ def populate_dir(seed_dir, files): with open(os.path.join(seed_dir, "context.sh"), "w") as fp: fp.write("# Context variables generated by OpenNebula\n") for (name, content) in files.iteritems(): - fp.write('%s="%s"\n' % (name.upper(), content)) + fp.write("%s='%s'\n" % (name.upper(), content.replace(r"'", r"'\''"))) fp.close() # vi: ts=4 expandtab -- cgit v1.2.3 From 8a2a88e0bb4520eabe99b6686413a548f3d59652 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Wed, 4 Sep 2013 16:36:00 +0200 Subject: Search for contextualization CDROM by LABEL=CONTEXT --- cloudinit/sources/DataSourceOpenNebula.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 0ab23b25..fef7beac 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -239,15 +239,11 @@ def find_candidate_devs(): """ Return a list of devices that may contain the context disk. """ - by_fstype = util.find_devs_with("TYPE=iso9660") - by_fstype.sort() - - by_label = util.find_devs_with("LABEL=CDROM") - by_label.sort() - - # combine list of items by putting by-label items first - # followed by fstype items, but with dupes removed - combined = (by_label + [d for d in by_fstype if d not in by_label]) + combined = [] + for f in ('LABEL=CONTEXT', 'LABEL=CDROM', 'TYPE=iso9660'): + for d in util.find_devs_with(f).sort(): + if d not in combined: + combined.append(d) return combined -- cgit v1.2.3 From d0e7898d4d0aef452fbfdd9b51fa2da6437397ff Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Thu, 5 Sep 2013 17:55:53 +0200 Subject: PEP8 and Pylint fixes. Move context.sh "parser" into separate function. Fix fetching user specified dsmode (from context). Rename context_sh->context. Reuse unittests.helpers.populate_dir. --- cloudinit/sources/DataSourceOpenNebula.py | 163 +++++++++++---------- tests/unittests/test_datasource/test_opennebula.py | 62 ++++---- 2 files changed, 119 insertions(+), 106 deletions(-) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index fef7beac..87ec51bd 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -3,7 +3,7 @@ # Copyright (C) 2012 Canonical Ltd. # Copyright (C) 2012 Yahoo! Inc. # Copyright (C) 2012-2013 CERIT Scientific Cloud -# Copyright (C) 2012 OpenNebula.org +# Copyright (C) 2012-2013 OpenNebula.org # # Author: Scott Moser # Author: Joshua Harlow @@ -45,45 +45,47 @@ class DataSourceOpenNebula(sources.DataSource): self.seed_dir = os.path.join(paths.seed_dir, 'opennebula') def __str__(self): - return "%s [seed=%s][dsmode=%s]" % \ - (util.obj_name(self), self.seed, self.dsmode) + root = sources.DataSource.__str__(self) + return "%s [seed=%s][dsmode=%s]" % (root, self.seed, self.dsmode) def get_data(self): defaults = { "instance-id": DEFAULT_IID, - "dsmode": self.dsmode} + "dsmode": self.dsmode + } - found = None - md = {} + seed = None results = {} + # first try to read local seed_dir if os.path.isdir(self.seed_dir): try: results = read_context_disk_dir(self.seed_dir) - found = self.seed_dir + seed = self.seed_dir except NonContextDiskDir: - util.logexc(LOG, "Failed reading context disk from %s", + util.logexc(LOG, "Failed reading context from %s", self.seed_dir) - # find candidate devices, try to mount them and - # read context script if present - if not found: + if not seed: + # then try to detect and mount candidate devices and + # read contextualization if present for dev in find_candidate_devs(): try: results = util.mount_cb(dev, read_context_disk_dir) - found = dev + seed = dev break except (NonContextDiskDir, util.MountFailedError): pass - if not found: + if not seed: return False + # merge fetched metadata with datasource defaults md = results['metadata'] - md = util.mergedict(md, defaults) + md = util.mergemanydict([md, defaults]) # check for valid user specified dsmode - user_dsmode = results.get('dsmode', None) + user_dsmode = results['metadata'].get('dsmode', None) if user_dsmode not in VALID_DSMODES + (None,): LOG.warn("user specified invalid mode: %s", user_dsmode) user_dsmode = None @@ -109,10 +111,9 @@ class DataSourceOpenNebula(sources.DataSource): LOG.debug("%s: not claiming datasource, dsmode=%s", self, dsmode) return False - self.seed = found + self.seed = seed self.metadata = md self.userdata_raw = results.get('userdata') - return True def get_hostname(self, fqdn=False, resolve_ip=None): @@ -139,9 +140,9 @@ class OpenNebulaNetwork(object): r'^\d+: (eth\d+):.*?link\/ether (..:..:..:..:..:..) ?', re.MULTILINE | re.DOTALL) - def __init__(self, ip, context_sh): + def __init__(self, ip, context): self.ip = ip - self.context_sh = context_sh + self.context = context self.ifaces = self.get_ifaces() def get_ifaces(self): @@ -153,50 +154,50 @@ class OpenNebulaNetwork(object): def get_ip(self, dev, components): var_name = dev + '_ip' - if var_name in self.context_sh: - return self.context_sh[var_name] + if var_name in self.context: + return self.context[var_name] else: return '.'.join(components) def get_mask(self, dev): var_name = dev + '_mask' - if var_name in self.context_sh: - return self.context_sh[var_name] + if var_name in self.context: + return self.context[var_name] else: return '255.255.255.0' def get_network(self, dev, components): var_name = dev + '_network' - if var_name in self.context_sh: - return self.context_sh[var_name] + if var_name in self.context: + return self.context[var_name] else: return '.'.join(components[:-1]) + '.0' def get_gateway(self, dev): var_name = dev + '_gateway' - if var_name in self.context_sh: - return self.context_sh[var_name] + if var_name in self.context: + return self.context[var_name] else: return None def get_dns(self, dev): var_name = dev + '_dns' - if var_name in self.context_sh: - return self.context_sh[var_name] + if var_name in self.context: + return self.context[var_name] else: return None def get_domain(self, dev): var_name = dev + '_domain' - if var_name in self.context_sh: - return self.context_sh[var_name] + if var_name in self.context: + return self.context[var_name] else: return None def gen_conf(self): global_dns = [] - if 'dns' in self.context_sh: - global_dns.append(self.context_sh['dns']) + if 'dns' in self.context: + global_dns.append(self.context['dns']) conf = [] conf.append('auto lo') @@ -241,101 +242,113 @@ def find_candidate_devs(): """ combined = [] for f in ('LABEL=CONTEXT', 'LABEL=CDROM', 'TYPE=iso9660'): - for d in util.find_devs_with(f).sort(): + devs = util.find_devs_with(f) + devs.sort() + for d in devs: if d not in combined: combined.append(d) return combined +def parse_context_data(data): + """ + parse_context_data(data) + parse context.sh variables provided as a single string. Uses + very simple matching RE. Returns None if nothing is matched. + """ + # RE groups: + # 1: key + # 2: single quoted value, respect '\'' + # 3: old double quoted value, but doesn't end with \" + context_reg = re.compile( + r"^([\w_]+)=(?:'((?:[^']|'\\'')*?)'|\"(.*?[^\\])\")$", + re.MULTILINE | re.DOTALL) + + found = context_reg.findall(data) + if not found: + return None + + variables = {} + for k, v1, v2 in found: + k = k.lower() + if v1: + # take single quoted variable 'xyz' + # (ON>=4) and unquote '\'' -> ' + variables[k] = v1.replace(r"'\''", r"'") + elif v2: + # take double quoted variable "xyz" + # (old ON<4) and unquote \" -> " + variables[k] = v2.replace(r'\"', r'"') + + return variables + + def read_context_disk_dir(source_dir): """ read_context_disk_dir(source_dir): read source_dir and return a tuple with metadata dict and user-data string populated. If not a valid dir, raise a NonContextDiskDir """ - found = {} for af in CONTEXT_DISK_FILES: fn = os.path.join(source_dir, af) if os.path.isfile(fn): found[af] = fn - if len(found) == 0: + if not found: raise NonContextDiskDir("%s: %s" % (source_dir, "no files found")) results = {'userdata': None, 'metadata': {}} - context_sh = {} + context = {} if "context.sh" in found: try: - f = open('%s/context.sh' % (source_dir), 'r') - text = f.read() - f.close() - - # lame matching: - # 1. group = key - # 2. group = single quoted value, respect '\'' - # 3. group = old double quoted value, but doesn't end with \" - context_reg = re.compile( - r"^([\w_]+)=(?:'((?:[^']|'\\'')*?)'|\"(.*?[^\\])\")$", - re.MULTILINE | re.DOTALL) - - variables = context_reg.findall(text) - if not variables: + with open(os.path.join(source_dir, 'context.sh'), 'r') as f: + context = parse_context_data(f.read()) + f.close() + if not context: raise NonContextDiskDir("No variables in context") - for k, v1, v2 in variables: - k = k.lower() - if v1: - # take single quoted variable 'xyz' - # (ON>=4) and unquote '\'' -> ' - context_sh[k] = v1.replace(r"'\''", r"'") - elif v2: - # take double quoted variable "xyz" - # (old ON<4) and unquote \" -> " - context_sh[k] = v2.replace(r'\"', r'"') - except (IOError, NonContextDiskDir) as e: raise NonContextDiskDir("Error reading context.sh: %s" % (e)) - results['metadata'] = context_sh + results['metadata'] = context else: raise NonContextDiskDir("Missing context.sh") # process single or multiple SSH keys ssh_key_var = None - - if "ssh_key" in context_sh: + if "ssh_key" in context: ssh_key_var = "ssh_key" - elif "ssh_public_key" in context_sh: + elif "ssh_public_key" in context: ssh_key_var = "ssh_public_key" if ssh_key_var: - lines = context_sh.get(ssh_key_var).splitlines() + lines = context.get(ssh_key_var).splitlines() results['metadata']['public-keys'] = [l for l in lines if len(l) and not l.startswith("#")] # custom hostname -- try hostname or leave cloud-init # itself create hostname from IP address later for k in ('hostname', 'public_ip', 'ip_public', 'eth0_ip'): - if k in context_sh: - results['metadata']['local-hostname'] = context_sh[k] + if k in context: + results['metadata']['local-hostname'] = context[k] break # raw user data - if "user_data" in context_sh: - results['userdata'] = context_sh["user_data"] - elif "userdata" in context_sh: - results['userdata'] = context_sh["userdata"] + if "user_data" in context: + results['userdata'] = context["user_data"] + elif "userdata" in context: + results['userdata'] = context["userdata"] # generate static /etc/network/interfaces # only if there are any required context variables # http://opennebula.org/documentation:rel3.8:cong#network_configuration - for k in context_sh.keys(): + for k in context.keys(): if re.match(r'^eth\d+_ip$', k): (out, _) = util.subp(['/sbin/ip', 'link']) - net = OpenNebulaNetwork(out, context_sh) + net = OpenNebulaNetwork(out, context) results['network-interfaces'] = net.gen_conf() break diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index 66a38e31..4b82a49c 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -1,8 +1,9 @@ -import os - from cloudinit.sources import DataSourceOpenNebula as ds from cloudinit import util from mocker import MockerTestCase +from tests.unittests.helpers import populate_dir + +import os TEST_VARS = { 'var1': 'single', @@ -13,7 +14,11 @@ TEST_VARS = { 'var6': "'multi\nline\n'", 'var7': 'single\\t', 'var8': 'double\\tword', - 'var9': 'multi\\t\nline\n'} + 'var9': 'multi\\t\nline\n', + 'var10': '\\', # expect \ + 'var11': '\'', # expect ' + 'var12': '$', # expect $ +} USER_DATA = '#cloud-config\napt_upgrade: true' SSH_KEY = 'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460-%i' @@ -38,17 +43,15 @@ class TestOpenNebulaDataSource(MockerTestCase): my_d = os.path.join(self.tmp, 'non-contextdisk') self.assertRaises(ds.NonContextDiskDir, ds.read_context_disk_dir, my_d) - def test_seed_dir_bad_context_sh(self): - my_d = os.path.join(self.tmp, 'bad-context-sh') + def test_seed_dir_bad_context(self): + my_d = os.path.join(self.tmp, 'bad-context') os.mkdir(my_d) - with open(os.path.join(my_d, "context.sh"), "w") as fp: - fp.write('/bin/false\n') - fp.close() + open(os.path.join(my_d, "context.sh"), "w").close() self.assertRaises(ds.NonContextDiskDir, ds.read_context_disk_dir, my_d) - def test_context_sh_parser(self): - my_d = os.path.join(self.tmp, 'context-sh-parser') - populate_dir(my_d, TEST_VARS) + def test_context_parser(self): + my_d = os.path.join(self.tmp, 'context-parser') + populate_context_dir(my_d, TEST_VARS) results = ds.read_context_disk_dir(my_d) self.assertTrue('metadata' in results) @@ -59,7 +62,7 @@ class TestOpenNebulaDataSource(MockerTestCase): for c in range(4): for k in ('SSH_KEY', 'SSH_PUBLIC_KEY'): my_d = os.path.join(self.tmp, "%s-%i" % (k, c)) - populate_dir(my_d, {k: '\n'.join(public_keys)}) + populate_context_dir(my_d, {k: '\n'.join(public_keys)}) results = ds.read_context_disk_dir(my_d) self.assertTrue('metadata' in results) @@ -72,7 +75,7 @@ class TestOpenNebulaDataSource(MockerTestCase): def test_user_data(self): for k in ('USER_DATA', 'USERDATA'): my_d = os.path.join(self.tmp, k) - populate_dir(my_d, {k: USER_DATA}) + populate_context_dir(my_d, {k: USER_DATA}) results = ds.read_context_disk_dir(my_d) self.assertTrue('userdata' in results) @@ -81,7 +84,7 @@ class TestOpenNebulaDataSource(MockerTestCase): def test_hostname(self): for k in ('HOSTNAME', 'PUBLIC_IP', 'IP_PUBLIC', 'ETH0_IP'): my_d = os.path.join(self.tmp, k) - populate_dir(my_d, {k: PUBLIC_IP}) + populate_context_dir(my_d, {k: PUBLIC_IP}) results = ds.read_context_disk_dir(my_d) self.assertTrue('metadata' in results) @@ -89,14 +92,12 @@ class TestOpenNebulaDataSource(MockerTestCase): self.assertEqual(PUBLIC_IP, results['metadata']['local-hostname']) def test_find_candidates(self): - devs_with_answers = { - "TYPE=iso9660": ["/dev/vdb"], - "LABEL=CDROM": ["/dev/sr0"], - "LABEL=CONTEXT": ["/dev/sdb"], - } - def my_devs_with(criteria): - return devs_with_answers[criteria] + return { + "LABEL=CONTEXT": ["/dev/sdb"], + "LABEL=CDROM": ["/dev/sr0"], + "TYPE=iso9660": ["/dev/vdb"], + }.get(criteria, []) try: orig_find_devs_with = util.find_devs_with @@ -133,16 +134,17 @@ iface eth0 inet static ''') def test_eth0_override(self): - context_sh = { + context = { 'dns': '1.2.3.8', 'eth0_ip': '1.2.3.4', 'eth0_network': '1.2.3.0', 'eth0_mask': '255.255.0.0', 'eth0_gateway': '1.2.3.5', 'eth0_domain': 'example.com', - 'eth0_dns': '1.2.3.6 1.2.3.7'} + 'eth0_dns': '1.2.3.6 1.2.3.7' + } - net = ds.OpenNebulaNetwork(CMD_IP_OUT, context_sh) + net = ds.OpenNebulaNetwork(CMD_IP_OUT, context) self.assertEqual(net.gen_conf(), u'''\ auto lo iface lo inet loopback @@ -158,12 +160,10 @@ iface eth0 inet static ''') -def populate_dir(seed_dir, files): - os.mkdir(seed_dir) - with open(os.path.join(seed_dir, "context.sh"), "w") as fp: - fp.write("# Context variables generated by OpenNebula\n") - for (name, content) in files.iteritems(): - fp.write("%s='%s'\n" % (name.upper(), content.replace(r"'", r"'\''"))) - fp.close() +def populate_context_dir(path, variables): + data = "# Context variables generated by OpenNebula\n" + for (k, v) in variables.iteritems(): + data += ("%s='%s'\n" % (k.upper(), v.replace(r"'", r"'\''"))) + populate_dir(path, {'context.sh': data}) # vi: ts=4 expandtab -- cgit v1.2.3 From 781d105a6ca9222e59909341dbd86a13bab5a67e Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Tue, 10 Sep 2013 00:40:16 +0200 Subject: Replace RE context.sh parser with Scott's rewrite of bash dumper. Upper case context variable names. --- cloudinit/sources/DataSourceOpenNebula.py | 178 ++++++++++++++------- tests/unittests/test_datasource/test_opennebula.py | 38 ++--- 2 files changed, 137 insertions(+), 79 deletions(-) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 87ec51bd..a1995aca 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -24,6 +24,8 @@ import os import re +import string +import subprocess from cloudinit import log as logging from cloudinit import sources @@ -50,8 +52,8 @@ class DataSourceOpenNebula(sources.DataSource): def get_data(self): defaults = { - "instance-id": DEFAULT_IID, - "dsmode": self.dsmode + "instance-id": DEFAULT_IID, #TODO:???? + "DSMODE": self.dsmode } seed = None @@ -85,7 +87,7 @@ class DataSourceOpenNebula(sources.DataSource): md = util.mergemanydict([md, defaults]) # check for valid user specified dsmode - user_dsmode = results['metadata'].get('dsmode', None) + user_dsmode = results['metadata'].get('DSMODE', None) if user_dsmode not in VALID_DSMODES + (None,): LOG.warn("user specified invalid mode: %s", user_dsmode) user_dsmode = None @@ -93,7 +95,7 @@ class DataSourceOpenNebula(sources.DataSource): # decide dsmode if user_dsmode: dsmode = user_dsmode - elif self.ds_cfg.get('dsmode'): + elif self.ds_cfg.get('dsmode'): #TODO: fakt se k tomu nekdy dostane? dsmode = self.ds_cfg.get('dsmode') else: dsmode = DEFAULT_MODE @@ -153,42 +155,42 @@ class OpenNebulaNetwork(object): return [str(int(c, 16)) for c in components] def get_ip(self, dev, components): - var_name = dev + '_ip' + var_name = dev.upper() + '_IP' if var_name in self.context: return self.context[var_name] else: return '.'.join(components) def get_mask(self, dev): - var_name = dev + '_mask' + var_name = dev.upper() + '_MASK' if var_name in self.context: return self.context[var_name] else: return '255.255.255.0' def get_network(self, dev, components): - var_name = dev + '_network' + var_name = dev.upper() + '_NETWORK' if var_name in self.context: return self.context[var_name] else: return '.'.join(components[:-1]) + '.0' def get_gateway(self, dev): - var_name = dev + '_gateway' + var_name = dev.upper() + '_GATEWAY' if var_name in self.context: return self.context[var_name] else: return None def get_dns(self, dev): - var_name = dev + '_dns' + var_name = dev.upper() + '_DNS' if var_name in self.context: return self.context[var_name] else: return None def get_domain(self, dev): - var_name = dev + '_domain' + var_name = dev.upper() + '_DOMAIN' if var_name in self.context: return self.context[var_name] else: @@ -196,8 +198,8 @@ class OpenNebulaNetwork(object): def gen_conf(self): global_dns = [] - if 'dns' in self.context: - global_dns.append(self.context['dns']) + if 'DNS' in self.context: + global_dns.append(self.context['DNS']) conf = [] conf.append('auto lo') @@ -251,37 +253,91 @@ def find_candidate_devs(): return combined -def parse_context_data(data): - """ - parse_context_data(data) - parse context.sh variables provided as a single string. Uses - very simple matching RE. Returns None if nothing is matched. - """ - # RE groups: - # 1: key - # 2: single quoted value, respect '\'' - # 3: old double quoted value, but doesn't end with \" - context_reg = re.compile( - r"^([\w_]+)=(?:'((?:[^']|'\\'')*?)'|\"(.*?[^\\])\")$", - re.MULTILINE | re.DOTALL) - - found = context_reg.findall(data) - if not found: - return None - - variables = {} - for k, v1, v2 in found: - k = k.lower() - if v1: - # take single quoted variable 'xyz' - # (ON>=4) and unquote '\'' -> ' - variables[k] = v1.replace(r"'\''", r"'") - elif v2: - # take double quoted variable "xyz" - # (old ON<4) and unquote \" -> " - variables[k] = v2.replace(r'\"', r'"') - - return variables +def parse_shell_config(content, keylist=None, bash=None, asuser=None): + + if isinstance(bash, str): + bash = [bash] + elif bash is None: + bash = ['bash', '-e'] + + # allvars expands to all existing variables by using '${!x*}' notation + # where x is lower or upper case letters or '_' + allvars = ["${!%s*}" % x for x in string.letters + "_"] + + keylist_in = keylist + if keylist is None: + keylist = allvars + keylist_in = [] + + setup = '\n'.join(('__v="";', '',)) + + def varprinter(vlist): + # output '\0'.join(['_start_', key=value NULL for vars in vlist] + return '\n'.join(( + 'printf "%s\\0" _start_', + 'for __v in %s; do' % ' '.join(vlist), + ' printf "%s=%s\\0" "$__v" "${!__v}";', + 'done', + '' + )) + + # the rendered 'bcmd' is bash syntax that does + # setup: declare variables we use (so they show up in 'all') + # varprinter(allvars): print all variables known at beginning + # content: execute the provided content + # varprinter(keylist): print all variables known after content + # + # output is then a null terminated array of: + # literal '_start_' + # key=value (for each preset variable) + # literal '_start_' + # key=value (for each post set variable) + bcmd = ('unset IFS\n' + + setup + + varprinter(allvars) + + '{\n%s\n\n} > /dev/null\n' % content + + 'unset IFS\n' + + varprinter(keylist) + "\n") + + cmd = [] + if asuser is not None: + cmd = ['sudo', '-u', asuser] + + cmd.extend(bash) + + sp = subprocess.Popen(cmd, stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + (output, error) = sp.communicate(input=bcmd) + + if sp.returncode != 0: + raise Exception("Process returned %d" % sp.returncode) + + # exclude vars in bash that change on their own or that we used + excluded = ("RANDOM", "LINENO", "_", "__v") + preset = {} + ret = {} + target = None + output = output[0:-1] # remove trailing null + + # go through output. First _start_ is for 'preset', second for 'target'. + # Add to target only things were changed and not in volitile + for line in output.split("\0"): + try: + (key, val) = line.split("=", 1) + if target is preset: + target[key] = val + elif (key not in excluded and + (key in keylist_in or preset.get(key) != val)): + ret[key] = val + except ValueError: + if line != "_start_": + raise + if target is None: + target = preset + elif target is preset: + target = ret + + return ret def read_context_disk_dir(source_dir): @@ -305,24 +361,26 @@ def read_context_disk_dir(source_dir): if "context.sh" in found: try: with open(os.path.join(source_dir, 'context.sh'), 'r') as f: - context = parse_context_data(f.read()) + content = f.read().strip() + if content: + context = parse_shell_config(content) f.close() - if not context: - raise NonContextDiskDir("No variables in context") - - except (IOError, NonContextDiskDir) as e: + except (IOError, Exception) as e: raise NonContextDiskDir("Error reading context.sh: %s" % (e)) - - results['metadata'] = context else: raise NonContextDiskDir("Missing context.sh") + if not context: + raise NonContextDiskDir("No context variables found") + + results['metadata'] = context + # process single or multiple SSH keys ssh_key_var = None - if "ssh_key" in context: - ssh_key_var = "ssh_key" - elif "ssh_public_key" in context: - ssh_key_var = "ssh_public_key" + if "SSH_KEY" in context: + ssh_key_var = "SSH_KEY" + elif "SSH_PUBLIC_KEY" in context: + ssh_key_var = "SSH_PUBLIC_KEY" if ssh_key_var: lines = context.get(ssh_key_var).splitlines() @@ -331,22 +389,22 @@ def read_context_disk_dir(source_dir): # custom hostname -- try hostname or leave cloud-init # itself create hostname from IP address later - for k in ('hostname', 'public_ip', 'ip_public', 'eth0_ip'): + for k in ('HOSTNAME', 'PUBLIC_IP', 'IP_PUBLIC', 'ETH0_IP'): if k in context: results['metadata']['local-hostname'] = context[k] break # raw user data - if "user_data" in context: - results['userdata'] = context["user_data"] - elif "userdata" in context: - results['userdata'] = context["userdata"] + if "USER_DATA" in context: + results['userdata'] = context["USER_DATA"] + elif "USERDATA" in context: + results['userdata'] = context["USERDATA"] # generate static /etc/network/interfaces # only if there are any required context variables # http://opennebula.org/documentation:rel3.8:cong#network_configuration for k in context.keys(): - if re.match(r'^eth\d+_ip$', k): + if re.match(r'^ETH\d+_ip$', k): (out, _) = util.subp(['/sbin/ip', 'link']) net = OpenNebulaNetwork(out, context) results['network-interfaces'] = net.gen_conf() diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index 4b82a49c..f544e145 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -6,18 +6,18 @@ from tests.unittests.helpers import populate_dir import os TEST_VARS = { - 'var1': 'single', - 'var2': 'double word', - 'var3': 'multi\nline\n', - 'var4': "'single'", - 'var5': "'double word'", - 'var6': "'multi\nline\n'", - 'var7': 'single\\t', - 'var8': 'double\\tword', - 'var9': 'multi\\t\nline\n', - 'var10': '\\', # expect \ - 'var11': '\'', # expect ' - 'var12': '$', # expect $ + 'VAR1': 'single', + 'VAR2': 'double word', + 'VAR3': 'multi\nline\n', + 'VAR4': "'single'", + 'VAR5': "'double word'", + 'VAR6': "'multi\nline\n'", + 'VAR7': 'single\\t', + 'VAR8': 'double\\tword', + 'VAR9': 'multi\\t\nline\n', + 'VAR10': '\\', # expect \ + 'VAR11': '\'', # expect ' + 'VAR12': '$', # expect $ } USER_DATA = '#cloud-config\napt_upgrade: true' @@ -135,13 +135,13 @@ iface eth0 inet static def test_eth0_override(self): context = { - 'dns': '1.2.3.8', - 'eth0_ip': '1.2.3.4', - 'eth0_network': '1.2.3.0', - 'eth0_mask': '255.255.0.0', - 'eth0_gateway': '1.2.3.5', - 'eth0_domain': 'example.com', - 'eth0_dns': '1.2.3.6 1.2.3.7' + 'DNS': '1.2.3.8', + 'ETH0_IP': '1.2.3.4', + 'ETH0_NETWORK': '1.2.3.0', + 'ETH0_MASK': '255.255.0.0', + 'ETH0_GATEWAY': '1.2.3.5', + 'ETH0_DOMAIN': 'example.com', + 'ETH0_DNS': '1.2.3.6 1.2.3.7' } net = ds.OpenNebulaNetwork(CMD_IP_OUT, context) -- cgit v1.2.3 From 7f3b1198da74430345d69e485326b03a3fa5b455 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Tue, 10 Sep 2013 00:46:37 +0200 Subject: Fix pylint complain on toks.split('.') --- cloudinit/sources/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index b0e43954..1dfdf9bf 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -173,7 +173,7 @@ class DataSource(object): toks = util.gethostbyaddr(lhost) if toks: - toks = toks.split('.') + toks = str(toks).split('.') else: toks = ["ip-%s" % lhost.replace(".", "-")] else: -- cgit v1.2.3 From 64a2f3c7d4eae1354134c021db232c117eb1c772 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Tue, 10 Sep 2013 15:37:15 +0200 Subject: Configurable OpenNebula::parseuser. Seed search dir+dev merge. Eat shell parser error output. Few tests for tests for get_data. --- cloudinit/sources/DataSourceOpenNebula.py | 78 +++++++++++------- tests/unittests/test_datasource/test_opennebula.py | 95 +++++++++++++++++++--- 2 files changed, 132 insertions(+), 41 deletions(-) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index a1995aca..1b419cfd 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -35,6 +35,7 @@ LOG = logging.getLogger(__name__) DEFAULT_IID = "iid-dsopennebula" DEFAULT_MODE = 'net' +DEFAULT_PARSEUSER = 'nobody' CONTEXT_DISK_FILES = ["context.sh"] VALID_DSMODES = ("local", "net", "disabled") @@ -51,33 +52,35 @@ class DataSourceOpenNebula(sources.DataSource): return "%s [seed=%s][dsmode=%s]" % (root, self.seed, self.dsmode) def get_data(self): - defaults = { - "instance-id": DEFAULT_IID, #TODO:???? - "DSMODE": self.dsmode - } - + defaults = {"instance-id": DEFAULT_IID} + results = None seed = None - results = {} - # first try to read local seed_dir - if os.path.isdir(self.seed_dir): + # decide parseuser for context.sh shell reader + parseuser = DEFAULT_PARSEUSER + if self.ds_cfg.get('parseuser'): + parseuser = self.ds_cfg.get('parseuser') + + candidates = [self.seed_dir] + candidates.extend(find_candidate_devs()) + for cdev in candidates: try: - results = read_context_disk_dir(self.seed_dir) - seed = self.seed_dir + if os.path.isdir(self.seed_dir): + results = read_context_disk_dir(cdev, asuser=parseuser) + elif cdev.startswith("/dev"): + results = util.mount_cb(cdev, read_context_disk_dir, + data=parseuser) except NonContextDiskDir: - util.logexc(LOG, "Failed reading context from %s", - self.seed_dir) + continue + except BrokenContextDiskDir as exc: + raise exc + except util.MountFailedError: + LOG.warn("%s was not mountable" % cdev) - if not seed: - # then try to detect and mount candidate devices and - # read contextualization if present - for dev in find_candidate_devs(): - try: - results = util.mount_cb(dev, read_context_disk_dir) - seed = dev - break - except (NonContextDiskDir, util.MountFailedError): - pass + if results: + seed = cdev + LOG.debug("found datasource in %s", cdev) + break if not seed: return False @@ -95,7 +98,7 @@ class DataSourceOpenNebula(sources.DataSource): # decide dsmode if user_dsmode: dsmode = user_dsmode - elif self.ds_cfg.get('dsmode'): #TODO: fakt se k tomu nekdy dostane? + elif self.ds_cfg.get('dsmode'): dsmode = self.ds_cfg.get('dsmode') else: dsmode = DEFAULT_MODE @@ -137,6 +140,10 @@ class NonContextDiskDir(Exception): pass +class BrokenContextDiskDir(Exception): + pass + + class OpenNebulaNetwork(object): REG_DEV_MAC = re.compile( r'^\d+: (eth\d+):.*?link\/ether (..:..:..:..:..:..) ?', @@ -306,7 +313,8 @@ def parse_shell_config(content, keylist=None, bash=None, asuser=None): cmd.extend(bash) sp = subprocess.Popen(cmd, stdin=subprocess.PIPE, - stdout=subprocess.PIPE) + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) (output, error) = sp.communicate(input=bcmd) if sp.returncode != 0: @@ -340,7 +348,7 @@ def parse_shell_config(content, keylist=None, bash=None, asuser=None): return ret -def read_context_disk_dir(source_dir): +def read_context_disk_dir(source_dir, asuser=None): """ read_context_disk_dir(source_dir): read source_dir and return a tuple with metadata dict and user-data @@ -355,23 +363,31 @@ def read_context_disk_dir(source_dir): if not found: raise NonContextDiskDir("%s: %s" % (source_dir, "no files found")) - results = {'userdata': None, 'metadata': {}} context = {} + results = {'userdata': None, 'metadata': {}} if "context.sh" in found: try: with open(os.path.join(source_dir, 'context.sh'), 'r') as f: - content = f.read().strip() - if content: - context = parse_shell_config(content) + content = f.read().strip() f.close() - except (IOError, Exception) as e: + + # don't pass empty context script + # to shell parser + non_empty = re.match(r'.*?^\s*([^# ]+)', content, + re.MULTILINE | re.DOTALL) + + if non_empty: + context = parse_shell_config(content, asuser=asuser) + except IOError as e: raise NonContextDiskDir("Error reading context.sh: %s" % (e)) + except Exception as e: + raise BrokenContextDiskDir("Error processing context.sh: %s" % (e)) else: raise NonContextDiskDir("Missing context.sh") if not context: - raise NonContextDiskDir("No context variables found") + return results results['metadata'] = context diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index f544e145..752638b6 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -1,4 +1,5 @@ from cloudinit.sources import DataSourceOpenNebula as ds +from cloudinit import helpers from cloudinit import util from mocker import MockerTestCase from tests.unittests.helpers import populate_dir @@ -20,6 +21,8 @@ TEST_VARS = { 'VAR12': '$', # expect $ } +INVALID_PARSEUSER = 'cloud-init-mocker-opennebula-invalid' +INVALID_CONTEXT = ';' USER_DATA = '#cloud-config\napt_upgrade: true' SSH_KEY = 'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460-%i' HOSTNAME = 'foo.example.com' @@ -38,21 +41,93 @@ class TestOpenNebulaDataSource(MockerTestCase): def setUp(self): super(TestOpenNebulaDataSource, self).setUp() self.tmp = self.makeDir() + self.paths = helpers.Paths({'cloud_dir': self.tmp}) + + # defaults for few tests + self.ds = ds.DataSourceOpenNebula + self.seed_dir = os.path.join(self.paths.seed_dir, "opennebula") + self.sys_cfg = {'datasource': {'OpenNebula': {'dsmode': 'local'}}} + + def test_get_data_non_contextdisk(self): + try: + # dont' try to lookup for CDs + orig_find_devs_with = util.find_devs_with + util.find_devs_with = lambda n: [] + + dsrc = self.ds(sys_cfg=self.sys_cfg, distro=None, paths=self.paths) + ret = dsrc.get_data() + self.assertFalse(ret) + finally: + util.find_devs_with = orig_find_devs_with + + def test_get_data_broken_contextdisk(self): + try: + # dont' try to lookup for CDs + orig_find_devs_with = util.find_devs_with + util.find_devs_with = lambda n: [] + + populate_dir(self.seed_dir, {'context.sh': INVALID_CONTEXT}) + dsrc = self.ds(sys_cfg=self.sys_cfg, distro=None, paths=self.paths) + self.assertRaises(ds.BrokenContextDiskDir, dsrc.get_data) + finally: + util.find_devs_with = orig_find_devs_with + + def test_get_data_invalid_identity(self): + try: + # dont' try to lookup for CDs + orig_find_devs_with = util.find_devs_with + util.find_devs_with = lambda n: [] + + sys_cfg = self.sys_cfg + sys_cfg['datasource']['OpenNebula']['parseuser'] = \ + INVALID_PARSEUSER + + populate_context_dir(self.seed_dir, {'KEY1': 'val1'}) + dsrc = self.ds(sys_cfg=sys_cfg, distro=None, paths=self.paths) + self.assertRaises(ds.BrokenContextDiskDir, dsrc.get_data) + finally: + util.find_devs_with = orig_find_devs_with + + def test_get_data(self): + try: + # dont' try to lookup for CDs + orig_find_devs_with = util.find_devs_with + util.find_devs_with = lambda n: [] + populate_context_dir(self.seed_dir, {'KEY1': 'val1'}) + dsrc = self.ds(sys_cfg=self.sys_cfg, distro=None, paths=self.paths) + ret = dsrc.get_data() + self.assertTrue(ret) + finally: + util.find_devs_with = orig_find_devs_with def test_seed_dir_non_contextdisk(self): - my_d = os.path.join(self.tmp, 'non-contextdisk') - self.assertRaises(ds.NonContextDiskDir, ds.read_context_disk_dir, my_d) + self.assertRaises(ds.NonContextDiskDir, ds.read_context_disk_dir, + self.seed_dir) + + def test_seed_dir_empty1_context(self): + populate_dir(self.seed_dir, {'context.sh': ''}) + results = ds.read_context_disk_dir(self.seed_dir) + + self.assertEqual(results['userdata'], None) + self.assertEqual(results['metadata'], {}) + + def test_seed_dir_empty2_context(self): + populate_context_dir(self.seed_dir, {}) + results = ds.read_context_disk_dir(self.seed_dir) + + self.assertEqual(results['userdata'], None) + self.assertEqual(results['metadata'], {}) + + def test_seed_dir_broken_context(self): + populate_dir(self.seed_dir, {'context.sh': INVALID_CONTEXT}) - def test_seed_dir_bad_context(self): - my_d = os.path.join(self.tmp, 'bad-context') - os.mkdir(my_d) - open(os.path.join(my_d, "context.sh"), "w").close() - self.assertRaises(ds.NonContextDiskDir, ds.read_context_disk_dir, my_d) + self.assertRaises(ds.BrokenContextDiskDir, + ds.read_context_disk_dir, + self.seed_dir) def test_context_parser(self): - my_d = os.path.join(self.tmp, 'context-parser') - populate_context_dir(my_d, TEST_VARS) - results = ds.read_context_disk_dir(my_d) + populate_context_dir(self.seed_dir, TEST_VARS) + results = ds.read_context_disk_dir(self.seed_dir) self.assertTrue('metadata' in results) self.assertEqual(TEST_VARS, results['metadata']) -- cgit v1.2.3 From 6e156e4877e420050307249345287cf48ff5d795 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Tue, 10 Sep 2013 15:57:18 +0200 Subject: Update OpenNebula documentation (parseuser, more fs. labels, K-V single quotes) --- doc/sources/opennebula/README.rst | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/doc/sources/opennebula/README.rst b/doc/sources/opennebula/README.rst index a84aebd4..4d7de27a 100644 --- a/doc/sources/opennebula/README.rst +++ b/doc/sources/opennebula/README.rst @@ -24,15 +24,24 @@ Datasource accepts following configuration options. Tells if this datasource will be processed in 'local' (pre-networking) or 'net' (post-networking) stage or even completely 'disabled'. +:: + + parseuser: + default: nobody + +Unprivileged system user used for contextualization script +processing. + Contextualization disk ~~~~~~~~~~~~~~~~~~~~~~ The following criteria are required: -1. Must be formatted with `iso9660`_ fs. or have fs. label of **CDROM** +1. Must be formatted with `iso9660`_ filesystem + or have a *filesystem* label of **CONTEXT** or **CDROM** 2. Must contain file *context.sh* with contextualization variables. - File is generated by OpenNebula, it has a KEY="VALUE" format and - can be easily read (via *source*) by shell + File is generated by OpenNebula, it has a KEY='VALUE' format and + can be easily read by bash Contextualization variables ~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -103,6 +112,7 @@ OpenNebula datasource only in 'net' mode. datasource: OpenNebula: dsmode: net + parseuser: nobody Example VM's context section ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- cgit v1.2.3 From 18ec61a5b5e2c67d84cbdcef1e47cc72b0ba6218 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Tue, 10 Sep 2013 16:47:15 +0200 Subject: Detect invalid system user for test --- tests/unittests/test_datasource/test_opennebula.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index 752638b6..f5103dc0 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -5,6 +5,7 @@ from mocker import MockerTestCase from tests.unittests.helpers import populate_dir import os +import pwd TEST_VARS = { 'VAR1': 'single', @@ -21,7 +22,6 @@ TEST_VARS = { 'VAR12': '$', # expect $ } -INVALID_PARSEUSER = 'cloud-init-mocker-opennebula-invalid' INVALID_CONTEXT = ';' USER_DATA = '#cloud-config\napt_upgrade: true' SSH_KEY = 'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460-%i' @@ -78,9 +78,16 @@ class TestOpenNebulaDataSource(MockerTestCase): orig_find_devs_with = util.find_devs_with util.find_devs_with = lambda n: [] + # generate non-existing system user name sys_cfg = self.sys_cfg - sys_cfg['datasource']['OpenNebula']['parseuser'] = \ - INVALID_PARSEUSER + invalid_user = 'invalid' + while not sys_cfg['datasource']['OpenNebula'].get('parseuser'): + try: + pwd.getpwnam(invalid_user) + invalid_user += 'X' + except KeyError: + sys_cfg['datasource']['OpenNebula']['parseuser'] = \ + invalid_user populate_context_dir(self.seed_dir, {'KEY1': 'val1'}) dsrc = self.ds(sys_cfg=sys_cfg, distro=None, paths=self.paths) -- cgit v1.2.3 From c0d1a59d96a16c080ad8b8278251294cccc21894 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Tue, 10 Sep 2013 18:35:19 +0200 Subject: Fix detection of ETHx_IP context variable, add test. --- cloudinit/sources/DataSourceOpenNebula.py | 3 +-- tests/unittests/test_datasource/test_opennebula.py | 6 ++++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 1b419cfd..141bd454 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -376,7 +376,6 @@ def read_context_disk_dir(source_dir, asuser=None): # to shell parser non_empty = re.match(r'.*?^\s*([^# ]+)', content, re.MULTILINE | re.DOTALL) - if non_empty: context = parse_shell_config(content, asuser=asuser) except IOError as e: @@ -420,7 +419,7 @@ def read_context_disk_dir(source_dir, asuser=None): # only if there are any required context variables # http://opennebula.org/documentation:rel3.8:cong#network_configuration for k in context.keys(): - if re.match(r'^ETH\d+_ip$', k): + if re.match(r'^ETH\d+_IP$', k): (out, _) = util.subp(['/sbin/ip', 'link']) net = OpenNebulaNetwork(out, context) results['network-interfaces'] = net.gen_conf() diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index f5103dc0..f2457657 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -173,6 +173,12 @@ class TestOpenNebulaDataSource(MockerTestCase): self.assertTrue('local-hostname' in results['metadata']) self.assertEqual(PUBLIC_IP, results['metadata']['local-hostname']) + def test_network_interfaces(self): + populate_context_dir(self.seed_dir, {'ETH0_IP': '1.2.3.4'}) + results = ds.read_context_disk_dir(self.seed_dir) + + self.assertTrue('network-interfaces' in results) + def test_find_candidates(self): def my_devs_with(criteria): return { -- cgit v1.2.3 From 8ed952a461ed81af4011e02f3df47add1066cd0f Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 10 Sep 2013 14:15:30 -0400 Subject: fix DataSource base class to set up ds_cfg for 'Net' sources When the base DataSource class would set 'ds_cfg' for the specific datasources' config, it would fail for the DataSources that are just named 'DataSourceFooNet' and we wanted to set configuration in 'Foo'. For example, both DataSourceOpenNebula and DataSourceOpenNebulaNet want to read datasource config from sources: OpenNebula: foo: bar But without this change, 'ds_cfg' would not be setup properly for OpenNebulaNet. --- cloudinit/sources/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 1dfdf9bf..7dc1fbde 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -53,9 +53,16 @@ class DataSource(object): self.userdata = None self.metadata = None self.userdata_raw = None + + # find the datasource config name. + # remove 'DataSource' from classname on front, and remove 'Net' on end. + # Both Foo and FooNet sources expect config in cfg['sources']['Foo'] name = type_utils.obj_name(self) if name.startswith(DS_PREFIX): name = name[len(DS_PREFIX):] + if name.endswith('Net'): + name = name[0:-3] + self.ds_cfg = util.get_cfg_by_path(self.sys_cfg, ("datasource", name), {}) if not ud_proc: -- cgit v1.2.3 From 7dff14c285d6562b9cd0b47876628607feac4a18 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 10 Sep 2013 14:18:58 -0400 Subject: some cleanups and changes * use util.subp from inside parse_shell_config, and adjust exception handling accordingly. * add 'switch_user_cmd' as a callback function to pass to parse_shell_config, which allows us to mock this to avoid 'sudo' when running test cases. Basically the test cases just return '[]' here. * fix some pylint * handle empty 'content' in parse_shell_config and remove the protection that was present. --- cloudinit/sources/DataSourceOpenNebula.py | 50 +++++++++++----------- tests/unittests/test_datasource/test_opennebula.py | 15 ++++++- 2 files changed, 40 insertions(+), 25 deletions(-) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 141bd454..07dc25ff 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -23,9 +23,9 @@ # along with this program. If not, see . import os +import pwd import re -import string -import subprocess +import string # pylint: disable=W0402 from cloudinit import log as logging from cloudinit import sources @@ -58,7 +58,7 @@ class DataSourceOpenNebula(sources.DataSource): # decide parseuser for context.sh shell reader parseuser = DEFAULT_PARSEUSER - if self.ds_cfg.get('parseuser'): + if 'parseuser' in self.ds_cfg: parseuser = self.ds_cfg.get('parseuser') candidates = [self.seed_dir] @@ -260,13 +260,21 @@ def find_candidate_devs(): return combined -def parse_shell_config(content, keylist=None, bash=None, asuser=None): +def switch_user_cmd(user): + return ['sudo', '-u', user] + + +def parse_shell_config(content, keylist=None, bash=None, asuser=None, + switch_user_cb=None): if isinstance(bash, str): bash = [bash] elif bash is None: bash = ['bash', '-e'] + if switch_user_cb is None: + switch_user_cb = switch_user_cmd + # allvars expands to all existing variables by using '${!x*}' notation # where x is lower or upper case letters or '_' allvars = ["${!%s*}" % x for x in string.letters + "_"] @@ -302,23 +310,17 @@ def parse_shell_config(content, keylist=None, bash=None, asuser=None): bcmd = ('unset IFS\n' + setup + varprinter(allvars) + - '{\n%s\n\n} > /dev/null\n' % content + + '{\n%s\n\n:\n} > /dev/null\n' % content + 'unset IFS\n' + varprinter(keylist) + "\n") cmd = [] if asuser is not None: - cmd = ['sudo', '-u', asuser] + cmd = switch_user_cb(asuser) cmd.extend(bash) - sp = subprocess.Popen(cmd, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - (output, error) = sp.communicate(input=bcmd) - - if sp.returncode != 0: - raise Exception("Process returned %d" % sp.returncode) + (output, _error) = util.subp(cmd, data=bcmd) # exclude vars in bash that change on their own or that we used excluded = ("RANDOM", "LINENO", "_", "__v") @@ -329,7 +331,7 @@ def parse_shell_config(content, keylist=None, bash=None, asuser=None): # go through output. First _start_ is for 'preset', second for 'target'. # Add to target only things were changed and not in volitile - for line in output.split("\0"): + for line in output.split("\x00"): try: (key, val) = line.split("=", 1) if target is preset: @@ -367,21 +369,21 @@ def read_context_disk_dir(source_dir, asuser=None): results = {'userdata': None, 'metadata': {}} if "context.sh" in found: + if asuser is not None: + try: + pwd.getpwnam(asuser) + except KeyError as e: + raise BrokenContextDiskDir("configured user '%s' " + "does not exist", asuser) try: with open(os.path.join(source_dir, 'context.sh'), 'r') as f: content = f.read().strip() - f.close() - - # don't pass empty context script - # to shell parser - non_empty = re.match(r'.*?^\s*([^# ]+)', content, - re.MULTILINE | re.DOTALL) - if non_empty: - context = parse_shell_config(content, asuser=asuser) + + context = parse_shell_config(content, asuser=asuser) + except util.ProcessExecutionError as e: + raise BrokenContextDiskDir("Error processing context.sh: %s" % (e)) except IOError as e: raise NonContextDiskDir("Error reading context.sh: %s" % (e)) - except Exception as e: - raise BrokenContextDiskDir("Error processing context.sh: %s" % (e)) else: raise NonContextDiskDir("Missing context.sh") diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index f2457657..9c7a644a 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -37,6 +37,7 @@ CMD_IP_OUT = '''\ class TestOpenNebulaDataSource(MockerTestCase): + parsed_user = None def setUp(self): super(TestOpenNebulaDataSource, self).setUp() @@ -48,6 +49,18 @@ class TestOpenNebulaDataSource(MockerTestCase): self.seed_dir = os.path.join(self.paths.seed_dir, "opennebula") self.sys_cfg = {'datasource': {'OpenNebula': {'dsmode': 'local'}}} + # we don't want 'sudo' called in tests. so we patch switch_user_cmd + def my_switch_user_cmd(user): + self.parsed_user = user + return [] + + self.switch_user_cmd_real = ds.switch_user_cmd + ds.switch_user_cmd = my_switch_user_cmd + + def tearDown(self): + ds.switch_user_cmd = self.switch_user_cmd_real + super(TestOpenNebulaDataSource, self).tearDown() + def test_get_data_non_contextdisk(self): try: # dont' try to lookup for CDs @@ -96,9 +109,9 @@ class TestOpenNebulaDataSource(MockerTestCase): util.find_devs_with = orig_find_devs_with def test_get_data(self): + orig_find_devs_with = util.find_devs_with try: # dont' try to lookup for CDs - orig_find_devs_with = util.find_devs_with util.find_devs_with = lambda n: [] populate_context_dir(self.seed_dir, {'KEY1': 'val1'}) dsrc = self.ds(sys_cfg=self.sys_cfg, distro=None, paths=self.paths) -- cgit v1.2.3 From 8020ff584b37e3786fb9575003cfc543a16344ce Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Wed, 11 Sep 2013 01:14:11 +0200 Subject: All fake util.find_devs_with set before try-finally section --- tests/unittests/test_datasource/test_opennebula.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index 9c7a644a..45256a86 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -62,11 +62,10 @@ class TestOpenNebulaDataSource(MockerTestCase): super(TestOpenNebulaDataSource, self).tearDown() def test_get_data_non_contextdisk(self): + orig_find_devs_with = util.find_devs_with try: # dont' try to lookup for CDs - orig_find_devs_with = util.find_devs_with util.find_devs_with = lambda n: [] - dsrc = self.ds(sys_cfg=self.sys_cfg, distro=None, paths=self.paths) ret = dsrc.get_data() self.assertFalse(ret) @@ -74,11 +73,10 @@ class TestOpenNebulaDataSource(MockerTestCase): util.find_devs_with = orig_find_devs_with def test_get_data_broken_contextdisk(self): + orig_find_devs_with = util.find_devs_with try: # dont' try to lookup for CDs - orig_find_devs_with = util.find_devs_with util.find_devs_with = lambda n: [] - populate_dir(self.seed_dir, {'context.sh': INVALID_CONTEXT}) dsrc = self.ds(sys_cfg=self.sys_cfg, distro=None, paths=self.paths) self.assertRaises(ds.BrokenContextDiskDir, dsrc.get_data) @@ -86,11 +84,8 @@ class TestOpenNebulaDataSource(MockerTestCase): util.find_devs_with = orig_find_devs_with def test_get_data_invalid_identity(self): + orig_find_devs_with = util.find_devs_with try: - # dont' try to lookup for CDs - orig_find_devs_with = util.find_devs_with - util.find_devs_with = lambda n: [] - # generate non-existing system user name sys_cfg = self.sys_cfg invalid_user = 'invalid' @@ -102,6 +97,8 @@ class TestOpenNebulaDataSource(MockerTestCase): sys_cfg['datasource']['OpenNebula']['parseuser'] = \ invalid_user + # dont' try to lookup for CDs + util.find_devs_with = lambda n: [] populate_context_dir(self.seed_dir, {'KEY1': 'val1'}) dsrc = self.ds(sys_cfg=sys_cfg, distro=None, paths=self.paths) self.assertRaises(ds.BrokenContextDiskDir, dsrc.get_data) @@ -200,8 +197,8 @@ class TestOpenNebulaDataSource(MockerTestCase): "TYPE=iso9660": ["/dev/vdb"], }.get(criteria, []) + orig_find_devs_with = util.find_devs_with try: - orig_find_devs_with = util.find_devs_with util.find_devs_with = my_devs_with self.assertEqual(["/dev/sdb", "/dev/sr0", "/dev/vdb"], ds.find_candidate_devs()) -- cgit v1.2.3 From d3a341dc6e2fcb4efd00a44d8f5a4524e64c4d27 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 10 Sep 2013 20:27:14 -0400 Subject: add entry to changelog --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index 4b2770a4..56f6ea68 100644 --- a/ChangeLog +++ b/ChangeLog @@ -15,6 +15,7 @@ which also reads from uptime. uptime is useful as clock may change during boot due to ntp. - prefer growpart resizer to 'parted resizepart' (LP: #1212492) + - add OpenNebula Datasource [Vlastimil Holer] 0.7.2: - add a debian watch file - add 'sudo' entry to ubuntu's default user (LP: #1080717) -- cgit v1.2.3 From 6b122985da19c08ea50a25226c726f2982680efb Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 11 Sep 2013 13:27:40 -0400 Subject: better checking and portability in read-dependencies and read-version --- tools/read-dependencies | 3 ++- tools/read-version | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/read-dependencies b/tools/read-dependencies index 87db5d83..3335f6a4 100755 --- a/tools/read-dependencies +++ b/tools/read-dependencies @@ -26,6 +26,7 @@ if [ ! -e "$REQUIRES" ]; then fi # Filter out comments and empty lines -DEPS=$(sed -n -e 's,#.*,,' -e '/./p' "$REQUIRES") || +DEPS=$(sed -n -e 's,#.*,,' -e '/./p' "$REQUIRES") && + [ -n "$DEPS" ] || fail "failed to read deps from '${REQUIRES}'" echo "$DEPS" | sort -d -f diff --git a/tools/read-version b/tools/read-version index c37228f8..3df0889b 100755 --- a/tools/read-version +++ b/tools/read-version @@ -25,7 +25,8 @@ if [ ! -e "$CHNG_LOG" ]; then fail "Unable to find 'ChangeLog' file located at '$CHNG_LOG'" fi -VERSION=$(sed -n '/^[0-9]\+[.][0-9]\+[.][0-9]\+:/ {s/://; p; :a;n; ba}' \ - "$CHNG_LOG") || +VERSION=$(sed -n '/^[0-9]\+[.][0-9]\+[.][0-9]\+:/ \ + {s/://; p; :a;n; ba; }' "$CHNG_LOG") && + [ -n "$VERSION" ] || fail "failed to get version from '$CHNG_LOG'" echo "$VERSION" -- cgit v1.2.3