From c59e06c6d20ce585927f336630e8ae3cca12c110 Mon Sep 17 00:00:00 2001
From: Vlastimil Holer <vlastimil.holer@gmail.com>
Date: Wed, 12 Sep 2012 10:59:19 +0200
Subject: Initial support for OpenNebula's contextualization disk.

---
 cloudinit/sources/DataSourceOpenNebula.py | 227 ++++++++++++++++++++++++++++++
 1 file changed, 227 insertions(+)
 create mode 100644 cloudinit/sources/DataSourceOpenNebula.py

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
new file mode 100644
index 00000000..03c0103e
--- /dev/null
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -0,0 +1,227 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2012 Canonical Ltd.
+#    Copyright (C) 2012 Yahoo! Inc.
+#    Copyright (C) 2012 CERIT-Scientific Cloud
+#
+#    Author: Scott Moser <scott.moser@canonical.com>
+#    Author: Joshua Harlow <harlowja@yahoo-inc.com>
+#    Author: Vlastimil Holer <xholer@mail.muni.cz>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import re
+import subprocess
+
+from cloudinit import log as logging
+from cloudinit import sources
+from cloudinit import util
+
+LOG = logging.getLogger(__name__)
+
+DEFAULT_IID = "iid-dsopennebula"
+CONTEXT_DISK_FILES = ["context.sh"]
+
+class DataSourceOpenNebula(sources.DataSource):
+    def __init__(self, sys_cfg, distro, paths):
+        sources.DataSource.__init__(self, sys_cfg, distro, paths)
+        self.dsmode = 'local'
+        self.seed = None
+        self.seed_dir = os.path.join(paths.seed_dir, 'opennebula')
+
+    def __str__(self):
+        mstr = "%s [seed=%s][dsmode=%s]" % (util.obj_name(self),
+                                            self.seed, self.dsmode)
+        return mstr
+
+    def get_data(self):
+        defaults = {
+            "instance-id": DEFAULT_IID,
+            "dsmode": self.dsmode,
+        }
+
+        found = None
+        md = {}
+        ud = ""
+
+        results = {}
+        if os.path.isdir(self.seed_dir):
+            try:
+                results=read_on_context_device_dir(self.seed_dir)
+                found = self.seed_dir
+            except NonContextDeviceDir:
+                util.logexc(LOG, "Failed reading context device from %s",
+                            self.seed_dir)
+        if not found:
+            devlist = find_candidate_devs()
+            for dev in devlist:
+                try:
+                    results = util.mount_cb(dev, read_context_disk_dir)
+                    found = dev
+                    break
+                except (NonConfigDriveDir, util.MountFailedError):
+                    pass
+                except BrokenConfigDriveDir:
+                    util.logexc(LOG, "broken config drive: %s", dev)
+
+        if not found:
+            return False
+
+        md = results['metadata']
+        md = util.mergedict(md, defaults)
+
+        # update interfaces and ifup only on the local datasource
+        # this way the DataSourceConfigDriveNet doesn't do it also.
+#       if 'network-interfaces' in md and self.dsmode == "local":
+#            if md['dsmode'] == "pass":
+#                log.info("updating network interfaces from configdrive")
+#            else:
+#                log.debug("updating network interfaces from configdrive")
+#
+#            util.write_file("/etc/network/interfaces",
+#                md['network-interfaces'])
+#            try:
+#                (out, err) = util.subp(['ifup', '--all'])
+#                if len(out) or len(err):
+#                    log.warn("ifup --all had stderr: %s" % err)
+#
+#            except subprocess.CalledProcessError as exc:
+#                log.warn("ifup --all failed: %s" % (exc.output[1]))
+#
+
+        if md['dsmode'] == self.dsmode:
+            self.seed = found
+            self.metadata = md
+            self.userdata_raw = ud
+            return True
+
+        LOG.debug("%s: not claiming datasource, dsmode=%s", self, md['dsmode'])
+        return False
+
+
+class DataSourceOpenNebulaNet(DataSourceOpenNebula):
+    dsmode = "net"
+
+
+class NonContextDeviceDir(Exception):
+    pass
+
+
+def find_candidate_devs():
+    """
+    Return a list of devices that may contain the context disk.
+    """
+    by_fstype = util.find_devs_with("TYPE=iso9660")
+    by_label = util.find_devs_with("LABEL=CDROM")
+
+    by_fstype.sort()
+    by_label.sort()
+
+    # combine list of items by putting by-label items first
+    # followed by fstype items, but with dupes removed
+    combined = (by_label + [d for d in by_fstype if d not in by_label])
+
+    # We are looking for block device (sda, not sda1), ignore partitions
+    combined = [d for d in combined if d[-1] not in "0123456789"]
+
+    return combined
+
+
+def read_context_disk_dir(source_dir):
+    """
+    read_context_disk_dir(source_dir):
+    read source_dir and return a tuple with metadata dict and user-data
+    string populated.  If not a valid dir, raise a NonContextDeviceDir
+    """
+
+    found = {}
+    for af in CONTEXT_DISK_FILES:
+        fn = os.path.join(source_dir, af)
+        if os.path.isfile(fn):
+            found[af] = fn
+
+    if len(found) == 0:
+        raise NonContextDeviceDir("%s: %s" % (source_dir, "no files found"))
+
+    context_sh = {}
+    results = {
+        'userdata':None,
+        'metadata':{},
+    }
+
+    if "context.sh" in found:
+        # let bash process the contextualization script;
+        # write out data in normalized output NAME=\$?'?VALUE'?
+        # TODO: don't trust context.sh! parse manually !!!
+        try:
+            BASH_CMD='VARS=`set | sort -u `;' \
+                '. %s/context.sh;' \
+                'comm -23 <(set | sort -u) <(echo "$VARS") | egrep -v "^(VARS|PIPESTATUS|_)="'
+
+            (out,err) = util.subp(['bash',
+                '--noprofile',
+                '--norc',
+                '-c',
+                BASH_CMD % (source_dir) ])
+
+            for (key,value) in [ l.split('=',1) for l in out.rstrip().split("\n") ]:
+                # with backslash escapes
+                r=re.match("^\$'(.*)'$",value)
+                if r:
+                    context_sh[key.lower()]=r.group(1).\
+                        replace('\\\\','\\').\
+                        replace('\\t','\t').\
+                        replace('\\n','\n').\
+                        replace("\\'","'")
+                else:
+                    # multiword values
+                    r=re.match("^'(.*)'$",value)
+                    if r:
+                        context_sh[key.lower()]=r.group(1)
+                    else:
+                        # simple values
+                        context_sh[key.lower()]=value
+        except subprocess.CalledProcessError as exc:
+            LOG.warn("context script faled to read" % (exc.output[1]))
+        results['metadata']=context_sh
+
+    # process single or multiple SSH keys
+    if "ssh_key" in context_sh:
+        lines = context_sh.get('ssh_key').splitlines()
+        results['metadata']['public-keys'] = [l for l in lines
+            if len(l) and not l.startswith("#")]
+
+    # custom hostname
+    if 'hostname' in context_sh:
+        results['metadata']['local-hostname'] = context_sh['hostname']
+
+    # raw user data
+    if "user_data" in context_sh:
+        results['userdata'] = context_sh["user_data"]
+    if "userdata" in context_sh:
+        results['userdata'] = context_sh["userdata"]
+
+    return results
+
+
+# Used to match classes to dependencies
+datasources = [
+    (DataSourceOpenNebula, (sources.DEP_FILESYSTEM, )),
+    (DataSourceOpenNebulaNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
+]
+
+
+# Return a list of data sources that match this set of dependencies
+def get_datasource_list(depends):
+    return sources.list_from_depends(depends, datasources)
-- 
cgit v1.2.3


From 34bebd8569e9319b791802f4fd551537967aec69 Mon Sep 17 00:00:00 2001
From: Vlastimil Holer <vlastimil.holer@gmail.com>
Date: Wed, 19 Sep 2012 15:31:10 +0200
Subject: Optionally resolve IPv4 hostname.

---
 cloudinit/sources/__init__.py | 10 ++++++++--
 cloudinit/util.py             |  7 +++++++
 2 files changed, 15 insertions(+), 2 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index 3f611d44..9b68f99e 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -149,7 +149,7 @@ class DataSource(object):
             return "iid-datasource"
         return str(self.metadata['instance-id'])
 
-    def get_hostname(self, fqdn=False):
+    def get_hostname(self, fqdn=False, resolve_ip=False):
         defdomain = "localdomain"
         defhost = "localhost"
         domain = defdomain
@@ -173,7 +173,13 @@ class DataSource(object):
             # make up a hostname (LP: #475354) in format ip-xx.xx.xx.xx
             lhost = self.metadata['local-hostname']
             if util.is_ipv4(lhost):
-                toks = "ip-%s" % lhost.replace(".", "-")
+                if resolve_ip:
+                    toks = util.gethostbyaddr(lhost)
+
+                if toks:
+                    toks = toks.split('.')
+                else:
+                    toks = "ip-%s" % lhost.replace(".", "-")
             else:
                 toks = lhost.split(".")
 
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 33da73eb..b25ded0d 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -874,6 +874,13 @@ def get_hostname():
     return hostname
 
 
+def gethostbyaddr(ip):
+    try:
+        return socket.gethostbyaddr(ip)[0]
+    except socket.herror:
+        return None
+
+
 def is_resolvable_url(url):
     """determine if this url is resolvable (existing or ip)."""
     return (is_resolvable(urlparse.urlparse(url).hostname))
-- 
cgit v1.2.3


From 0659e75ee2a2c25c113e8e652a3366433e76eff0 Mon Sep 17 00:00:00 2001
From: Vlastimil Holer <vlastimil.holer@gmail.com>
Date: Wed, 19 Sep 2012 15:32:10 +0200
Subject: Use and resolve PUBLIC_IP from context disk if no HOSTNAME is
 specified.

---
 cloudinit/sources/DataSourceOpenNebula.py | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index 03c0103e..e8e79c7e 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -109,6 +109,8 @@ class DataSourceOpenNebula(sources.DataSource):
         LOG.debug("%s: not claiming datasource, dsmode=%s", self, md['dsmode'])
         return False
 
+    def get_hostname(self, fqdn=False, resolve_ip=True):
+        return sources.DataSource.get_hostname(self, fqdn, resolve_ip)
 
 class DataSourceOpenNebulaNet(DataSourceOpenNebula):
     dsmode = "net"
@@ -205,6 +207,8 @@ def read_context_disk_dir(source_dir):
     # custom hostname
     if 'hostname' in context_sh:
         results['metadata']['local-hostname'] = context_sh['hostname']
+    elif 'public_ip'in context_sh:
+        results['metadata']['local-hostname'] = context_sh['public_ip']
 
     # raw user data
     if "user_data" in context_sh:
-- 
cgit v1.2.3


From 17ac62b5d4d0cb3f25f431ff85411a7cca860d12 Mon Sep 17 00:00:00 2001
From: Vlastimil Holer <vlastimil.holer@gmail.com>
Date: Thu, 20 Sep 2012 18:39:54 +0200
Subject: Initialize toks variable. Fix EC2-like hostname generation based on
 IPv4.

---
 cloudinit/sources/__init__.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index 9b68f99e..a89f4703 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -173,13 +173,14 @@ class DataSource(object):
             # make up a hostname (LP: #475354) in format ip-xx.xx.xx.xx
             lhost = self.metadata['local-hostname']
             if util.is_ipv4(lhost):
+                toks = []
                 if resolve_ip:
                     toks = util.gethostbyaddr(lhost)
 
                 if toks:
                     toks = toks.split('.')
                 else:
-                    toks = "ip-%s" % lhost.replace(".", "-")
+                    toks = ["ip-%s" % lhost.replace(".", "-")]
             else:
                 toks = lhost.split(".")
 
-- 
cgit v1.2.3


From ea42bf67999c1ef61fa46ab805b0093b95dc575f Mon Sep 17 00:00:00 2001
From: Vlastimil Holer <vlastimil.holer@gmail.com>
Date: Thu, 20 Sep 2012 18:42:18 +0200
Subject: Configurable dsmode. Resolve IPv4 hostname only with dsmode=net.

---
 cloudinit/sources/DataSourceOpenNebula.py | 42 ++++++++++++++++++++++++-------
 1 file changed, 33 insertions(+), 9 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index e8e79c7e..0b498a54 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -32,6 +32,7 @@ LOG = logging.getLogger(__name__)
 
 DEFAULT_IID = "iid-dsopennebula"
 CONTEXT_DISK_FILES = ["context.sh"]
+VALID_DSMODES = ("local", "net", "disabled")
 
 class DataSourceOpenNebula(sources.DataSource):
     def __init__(self, sys_cfg, distro, paths):
@@ -81,6 +82,20 @@ class DataSourceOpenNebula(sources.DataSource):
         md = results['metadata']
         md = util.mergedict(md, defaults)
 
+        dsmode = results.get('dsmode', None)
+        if dsmode not in VALID_DSMODES + (None,):
+            LOG.warn("user specified invalid mode: %s" % dsmode)
+            dsmode = None
+
+        if (dsmode is None) and self.ds_cfg.get('dsmode'):
+            dsmode = self.ds_cfg.get('dsmode')
+        else:
+            dsmode = self.dsmode
+
+        if dsmode == "disabled":
+            # most likely user specified
+            return False
+
         # update interfaces and ifup only on the local datasource
         # this way the DataSourceConfigDriveNet doesn't do it also.
 #       if 'network-interfaces' in md and self.dsmode == "local":
@@ -100,20 +115,29 @@ class DataSourceOpenNebula(sources.DataSource):
 #                log.warn("ifup --all failed: %s" % (exc.output[1]))
 #
 
-        if md['dsmode'] == self.dsmode:
-            self.seed = found
-            self.metadata = md
-            self.userdata_raw = ud
-            return True
+        if dsmode != self.dsmode:
+            LOG.debug("%s: not claiming datasource, dsmode=%s", self, dsmode)
+            return False
+
+        self.seed = found
+        self.metadata = md
+        self.userdata_raw = ud
 
-        LOG.debug("%s: not claiming datasource, dsmode=%s", self, md['dsmode'])
-        return False
+        return True
 
-    def get_hostname(self, fqdn=False, resolve_ip=True):
+    def get_hostname(self, fqdn=False, resolve_ip=None):
+        if resolve_ip is None:
+            if self.dsmode == 'net':
+                resolve_ip = True
+            else:
+                resolve_ip = False
         return sources.DataSource.get_hostname(self, fqdn, resolve_ip)
 
+
 class DataSourceOpenNebulaNet(DataSourceOpenNebula):
-    dsmode = "net"
+    def __init__(self, sys_cfg, distro, paths):
+        DataSourceOpenNebula.__init__(self, sys_cfg, distro, paths)
+        self.dsmode = 'net'
 
 
 class NonContextDeviceDir(Exception):
-- 
cgit v1.2.3


From 9eaafc7379e0c6ded70a851a703a5bc5c7e56e42 Mon Sep 17 00:00:00 2001
From: Vlastimil Holer <vlastimil.holer@gmail.com>
Date: Mon, 24 Sep 2012 13:28:51 +0200
Subject: Process userdata, ignored by mistake.

---
 cloudinit/sources/DataSourceOpenNebula.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index 0b498a54..ad65a36e 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -54,7 +54,6 @@ class DataSourceOpenNebula(sources.DataSource):
 
         found = None
         md = {}
-        ud = ""
 
         results = {}
         if os.path.isdir(self.seed_dir):
@@ -121,7 +120,7 @@ class DataSourceOpenNebula(sources.DataSource):
 
         self.seed = found
         self.metadata = md
-        self.userdata_raw = ud
+        self.userdata_raw = results.get('userdata')
 
         return True
 
@@ -237,7 +236,7 @@ def read_context_disk_dir(source_dir):
     # raw user data
     if "user_data" in context_sh:
         results['userdata'] = context_sh["user_data"]
-    if "userdata" in context_sh:
+    elif "userdata" in context_sh:
         results['userdata'] = context_sh["userdata"]
 
     return results
-- 
cgit v1.2.3


From 1567b1bc3b2948aa80e0b150d34542e2ff02428f Mon Sep 17 00:00:00 2001
From: Vlastimil Holer <vlastimil.holer@gmail.com>
Date: Wed, 3 Oct 2012 13:26:36 +0200
Subject: Delete old ConfigDrive's code.

---
 cloudinit/sources/DataSourceOpenNebula.py | 19 -------------------
 1 file changed, 19 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index ad65a36e..325269f6 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -95,25 +95,6 @@ class DataSourceOpenNebula(sources.DataSource):
             # most likely user specified
             return False
 
-        # update interfaces and ifup only on the local datasource
-        # this way the DataSourceConfigDriveNet doesn't do it also.
-#       if 'network-interfaces' in md and self.dsmode == "local":
-#            if md['dsmode'] == "pass":
-#                log.info("updating network interfaces from configdrive")
-#            else:
-#                log.debug("updating network interfaces from configdrive")
-#
-#            util.write_file("/etc/network/interfaces",
-#                md['network-interfaces'])
-#            try:
-#                (out, err) = util.subp(['ifup', '--all'])
-#                if len(out) or len(err):
-#                    log.warn("ifup --all had stderr: %s" % err)
-#
-#            except subprocess.CalledProcessError as exc:
-#                log.warn("ifup --all failed: %s" % (exc.output[1]))
-#
-
         if dsmode != self.dsmode:
             LOG.debug("%s: not claiming datasource, dsmode=%s", self, dsmode)
             return False
-- 
cgit v1.2.3


From 0e7ce376fdb84345c97f936f5ccb1fee964a4c03 Mon Sep 17 00:00:00 2001
From: Vlastimil Holer <vlastimil.holer@gmail.com>
Date: Wed, 3 Oct 2012 13:29:24 +0200
Subject: Delete hyphen.

---
 cloudinit/sources/DataSourceOpenNebula.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index 325269f6..00c076e6 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -2,7 +2,7 @@
 #
 #    Copyright (C) 2012 Canonical Ltd.
 #    Copyright (C) 2012 Yahoo! Inc.
-#    Copyright (C) 2012 CERIT-Scientific Cloud
+#    Copyright (C) 2012 CERIT Scientific Cloud
 #
 #    Author: Scott Moser <scott.moser@canonical.com>
 #    Author: Joshua Harlow <harlowja@yahoo-inc.com>
-- 
cgit v1.2.3


From 268cdf3458849eb7be41253c38a86c68bbddde59 Mon Sep 17 00:00:00 2001
From: Vlastimil Holer <vlastimil.holer@gmail.com>
Date: Wed, 19 Dec 2012 18:12:37 +0100
Subject: Remove conflicting merge block

---
 cloudinit/sources/__init__.py | 4 ----
 1 file changed, 4 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index f98493de..0bad4c8b 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -163,7 +163,6 @@ class DataSource(object):
             # make up a hostname (LP: #475354) in format ip-xx.xx.xx.xx
             lhost = self.metadata['local-hostname']
             if util.is_ipv4(lhost):
-<<<<<<< TREE
                 toks = []
                 if resolve_ip:
                     toks = util.gethostbyaddr(lhost)
@@ -172,9 +171,6 @@ class DataSource(object):
                     toks = toks.split('.')
                 else:
                     toks = ["ip-%s" % lhost.replace(".", "-")]
-=======
-                toks = ["ip-%s" % lhost.replace(".", "-")]
->>>>>>> MERGE-SOURCE
             else:
                 toks = lhost.split(".")
 
-- 
cgit v1.2.3


From 5746598d3cffe07dbae155347eaa44aae48ff572 Mon Sep 17 00:00:00 2001
From: Vlastimil Holer <vlastimil.holer@gmail.com>
Date: Thu, 20 Dec 2012 13:20:57 +0100
Subject: Change context variables .replace with .decode

---
 cloudinit/sources/DataSourceOpenNebula.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index 00c076e6..1622a66e 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -185,11 +185,7 @@ def read_context_disk_dir(source_dir):
                 # with backslash escapes
                 r=re.match("^\$'(.*)'$",value)
                 if r:
-                    context_sh[key.lower()]=r.group(1).\
-                        replace('\\\\','\\').\
-                        replace('\\t','\t').\
-                        replace('\\n','\n').\
-                        replace("\\'","'")
+                    context_sh[key.lower()]=r.group(1).decode('string_escape')
                 else:
                     # multiword values
                     r=re.match("^'(.*)'$",value)
-- 
cgit v1.2.3


From 21d3a5af7d2e1884009cfd1ad650a937438f5991 Mon Sep 17 00:00:00 2001
From: Vlastimil Holer <vlastimil.holer@gmail.com>
Date: Thu, 20 Dec 2012 13:57:57 +0100
Subject: Add explanation on how context variables parsing works.

---
 cloudinit/sources/DataSourceOpenNebula.py | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index 1622a66e..a50b0c10 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -167,10 +167,24 @@ def read_context_disk_dir(source_dir):
     }
 
     if "context.sh" in found:
-        # let bash process the contextualization script;
-        # write out data in normalized output NAME=\$?'?VALUE'?
-        # TODO: don't trust context.sh! parse manually !!!
         try:
+            # Note: context.sh is a "shell" script with defined context
+            # variables, like: X="Y" . It's ready to use as a shell source
+            # e.g.: ". context.sh" and as a shell script it can also reference
+            # to already defined shell variables. So to have same context var.
+            # values as we can have in custom shell script, we use bash itself
+            # to read context.sh and dump variables in easily parsable way.
+            #
+            # normalized variables dump format (get by cmd "set"):
+            # 1. simple single word assignment ........ X=Y
+            # 2. multiword assignment ................. X='Y Z'
+            # 3. assignments with backslash escapes ... X=$'Y\nZ'
+            #
+            # how context variables are read:
+            # 1. list existing ("old") shell variables and store into $VARS
+            # 2. read context variables
+            # 3. use comm to filter "old" variables from all current
+            #    variables and excl. few other vars with grep
             BASH_CMD='VARS=`set | sort -u `;' \
                 '. %s/context.sh;' \
                 'comm -23 <(set | sort -u) <(echo "$VARS") | egrep -v "^(VARS|PIPESTATUS|_)="'
-- 
cgit v1.2.3


From 46e646fabf3a0f2593dc9d9f231fd075134de36f Mon Sep 17 00:00:00 2001
From: Vlastimil Holer <vlastimil.holer@gmail.com>
Date: Thu, 20 Dec 2012 15:17:32 +0100
Subject: Change subp exception handling to util.ProcessExecutionError

---
 cloudinit/sources/DataSourceOpenNebula.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index a50b0c10..967d7170 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -208,8 +208,8 @@ def read_context_disk_dir(source_dir):
                     else:
                         # simple values
                         context_sh[key.lower()]=value
-        except subprocess.CalledProcessError as exc:
-            LOG.warn("context script faled to read" % (exc.output[1]))
+        except util.ProcessExecutionError, _err:
+            LOG.warn("Failed to read context variables: %s" % (_err.message))
         results['metadata']=context_sh
 
     # process single or multiple SSH keys
-- 
cgit v1.2.3


From 2efdb4b8d1c17eea352ae0dc022d2ddca80833da Mon Sep 17 00:00:00 2001
From: Vlastimil Holer <vlastimil.holer@gmail.com>
Date: Thu, 20 Dec 2012 17:16:47 +0100
Subject: Fix exception handlers for data read. Fix name read_context_disk_dir.

---
 cloudinit/sources/DataSourceOpenNebula.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index 967d7170..2f1ce459 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -58,7 +58,7 @@ class DataSourceOpenNebula(sources.DataSource):
         results = {}
         if os.path.isdir(self.seed_dir):
             try:
-                results=read_on_context_device_dir(self.seed_dir)
+                results=read_context_disk_dir(self.seed_dir)
                 found = self.seed_dir
             except NonContextDeviceDir:
                 util.logexc(LOG, "Failed reading context device from %s",
@@ -70,10 +70,8 @@ class DataSourceOpenNebula(sources.DataSource):
                     results = util.mount_cb(dev, read_context_disk_dir)
                     found = dev
                     break
-                except (NonConfigDriveDir, util.MountFailedError):
+                except (NonContextDeviceDir, util.MountFailedError):
                     pass
-                except BrokenConfigDriveDir:
-                    util.logexc(LOG, "broken config drive: %s", dev)
 
         if not found:
             return False
@@ -211,6 +209,8 @@ def read_context_disk_dir(source_dir):
         except util.ProcessExecutionError, _err:
             LOG.warn("Failed to read context variables: %s" % (_err.message))
         results['metadata']=context_sh
+    else:
+        raise NonContextDeviceDir("Missing context.sh")
 
     # process single or multiple SSH keys
     if "ssh_key" in context_sh:
-- 
cgit v1.2.3


From 45641d2e31ffa1b7235a2f36d234e804484ff4a3 Mon Sep 17 00:00:00 2001
From: Javi Fontan <jfontan@opennebula.org>
Date: Fri, 21 Dec 2012 12:47:59 +0100
Subject: Add OpenNebula contextualization options to cloud-init

---
 cloudinit/sources/DataSourceOpenNebula.py | 115 ++++++++++++++++++++++++++++--
 1 file changed, 111 insertions(+), 4 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index 2f1ce459..8af6ad3d 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -101,6 +101,12 @@ class DataSourceOpenNebula(sources.DataSource):
         self.metadata = md
         self.userdata_raw = results.get('userdata')
 
+        if 'network-interfaces' in results:
+            self.distro.apply_network(results['network-interfaces'])
+
+        if 'dns' in results:
+            self.distro.apply_resolv_conf(results['dns'])
+
         return True
 
     def get_hostname(self, fqdn=False, resolve_ip=None):
@@ -122,6 +128,93 @@ class NonContextDeviceDir(Exception):
     pass
 
 
+class OpenNebulaNetwork(object):
+    REG_ETH=re.compile('^eth')
+    REG_DEV_MAC=re.compile('^(eth\d+).*HWaddr (..:..:..:..:..:..)')
+
+    def __init__(self, ifconfig, context_sh):
+        self.ifconfig=ifconfig
+        self.context_sh=context_sh
+        self.ifaces=self.get_ifaces()
+
+    def get_ifaces(self):
+        return [self.REG_DEV_MAC.search(f).groups() for f in self.ifconfig.split("\n") if self.REG_ETH.match(f)]
+
+    def mac2ip(self, mac):
+        components=mac.split(':')[2:]
+
+        return [str(int(c, 16)) for c in components]
+        
+    def get_ip(self, dev, components):
+        var_name=dev+'_ip'
+        if var_name in self.context_sh:
+            return self.context_sh[var_name]
+        else:
+            return '.'.join(components)
+
+    def get_mask(self, dev, components):
+        var_name=dev+'_mask'
+        if var_name in self.context_sh:
+            return self.context_sh[var_name]
+        else:
+            return '255.255.255.0'
+
+    def get_network(self, dev, components):
+        var_name=dev+'_network'
+        if var_name in self.context_sh:
+            return self.context_sh[var_name]
+        else:
+            return '.'.join(components[:-1])+'.0'
+
+    def get_gateway(self, dev, components):
+        var_name=dev+'_gateway'
+        if var_name in self.context_sh:
+            return self.context_sh[var_name]
+        else:
+            None
+
+    def gen_conf(self):
+        conf=[]
+        conf.append('auto lo')
+        conf.append('iface lo inet loopback')
+        conf.append('')
+
+        for i in self.ifaces:
+            dev=i[0]
+            mac=i[1]
+            ip_components=self.mac2ip(mac)
+
+            conf.append('auto '+dev)
+            conf.append('iface '+dev+' inet static')
+            conf.append('  address '+self.get_ip(dev, ip_components))
+            conf.append('  network '+self.get_network(dev, ip_components))
+            conf.append('  netmask '+self.get_mask(dev, ip_components))
+
+            gateway=self.get_gateway(dev, ip_components)
+            if gateway:
+                conf.append('  gateway '+gateway)
+
+            conf.append('')
+
+        return "\n".join(conf)
+
+    def gen_dns(self):
+        dnss=[]
+
+        if 'dns' in self.context_sh:
+            dnss.append('nameserver '+self.context_sh['dns'])
+
+        keys=[d for d in self.context_sh.keys() if re.match('^eth\d+_dns$', d)]
+
+        for k in sorted(keys):
+            dnss.append('nameserver '+self.context_sh[k])
+
+        if not dnss:
+            return None
+        else:
+            return "\n".join(dnss)+"\n"
+
+
 def find_candidate_devs():
     """
     Return a list of devices that may contain the context disk.
@@ -136,9 +229,6 @@ def find_candidate_devs():
     # followed by fstype items, but with dupes removed
     combined = (by_label + [d for d in by_fstype if d not in by_label])
 
-    # We are looking for block device (sda, not sda1), ignore partitions
-    combined = [d for d in combined if d[-1] not in "0123456789"]
-
     return combined
 
 
@@ -213,8 +303,15 @@ def read_context_disk_dir(source_dir):
         raise NonContextDeviceDir("Missing context.sh")
 
     # process single or multiple SSH keys
+    ssh_key_var=None
+
     if "ssh_key" in context_sh:
-        lines = context_sh.get('ssh_key').splitlines()
+        ssh_key_var="ssh_key"
+    elif "ssh_public_key" in context_sh:
+        ssh_key_var="ssh_public_key"
+
+    if ssh_key_var:
+        lines = context_sh.get(ssh_key_var).splitlines()
         results['metadata']['public-keys'] = [l for l in lines
             if len(l) and not l.startswith("#")]
 
@@ -223,6 +320,8 @@ def read_context_disk_dir(source_dir):
         results['metadata']['local-hostname'] = context_sh['hostname']
     elif 'public_ip'in context_sh:
         results['metadata']['local-hostname'] = context_sh['public_ip']
+    elif 'eth0_ip' in context_sh:
+        results['metadata']['local-hostname'] = context_sh['eth0_ip']
 
     # raw user data
     if "user_data" in context_sh:
@@ -230,6 +329,14 @@ def read_context_disk_dir(source_dir):
     elif "userdata" in context_sh:
         results['userdata'] = context_sh["userdata"]
 
+    (out, err) = util.subp(['/sbin/ifconfig', '-a'])
+    net=OpenNebulaNetwork(out, context_sh)
+    results['network-interfaces']=net.gen_conf()
+
+    dns=net.gen_dns()
+    if dns:
+        results['dns']=dns
+
     return results
 
 
-- 
cgit v1.2.3


From 5aab64f71c1f07670b59dd7be18d704611dc0ab5 Mon Sep 17 00:00:00 2001
From: Vlastimil Holer <vlastimil.holer@gmail.com>
Date: Wed, 2 Jan 2013 15:20:33 +0100
Subject: Add OpenNebula.org copyright.

---
 cloudinit/sources/DataSourceOpenNebula.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index 8af6ad3d..57b0c62c 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -2,11 +2,13 @@
 #
 #    Copyright (C) 2012 Canonical Ltd.
 #    Copyright (C) 2012 Yahoo! Inc.
-#    Copyright (C) 2012 CERIT Scientific Cloud
+#    Copyright (C) 2012-2013 CERIT Scientific Cloud
+#    Copyright (C) 2012 OpenNebula.org
 #
 #    Author: Scott Moser <scott.moser@canonical.com>
 #    Author: Joshua Harlow <harlowja@yahoo-inc.com>
 #    Author: Vlastimil Holer <xholer@mail.muni.cz>
+#    Author: Javier Fontan <jfontan@opennebula.org>
 #
 #    This program is free software: you can redistribute it and/or modify
 #    it under the terms of the GNU General Public License version 3, as
-- 
cgit v1.2.3


From 5bfbf7a81872afd426271b0ed3ed65e76b9a584e Mon Sep 17 00:00:00 2001
From: Vlastimil Holer <vlastimil.holer@gmail.com>
Date: Fri, 4 Jan 2013 15:26:11 +0100
Subject: Append DNS related stuff to network interfaces configuration. Minor
 cleanups.

---
 cloudinit/sources/DataSourceOpenNebula.py | 76 +++++++++++++++++--------------
 1 file changed, 41 insertions(+), 35 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index 57b0c62c..52474675 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -103,12 +103,11 @@ class DataSourceOpenNebula(sources.DataSource):
         self.metadata = md
         self.userdata_raw = results.get('userdata')
 
-        if 'network-interfaces' in results:
+        # apply static network configuration only in 'local' dsmode
+        if ('network-interfaces' in results and self.dsmode == "local"):
+            LOG.debug("Updating network interfaces from %s", self)
             self.distro.apply_network(results['network-interfaces'])
 
-        if 'dns' in results:
-            self.distro.apply_resolv_conf(results['dns'])
-
         return True
 
     def get_hostname(self, fqdn=False, resolve_ip=None):
@@ -175,7 +174,25 @@ class OpenNebulaNetwork(object):
         else:
             None
 
+    def get_dns(self, dev, components):
+        var_name=dev+'_dns'
+        if var_name in self.context_sh:
+            return self.context_sh[var_name]
+        else:
+            None
+
+    def get_domain(self, dev, components):
+        var_name=dev+'_domain'
+        if var_name in self.context_sh:
+            return self.context_sh[var_name]
+        else:
+            None
+
     def gen_conf(self):
+        global_dns=[]
+        if 'dns' in self.context_sh:
+            global_dns.append(self.context_sh['dns'])
+
         conf=[]
         conf.append('auto lo')
         conf.append('iface lo inet loopback')
@@ -196,25 +213,21 @@ class OpenNebulaNetwork(object):
             if gateway:
                 conf.append('  gateway '+gateway)
 
-            conf.append('')
-
-        return "\n".join(conf)
-
-    def gen_dns(self):
-        dnss=[]
+            domain=self.get_domain(dev, ip_components)
+            if domain:
+                conf.append('  dns-search '+domain)
 
-        if 'dns' in self.context_sh:
-            dnss.append('nameserver '+self.context_sh['dns'])
+            # add global DNS servers to all interfaces
+            dns=self.get_dns(dev, ip_components)
+            if global_dns or dns:
+                all_dns=global_dns
+                if dns:
+                    all_dns.append(dns)
+                conf.append('  dns-nameservers '+' '.join(all_dns))
 
-        keys=[d for d in self.context_sh.keys() if re.match('^eth\d+_dns$', d)]
-
-        for k in sorted(keys):
-            dnss.append('nameserver '+self.context_sh[k])
+            conf.append('')
 
-        if not dnss:
-            return None
-        else:
-            return "\n".join(dnss)+"\n"
+        return "\n".join(conf)
 
 
 def find_candidate_devs():
@@ -280,10 +293,8 @@ def read_context_disk_dir(source_dir):
                 'comm -23 <(set | sort -u) <(echo "$VARS") | egrep -v "^(VARS|PIPESTATUS|_)="'
 
             (out,err) = util.subp(['bash',
-                '--noprofile',
-                '--norc',
-                '-c',
-                BASH_CMD % (source_dir) ])
+                '--noprofile', '--norc',
+                '-c', BASH_CMD % (source_dir) ])
 
             for (key,value) in [ l.split('=',1) for l in out.rstrip().split("\n") ]:
                 # with backslash escapes
@@ -317,13 +328,12 @@ def read_context_disk_dir(source_dir):
         results['metadata']['public-keys'] = [l for l in lines
             if len(l) and not l.startswith("#")]
 
-    # custom hostname
-    if 'hostname' in context_sh:
-        results['metadata']['local-hostname'] = context_sh['hostname']
-    elif 'public_ip'in context_sh:
-        results['metadata']['local-hostname'] = context_sh['public_ip']
-    elif 'eth0_ip' in context_sh:
-        results['metadata']['local-hostname'] = context_sh['eth0_ip']
+    # custom hostname -- try hostname or leave cloud-init
+    # itself create hostname from IP address later
+    for k in ('hostname','public_ip','ip_public','eth0_ip'):
+        if k in context_sh:
+            results['metadata']['local-hostname'] = context_sh[k]
+            break
 
     # raw user data
     if "user_data" in context_sh:
@@ -335,10 +345,6 @@ def read_context_disk_dir(source_dir):
     net=OpenNebulaNetwork(out, context_sh)
     results['network-interfaces']=net.gen_conf()
 
-    dns=net.gen_dns()
-    if dns:
-        results['dns']=dns
-
     return results
 
 
-- 
cgit v1.2.3


From 54f6ccccfb4f75dc6877b04b42987e834b7a0015 Mon Sep 17 00:00:00 2001
From: Vlastimil Holer <vlastimil.holer@gmail.com>
Date: Tue, 19 Feb 2013 12:35:12 +0100
Subject: Change network ifaces detection from ifconfig to ip command.

---
 cloudinit/sources/DataSourceOpenNebula.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index 52474675..9b1b5f9c 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -130,16 +130,15 @@ class NonContextDeviceDir(Exception):
 
 
 class OpenNebulaNetwork(object):
-    REG_ETH=re.compile('^eth')
-    REG_DEV_MAC=re.compile('^(eth\d+).*HWaddr (..:..:..:..:..:..)')
+    REG_DEV_MAC=re.compile('^\d+: (eth\d+):.*link\/ether (..:..:..:..:..:..) ')
 
-    def __init__(self, ifconfig, context_sh):
-        self.ifconfig=ifconfig
+    def __init__(self, ip, context_sh):
+        self.ip=ip
         self.context_sh=context_sh
         self.ifaces=self.get_ifaces()
 
     def get_ifaces(self):
-        return [self.REG_DEV_MAC.search(f).groups() for f in self.ifconfig.split("\n") if self.REG_ETH.match(f)]
+        return [self.REG_DEV_MAC.search(f).groups() for f in self.ip.split("\n") if self.REG_DEV_MAC.match(f)]
 
     def mac2ip(self, mac):
         components=mac.split(':')[2:]
@@ -341,7 +340,7 @@ def read_context_disk_dir(source_dir):
     elif "userdata" in context_sh:
         results['userdata'] = context_sh["userdata"]
 
-    (out, err) = util.subp(['/sbin/ifconfig', '-a'])
+    (out, err) = util.subp(['/sbin/ip', '-o', 'link'])
     net=OpenNebulaNetwork(out, context_sh)
     results['network-interfaces']=net.gen_conf()
 
-- 
cgit v1.2.3


From eda36b6116a79433cc29b778b0008cf35d6a2afa Mon Sep 17 00:00:00 2001
From: Vlastimil Holer <vlastimil.holer@gmail.com>
Date: Tue, 19 Feb 2013 14:21:42 +0100
Subject: Name unification Context.*Device -> Context.*Disk

---
 cloudinit/sources/DataSourceOpenNebula.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index 9b1b5f9c..bc199461 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -62,8 +62,8 @@ class DataSourceOpenNebula(sources.DataSource):
             try:
                 results=read_context_disk_dir(self.seed_dir)
                 found = self.seed_dir
-            except NonContextDeviceDir:
-                util.logexc(LOG, "Failed reading context device from %s",
+            except NonContextDiskDir:
+                util.logexc(LOG, "Failed reading context disk from %s",
                             self.seed_dir)
         if not found:
             devlist = find_candidate_devs()
@@ -72,7 +72,7 @@ class DataSourceOpenNebula(sources.DataSource):
                     results = util.mount_cb(dev, read_context_disk_dir)
                     found = dev
                     break
-                except (NonContextDeviceDir, util.MountFailedError):
+                except (NonContextDiskDir, util.MountFailedError):
                     pass
 
         if not found:
@@ -125,7 +125,7 @@ class DataSourceOpenNebulaNet(DataSourceOpenNebula):
         self.dsmode = 'net'
 
 
-class NonContextDeviceDir(Exception):
+class NonContextDiskDir(Exception):
     pass
 
 
@@ -250,7 +250,7 @@ def read_context_disk_dir(source_dir):
     """
     read_context_disk_dir(source_dir):
     read source_dir and return a tuple with metadata dict and user-data
-    string populated.  If not a valid dir, raise a NonContextDeviceDir
+    string populated.  If not a valid dir, raise a NonContextDiskDir
     """
 
     found = {}
@@ -260,7 +260,7 @@ def read_context_disk_dir(source_dir):
             found[af] = fn
 
     if len(found) == 0:
-        raise NonContextDeviceDir("%s: %s" % (source_dir, "no files found"))
+        raise NonContextDiskDir("%s: %s" % (source_dir, "no files found"))
 
     context_sh = {}
     results = {
@@ -312,7 +312,7 @@ def read_context_disk_dir(source_dir):
             LOG.warn("Failed to read context variables: %s" % (_err.message))
         results['metadata']=context_sh
     else:
-        raise NonContextDeviceDir("Missing context.sh")
+        raise NonContextDiskDir("Missing context.sh")
 
     # process single or multiple SSH keys
     ssh_key_var=None
-- 
cgit v1.2.3


From a8e39f502cffdf8639263fb1dcb0ad36158c2d83 Mon Sep 17 00:00:00 2001
From: Vlastimil Holer <vlastimil.holer@gmail.com>
Date: Tue, 19 Feb 2013 16:00:16 +0100
Subject: Context.sh parsing cleanups, fix single quotes handling in multiword
 variables.

---
 cloudinit/sources/DataSourceOpenNebula.py | 29 ++++++++++++++++++-----------
 1 file changed, 18 insertions(+), 11 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index bc199461..dad64bd4 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -288,28 +288,35 @@ def read_context_disk_dir(source_dir):
             # 3. use comm to filter "old" variables from all current
             #    variables and excl. few other vars with grep
             BASH_CMD='VARS=`set | sort -u `;' \
-                '. %s/context.sh;' \
+                'source %s/context.sh;' \
                 'comm -23 <(set | sort -u) <(echo "$VARS") | egrep -v "^(VARS|PIPESTATUS|_)="'
 
-            (out,err) = util.subp(['bash',
-                '--noprofile', '--norc',
+            (out,err) = util.subp(['bash','--noprofile', '--norc',
                 '-c', BASH_CMD % (source_dir) ])
 
             for (key,value) in [ l.split('=',1) for l in out.rstrip().split("\n") ]:
-                # with backslash escapes
+                k=key.lower()
+
+                # with backslash escapes, e.g.
+                # X=$'Y\nZ'
                 r=re.match("^\$'(.*)'$",value)
                 if r:
-                    context_sh[key.lower()]=r.group(1).decode('string_escape')
+                    context_sh[k]=r.group(1).decode('string_escape')
                 else:
-                    # multiword values
+                    # multiword values, e.g.:
+                    # X='Y Z' 
+                    # X='Y'\''Z' for "Y'Z"
                     r=re.match("^'(.*)'$",value)
                     if r:
-                        context_sh[key.lower()]=r.group(1)
+                        context_sh[k]=r.group(1).replace("'\\''","'")
                     else:
-                        # simple values
-                        context_sh[key.lower()]=value
-        except util.ProcessExecutionError, _err:
-            LOG.warn("Failed to read context variables: %s" % (_err.message))
+                        # simple values, e.g.:
+                        # X=Y 
+                        context_sh[k]=value
+
+        except util.ProcessExecutionError as e:
+            raise NonContextDiskDir("Error reading context.sh: %s" % (e))
+
         results['metadata']=context_sh
     else:
         raise NonContextDiskDir("Missing context.sh")
-- 
cgit v1.2.3


From 5e42d558c2589701c8ad22cd77fd7060ab3f1c02 Mon Sep 17 00:00:00 2001
From: Vlastimil Holer <vlastimil.holer@gmail.com>
Date: Wed, 20 Feb 2013 14:19:21 +0100
Subject: Minor OpenNebula DS cleanups (style, dsmode, static network).

---
 cloudinit/sources/DataSourceOpenNebula.py | 70 +++++++++++++++++--------------
 1 file changed, 39 insertions(+), 31 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index dad64bd4..bfa7eeaf 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -33,6 +33,7 @@ from cloudinit import util
 LOG = logging.getLogger(__name__)
 
 DEFAULT_IID = "iid-dsopennebula"
+DEFAULT_MODE = 'net'
 CONTEXT_DISK_FILES = ["context.sh"]
 VALID_DSMODES = ("local", "net", "disabled")
 
@@ -44,30 +45,29 @@ class DataSourceOpenNebula(sources.DataSource):
         self.seed_dir = os.path.join(paths.seed_dir, 'opennebula')
 
     def __str__(self):
-        mstr = "%s [seed=%s][dsmode=%s]" % (util.obj_name(self),
-                                            self.seed, self.dsmode)
-        return mstr
+        return "%s [seed=%s][dsmode=%s]" % \
+            (util.obj_name(self), self.seed, self.dsmode)
 
     def get_data(self):
         defaults = {
             "instance-id": DEFAULT_IID,
-            "dsmode": self.dsmode,
-        }
+            "dsmode": self.dsmode }
 
         found = None
         md = {}
-
         results = {}
+
         if os.path.isdir(self.seed_dir):
             try:
-                results=read_context_disk_dir(self.seed_dir)
+                results = read_context_disk_dir(self.seed_dir)
                 found = self.seed_dir
             except NonContextDiskDir:
-                util.logexc(LOG, "Failed reading context disk from %s",
-                            self.seed_dir)
+                util.logexc(LOG, "Failed reading context disk from %s", self.seed_dir)
+
+        # find candidate devices, try to mount them and
+        # read context script if present
         if not found:
-            devlist = find_candidate_devs()
-            for dev in devlist:
+            for dev in find_candidate_devs():
                 try:
                     results = util.mount_cb(dev, read_context_disk_dir)
                     found = dev
@@ -81,20 +81,30 @@ class DataSourceOpenNebula(sources.DataSource):
         md = results['metadata']
         md = util.mergedict(md, defaults)
 
-        dsmode = results.get('dsmode', None)
-        if dsmode not in VALID_DSMODES + (None,):
-            LOG.warn("user specified invalid mode: %s" % dsmode)
-            dsmode = None
+        # check for valid user specified dsmode
+        user_dsmode = results.get('dsmode', None)
+        if user_dsmode not in VALID_DSMODES + (None,):
+            LOG.warn("user specified invalid mode: %s" % user_dsmode)
+            user_dsmode = None
 
-        if (dsmode is None) and self.ds_cfg.get('dsmode'):
+        # decide dsmode
+        if user_dsmode:
+            dsmode = user_dsmode
+        elif self.ds_cfg.get('dsmode'):
             dsmode = self.ds_cfg.get('dsmode')
         else:
-            dsmode = self.dsmode
+            dsmode = DEFAULT_MODE
 
         if dsmode == "disabled":
             # most likely user specified
             return False
 
+        # apply static network configuration only in 'local' dsmode
+        # TODO: first boot?
+        if ('network-interfaces' in results and self.dsmode == "local"):
+            LOG.debug("Updating network interfaces from %s", self)
+            self.distro.apply_network(results['network-interfaces'])
+
         if dsmode != self.dsmode:
             LOG.debug("%s: not claiming datasource, dsmode=%s", self, dsmode)
             return False
@@ -103,11 +113,6 @@ class DataSourceOpenNebula(sources.DataSource):
         self.metadata = md
         self.userdata_raw = results.get('userdata')
 
-        # apply static network configuration only in 'local' dsmode
-        if ('network-interfaces' in results and self.dsmode == "local"):
-            LOG.debug("Updating network interfaces from %s", self)
-            self.distro.apply_network(results['network-interfaces'])
-
         return True
 
     def get_hostname(self, fqdn=False, resolve_ip=None):
@@ -234,9 +239,9 @@ def find_candidate_devs():
     Return a list of devices that may contain the context disk.
     """
     by_fstype = util.find_devs_with("TYPE=iso9660")
-    by_label = util.find_devs_with("LABEL=CDROM")
-
     by_fstype.sort()
+
+    by_label = util.find_devs_with("LABEL=CDROM")
     by_label.sort()
 
     # combine list of items by putting by-label items first
@@ -262,11 +267,8 @@ def read_context_disk_dir(source_dir):
     if len(found) == 0:
         raise NonContextDiskDir("%s: %s" % (source_dir, "no files found"))
 
+    results = {'userdata':None, 'metadata':{}}
     context_sh = {}
-    results = {
-        'userdata':None,
-        'metadata':{},
-    }
 
     if "context.sh" in found:
         try:
@@ -347,9 +349,15 @@ def read_context_disk_dir(source_dir):
     elif "userdata" in context_sh:
         results['userdata'] = context_sh["userdata"]
 
-    (out, err) = util.subp(['/sbin/ip', '-o', 'link'])
-    net=OpenNebulaNetwork(out, context_sh)
-    results['network-interfaces']=net.gen_conf()
+    # generate static /etc/network/interfaces
+    # only if there are any required context variables
+    # http://opennebula.org/documentation:rel3.8:cong#network_configuration
+    for k in context_sh.keys():
+        if re.match('^eth\d+_ip$',k):
+            (out, err) = util.subp(['/sbin/ip', '-o', 'link'])
+            net=OpenNebulaNetwork(out, context_sh)
+            results['network-interfaces']=net.gen_conf()
+            break
 
     return results
 
-- 
cgit v1.2.3


From b83c0fb7a35ea5c4f61ef4ce94d037a4f10f3c1e Mon Sep 17 00:00:00 2001
From: Vlastimil Holer <vlastimil.holer@gmail.com>
Date: Thu, 21 Feb 2013 14:45:34 +0100
Subject: Remove TODO

---
 cloudinit/sources/DataSourceOpenNebula.py | 1 -
 1 file changed, 1 deletion(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index bfa7eeaf..b22c8aed 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -100,7 +100,6 @@ class DataSourceOpenNebula(sources.DataSource):
             return False
 
         # apply static network configuration only in 'local' dsmode
-        # TODO: first boot?
         if ('network-interfaces' in results and self.dsmode == "local"):
             LOG.debug("Updating network interfaces from %s", self)
             self.distro.apply_network(results['network-interfaces'])
-- 
cgit v1.2.3


From 1d35141960f2c0183d3f306d4604011ca9d5d2e8 Mon Sep 17 00:00:00 2001
From: Vlastimil Holer <vlastimil.holer@gmail.com>
Date: Tue, 2 Apr 2013 19:55:15 +0200
Subject: PEP8 fixes.

---
 cloudinit/sources/DataSourceOpenNebula.py | 109 +++++++++++++++---------------
 1 file changed, 55 insertions(+), 54 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index b22c8aed..ff8e6b6b 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -37,6 +37,7 @@ DEFAULT_MODE = 'net'
 CONTEXT_DISK_FILES = ["context.sh"]
 VALID_DSMODES = ("local", "net", "disabled")
 
+
 class DataSourceOpenNebula(sources.DataSource):
     def __init__(self, sys_cfg, distro, paths):
         sources.DataSource.__init__(self, sys_cfg, distro, paths)
@@ -51,7 +52,7 @@ class DataSourceOpenNebula(sources.DataSource):
     def get_data(self):
         defaults = {
             "instance-id": DEFAULT_IID,
-            "dsmode": self.dsmode }
+            "dsmode": self.dsmode}
 
         found = None
         md = {}
@@ -62,7 +63,8 @@ class DataSourceOpenNebula(sources.DataSource):
                 results = read_context_disk_dir(self.seed_dir)
                 found = self.seed_dir
             except NonContextDiskDir:
-                util.logexc(LOG, "Failed reading context disk from %s", self.seed_dir)
+                util.logexc(LOG, "Failed reading context disk from %s",
+                            self.seed_dir)
 
         # find candidate devices, try to mount them and
         # read context script if present
@@ -134,99 +136,98 @@ class NonContextDiskDir(Exception):
 
 
 class OpenNebulaNetwork(object):
-    REG_DEV_MAC=re.compile('^\d+: (eth\d+):.*link\/ether (..:..:..:..:..:..) ')
+    REG_DEV_MAC = re.compile('^\d+: (eth\d+):.*link\/ether (..:..:..:..:..:..) ')
 
     def __init__(self, ip, context_sh):
-        self.ip=ip
-        self.context_sh=context_sh
-        self.ifaces=self.get_ifaces()
+        self.ip = ip
+        self.context_sh = context_sh
+        self.ifaces = self.get_ifaces()
 
     def get_ifaces(self):
         return [self.REG_DEV_MAC.search(f).groups() for f in self.ip.split("\n") if self.REG_DEV_MAC.match(f)]
 
     def mac2ip(self, mac):
-        components=mac.split(':')[2:]
-
+        components = mac.split(':')[2:]
         return [str(int(c, 16)) for c in components]
-        
+
     def get_ip(self, dev, components):
-        var_name=dev+'_ip'
+        var_name = dev + '_ip'
         if var_name in self.context_sh:
             return self.context_sh[var_name]
         else:
             return '.'.join(components)
 
     def get_mask(self, dev, components):
-        var_name=dev+'_mask'
+        var_name = dev + '_mask'
         if var_name in self.context_sh:
             return self.context_sh[var_name]
         else:
             return '255.255.255.0'
 
     def get_network(self, dev, components):
-        var_name=dev+'_network'
+        var_name = dev + '_network'
         if var_name in self.context_sh:
             return self.context_sh[var_name]
         else:
-            return '.'.join(components[:-1])+'.0'
+            return '.'.join(components[:-1]) + '.0'
 
     def get_gateway(self, dev, components):
-        var_name=dev+'_gateway'
+        var_name = dev + '_gateway'
         if var_name in self.context_sh:
             return self.context_sh[var_name]
         else:
             None
 
     def get_dns(self, dev, components):
-        var_name=dev+'_dns'
+        var_name = dev + '_dns'
         if var_name in self.context_sh:
             return self.context_sh[var_name]
         else:
             None
 
     def get_domain(self, dev, components):
-        var_name=dev+'_domain'
+        var_name = dev + '_domain'
         if var_name in self.context_sh:
             return self.context_sh[var_name]
         else:
             None
 
     def gen_conf(self):
-        global_dns=[]
+        global_dns = []
         if 'dns' in self.context_sh:
             global_dns.append(self.context_sh['dns'])
 
-        conf=[]
+        conf = []
         conf.append('auto lo')
         conf.append('iface lo inet loopback')
         conf.append('')
 
         for i in self.ifaces:
-            dev=i[0]
-            mac=i[1]
-            ip_components=self.mac2ip(mac)
+            dev = i[0]
+            mac = i[1]
+            ip_components = self.mac2ip(mac)
 
-            conf.append('auto '+dev)
-            conf.append('iface '+dev+' inet static')
-            conf.append('  address '+self.get_ip(dev, ip_components))
-            conf.append('  network '+self.get_network(dev, ip_components))
-            conf.append('  netmask '+self.get_mask(dev, ip_components))
+            conf.append('auto ' + dev)
+            conf.append('iface ' + dev + ' inet static')
+            conf.append('  address ' + self.get_ip(dev, ip_components))
+            conf.append('  network ' + self.get_network(dev, ip_components))
+            conf.append('  netmask ' + self.get_mask(dev, ip_components))
 
-            gateway=self.get_gateway(dev, ip_components)
+            gateway = self.get_gateway(dev, ip_components)
             if gateway:
-                conf.append('  gateway '+gateway)
+                conf.append('  gateway ' + gateway)
 
-            domain=self.get_domain(dev, ip_components)
+            domain = self.get_domain(dev, ip_components)
             if domain:
-                conf.append('  dns-search '+domain)
+                conf.append('  dns-search ' + domain)
 
             # add global DNS servers to all interfaces
-            dns=self.get_dns(dev, ip_components)
+            dns = self.get_dns(dev, ip_components)
             if global_dns or dns:
-                all_dns=global_dns
+                all_dns = global_dns
                 if dns:
                     all_dns.append(dns)
-                conf.append('  dns-nameservers '+' '.join(all_dns))
+                conf.append('  dns-nameservers ' + ' '.join(all_dns))
 
             conf.append('')
 
@@ -266,7 +267,7 @@ def read_context_disk_dir(source_dir):
     if len(found) == 0:
         raise NonContextDiskDir("%s: %s" % (source_dir, "no files found"))
 
-    results = {'userdata':None, 'metadata':{}}
+    results = {'userdata': None, 'metadata': {}}
     context_sh = {}
 
     if "context.sh" in found:
@@ -288,47 +289,47 @@ def read_context_disk_dir(source_dir):
             # 2. read context variables
             # 3. use comm to filter "old" variables from all current
             #    variables and excl. few other vars with grep
-            BASH_CMD='VARS=`set | sort -u `;' \
+            BASH_CMD = 'VARS=`set | sort -u `;' \
                 'source %s/context.sh;' \
                 'comm -23 <(set | sort -u) <(echo "$VARS") | egrep -v "^(VARS|PIPESTATUS|_)="'
 
-            (out,err) = util.subp(['bash','--noprofile', '--norc',
-                '-c', BASH_CMD % (source_dir) ])
+            (out, err) = util.subp(['bash', '--noprofile', '--norc',
+                                   '-c', BASH_CMD % (source_dir)])
 
-            for (key,value) in [ l.split('=',1) for l in out.rstrip().split("\n") ]:
-                k=key.lower()
+            for (key, value) in [l.split('=', 1) for l in out.rstrip().split("\n")]:
+                k = key.lower()
 
                 # with backslash escapes, e.g.
                 # X=$'Y\nZ'
-                r=re.match("^\$'(.*)'$",value)
+                r = re.match("^\$'(.*)'$", value)
                 if r:
-                    context_sh[k]=r.group(1).decode('string_escape')
+                    context_sh[k] = r.group(1).decode('string_escape')
                 else:
                     # multiword values, e.g.:
-                    # X='Y Z' 
+                    # X='Y Z'
                     # X='Y'\''Z' for "Y'Z"
-                    r=re.match("^'(.*)'$",value)
+                    r = re.match("^'(.*)'$", value)
                     if r:
-                        context_sh[k]=r.group(1).replace("'\\''","'")
+                        context_sh[k] = r.group(1).replace("'\\''", "'")
                     else:
                         # simple values, e.g.:
-                        # X=Y 
-                        context_sh[k]=value
+                        # X=Y
+                        context_sh[k] = value
 
         except util.ProcessExecutionError as e:
             raise NonContextDiskDir("Error reading context.sh: %s" % (e))
 
-        results['metadata']=context_sh
+        results['metadata'] = context_sh
     else:
         raise NonContextDiskDir("Missing context.sh")
 
     # process single or multiple SSH keys
-    ssh_key_var=None
+    ssh_key_var = None
 
     if "ssh_key" in context_sh:
-        ssh_key_var="ssh_key"
+        ssh_key_var = "ssh_key"
     elif "ssh_public_key" in context_sh:
-        ssh_key_var="ssh_public_key"
+        ssh_key_var = "ssh_public_key"
 
     if ssh_key_var:
         lines = context_sh.get(ssh_key_var).splitlines()
@@ -337,7 +338,7 @@ def read_context_disk_dir(source_dir):
 
     # custom hostname -- try hostname or leave cloud-init
     # itself create hostname from IP address later
-    for k in ('hostname','public_ip','ip_public','eth0_ip'):
+    for k in ('hostname', 'public_ip', 'ip_public', 'eth0_ip'):
         if k in context_sh:
             results['metadata']['local-hostname'] = context_sh[k]
             break
@@ -352,10 +353,10 @@ def read_context_disk_dir(source_dir):
     # only if there are any required context variables
     # http://opennebula.org/documentation:rel3.8:cong#network_configuration
     for k in context_sh.keys():
-        if re.match('^eth\d+_ip$',k):
+        if re.match('^eth\d+_ip$', k):
             (out, err) = util.subp(['/sbin/ip', '-o', 'link'])
-            net=OpenNebulaNetwork(out, context_sh)
-            results['network-interfaces']=net.gen_conf()
+            net = OpenNebulaNetwork(out, context_sh)
+            results['network-interfaces'] = net.gen_conf()
             break
 
     return results
-- 
cgit v1.2.3


From 77c8798388637fcb2f7dbc057cad81e8fd5fbe58 Mon Sep 17 00:00:00 2001
From: Vlastimil Holer <vlastimil.holer@gmail.com>
Date: Wed, 4 Sep 2013 14:32:24 +0200
Subject: Apply parse.diff by Javier Fontan <jfontan@opennebula.org>

---
 cloudinit/sources/DataSourceOpenNebula.py          | 96 ++++++++--------------
 tests/unittests/helpers.py                         |  1 +
 tests/unittests/test_datasource/test_opennebula.py | 16 ++--
 3 files changed, 45 insertions(+), 68 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index ff8e6b6b..d2ab348f 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -24,7 +24,6 @@
 
 import os
 import re
-import subprocess
 
 from cloudinit import log as logging
 from cloudinit import sources
@@ -86,7 +85,7 @@ class DataSourceOpenNebula(sources.DataSource):
         # check for valid user specified dsmode
         user_dsmode = results.get('dsmode', None)
         if user_dsmode not in VALID_DSMODES + (None,):
-            LOG.warn("user specified invalid mode: %s" % user_dsmode)
+            LOG.warn("user specified invalid mode: %s", user_dsmode)
             user_dsmode = None
 
         # decide dsmode
@@ -136,7 +135,9 @@ class NonContextDiskDir(Exception):
 
 
 class OpenNebulaNetwork(object):
-    REG_DEV_MAC = re.compile('^\d+: (eth\d+):.*link\/ether (..:..:..:..:..:..) ')
+    REG_DEV_MAC = re.compile(
+                    r'^\d+: (eth\d+):.*?link\/ether (..:..:..:..:..:..) ?',
+                    re.MULTILINE | re.DOTALL)
 
     def __init__(self, ip, context_sh):
         self.ip = ip
@@ -144,7 +145,7 @@ class OpenNebulaNetwork(object):
         self.ifaces = self.get_ifaces()
 
     def get_ifaces(self):
-        return [self.REG_DEV_MAC.search(f).groups() for f in self.ip.split("\n") if self.REG_DEV_MAC.match(f)]
+        return self.REG_DEV_MAC.findall(self.ip)
 
     def mac2ip(self, mac):
         components = mac.split(':')[2:]
@@ -157,7 +158,7 @@ class OpenNebulaNetwork(object):
         else:
             return '.'.join(components)
 
-    def get_mask(self, dev, components):
+    def get_mask(self, dev):
         var_name = dev + '_mask'
         if var_name in self.context_sh:
             return self.context_sh[var_name]
@@ -171,26 +172,26 @@ class OpenNebulaNetwork(object):
         else:
             return '.'.join(components[:-1]) + '.0'
 
-    def get_gateway(self, dev, components):
+    def get_gateway(self, dev):
         var_name = dev + '_gateway'
         if var_name in self.context_sh:
             return self.context_sh[var_name]
         else:
-            None
+            return None
 
-    def get_dns(self, dev, components):
+    def get_dns(self, dev):
         var_name = dev + '_dns'
         if var_name in self.context_sh:
             return self.context_sh[var_name]
         else:
-            None
+            return None
 
-    def get_domain(self, dev, components):
+    def get_domain(self, dev):
         var_name = dev + '_domain'
         if var_name in self.context_sh:
             return self.context_sh[var_name]
         else:
-            None
+            return None
 
     def gen_conf(self):
         global_dns = []
@@ -211,18 +212,18 @@ class OpenNebulaNetwork(object):
             conf.append('iface ' + dev + ' inet static')
             conf.append('  address ' + self.get_ip(dev, ip_components))
             conf.append('  network ' + self.get_network(dev, ip_components))
-            conf.append('  netmask ' + self.get_mask(dev, ip_components))
+            conf.append('  netmask ' + self.get_mask(dev))
 
-            gateway = self.get_gateway(dev, ip_components)
+            gateway = self.get_gateway(dev)
             if gateway:
                 conf.append('  gateway ' + gateway)
 
-            domain = self.get_domain(dev, ip_components)
+            domain = self.get_domain(dev)
             if domain:
                 conf.append('  dns-search ' + domain)
 
             # add global DNS servers to all interfaces
-            dns = self.get_dns(dev, ip_components)
+            dns = self.get_dns(dev)
             if global_dns or dns:
                 all_dns = global_dns
                 if dns:
@@ -272,51 +273,22 @@ def read_context_disk_dir(source_dir):
 
     if "context.sh" in found:
         try:
-            # Note: context.sh is a "shell" script with defined context
-            # variables, like: X="Y" . It's ready to use as a shell source
-            # e.g.: ". context.sh" and as a shell script it can also reference
-            # to already defined shell variables. So to have same context var.
-            # values as we can have in custom shell script, we use bash itself
-            # to read context.sh and dump variables in easily parsable way.
-            #
-            # normalized variables dump format (get by cmd "set"):
-            # 1. simple single word assignment ........ X=Y
-            # 2. multiword assignment ................. X='Y Z'
-            # 3. assignments with backslash escapes ... X=$'Y\nZ'
-            #
-            # how context variables are read:
-            # 1. list existing ("old") shell variables and store into $VARS
-            # 2. read context variables
-            # 3. use comm to filter "old" variables from all current
-            #    variables and excl. few other vars with grep
-            BASH_CMD = 'VARS=`set | sort -u `;' \
-                'source %s/context.sh;' \
-                'comm -23 <(set | sort -u) <(echo "$VARS") | egrep -v "^(VARS|PIPESTATUS|_)="'
-
-            (out, err) = util.subp(['bash', '--noprofile', '--norc',
-                                   '-c', BASH_CMD % (source_dir)])
-
-            for (key, value) in [l.split('=', 1) for l in out.rstrip().split("\n")]:
-                k = key.lower()
-
-                # with backslash escapes, e.g.
-                # X=$'Y\nZ'
-                r = re.match("^\$'(.*)'$", value)
-                if r:
-                    context_sh[k] = r.group(1).decode('string_escape')
-                else:
-                    # multiword values, e.g.:
-                    # X='Y Z'
-                    # X='Y'\''Z' for "Y'Z"
-                    r = re.match("^'(.*)'$", value)
-                    if r:
-                        context_sh[k] = r.group(1).replace("'\\''", "'")
-                    else:
-                        # simple values, e.g.:
-                        # X=Y
-                        context_sh[k] = value
-
-        except util.ProcessExecutionError as e:
+            f = open('%s/context.sh' % (source_dir), 'r')
+            text = f.read()
+            f.close()
+
+            context_reg = re.compile(r"^([\w_]+)=['\"](.*?[^\\])['\"]$",
+                                     re.MULTILINE | re.DOTALL)
+            variables = context_reg.findall(text)
+
+            if not variables:
+                raise NonContextDiskDir("No variables in context")
+
+            context_sh = {}
+            for k, v in variables:
+                context_sh[k.lower()] = v
+
+        except (IOError, NonContextDiskDir) as e:
             raise NonContextDiskDir("Error reading context.sh: %s" % (e))
 
         results['metadata'] = context_sh
@@ -353,8 +325,8 @@ def read_context_disk_dir(source_dir):
     # only if there are any required context variables
     # http://opennebula.org/documentation:rel3.8:cong#network_configuration
     for k in context_sh.keys():
-        if re.match('^eth\d+_ip$', k):
-            (out, err) = util.subp(['/sbin/ip', '-o', 'link'])
+        if re.match(r'^eth\d+_ip$', k):
+            (out, _) = util.subp(['/sbin/ip', '-o', 'link'])
             net = OpenNebulaNetwork(out, context_sh)
             results['network-interfaces'] = net.gen_conf()
             break
diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py
index 91a50e18..904677f1 100644
--- a/tests/unittests/helpers.py
+++ b/tests/unittests/helpers.py
@@ -183,6 +183,7 @@ class FilesystemMockingTestCase(ResourceUsingTestCase):
                 setattr(mod, f, trap_func)
                 self.patched_funcs.append((mod, f, func))
 
+
 def populate_dir(path, files):
     os.makedirs(path)
     for (name, content) in files.iteritems():
diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py
index bc6c4b73..27725930 100644
--- a/tests/unittests/test_datasource/test_opennebula.py
+++ b/tests/unittests/test_datasource/test_opennebula.py
@@ -19,9 +19,11 @@ SSH_KEY = 'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460-%i'
 HOSTNAME = 'foo.example.com'
 PUBLIC_IP = '10.0.0.3'
 
-CMD_IP_OUT = '''\
-1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue state UNKNOWN \    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
-2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP qlen 1000\    link/ether 02:00:0a:12:01:01 brd ff:ff:ff:ff:ff:ff
+CMD_IP_OUT = '''
+1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue state UNKNOWN
+    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
+2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP qlen 1000
+    link/ether 02:00:0a:12:01:01 brd ff:ff:ff:ff:ff:ff
 '''
 
 
@@ -52,7 +54,7 @@ class TestOpenNebulaDataSource(MockerTestCase):
         self.assertEqual(TEST_VARS, results['metadata'])
 
     def test_ssh_key(self):
-        public_keys = []
+        public_keys = ['first key', 'second key']
         for c in range(4):
             for k in ('SSH_KEY', 'SSH_PUBLIC_KEY'):
                 my_d = os.path.join(self.tmp, "%s-%i" % (k, c))
@@ -61,7 +63,8 @@ class TestOpenNebulaDataSource(MockerTestCase):
 
                 self.assertTrue('metadata' in results)
                 self.assertTrue('public-keys' in results['metadata'])
-                self.assertEqual(public_keys, results['metadata']['public-keys'])
+                self.assertEqual(public_keys,
+                                 results['metadata']['public-keys'])
 
             public_keys.append(SSH_KEY % (c + 1,))
 
@@ -96,7 +99,8 @@ class TestOpenNebulaDataSource(MockerTestCase):
         try:
             orig_find_devs_with = util.find_devs_with
             util.find_devs_with = my_devs_with
-            self.assertEqual(["/dev/sr0", "/dev/vdb"], ds.find_candidate_devs())
+            self.assertEqual(["/dev/sr0", "/dev/vdb"],
+                             ds.find_candidate_devs())
         finally:
             util.find_devs_with = orig_find_devs_with
 
-- 
cgit v1.2.3


From 1adc68b643b2f73a2a08ca7d19c3fc8e759f06c2 Mon Sep 17 00:00:00 2001
From: Vlastimil Holer <vlastimil.holer@gmail.com>
Date: Wed, 4 Sep 2013 16:19:54 +0200
Subject: Fix RE matching context variables. Test cleanups.

---
 cloudinit/sources/DataSourceOpenNebula.py          | 26 ++++++++++++++++------
 tests/unittests/test_datasource/test_opennebula.py | 12 +++++-----
 2 files changed, 26 insertions(+), 12 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index d2ab348f..0ab23b25 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -277,16 +277,28 @@ def read_context_disk_dir(source_dir):
             text = f.read()
             f.close()
 
-            context_reg = re.compile(r"^([\w_]+)=['\"](.*?[^\\])['\"]$",
-                                     re.MULTILINE | re.DOTALL)
-            variables = context_reg.findall(text)
+            # lame matching:
+            # 1. group = key
+            # 2. group = single quoted value, respect '\''
+            # 3. group = old double quoted value, but doesn't end with \"
+            context_reg = re.compile(
+                r"^([\w_]+)=(?:'((?:[^']|'\\'')*?)'|\"(.*?[^\\])\")$",
+                re.MULTILINE | re.DOTALL)
 
+            variables = context_reg.findall(text)
             if not variables:
                 raise NonContextDiskDir("No variables in context")
 
-            context_sh = {}
-            for k, v in variables:
-                context_sh[k.lower()] = v
+            for k, v1, v2 in variables:
+                k = k.lower()
+                if v1:
+                    # take single quoted variable 'xyz'
+                    # (ON>=4) and unquote '\'' -> '
+                    context_sh[k] = v1.replace(r"'\''", r"'")
+                elif v2:
+                    # take double quoted variable "xyz"
+                    # (old ON<4) and unquote \" -> "
+                    context_sh[k] = v2.replace(r'\"', r'"')
 
         except (IOError, NonContextDiskDir) as e:
             raise NonContextDiskDir("Error reading context.sh: %s" % (e))
@@ -326,7 +338,7 @@ def read_context_disk_dir(source_dir):
     # http://opennebula.org/documentation:rel3.8:cong#network_configuration
     for k in context_sh.keys():
         if re.match(r'^eth\d+_ip$', k):
-            (out, _) = util.subp(['/sbin/ip', '-o', 'link'])
+            (out, _) = util.subp(['/sbin/ip', 'link'])
             net = OpenNebulaNetwork(out, context_sh)
             results['network-interfaces'] = net.gen_conf()
             break
diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py
index 27725930..66a38e31 100644
--- a/tests/unittests/test_datasource/test_opennebula.py
+++ b/tests/unittests/test_datasource/test_opennebula.py
@@ -1,7 +1,8 @@
 import os
-from mocker import MockerTestCase
-from cloudinit import util
+
 from cloudinit.sources import DataSourceOpenNebula as ds
+from cloudinit import util
+from mocker import MockerTestCase
 
 TEST_VARS = {
     'var1': 'single',
@@ -19,7 +20,7 @@ SSH_KEY = 'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460-%i'
 HOSTNAME = 'foo.example.com'
 PUBLIC_IP = '10.0.0.3'
 
-CMD_IP_OUT = '''
+CMD_IP_OUT = '''\
 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue state UNKNOWN
     link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
 2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP qlen 1000
@@ -91,6 +92,7 @@ class TestOpenNebulaDataSource(MockerTestCase):
         devs_with_answers = {
             "TYPE=iso9660": ["/dev/vdb"],
             "LABEL=CDROM": ["/dev/sr0"],
+            "LABEL=CONTEXT": ["/dev/sdb"],
         }
 
         def my_devs_with(criteria):
@@ -99,7 +101,7 @@ class TestOpenNebulaDataSource(MockerTestCase):
         try:
             orig_find_devs_with = util.find_devs_with
             util.find_devs_with = my_devs_with
-            self.assertEqual(["/dev/sr0", "/dev/vdb"],
+            self.assertEqual(["/dev/sdb", "/dev/sr0", "/dev/vdb"],
                              ds.find_candidate_devs())
         finally:
             util.find_devs_with = orig_find_devs_with
@@ -161,7 +163,7 @@ def populate_dir(seed_dir, files):
     with open(os.path.join(seed_dir, "context.sh"), "w") as fp:
         fp.write("# Context variables generated by OpenNebula\n")
         for (name, content) in files.iteritems():
-            fp.write('%s="%s"\n' % (name.upper(), content))
+            fp.write("%s='%s'\n" % (name.upper(), content.replace(r"'", r"'\''")))
         fp.close()
 
 # vi: ts=4 expandtab
-- 
cgit v1.2.3


From 8a2a88e0bb4520eabe99b6686413a548f3d59652 Mon Sep 17 00:00:00 2001
From: Vlastimil Holer <vlastimil.holer@gmail.com>
Date: Wed, 4 Sep 2013 16:36:00 +0200
Subject: Search for contextualization CDROM by LABEL=CONTEXT

---
 cloudinit/sources/DataSourceOpenNebula.py | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index 0ab23b25..fef7beac 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -239,15 +239,11 @@ def find_candidate_devs():
     """
     Return a list of devices that may contain the context disk.
     """
-    by_fstype = util.find_devs_with("TYPE=iso9660")
-    by_fstype.sort()
-
-    by_label = util.find_devs_with("LABEL=CDROM")
-    by_label.sort()
-
-    # combine list of items by putting by-label items first
-    # followed by fstype items, but with dupes removed
-    combined = (by_label + [d for d in by_fstype if d not in by_label])
+    combined = []
+    for f in ('LABEL=CONTEXT', 'LABEL=CDROM', 'TYPE=iso9660'):
+        for d in util.find_devs_with(f).sort():
+            if d not in combined:
+                combined.append(d)
 
     return combined
 
-- 
cgit v1.2.3


From d0e7898d4d0aef452fbfdd9b51fa2da6437397ff Mon Sep 17 00:00:00 2001
From: Vlastimil Holer <vlastimil.holer@gmail.com>
Date: Thu, 5 Sep 2013 17:55:53 +0200
Subject: PEP8 and Pylint fixes. Move context.sh "parser" into separate
 function. Fix fetching user specified dsmode (from context). Rename
 context_sh->context. Reuse unittests.helpers.populate_dir.

---
 cloudinit/sources/DataSourceOpenNebula.py          | 163 +++++++++++----------
 tests/unittests/test_datasource/test_opennebula.py |  62 ++++----
 2 files changed, 119 insertions(+), 106 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index fef7beac..87ec51bd 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -3,7 +3,7 @@
 #    Copyright (C) 2012 Canonical Ltd.
 #    Copyright (C) 2012 Yahoo! Inc.
 #    Copyright (C) 2012-2013 CERIT Scientific Cloud
-#    Copyright (C) 2012 OpenNebula.org
+#    Copyright (C) 2012-2013 OpenNebula.org
 #
 #    Author: Scott Moser <scott.moser@canonical.com>
 #    Author: Joshua Harlow <harlowja@yahoo-inc.com>
@@ -45,45 +45,47 @@ class DataSourceOpenNebula(sources.DataSource):
         self.seed_dir = os.path.join(paths.seed_dir, 'opennebula')
 
     def __str__(self):
-        return "%s [seed=%s][dsmode=%s]" % \
-            (util.obj_name(self), self.seed, self.dsmode)
+        root = sources.DataSource.__str__(self)
+        return "%s [seed=%s][dsmode=%s]" % (root, self.seed, self.dsmode)
 
     def get_data(self):
         defaults = {
             "instance-id": DEFAULT_IID,
-            "dsmode": self.dsmode}
+            "dsmode": self.dsmode
+        }
 
-        found = None
-        md = {}
+        seed = None
         results = {}
 
+        # first try to read local seed_dir
         if os.path.isdir(self.seed_dir):
             try:
                 results = read_context_disk_dir(self.seed_dir)
-                found = self.seed_dir
+                seed = self.seed_dir
             except NonContextDiskDir:
-                util.logexc(LOG, "Failed reading context disk from %s",
+                util.logexc(LOG, "Failed reading context from %s",
                             self.seed_dir)
 
-        # find candidate devices, try to mount them and
-        # read context script if present
-        if not found:
+        if not seed:
+            # then try to detect and mount candidate devices and
+            # read contextualization if present
             for dev in find_candidate_devs():
                 try:
                     results = util.mount_cb(dev, read_context_disk_dir)
-                    found = dev
+                    seed = dev
                     break
                 except (NonContextDiskDir, util.MountFailedError):
                     pass
 
-        if not found:
+        if not seed:
             return False
 
+        # merge fetched metadata with datasource defaults
         md = results['metadata']
-        md = util.mergedict(md, defaults)
+        md = util.mergemanydict([md, defaults])
 
         # check for valid user specified dsmode
-        user_dsmode = results.get('dsmode', None)
+        user_dsmode = results['metadata'].get('dsmode', None)
         if user_dsmode not in VALID_DSMODES + (None,):
             LOG.warn("user specified invalid mode: %s", user_dsmode)
             user_dsmode = None
@@ -109,10 +111,9 @@ class DataSourceOpenNebula(sources.DataSource):
             LOG.debug("%s: not claiming datasource, dsmode=%s", self, dsmode)
             return False
 
-        self.seed = found
+        self.seed = seed
         self.metadata = md
         self.userdata_raw = results.get('userdata')
-
         return True
 
     def get_hostname(self, fqdn=False, resolve_ip=None):
@@ -139,9 +140,9 @@ class OpenNebulaNetwork(object):
                     r'^\d+: (eth\d+):.*?link\/ether (..:..:..:..:..:..) ?',
                     re.MULTILINE | re.DOTALL)
 
-    def __init__(self, ip, context_sh):
+    def __init__(self, ip, context):
         self.ip = ip
-        self.context_sh = context_sh
+        self.context = context
         self.ifaces = self.get_ifaces()
 
     def get_ifaces(self):
@@ -153,50 +154,50 @@ class OpenNebulaNetwork(object):
 
     def get_ip(self, dev, components):
         var_name = dev + '_ip'
-        if var_name in self.context_sh:
-            return self.context_sh[var_name]
+        if var_name in self.context:
+            return self.context[var_name]
         else:
             return '.'.join(components)
 
     def get_mask(self, dev):
         var_name = dev + '_mask'
-        if var_name in self.context_sh:
-            return self.context_sh[var_name]
+        if var_name in self.context:
+            return self.context[var_name]
         else:
             return '255.255.255.0'
 
     def get_network(self, dev, components):
         var_name = dev + '_network'
-        if var_name in self.context_sh:
-            return self.context_sh[var_name]
+        if var_name in self.context:
+            return self.context[var_name]
         else:
             return '.'.join(components[:-1]) + '.0'
 
     def get_gateway(self, dev):
         var_name = dev + '_gateway'
-        if var_name in self.context_sh:
-            return self.context_sh[var_name]
+        if var_name in self.context:
+            return self.context[var_name]
         else:
             return None
 
     def get_dns(self, dev):
         var_name = dev + '_dns'
-        if var_name in self.context_sh:
-            return self.context_sh[var_name]
+        if var_name in self.context:
+            return self.context[var_name]
         else:
             return None
 
     def get_domain(self, dev):
         var_name = dev + '_domain'
-        if var_name in self.context_sh:
-            return self.context_sh[var_name]
+        if var_name in self.context:
+            return self.context[var_name]
         else:
             return None
 
     def gen_conf(self):
         global_dns = []
-        if 'dns' in self.context_sh:
-            global_dns.append(self.context_sh['dns'])
+        if 'dns' in self.context:
+            global_dns.append(self.context['dns'])
 
         conf = []
         conf.append('auto lo')
@@ -241,101 +242,113 @@ def find_candidate_devs():
     """
     combined = []
     for f in ('LABEL=CONTEXT', 'LABEL=CDROM', 'TYPE=iso9660'):
-        for d in util.find_devs_with(f).sort():
+        devs = util.find_devs_with(f)
+        devs.sort()
+        for d in devs:
             if d not in combined:
                 combined.append(d)
 
     return combined
 
 
+def parse_context_data(data):
+    """
+    parse_context_data(data)
+    parse context.sh variables provided as a single string. Uses
+    very simple matching RE. Returns None if nothing is matched.
+    """
+    # RE groups:
+    # 1: key
+    # 2: single quoted value, respect '\''
+    # 3: old double quoted value, but doesn't end with \"
+    context_reg = re.compile(
+        r"^([\w_]+)=(?:'((?:[^']|'\\'')*?)'|\"(.*?[^\\])\")$",
+        re.MULTILINE | re.DOTALL)
+
+    found = context_reg.findall(data)
+    if not found:
+        return None
+
+    variables = {}
+    for k, v1, v2 in found:
+        k = k.lower()
+        if v1:
+            # take single quoted variable 'xyz'
+            # (ON>=4) and unquote '\'' -> '
+            variables[k] = v1.replace(r"'\''", r"'")
+        elif v2:
+            # take double quoted variable "xyz"
+            # (old ON<4) and unquote \" -> "
+            variables[k] = v2.replace(r'\"', r'"')
+
+    return variables
+
+
 def read_context_disk_dir(source_dir):
     """
     read_context_disk_dir(source_dir):
     read source_dir and return a tuple with metadata dict and user-data
     string populated.  If not a valid dir, raise a NonContextDiskDir
     """
-
     found = {}
     for af in CONTEXT_DISK_FILES:
         fn = os.path.join(source_dir, af)
         if os.path.isfile(fn):
             found[af] = fn
 
-    if len(found) == 0:
+    if not found:
         raise NonContextDiskDir("%s: %s" % (source_dir, "no files found"))
 
     results = {'userdata': None, 'metadata': {}}
-    context_sh = {}
+    context = {}
 
     if "context.sh" in found:
         try:
-            f = open('%s/context.sh' % (source_dir), 'r')
-            text = f.read()
-            f.close()
-
-            # lame matching:
-            # 1. group = key
-            # 2. group = single quoted value, respect '\''
-            # 3. group = old double quoted value, but doesn't end with \"
-            context_reg = re.compile(
-                r"^([\w_]+)=(?:'((?:[^']|'\\'')*?)'|\"(.*?[^\\])\")$",
-                re.MULTILINE | re.DOTALL)
-
-            variables = context_reg.findall(text)
-            if not variables:
+            with open(os.path.join(source_dir, 'context.sh'), 'r') as f:
+                context = parse_context_data(f.read())
+                f.close()
+            if not context:
                 raise NonContextDiskDir("No variables in context")
 
-            for k, v1, v2 in variables:
-                k = k.lower()
-                if v1:
-                    # take single quoted variable 'xyz'
-                    # (ON>=4) and unquote '\'' -> '
-                    context_sh[k] = v1.replace(r"'\''", r"'")
-                elif v2:
-                    # take double quoted variable "xyz"
-                    # (old ON<4) and unquote \" -> "
-                    context_sh[k] = v2.replace(r'\"', r'"')
-
         except (IOError, NonContextDiskDir) as e:
             raise NonContextDiskDir("Error reading context.sh: %s" % (e))
 
-        results['metadata'] = context_sh
+        results['metadata'] = context
     else:
         raise NonContextDiskDir("Missing context.sh")
 
     # process single or multiple SSH keys
     ssh_key_var = None
-
-    if "ssh_key" in context_sh:
+    if "ssh_key" in context:
         ssh_key_var = "ssh_key"
-    elif "ssh_public_key" in context_sh:
+    elif "ssh_public_key" in context:
         ssh_key_var = "ssh_public_key"
 
     if ssh_key_var:
-        lines = context_sh.get(ssh_key_var).splitlines()
+        lines = context.get(ssh_key_var).splitlines()
         results['metadata']['public-keys'] = [l for l in lines
             if len(l) and not l.startswith("#")]
 
     # custom hostname -- try hostname or leave cloud-init
     # itself create hostname from IP address later
     for k in ('hostname', 'public_ip', 'ip_public', 'eth0_ip'):
-        if k in context_sh:
-            results['metadata']['local-hostname'] = context_sh[k]
+        if k in context:
+            results['metadata']['local-hostname'] = context[k]
             break
 
     # raw user data
-    if "user_data" in context_sh:
-        results['userdata'] = context_sh["user_data"]
-    elif "userdata" in context_sh:
-        results['userdata'] = context_sh["userdata"]
+    if "user_data" in context:
+        results['userdata'] = context["user_data"]
+    elif "userdata" in context:
+        results['userdata'] = context["userdata"]
 
     # generate static /etc/network/interfaces
     # only if there are any required context variables
     # http://opennebula.org/documentation:rel3.8:cong#network_configuration
-    for k in context_sh.keys():
+    for k in context.keys():
         if re.match(r'^eth\d+_ip$', k):
             (out, _) = util.subp(['/sbin/ip', 'link'])
-            net = OpenNebulaNetwork(out, context_sh)
+            net = OpenNebulaNetwork(out, context)
             results['network-interfaces'] = net.gen_conf()
             break
 
diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py
index 66a38e31..4b82a49c 100644
--- a/tests/unittests/test_datasource/test_opennebula.py
+++ b/tests/unittests/test_datasource/test_opennebula.py
@@ -1,8 +1,9 @@
-import os
-
 from cloudinit.sources import DataSourceOpenNebula as ds
 from cloudinit import util
 from mocker import MockerTestCase
+from tests.unittests.helpers import populate_dir
+
+import os
 
 TEST_VARS = {
     'var1': 'single',
@@ -13,7 +14,11 @@ TEST_VARS = {
     'var6': "'multi\nline\n'",
     'var7': 'single\\t',
     'var8': 'double\\tword',
-    'var9': 'multi\\t\nline\n'}
+    'var9': 'multi\\t\nline\n',
+    'var10': '\\',  # expect \
+    'var11': '\'',  # expect '
+    'var12': '$',   # expect $
+}
 
 USER_DATA = '#cloud-config\napt_upgrade: true'
 SSH_KEY = 'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460-%i'
@@ -38,17 +43,15 @@ class TestOpenNebulaDataSource(MockerTestCase):
         my_d = os.path.join(self.tmp, 'non-contextdisk')
         self.assertRaises(ds.NonContextDiskDir, ds.read_context_disk_dir, my_d)
 
-    def test_seed_dir_bad_context_sh(self):
-        my_d = os.path.join(self.tmp, 'bad-context-sh')
+    def test_seed_dir_bad_context(self):
+        my_d = os.path.join(self.tmp, 'bad-context')
         os.mkdir(my_d)
-        with open(os.path.join(my_d, "context.sh"), "w") as fp:
-            fp.write('/bin/false\n')
-            fp.close()
+        open(os.path.join(my_d, "context.sh"), "w").close()
         self.assertRaises(ds.NonContextDiskDir, ds.read_context_disk_dir, my_d)
 
-    def test_context_sh_parser(self):
-        my_d = os.path.join(self.tmp, 'context-sh-parser')
-        populate_dir(my_d, TEST_VARS)
+    def test_context_parser(self):
+        my_d = os.path.join(self.tmp, 'context-parser')
+        populate_context_dir(my_d, TEST_VARS)
         results = ds.read_context_disk_dir(my_d)
 
         self.assertTrue('metadata' in results)
@@ -59,7 +62,7 @@ class TestOpenNebulaDataSource(MockerTestCase):
         for c in range(4):
             for k in ('SSH_KEY', 'SSH_PUBLIC_KEY'):
                 my_d = os.path.join(self.tmp, "%s-%i" % (k, c))
-                populate_dir(my_d, {k: '\n'.join(public_keys)})
+                populate_context_dir(my_d, {k: '\n'.join(public_keys)})
                 results = ds.read_context_disk_dir(my_d)
 
                 self.assertTrue('metadata' in results)
@@ -72,7 +75,7 @@ class TestOpenNebulaDataSource(MockerTestCase):
     def test_user_data(self):
         for k in ('USER_DATA', 'USERDATA'):
             my_d = os.path.join(self.tmp, k)
-            populate_dir(my_d, {k: USER_DATA})
+            populate_context_dir(my_d, {k: USER_DATA})
             results = ds.read_context_disk_dir(my_d)
 
             self.assertTrue('userdata' in results)
@@ -81,7 +84,7 @@ class TestOpenNebulaDataSource(MockerTestCase):
     def test_hostname(self):
         for k in ('HOSTNAME', 'PUBLIC_IP', 'IP_PUBLIC', 'ETH0_IP'):
             my_d = os.path.join(self.tmp, k)
-            populate_dir(my_d, {k: PUBLIC_IP})
+            populate_context_dir(my_d, {k: PUBLIC_IP})
             results = ds.read_context_disk_dir(my_d)
 
             self.assertTrue('metadata' in results)
@@ -89,14 +92,12 @@ class TestOpenNebulaDataSource(MockerTestCase):
             self.assertEqual(PUBLIC_IP, results['metadata']['local-hostname'])
 
     def test_find_candidates(self):
-        devs_with_answers = {
-            "TYPE=iso9660": ["/dev/vdb"],
-            "LABEL=CDROM": ["/dev/sr0"],
-            "LABEL=CONTEXT": ["/dev/sdb"],
-        }
-
         def my_devs_with(criteria):
-            return devs_with_answers[criteria]
+            return {
+                "LABEL=CONTEXT": ["/dev/sdb"],
+                "LABEL=CDROM": ["/dev/sr0"],
+                "TYPE=iso9660": ["/dev/vdb"],
+            }.get(criteria, [])
 
         try:
             orig_find_devs_with = util.find_devs_with
@@ -133,16 +134,17 @@ iface eth0 inet static
 ''')
 
     def test_eth0_override(self):
-        context_sh = {
+        context = {
             'dns': '1.2.3.8',
             'eth0_ip': '1.2.3.4',
             'eth0_network': '1.2.3.0',
             'eth0_mask': '255.255.0.0',
             'eth0_gateway': '1.2.3.5',
             'eth0_domain': 'example.com',
-            'eth0_dns': '1.2.3.6 1.2.3.7'}
+            'eth0_dns': '1.2.3.6 1.2.3.7'
+        }
 
-        net = ds.OpenNebulaNetwork(CMD_IP_OUT, context_sh)
+        net = ds.OpenNebulaNetwork(CMD_IP_OUT, context)
         self.assertEqual(net.gen_conf(), u'''\
 auto lo
 iface lo inet loopback
@@ -158,12 +160,10 @@ iface eth0 inet static
 ''')
 
 
-def populate_dir(seed_dir, files):
-    os.mkdir(seed_dir)
-    with open(os.path.join(seed_dir, "context.sh"), "w") as fp:
-        fp.write("# Context variables generated by OpenNebula\n")
-        for (name, content) in files.iteritems():
-            fp.write("%s='%s'\n" % (name.upper(), content.replace(r"'", r"'\''")))
-        fp.close()
+def populate_context_dir(path, variables):
+    data = "# Context variables generated by OpenNebula\n"
+    for (k, v) in variables.iteritems():
+        data += ("%s='%s'\n" % (k.upper(), v.replace(r"'", r"'\''")))
+    populate_dir(path, {'context.sh': data})
 
 # vi: ts=4 expandtab
-- 
cgit v1.2.3


From 781d105a6ca9222e59909341dbd86a13bab5a67e Mon Sep 17 00:00:00 2001
From: Vlastimil Holer <vlastimil.holer@gmail.com>
Date: Tue, 10 Sep 2013 00:40:16 +0200
Subject: Replace RE context.sh parser with Scott's rewrite of bash dumper.
 Upper case context variable names.

---
 cloudinit/sources/DataSourceOpenNebula.py          | 178 ++++++++++++++-------
 tests/unittests/test_datasource/test_opennebula.py |  38 ++---
 2 files changed, 137 insertions(+), 79 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index 87ec51bd..a1995aca 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -24,6 +24,8 @@
 
 import os
 import re
+import string
+import subprocess
 
 from cloudinit import log as logging
 from cloudinit import sources
@@ -50,8 +52,8 @@ class DataSourceOpenNebula(sources.DataSource):
 
     def get_data(self):
         defaults = {
-            "instance-id": DEFAULT_IID,
-            "dsmode": self.dsmode
+            "instance-id": DEFAULT_IID, #TODO:????
+            "DSMODE": self.dsmode
         }
 
         seed = None
@@ -85,7 +87,7 @@ class DataSourceOpenNebula(sources.DataSource):
         md = util.mergemanydict([md, defaults])
 
         # check for valid user specified dsmode
-        user_dsmode = results['metadata'].get('dsmode', None)
+        user_dsmode = results['metadata'].get('DSMODE', None)
         if user_dsmode not in VALID_DSMODES + (None,):
             LOG.warn("user specified invalid mode: %s", user_dsmode)
             user_dsmode = None
@@ -93,7 +95,7 @@ class DataSourceOpenNebula(sources.DataSource):
         # decide dsmode
         if user_dsmode:
             dsmode = user_dsmode
-        elif self.ds_cfg.get('dsmode'):
+        elif self.ds_cfg.get('dsmode'): #TODO: fakt se k tomu nekdy dostane?
             dsmode = self.ds_cfg.get('dsmode')
         else:
             dsmode = DEFAULT_MODE
@@ -153,42 +155,42 @@ class OpenNebulaNetwork(object):
         return [str(int(c, 16)) for c in components]
 
     def get_ip(self, dev, components):
-        var_name = dev + '_ip'
+        var_name = dev.upper() + '_IP'
         if var_name in self.context:
             return self.context[var_name]
         else:
             return '.'.join(components)
 
     def get_mask(self, dev):
-        var_name = dev + '_mask'
+        var_name = dev.upper() + '_MASK'
         if var_name in self.context:
             return self.context[var_name]
         else:
             return '255.255.255.0'
 
     def get_network(self, dev, components):
-        var_name = dev + '_network'
+        var_name = dev.upper() + '_NETWORK'
         if var_name in self.context:
             return self.context[var_name]
         else:
             return '.'.join(components[:-1]) + '.0'
 
     def get_gateway(self, dev):
-        var_name = dev + '_gateway'
+        var_name = dev.upper() + '_GATEWAY'
         if var_name in self.context:
             return self.context[var_name]
         else:
             return None
 
     def get_dns(self, dev):
-        var_name = dev + '_dns'
+        var_name = dev.upper() + '_DNS'
         if var_name in self.context:
             return self.context[var_name]
         else:
             return None
 
     def get_domain(self, dev):
-        var_name = dev + '_domain'
+        var_name = dev.upper() + '_DOMAIN'
         if var_name in self.context:
             return self.context[var_name]
         else:
@@ -196,8 +198,8 @@ class OpenNebulaNetwork(object):
 
     def gen_conf(self):
         global_dns = []
-        if 'dns' in self.context:
-            global_dns.append(self.context['dns'])
+        if 'DNS' in self.context:
+            global_dns.append(self.context['DNS'])
 
         conf = []
         conf.append('auto lo')
@@ -251,37 +253,91 @@ def find_candidate_devs():
     return combined
 
 
-def parse_context_data(data):
-    """
-    parse_context_data(data)
-    parse context.sh variables provided as a single string. Uses
-    very simple matching RE. Returns None if nothing is matched.
-    """
-    # RE groups:
-    # 1: key
-    # 2: single quoted value, respect '\''
-    # 3: old double quoted value, but doesn't end with \"
-    context_reg = re.compile(
-        r"^([\w_]+)=(?:'((?:[^']|'\\'')*?)'|\"(.*?[^\\])\")$",
-        re.MULTILINE | re.DOTALL)
-
-    found = context_reg.findall(data)
-    if not found:
-        return None
-
-    variables = {}
-    for k, v1, v2 in found:
-        k = k.lower()
-        if v1:
-            # take single quoted variable 'xyz'
-            # (ON>=4) and unquote '\'' -> '
-            variables[k] = v1.replace(r"'\''", r"'")
-        elif v2:
-            # take double quoted variable "xyz"
-            # (old ON<4) and unquote \" -> "
-            variables[k] = v2.replace(r'\"', r'"')
-
-    return variables
+def parse_shell_config(content, keylist=None, bash=None, asuser=None):
+
+    if isinstance(bash, str):
+        bash = [bash]
+    elif bash is None:
+        bash = ['bash', '-e']
+
+    # allvars expands to all existing variables by using '${!x*}' notation
+    # where x is lower or upper case letters or '_'
+    allvars = ["${!%s*}" % x for x in string.letters + "_"]
+
+    keylist_in = keylist
+    if keylist is None:
+        keylist = allvars
+        keylist_in = []
+
+    setup = '\n'.join(('__v="";', '',))
+
+    def varprinter(vlist):
+        # output '\0'.join(['_start_', key=value NULL for vars in vlist]
+        return '\n'.join((
+            'printf "%s\\0" _start_',
+            'for __v in %s; do' % ' '.join(vlist),
+            '   printf "%s=%s\\0" "$__v" "${!__v}";',
+            'done',
+            ''
+        ))
+
+    # the rendered 'bcmd' is bash syntax that does
+    # setup: declare variables we use (so they show up in 'all')
+    # varprinter(allvars): print all variables known at beginning
+    # content: execute the provided content
+    # varprinter(keylist): print all variables known after content
+    #
+    # output is then a null terminated array of:
+    #   literal '_start_'
+    #   key=value (for each preset variable)
+    #   literal '_start_'
+    #   key=value (for each post set variable)
+    bcmd = ('unset IFS\n' +
+            setup +
+            varprinter(allvars) +
+            '{\n%s\n\n} > /dev/null\n' % content +
+            'unset IFS\n' +
+            varprinter(keylist) + "\n")
+
+    cmd = []
+    if asuser is not None:
+        cmd = ['sudo', '-u', asuser]
+
+    cmd.extend(bash)
+
+    sp = subprocess.Popen(cmd, stdin=subprocess.PIPE,
+                          stdout=subprocess.PIPE)
+    (output, error) = sp.communicate(input=bcmd)
+
+    if sp.returncode != 0:
+        raise Exception("Process returned %d" % sp.returncode)
+
+    # exclude vars in bash that change on their own or that we used
+    excluded = ("RANDOM", "LINENO", "_", "__v")
+    preset = {}
+    ret = {}
+    target = None
+    output = output[0:-1]  # remove trailing null
+
+    # go through output.  First _start_ is for 'preset', second for 'target'.
+    # Add to target only things were changed and not in volitile
+    for line in output.split("\0"):
+        try:
+            (key, val) = line.split("=", 1)
+            if target is preset:
+                target[key] = val
+            elif (key not in excluded and
+                  (key in keylist_in or preset.get(key) != val)):
+                ret[key] = val
+        except ValueError:
+            if line != "_start_":
+                raise
+            if target is None:
+                target = preset
+            elif target is preset:
+                target = ret
+
+    return ret
 
 
 def read_context_disk_dir(source_dir):
@@ -305,24 +361,26 @@ def read_context_disk_dir(source_dir):
     if "context.sh" in found:
         try:
             with open(os.path.join(source_dir, 'context.sh'), 'r') as f:
-                context = parse_context_data(f.read())
+                content = f.read().strip()  
+                if content:
+                    context = parse_shell_config(content)
                 f.close()
-            if not context:
-                raise NonContextDiskDir("No variables in context")
-
-        except (IOError, NonContextDiskDir) as e:
+        except (IOError, Exception) as e:
             raise NonContextDiskDir("Error reading context.sh: %s" % (e))
-
-        results['metadata'] = context
     else:
         raise NonContextDiskDir("Missing context.sh")
 
+    if not context:
+        raise NonContextDiskDir("No context variables found")
+
+    results['metadata'] = context
+
     # process single or multiple SSH keys
     ssh_key_var = None
-    if "ssh_key" in context:
-        ssh_key_var = "ssh_key"
-    elif "ssh_public_key" in context:
-        ssh_key_var = "ssh_public_key"
+    if "SSH_KEY" in context:
+        ssh_key_var = "SSH_KEY"
+    elif "SSH_PUBLIC_KEY" in context:
+        ssh_key_var = "SSH_PUBLIC_KEY"
 
     if ssh_key_var:
         lines = context.get(ssh_key_var).splitlines()
@@ -331,22 +389,22 @@ def read_context_disk_dir(source_dir):
 
     # custom hostname -- try hostname or leave cloud-init
     # itself create hostname from IP address later
-    for k in ('hostname', 'public_ip', 'ip_public', 'eth0_ip'):
+    for k in ('HOSTNAME', 'PUBLIC_IP', 'IP_PUBLIC', 'ETH0_IP'):
         if k in context:
             results['metadata']['local-hostname'] = context[k]
             break
 
     # raw user data
-    if "user_data" in context:
-        results['userdata'] = context["user_data"]
-    elif "userdata" in context:
-        results['userdata'] = context["userdata"]
+    if "USER_DATA" in context:
+        results['userdata'] = context["USER_DATA"]
+    elif "USERDATA" in context:
+        results['userdata'] = context["USERDATA"]
 
     # generate static /etc/network/interfaces
     # only if there are any required context variables
     # http://opennebula.org/documentation:rel3.8:cong#network_configuration
     for k in context.keys():
-        if re.match(r'^eth\d+_ip$', k):
+        if re.match(r'^ETH\d+_ip$', k):
             (out, _) = util.subp(['/sbin/ip', 'link'])
             net = OpenNebulaNetwork(out, context)
             results['network-interfaces'] = net.gen_conf()
diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py
index 4b82a49c..f544e145 100644
--- a/tests/unittests/test_datasource/test_opennebula.py
+++ b/tests/unittests/test_datasource/test_opennebula.py
@@ -6,18 +6,18 @@ from tests.unittests.helpers import populate_dir
 import os
 
 TEST_VARS = {
-    'var1': 'single',
-    'var2': 'double word',
-    'var3': 'multi\nline\n',
-    'var4': "'single'",
-    'var5': "'double word'",
-    'var6': "'multi\nline\n'",
-    'var7': 'single\\t',
-    'var8': 'double\\tword',
-    'var9': 'multi\\t\nline\n',
-    'var10': '\\',  # expect \
-    'var11': '\'',  # expect '
-    'var12': '$',   # expect $
+    'VAR1': 'single',
+    'VAR2': 'double word',
+    'VAR3': 'multi\nline\n',
+    'VAR4': "'single'",
+    'VAR5': "'double word'",
+    'VAR6': "'multi\nline\n'",
+    'VAR7': 'single\\t',
+    'VAR8': 'double\\tword',
+    'VAR9': 'multi\\t\nline\n',
+    'VAR10': '\\',  # expect \
+    'VAR11': '\'',  # expect '
+    'VAR12': '$',   # expect $
 }
 
 USER_DATA = '#cloud-config\napt_upgrade: true'
@@ -135,13 +135,13 @@ iface eth0 inet static
 
     def test_eth0_override(self):
         context = {
-            'dns': '1.2.3.8',
-            'eth0_ip': '1.2.3.4',
-            'eth0_network': '1.2.3.0',
-            'eth0_mask': '255.255.0.0',
-            'eth0_gateway': '1.2.3.5',
-            'eth0_domain': 'example.com',
-            'eth0_dns': '1.2.3.6 1.2.3.7'
+            'DNS': '1.2.3.8',
+            'ETH0_IP': '1.2.3.4',
+            'ETH0_NETWORK': '1.2.3.0',
+            'ETH0_MASK': '255.255.0.0',
+            'ETH0_GATEWAY': '1.2.3.5',
+            'ETH0_DOMAIN': 'example.com',
+            'ETH0_DNS': '1.2.3.6 1.2.3.7'
         }
 
         net = ds.OpenNebulaNetwork(CMD_IP_OUT, context)
-- 
cgit v1.2.3


From 7f3b1198da74430345d69e485326b03a3fa5b455 Mon Sep 17 00:00:00 2001
From: Vlastimil Holer <vlastimil.holer@gmail.com>
Date: Tue, 10 Sep 2013 00:46:37 +0200
Subject: Fix pylint complain on toks.split('.')

---
 cloudinit/sources/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index b0e43954..1dfdf9bf 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -173,7 +173,7 @@ class DataSource(object):
                     toks = util.gethostbyaddr(lhost)
 
                 if toks:
-                    toks = toks.split('.')
+                    toks = str(toks).split('.')
                 else:
                     toks = ["ip-%s" % lhost.replace(".", "-")]
             else:
-- 
cgit v1.2.3


From 64a2f3c7d4eae1354134c021db232c117eb1c772 Mon Sep 17 00:00:00 2001
From: Vlastimil Holer <vlastimil.holer@gmail.com>
Date: Tue, 10 Sep 2013 15:37:15 +0200
Subject: Configurable OpenNebula::parseuser. Seed search dir+dev merge. Eat
 shell parser error output. Few tests for tests for get_data.

---
 cloudinit/sources/DataSourceOpenNebula.py          | 78 +++++++++++-------
 tests/unittests/test_datasource/test_opennebula.py | 95 +++++++++++++++++++---
 2 files changed, 132 insertions(+), 41 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index a1995aca..1b419cfd 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -35,6 +35,7 @@ LOG = logging.getLogger(__name__)
 
 DEFAULT_IID = "iid-dsopennebula"
 DEFAULT_MODE = 'net'
+DEFAULT_PARSEUSER = 'nobody'
 CONTEXT_DISK_FILES = ["context.sh"]
 VALID_DSMODES = ("local", "net", "disabled")
 
@@ -51,33 +52,35 @@ class DataSourceOpenNebula(sources.DataSource):
         return "%s [seed=%s][dsmode=%s]" % (root, self.seed, self.dsmode)
 
     def get_data(self):
-        defaults = {
-            "instance-id": DEFAULT_IID, #TODO:????
-            "DSMODE": self.dsmode
-        }
-
+        defaults = {"instance-id": DEFAULT_IID}
+        results = None
         seed = None
-        results = {}
 
-        # first try to read local seed_dir
-        if os.path.isdir(self.seed_dir):
+        # decide parseuser for context.sh shell reader
+        parseuser = DEFAULT_PARSEUSER
+        if self.ds_cfg.get('parseuser'):
+            parseuser = self.ds_cfg.get('parseuser')
+
+        candidates = [self.seed_dir]
+        candidates.extend(find_candidate_devs())
+        for cdev in candidates:
             try:
-                results = read_context_disk_dir(self.seed_dir)
-                seed = self.seed_dir
+                if os.path.isdir(self.seed_dir):
+                    results = read_context_disk_dir(cdev, asuser=parseuser)
+                elif cdev.startswith("/dev"):
+                    results = util.mount_cb(cdev, read_context_disk_dir,
+                                            data=parseuser)
             except NonContextDiskDir:
-                util.logexc(LOG, "Failed reading context from %s",
-                            self.seed_dir)
+                continue
+            except BrokenContextDiskDir as exc:
+                raise exc
+            except util.MountFailedError:
+                LOG.warn("%s was not mountable" % cdev)
 
-        if not seed:
-            # then try to detect and mount candidate devices and
-            # read contextualization if present
-            for dev in find_candidate_devs():
-                try:
-                    results = util.mount_cb(dev, read_context_disk_dir)
-                    seed = dev
-                    break
-                except (NonContextDiskDir, util.MountFailedError):
-                    pass
+            if results:
+                seed = cdev
+                LOG.debug("found datasource in %s", cdev)
+                break
 
         if not seed:
             return False
@@ -95,7 +98,7 @@ class DataSourceOpenNebula(sources.DataSource):
         # decide dsmode
         if user_dsmode:
             dsmode = user_dsmode
-        elif self.ds_cfg.get('dsmode'): #TODO: fakt se k tomu nekdy dostane?
+        elif self.ds_cfg.get('dsmode'):
             dsmode = self.ds_cfg.get('dsmode')
         else:
             dsmode = DEFAULT_MODE
@@ -137,6 +140,10 @@ class NonContextDiskDir(Exception):
     pass
 
 
+class BrokenContextDiskDir(Exception):
+    pass
+
+
 class OpenNebulaNetwork(object):
     REG_DEV_MAC = re.compile(
                     r'^\d+: (eth\d+):.*?link\/ether (..:..:..:..:..:..) ?',
@@ -306,7 +313,8 @@ def parse_shell_config(content, keylist=None, bash=None, asuser=None):
     cmd.extend(bash)
 
     sp = subprocess.Popen(cmd, stdin=subprocess.PIPE,
-                          stdout=subprocess.PIPE)
+                          stdout=subprocess.PIPE,
+                          stderr=subprocess.PIPE)
     (output, error) = sp.communicate(input=bcmd)
 
     if sp.returncode != 0:
@@ -340,7 +348,7 @@ def parse_shell_config(content, keylist=None, bash=None, asuser=None):
     return ret
 
 
-def read_context_disk_dir(source_dir):
+def read_context_disk_dir(source_dir, asuser=None):
     """
     read_context_disk_dir(source_dir):
     read source_dir and return a tuple with metadata dict and user-data
@@ -355,23 +363,31 @@ def read_context_disk_dir(source_dir):
     if not found:
         raise NonContextDiskDir("%s: %s" % (source_dir, "no files found"))
 
-    results = {'userdata': None, 'metadata': {}}
     context = {}
+    results = {'userdata': None, 'metadata': {}}
 
     if "context.sh" in found:
         try:
             with open(os.path.join(source_dir, 'context.sh'), 'r') as f:
-                content = f.read().strip()  
-                if content:
-                    context = parse_shell_config(content)
+                content = f.read().strip()
                 f.close()
-        except (IOError, Exception) as e:
+
+                # don't pass empty context script
+                # to shell parser
+                non_empty = re.match(r'.*?^\s*([^# ]+)', content,
+                                     re.MULTILINE | re.DOTALL)
+
+                if non_empty:
+                    context = parse_shell_config(content, asuser=asuser)
+        except IOError as e:
             raise NonContextDiskDir("Error reading context.sh: %s" % (e))
+        except Exception as e:
+            raise BrokenContextDiskDir("Error processing context.sh: %s" % (e))
     else:
         raise NonContextDiskDir("Missing context.sh")
 
     if not context:
-        raise NonContextDiskDir("No context variables found")
+        return results
 
     results['metadata'] = context
 
diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py
index f544e145..752638b6 100644
--- a/tests/unittests/test_datasource/test_opennebula.py
+++ b/tests/unittests/test_datasource/test_opennebula.py
@@ -1,4 +1,5 @@
 from cloudinit.sources import DataSourceOpenNebula as ds
+from cloudinit import helpers
 from cloudinit import util
 from mocker import MockerTestCase
 from tests.unittests.helpers import populate_dir
@@ -20,6 +21,8 @@ TEST_VARS = {
     'VAR12': '$',   # expect $
 }
 
+INVALID_PARSEUSER = 'cloud-init-mocker-opennebula-invalid'
+INVALID_CONTEXT = ';'
 USER_DATA = '#cloud-config\napt_upgrade: true'
 SSH_KEY = 'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460-%i'
 HOSTNAME = 'foo.example.com'
@@ -38,21 +41,93 @@ class TestOpenNebulaDataSource(MockerTestCase):
     def setUp(self):
         super(TestOpenNebulaDataSource, self).setUp()
         self.tmp = self.makeDir()
+        self.paths = helpers.Paths({'cloud_dir': self.tmp})
+
+        # defaults for few tests
+        self.ds = ds.DataSourceOpenNebula
+        self.seed_dir = os.path.join(self.paths.seed_dir, "opennebula")
+        self.sys_cfg = {'datasource': {'OpenNebula': {'dsmode': 'local'}}}
+
+    def test_get_data_non_contextdisk(self):
+        try:
+            # dont' try to lookup for CDs
+            orig_find_devs_with = util.find_devs_with
+            util.find_devs_with = lambda n: []
+
+            dsrc = self.ds(sys_cfg=self.sys_cfg, distro=None, paths=self.paths)
+            ret = dsrc.get_data()
+            self.assertFalse(ret)
+        finally:
+            util.find_devs_with = orig_find_devs_with
+
+    def test_get_data_broken_contextdisk(self):
+        try:
+            # dont' try to lookup for CDs
+            orig_find_devs_with = util.find_devs_with
+            util.find_devs_with = lambda n: []
+
+            populate_dir(self.seed_dir, {'context.sh': INVALID_CONTEXT})
+            dsrc = self.ds(sys_cfg=self.sys_cfg, distro=None, paths=self.paths)
+            self.assertRaises(ds.BrokenContextDiskDir, dsrc.get_data)
+        finally:
+            util.find_devs_with = orig_find_devs_with
+
+    def test_get_data_invalid_identity(self):
+        try:
+            # dont' try to lookup for CDs
+            orig_find_devs_with = util.find_devs_with
+            util.find_devs_with = lambda n: []
+
+            sys_cfg = self.sys_cfg
+            sys_cfg['datasource']['OpenNebula']['parseuser'] = \
+                INVALID_PARSEUSER
+
+            populate_context_dir(self.seed_dir, {'KEY1': 'val1'})
+            dsrc = self.ds(sys_cfg=sys_cfg, distro=None, paths=self.paths)
+            self.assertRaises(ds.BrokenContextDiskDir, dsrc.get_data)
+        finally:
+            util.find_devs_with = orig_find_devs_with
+
+    def test_get_data(self):
+        try:
+            # dont' try to lookup for CDs
+            orig_find_devs_with = util.find_devs_with
+            util.find_devs_with = lambda n: []
+            populate_context_dir(self.seed_dir, {'KEY1': 'val1'})
+            dsrc = self.ds(sys_cfg=self.sys_cfg, distro=None, paths=self.paths)
+            ret = dsrc.get_data()
+            self.assertTrue(ret)
+        finally:
+            util.find_devs_with = orig_find_devs_with
 
     def test_seed_dir_non_contextdisk(self):
-        my_d = os.path.join(self.tmp, 'non-contextdisk')
-        self.assertRaises(ds.NonContextDiskDir, ds.read_context_disk_dir, my_d)
+        self.assertRaises(ds.NonContextDiskDir, ds.read_context_disk_dir,
+                          self.seed_dir)
+
+    def test_seed_dir_empty1_context(self):
+        populate_dir(self.seed_dir, {'context.sh': ''})
+        results = ds.read_context_disk_dir(self.seed_dir)
+
+        self.assertEqual(results['userdata'], None)
+        self.assertEqual(results['metadata'], {})
+
+    def test_seed_dir_empty2_context(self):
+        populate_context_dir(self.seed_dir, {})
+        results = ds.read_context_disk_dir(self.seed_dir)
+
+        self.assertEqual(results['userdata'], None)
+        self.assertEqual(results['metadata'], {})
+
+    def test_seed_dir_broken_context(self):
+        populate_dir(self.seed_dir, {'context.sh': INVALID_CONTEXT})
 
-    def test_seed_dir_bad_context(self):
-        my_d = os.path.join(self.tmp, 'bad-context')
-        os.mkdir(my_d)
-        open(os.path.join(my_d, "context.sh"), "w").close()
-        self.assertRaises(ds.NonContextDiskDir, ds.read_context_disk_dir, my_d)
+        self.assertRaises(ds.BrokenContextDiskDir,
+                          ds.read_context_disk_dir,
+                          self.seed_dir)
 
     def test_context_parser(self):
-        my_d = os.path.join(self.tmp, 'context-parser')
-        populate_context_dir(my_d, TEST_VARS)
-        results = ds.read_context_disk_dir(my_d)
+        populate_context_dir(self.seed_dir, TEST_VARS)
+        results = ds.read_context_disk_dir(self.seed_dir)
 
         self.assertTrue('metadata' in results)
         self.assertEqual(TEST_VARS, results['metadata'])
-- 
cgit v1.2.3


From c0d1a59d96a16c080ad8b8278251294cccc21894 Mon Sep 17 00:00:00 2001
From: Vlastimil Holer <vlastimil.holer@gmail.com>
Date: Tue, 10 Sep 2013 18:35:19 +0200
Subject: Fix detection of ETHx_IP context variable, add test.

---
 cloudinit/sources/DataSourceOpenNebula.py          | 3 +--
 tests/unittests/test_datasource/test_opennebula.py | 6 ++++++
 2 files changed, 7 insertions(+), 2 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index 1b419cfd..141bd454 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -376,7 +376,6 @@ def read_context_disk_dir(source_dir, asuser=None):
                 # to shell parser
                 non_empty = re.match(r'.*?^\s*([^# ]+)', content,
                                      re.MULTILINE | re.DOTALL)
-
                 if non_empty:
                     context = parse_shell_config(content, asuser=asuser)
         except IOError as e:
@@ -420,7 +419,7 @@ def read_context_disk_dir(source_dir, asuser=None):
     # only if there are any required context variables
     # http://opennebula.org/documentation:rel3.8:cong#network_configuration
     for k in context.keys():
-        if re.match(r'^ETH\d+_ip$', k):
+        if re.match(r'^ETH\d+_IP$', k):
             (out, _) = util.subp(['/sbin/ip', 'link'])
             net = OpenNebulaNetwork(out, context)
             results['network-interfaces'] = net.gen_conf()
diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py
index f5103dc0..f2457657 100644
--- a/tests/unittests/test_datasource/test_opennebula.py
+++ b/tests/unittests/test_datasource/test_opennebula.py
@@ -173,6 +173,12 @@ class TestOpenNebulaDataSource(MockerTestCase):
             self.assertTrue('local-hostname' in results['metadata'])
             self.assertEqual(PUBLIC_IP, results['metadata']['local-hostname'])
 
+    def test_network_interfaces(self):
+        populate_context_dir(self.seed_dir, {'ETH0_IP': '1.2.3.4'})
+        results = ds.read_context_disk_dir(self.seed_dir)
+
+        self.assertTrue('network-interfaces' in results)
+
     def test_find_candidates(self):
         def my_devs_with(criteria):
             return {
-- 
cgit v1.2.3


From 8ed952a461ed81af4011e02f3df47add1066cd0f Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Tue, 10 Sep 2013 14:15:30 -0400
Subject: fix DataSource base class to set up ds_cfg for 'Net' sources

When the base DataSource class would set 'ds_cfg' for the specific
datasources' config, it would fail for the DataSources that are just named
'DataSourceFooNet' and we wanted to set configuration in 'Foo'.

For example, both DataSourceOpenNebula and DataSourceOpenNebulaNet want to
read datasource config from
 sources:
  OpenNebula:
   foo: bar

But without this change, 'ds_cfg' would not be setup properly for
OpenNebulaNet.
---
 cloudinit/sources/__init__.py | 7 +++++++
 1 file changed, 7 insertions(+)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index 1dfdf9bf..7dc1fbde 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -53,9 +53,16 @@ class DataSource(object):
         self.userdata = None
         self.metadata = None
         self.userdata_raw = None
+
+        # find the datasource config name.
+        # remove 'DataSource' from classname on front, and remove 'Net' on end.
+        # Both Foo and FooNet sources expect config in cfg['sources']['Foo']
         name = type_utils.obj_name(self)
         if name.startswith(DS_PREFIX):
             name = name[len(DS_PREFIX):]
+        if name.endswith('Net'):
+            name = name[0:-3]
+
         self.ds_cfg = util.get_cfg_by_path(self.sys_cfg,
                                           ("datasource", name), {})
         if not ud_proc:
-- 
cgit v1.2.3


From 7dff14c285d6562b9cd0b47876628607feac4a18 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Tue, 10 Sep 2013 14:18:58 -0400
Subject: some cleanups and changes

 * use util.subp from inside parse_shell_config,
   and adjust exception handling accordingly.
 * add 'switch_user_cmd' as a callback function to pass
   to parse_shell_config, which allows us to mock this to avoid
   'sudo' when running test cases.
   Basically the test cases just return '[]' here.
 * fix some pylint
 * handle empty 'content' in parse_shell_config and remove
   the protection that was present.
---
 cloudinit/sources/DataSourceOpenNebula.py          | 50 +++++++++++-----------
 tests/unittests/test_datasource/test_opennebula.py | 15 ++++++-
 2 files changed, 40 insertions(+), 25 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index 141bd454..07dc25ff 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -23,9 +23,9 @@
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
+import pwd
 import re
-import string
-import subprocess
+import string  # pylint: disable=W0402
 
 from cloudinit import log as logging
 from cloudinit import sources
@@ -58,7 +58,7 @@ class DataSourceOpenNebula(sources.DataSource):
 
         # decide parseuser for context.sh shell reader
         parseuser = DEFAULT_PARSEUSER
-        if self.ds_cfg.get('parseuser'):
+        if 'parseuser' in self.ds_cfg:
             parseuser = self.ds_cfg.get('parseuser')
 
         candidates = [self.seed_dir]
@@ -260,13 +260,21 @@ def find_candidate_devs():
     return combined
 
 
-def parse_shell_config(content, keylist=None, bash=None, asuser=None):
+def switch_user_cmd(user):
+    return ['sudo', '-u', user]
+
+
+def parse_shell_config(content, keylist=None, bash=None, asuser=None,
+                       switch_user_cb=None):
 
     if isinstance(bash, str):
         bash = [bash]
     elif bash is None:
         bash = ['bash', '-e']
 
+    if switch_user_cb is None:
+        switch_user_cb = switch_user_cmd
+
     # allvars expands to all existing variables by using '${!x*}' notation
     # where x is lower or upper case letters or '_'
     allvars = ["${!%s*}" % x for x in string.letters + "_"]
@@ -302,23 +310,17 @@ def parse_shell_config(content, keylist=None, bash=None, asuser=None):
     bcmd = ('unset IFS\n' +
             setup +
             varprinter(allvars) +
-            '{\n%s\n\n} > /dev/null\n' % content +
+            '{\n%s\n\n:\n} > /dev/null\n' % content +
             'unset IFS\n' +
             varprinter(keylist) + "\n")
 
     cmd = []
     if asuser is not None:
-        cmd = ['sudo', '-u', asuser]
+        cmd = switch_user_cb(asuser)
 
     cmd.extend(bash)
 
-    sp = subprocess.Popen(cmd, stdin=subprocess.PIPE,
-                          stdout=subprocess.PIPE,
-                          stderr=subprocess.PIPE)
-    (output, error) = sp.communicate(input=bcmd)
-
-    if sp.returncode != 0:
-        raise Exception("Process returned %d" % sp.returncode)
+    (output, _error) = util.subp(cmd, data=bcmd)
 
     # exclude vars in bash that change on their own or that we used
     excluded = ("RANDOM", "LINENO", "_", "__v")
@@ -329,7 +331,7 @@ def parse_shell_config(content, keylist=None, bash=None, asuser=None):
 
     # go through output.  First _start_ is for 'preset', second for 'target'.
     # Add to target only things were changed and not in volitile
-    for line in output.split("\0"):
+    for line in output.split("\x00"):
         try:
             (key, val) = line.split("=", 1)
             if target is preset:
@@ -367,21 +369,21 @@ def read_context_disk_dir(source_dir, asuser=None):
     results = {'userdata': None, 'metadata': {}}
 
     if "context.sh" in found:
+        if asuser is not None:
+            try:
+                pwd.getpwnam(asuser)
+            except KeyError as e:
+                raise BrokenContextDiskDir("configured user '%s' "
+                                           "does not exist", asuser)
         try:
             with open(os.path.join(source_dir, 'context.sh'), 'r') as f:
                 content = f.read().strip()
-                f.close()
-
-                # don't pass empty context script
-                # to shell parser
-                non_empty = re.match(r'.*?^\s*([^# ]+)', content,
-                                     re.MULTILINE | re.DOTALL)
-                if non_empty:
-                    context = parse_shell_config(content, asuser=asuser)
+
+            context = parse_shell_config(content, asuser=asuser)
+        except util.ProcessExecutionError as e:
+            raise BrokenContextDiskDir("Error processing context.sh: %s" % (e))
         except IOError as e:
             raise NonContextDiskDir("Error reading context.sh: %s" % (e))
-        except Exception as e:
-            raise BrokenContextDiskDir("Error processing context.sh: %s" % (e))
     else:
         raise NonContextDiskDir("Missing context.sh")
 
diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py
index f2457657..9c7a644a 100644
--- a/tests/unittests/test_datasource/test_opennebula.py
+++ b/tests/unittests/test_datasource/test_opennebula.py
@@ -37,6 +37,7 @@ CMD_IP_OUT = '''\
 
 
 class TestOpenNebulaDataSource(MockerTestCase):
+    parsed_user = None
 
     def setUp(self):
         super(TestOpenNebulaDataSource, self).setUp()
@@ -48,6 +49,18 @@ class TestOpenNebulaDataSource(MockerTestCase):
         self.seed_dir = os.path.join(self.paths.seed_dir, "opennebula")
         self.sys_cfg = {'datasource': {'OpenNebula': {'dsmode': 'local'}}}
 
+        # we don't want 'sudo' called in tests. so we patch switch_user_cmd
+        def my_switch_user_cmd(user):
+            self.parsed_user = user
+            return []
+
+        self.switch_user_cmd_real = ds.switch_user_cmd
+        ds.switch_user_cmd = my_switch_user_cmd
+
+    def tearDown(self):
+        ds.switch_user_cmd = self.switch_user_cmd_real
+        super(TestOpenNebulaDataSource, self).tearDown()
+
     def test_get_data_non_contextdisk(self):
         try:
             # dont' try to lookup for CDs
@@ -96,9 +109,9 @@ class TestOpenNebulaDataSource(MockerTestCase):
             util.find_devs_with = orig_find_devs_with
 
     def test_get_data(self):
+        orig_find_devs_with = util.find_devs_with
         try:
             # dont' try to lookup for CDs
-            orig_find_devs_with = util.find_devs_with
             util.find_devs_with = lambda n: []
             populate_context_dir(self.seed_dir, {'KEY1': 'val1'})
             dsrc = self.ds(sys_cfg=self.sys_cfg, distro=None, paths=self.paths)
-- 
cgit v1.2.3