Add initial DataSourceMaaS.

Tests at this point seem to indicate that seed-dir would work.

2 files changed, 420 insertions, 0 deletions

diff --git a/cloudinit/DataSourceMaaS.py b/cloudinit/DataSourceMaaS.py
new file mode 100644
index 00000000..2bc1f71f
--- /dev/null
+++ b/cloudinit/DataSourceMaaS.py
@@ -0,0 +1,308 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2009-2010 Canonical Ltd.
+#    Copyright (C) 2012 Hewlett-Packard Development Company, L.P.
+#
+#    Author: Scott Moser <scott.moser@canonical.com>
+#    Author: Juerg Hafliger <juerg.haefliger@hp.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import cloudinit.DataSource as DataSource
+
+from cloudinit import seeddir as base_seeddir
+from cloudinit import log
+import cloudinit.util as util
+import errno
+import oauth.oauth as oauth
+import os.path
+import socket
+import urllib2
+import time
+
+
+class DataSourceMaaS(DataSource.DataSource):
+    """
+    DataSourceMaaS reads instance information from MaaS.
+    Given a config metadata_url, and oauth tokens, it expects to find
+    files under the root named:
+      instance-id
+      user-data
+      hostname
+    """
+    seeddir = base_seeddir + '/maas'
+    baseurl = None
+
+    def __str__(self):
+        return("DataSourceMaaS[%s]" % self.baseurl)
+
+    def get_data(self):
+        mcfg = self.ds_cfg
+
+        try:
+            (userdata, metadata) = read_maas_seed_dir(self.seeddir)
+            self.userdata_raw = userdata
+            self.metadata = metadata
+            self.baseurl = self.seeddir
+            return True
+        except MaasSeedDirNone:
+            pass
+        except MaasSeedDirMalformed as exc:
+            log.warn("%s was malformed: %s\n" % (self.seeddir, exc))
+            raise
+
+        try:
+            # if there is no metadata_url, then we're not configured
+            url = mcfg.get('metadata_url', None)
+            if url == None:
+                return False
+
+            if not self.wait_for_metadata_service(url):
+                return False
+
+            self.baseurl = url
+
+            (userdata, metadata) = read_maas_seed_url(self.baseurl,
+                self.md_headers)
+            return True
+        except Exception:
+            util.logexc(log)
+            return False
+
+    def md_headers(self, url):
+        mcfg = self.ds_cfg
+
+        # if we are missing token_key, token_secret or consumer_key
+        # then just do non-authed requests
+        for required in ('token_key', 'token_secret', 'consumer_key'):
+            if required not in mcfg:
+                return({})
+
+        consumer_secret = mcfg.get('consumer_secret', "")
+
+        return(oauth_headers(url=url, consumer_key=mcfg['consumer_key'],
+            token_key=mcfg['token_key'], token_secret=mcfg['token_secret'],
+            consumer_secret=consumer_secret))
+
+    def wait_for_metadata_service(self, url):
+        mcfg = self.ds_cfg
+
+        max_wait = 120
+        try:
+            max_wait = int(mcfg.get("max_wait", max_wait))
+        except Exception:
+            util.logexc(log)
+            log.warn("Failed to get max wait. using %s" % max_wait)
+
+        if max_wait == 0:
+            return False
+
+        timeout = 50
+        try:
+            timeout = int(mcfg.get("timeout", timeout))
+        except Exception:
+            util.logexc(log)
+            log.warn("Failed to get timeout, using %s" % timeout)
+
+        starttime = time.time()
+        check_url = "%s/instance-id" % url
+        url = wait_for_metadata_service(urls=[check_url], max_wait=max_wait,
+            timeout=timeout, status_cb=log.warn,
+            headers_cb=self.md_headers)
+
+        if url:
+            log.debug("Using metadata source: '%s'" % url)
+        else:
+            log.critical("giving up on md after %i seconds\n" %
+                         int(time.time() - starttime))
+
+        return (bool(url))
+
+
+def wait_for_metadata_service(urls, max_wait=None, timeout=None,
+                              status_cb=None, headers_cb=None):
+    """
+    urls:      a list of urls to try
+    max_wait:  roughly the maximum time to wait before giving up
+               The max time is *actually* len(urls)*timeout as each url will
+               be tried once and given the timeout provided.
+    timeout:   the timeout provided to urllib2.urlopen
+    status_cb: call method with string message when a url is not available
+
+    the idea of this routine is to wait for the EC2 metdata service to
+    come up.  On both Eucalyptus and EC2 we have seen the case where
+    the instance hit the MD before the MD service was up.  EC2 seems
+    to have permenantely fixed this, though.
+
+    In openstack, the metadata service might be painfully slow, and
+    unable to avoid hitting a timeout of even up to 10 seconds or more
+    (LP: #894279) for a simple GET.
+
+    Offset those needs with the need to not hang forever (and block boot)
+    on a system where cloud-init is configured to look for EC2 Metadata
+    service but is not going to find one.  It is possible that the instance
+    data host (169.254.169.254) may be firewalled off Entirely for a sytem,
+    meaning that the connection will block forever unless a timeout is set.
+    """
+    starttime = time.time()
+
+    sleeptime = 1
+
+    def nullstatus_cb(msg):
+        return
+
+    if status_cb == None:
+        status_cb = nullstatus_cb
+
+    def timeup(max_wait, starttime):
+        return((max_wait <= 0 or max_wait == None) or
+               (time.time() - starttime > max_wait))
+
+    loop_n = 0
+    while True:
+        sleeptime = int(loop_n / 5) + 1
+        for url in urls:
+            now = time.time()
+            if loop_n != 0:
+                if timeup(max_wait, starttime):
+                    break
+                if timeout and (now + timeout > (starttime + max_wait)):
+                    # shorten timeout to not run way over max_time
+                    timeout = int((starttime + max_wait) - now)
+
+            reason = ""
+            try:
+                if headers_cb != None:
+                    headers = headers_cb(url)
+                else:
+                    headers = {}
+
+                req = urllib2.Request(url, data=None, headers=headers)
+                resp = urllib2.urlopen(req, timeout=timeout)
+                if resp.read() != "":
+                    return url
+                reason = "empty data [%s]" % resp.getcode()
+            except urllib2.HTTPError as e:
+                reason = "http error [%s]" % e.code
+            except urllib2.URLError as e:
+                reason = "url error [%s]" % e.reason
+            except socket.timeout as e:
+                reason = "socket timeout [%s]" % e
+            except Exception as e:
+                reason = "unexpected error [%s]" % e
+
+            if log:
+                status_cb("'%s' failed [%s/%ss]: %s" %
+                          (url, int(time.time() - starttime), max_wait,
+                           reason))
+
+        if timeup(max_wait, starttime):
+            break
+
+        loop_n = loop_n + 1
+        time.sleep(sleeptime)
+
+    return False
+
+
+def read_maas_seed_dir(seed_d):
+    """
+    Return user-data and metadata for a maas seed dir in seed_d.
+    Expected format of seed_d are the following files:
+      * instance-id
+      * hostname
+      * user-data
+    """
+    md_required = set(('hostname', 'instance-id'))
+    files = md_required.union(set(('userdata',)))
+    userdata = None
+    md = {}
+
+    if not os.path.isdir(seed_d):
+        raise MaasSeedDirNone("%s: not a directory")
+
+    for fname in files:
+        try:
+            with open(os.path.join(seed_d, fname)) as fp:
+                if fname == 'userdata':
+                    userdata = fp.read()
+                else:
+                    md[fname] = fp.read()
+                fp.close()
+        except IOError as e:
+            if e.errno != errno.ENOENT:
+                raise
+
+    if userdata == None and len(md) == 0:
+        raise MaasSeedDirNone("%s: no data files found" % seed_d)
+
+    if userdata == None:
+        raise MaasSeedDirMalformed("%s: missing userdata" % seed_d)
+
+    missing = md_required - set(md.keys())
+    if len(missing):
+        raise MaasSeedDirMalformed("%s: missing files %s" %
+            (seed_d, str(missing)))
+
+    return(userdata, md)
+
+
+def read_maas_seed_url(seed_url, header_cb=None):
+    """
+    Read the maas datasource at seed_url.
+    header_cb is a method that should return a headers dictionary that will
+    be given to urllib2.Request()
+
+    Expected format of seed_url is are the following files:
+      * <seed_url>/instance-id
+      * <seed_url>/hostname
+      * <seed_url>/user-data
+    """
+    userdata = ""
+    metadata = {'instance-id': 'i-maas-url', 'hostname': 'maas-url-hostname'}
+
+    return(userdata, metadata)
+
+
+def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret):
+    consumer = oauth.OAuthConsumer(consumer_key, consumer_secret)
+    token = oauth.OAuthToken(token_key, token_secret)
+    params = {
+        'oauth_version': "1.0",
+        'oauth_nonce': oauth.generate_nonce(),
+        'oauth_timestamp': int(time.time()),
+        'oauth_token': token.key,
+        'oauth_consumer_key': consumer.key,
+    }
+    req = oauth.OAuthRequest(http_url=url, parameters=params)
+    req.sign_request(oauth.OAuthSignatureMethod_PLAINTEXT(),
+        consumer, token)
+    return(req.to_header())
+
+
+class MaasSeedDirNone(Exception):
+    pass
+
+
+class MaasSeedDirMalformed(Exception):
+    pass
+
+
+datasources = [
+  (DataSourceMaaS, (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)),
+]
+
+
+# return a list of data sources that match this set of dependencies
+def get_datasource_list(depends):
+    return(DataSource.list_from_depends(depends, datasources))
diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py
new file mode 100644
index 00000000..ad169b97
--- /dev/null
+++ b/tests/unittests/test_datasource/test_maas.py
@@ -0,0 +1,112 @@
+from unittest import TestCase
+from tempfile import mkdtemp
+from shutil import rmtree
+import os
+from copy import copy
+from cloudinit.DataSourceMaaS import (
+    MaasSeedDirNone,
+    MaasSeedDirMalformed,
+    read_maas_seed_dir,
+)
+
+
+class TestMaasDataSource(TestCase):
+
+    def setUp(self):
+        super(TestMaasDataSource, self).setUp()
+        # Make a temp directoy for tests to use.
+        self.tmp = mkdtemp(prefix="unittest_")
+
+    def tearDown(self):
+        super(TestMaasDataSource, self).tearDown()
+        # Clean up temp directory
+        rmtree(self.tmp)
+
+    def test_seed_dir_valid(self):
+        """Verify a valid seeddir is read as such"""
+
+        data = {'instance-id': 'i-valid01', 'hostname': 'valid01-hostname',
+            'userdata': 'valid01-userdata'}
+
+        my_d = os.path.join(self.tmp, "valid")
+        populate_dir(my_d, data)
+
+        (userdata, metadata) = read_maas_seed_dir(my_d)
+
+        self.assertEqual(userdata, data['userdata'])
+        for key in ('instance-id', 'hostname'):
+            self.assertEqual(data[key], metadata[key])
+        
+    def test_seed_dir_valid_extra(self):
+        """Verify extra files do not affect seed_dir validity """
+
+        data = {'instance-id': 'i-valid-extra',
+            'hostname': 'valid-extra-hostname',
+            'userdata': 'valid-extra-userdata', 'foo': 'bar'}
+
+        my_d = os.path.join(self.tmp, "valid_extra")
+        populate_dir(my_d, data)
+
+        (userdata, metadata) = read_maas_seed_dir(my_d)
+
+        self.assertEqual(userdata, data['userdata'])
+        for key in ('instance-id', 'hostname'):
+            self.assertEqual(data[key], metadata[key])
+
+        # additional files should not just appear as keys in metadata atm
+        self.assertFalse(('foo' in metadata))
+
+    def test_seed_dir_invalid(self):
+        """Verify that invalid seed_dir raises MaasSeedDirMalformed"""
+
+        valid = {'instance-id': 'i-instanceid',
+            'hostname': 'test-hostname', 'userdata': ''}
+
+        my_based = os.path.join(self.tmp, "valid_extra")
+
+        # missing 'userdata' file
+        my_d = "%s-01" % my_based
+        invalid_data = copy(valid)
+        del invalid_data['userdata']
+        populate_dir(my_d, invalid_data)
+        self.assertRaises(MaasSeedDirMalformed, read_maas_seed_dir, my_d)
+
+        # missing 'instance-id'
+        my_d = "%s-02" % my_based
+        invalid_data = copy(valid)
+        del invalid_data['instance-id']
+        populate_dir(my_d, invalid_data)
+        self.assertRaises(MaasSeedDirMalformed, read_maas_seed_dir, my_d)
+
+    def test_seed_dir_none(self):
+        """Verify that empty seed_dir raises MaasSeedDirNone"""
+
+        my_d = os.path.join(self.tmp, "valid_empty")
+        self.assertRaises(MaasSeedDirNone, read_maas_seed_dir, my_d)
+
+    def test_seed_dir_missing(self):
+        """Verify that missing seed_dir raises MaasSeedDirNone"""
+        self.assertRaises(MaasSeedDirNone, read_maas_seed_dir,
+            os.path.join(self.tmp, "nonexistantdirectory"))
+
+    def test_seed_url_valid(self):
+        """Verify that valid seed_url is read as such"""
+        pass
+
+    def test_seed_url_invalid(self):
+        """Verify that invalid seed_url raises MaasSeedDirMalformed"""
+        pass
+
+    def test_seed_url_missing(self):
+        """Verify seed_url with no found entries raises MaasSeedDirNone"""
+        pass
+
+
+def populate_dir(seed_dir, files):
+    os.mkdir(seed_dir)
+    for (name, content) in files.iteritems():
+        with open(os.path.join(seed_dir, name), "w") as fp:
+            fp.write(content)
+            fp.close()
+
+# vi: ts=4 expandtab