Add Digital Ocean Datasource.

This adds a DataSource for DigitalOcean's metadata service. The service is
documented at https://developers.digitalocean.com/metadata/ .

5 files changed, 252 insertions, 1 deletions

diff --git a/ChangeLog b/ChangeLog
index c33a45b9..d2fc2d69 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,6 @@
 0.7.7:
  - open 0.7.7
+ - Digital Ocean: add datasource for Digital Ocean. [Neal Shrader]
 0.7.6:
  - open 0.7.6
  - Enable vendordata on CloudSigma datasource (LP: #1303986)
diff --git a/cloudinit/sources/DataSourceDigitalOcean.py b/cloudinit/sources/DataSourceDigitalOcean.py
new file mode 100644
index 00000000..069bdb41
--- /dev/null
+++ b/cloudinit/sources/DataSourceDigitalOcean.py
@@ -0,0 +1,104 @@
+# vi: ts=4 expandtab
+#
+#    Author: Neal Shrader <neal@digitalocean.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from cloudinit import log as logging
+from cloudinit import util
+from cloudinit import sources
+from cloudinit import ec2_utils
+from types import StringType
+import functools
+
+
+LOG = logging.getLogger(__name__)
+
+BUILTIN_DS_CONFIG = {
+    'metadata_url': 'http://169.254.169.254/metadata/v1/',
+    'mirrors_url': 'http://mirrors.digitalocean.com/'
+}
+MD_RETRIES = 0
+MD_TIMEOUT = 1
+
+class DataSourceDigitalOcean(sources.DataSource):
+    def __init__(self, sys_cfg, distro, paths):
+        sources.DataSource.__init__(self, sys_cfg, distro, paths)
+        self.metadata = dict()
+        self.ds_cfg = util.mergemanydict([
+            util.get_cfg_by_path(sys_cfg, ["datasource", "DigitalOcean"], {}),
+            BUILTIN_DS_CONFIG])
+        self.metadata_address = self.ds_cfg['metadata_url']
+
+        if self.ds_cfg.get('retries'):
+            self.retries = self.ds_cfg['retries']
+        else:
+            self.retries = MD_RETRIES
+
+        if self.ds_cfg.get('timeout'):
+            self.timeout = self.ds_cfg['timeout']
+        else:
+            self.timeout = MD_TIMEOUT
+
+    def get_data(self):
+        caller = functools.partial(util.read_file_or_url, timeout=self.timeout, 
+                                   retries=self.retries)
+        md = ec2_utils.MetadataMaterializer(str(caller(self.metadata_address)),
+                                            base_url=self.metadata_address, 
+                                            caller=caller)
+
+        self.metadata = md.materialize()
+
+        if self.metadata.get('id'):
+            return True
+        else:
+            return False
+
+    def get_userdata_raw(self):
+        return "\n".join(self.metadata['user-data'])
+
+    def get_vendordata_raw(self):
+        return "\n".join(self.metadata['vendor-data'])
+
+    def get_public_ssh_keys(self):
+        if type(self.metadata['public-keys']) is StringType:
+           return [self.metadata['public-keys']]
+        else:
+           return self.metadata['public-keys']
+
+    @property
+    def availability_zone(self):
+        return self.metadata['region']
+
+    def get_instance_id(self):
+        return self.metadata['id']
+
+    def get_hostname(self, fqdn=False):
+        return self.metadata['hostname']
+
+    def get_package_mirror_info(self):
+        return self.ds_cfg['mirrors_url']
+
+    @property
+    def launch_index(self):
+        return None
+
+# Used to match classes to dependencies
+datasources = [
+  (DataSourceDigitalOcean, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
+  ]
+
+
+# Return a list of data sources that match this set of dependencies
+def get_datasource_list(depends):
+    return sources.list_from_depends(depends, datasources)
diff --git a/doc/sources/digitalocean/README.rst b/doc/sources/digitalocean/README.rst
new file mode 100644
index 00000000..1bb89fe1
--- /dev/null
+++ b/doc/sources/digitalocean/README.rst
@@ -0,0 +1,21 @@
+ The `DigitalOcean`_ datasource consumes the content served from DigitalOcean's `metadata service`_.  This
+metadata service serves information about the running droplet via HTTP over the link local address
+169.254.169.254.  The metadata API endpoints are fully described at 
+`https://developers.digitalocean.com/metadata/ <https://developers.digitalocean.com/metadata/>`_.
+
+Configuration
+~~~~~~~~~~~~~
+
+DigitalOcean's datasource can be configured as follows:
+
+  datasource:
+    DigitalOcean:
+      retries: 3
+      timeout: 2
+
+- *retries*: Determines the number of times to attempt to connect to the metadata service
+- *timeout*: Determines the timeout in seconds to wait for a response from the metadata service
+
+.. _DigitalOcean: http://digitalocean.com/
+.. _metadata service: https://developers.digitalocean.com/metadata/
+.. _Full documentation: https://developers.digitalocean.com/metadata/
diff --git a/tests/unittests/test_datasource/test_digitalocean.py b/tests/unittests/test_datasource/test_digitalocean.py
new file mode 100644
index 00000000..04bee340
--- /dev/null
+++ b/tests/unittests/test_datasource/test_digitalocean.py
@@ -0,0 +1,126 @@
+#
+#    Copyright (C) 2014 Neal Shrader
+#
+#    Author: Neal Shrader <neal@digitalocean.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import httpretty
+import re
+
+from types import ListType
+from urlparse import urlparse
+
+from cloudinit import settings
+from cloudinit import helpers
+from cloudinit.sources import DataSourceDigitalOcean
+
+from .. import helpers as test_helpers
+
+# Abbreviated for the test
+DO_INDEX = """id
+           hostname
+           user-data
+           vendor-data
+           public-keys
+           region"""
+
+DO_MULTIPLE_KEYS = """ssh-rsa AAAAB3NzaC1yc2EAAAA... neal@digitalocean.com
+                   ssh-rsa AAAAB3NzaC1yc2EAAAA... neal2@digitalocean.com"""
+DO_SINGLE_KEY = "ssh-rsa AAAAB3NzaC1yc2EAAAA... neal@digitalocean.com"
+
+DO_META = {
+    '': DO_INDEX,
+    'user-data': '#!/bin/bash\necho "user-data"',
+    'vendor-data': '#!/bin/bash\necho "vendor-data"',
+    'public-keys': DO_SINGLE_KEY,
+    'region': 'nyc3',
+    'id': '2000000',
+    'hostname': 'cloudinit-test',
+}
+
+MD_URL_RE = re.compile(r'http://169.254.169.254/metadata/v1/.*')
+
+def _request_callback(method, uri, headers):
+    url_path = urlparse(uri).path
+    if url_path.startswith('/metadata/v1/'):
+        path = url_path.split('/metadata/v1/')[1:][0]
+    else:
+        path = None
+    if path in DO_META:
+        return (200, headers, DO_META.get(path))
+    else:
+        return (404, headers, '')
+
+
+class TestDataSourceDigitalOcean(test_helpers.HttprettyTestCase):
+
+    def setUp(self):
+        self.ds = DataSourceDigitalOcean.DataSourceDigitalOcean(
+            settings.CFG_BUILTIN, None,
+            helpers.Paths({}))
+        super(TestDataSourceDigitalOcean, self).setUp()
+
+    @httpretty.activate
+    def test_connection(self):
+        httpretty.register_uri(
+            httpretty.GET, MD_URL_RE,
+            body=_request_callback)
+
+        success = self.ds.get_data()
+        self.assertTrue(success)
+
+    @httpretty.activate
+    def test_metadata(self):
+        httpretty.register_uri(
+            httpretty.GET, MD_URL_RE,
+            body=_request_callback)
+        self.ds.get_data()
+
+        self.assertEqual(DO_META.get('user-data'),
+                         self.ds.get_userdata_raw())
+
+        self.assertEqual(DO_META.get('vendor-data'),
+                         self.ds.get_vendordata_raw())
+
+        self.assertEqual(DO_META.get('region'),
+                         self.ds.availability_zone)
+
+        self.assertEqual(DO_META.get('id'),
+                         self.ds.get_instance_id())
+
+        self.assertEqual(DO_META.get('hostname'),
+                         self.ds.get_hostname())
+
+        self.assertEqual('http://mirrors.digitalocean.com/',
+                         self.ds.get_package_mirror_info())
+
+        # Single key
+        self.assertEqual([DO_META.get('public-keys')],
+                         self.ds.get_public_ssh_keys())
+
+        self.assertIs(type(self.ds.get_public_ssh_keys()), ListType)
+
+    @httpretty.activate
+    def test_multiple_ssh_keys(self):
+        DO_META['public_keys'] = DO_MULTIPLE_KEYS
+        httpretty.register_uri(
+            httpretty.GET, MD_URL_RE,
+            body=_request_callback)
+        self.ds.get_data()
+
+        # Multiple keys
+        self.assertEqual(DO_META.get('public-keys').splitlines(),
+                         self.ds.get_public_ssh_keys())
+
+        self.assertIs(type(self.ds.get_public_ssh_keys()), ListType)
diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py
index 8becbdd2..49894e51 100644
--- a/tests/unittests/test_datasource/test_openstack.py
+++ b/tests/unittests/test_datasource/test_openstack.py
@@ -19,7 +19,6 @@
 import copy
 import json
 import re
-import unittest
 
 from StringIO import StringIO