Add a openstack specific datasource

Openstack has a unique derivative datasource
that is gaining usage. Previously the config
drive datasource provided part of this functionality
as well as the ec2 datasource, but since new
functionality is being added to openstack is
seems benefical to combine the used parts into
one datasource just made for handling openstack
deployments.

This patch factors out the common logic shared
between the config drive and the openstack
metadata datasource and places that in a shared
helper file and then creates a new openstack
datasource that readers from the openstack metadata
service and refactors the config drive datasource
to use this common logic.

2 files changed, 37 insertions, 33 deletions

diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py
index d5935294..bd6cdd5d 100644
--- a/tests/unittests/test_datasource/test_configdrive.py
+++ b/tests/unittests/test_datasource/test_configdrive.py
@@ -9,6 +9,7 @@ from mocker import MockerTestCase
 from cloudinit import helpers
 from cloudinit import settings
 from cloudinit.sources import DataSourceConfigDrive as ds
+from cloudinit.sources.helpers import openstack
 from cloudinit import util
 
 from tests.unittests import helpers as unit_helpers
@@ -71,7 +72,7 @@ class TestConfigDriveDataSource(MockerTestCase):
 
     def test_ec2_metadata(self):
         populate_dir(self.tmp, CFG_DRIVE_FILES_V2)
-        found = ds.read_config_drive_dir(self.tmp)
+        found = ds.read_config_drive(self.tmp)
         self.assertTrue('ec2-metadata' in found)
         ec2_md = found['ec2-metadata']
         self.assertEqual(EC2_META, ec2_md)
@@ -81,7 +82,7 @@ class TestConfigDriveDataSource(MockerTestCase):
         cfg_ds = ds.DataSourceConfigDrive(settings.CFG_BUILTIN,
                                           None,
                                           helpers.Paths({}))
-        found = ds.read_config_drive_dir(self.tmp)
+        found = ds.read_config_drive(self.tmp)
         cfg_ds.metadata = found['metadata']
         name_tests = {
             'ami': '/dev/vda1',
@@ -112,7 +113,7 @@ class TestConfigDriveDataSource(MockerTestCase):
         cfg_ds = ds.DataSourceConfigDrive(settings.CFG_BUILTIN,
                                           None,
                                           helpers.Paths({}))
-        found = ds.read_config_drive_dir(self.tmp)
+        found = ds.read_config_drive(self.tmp)
         os_md = found['metadata']
         cfg_ds.metadata = os_md
         name_tests = {
@@ -140,7 +141,7 @@ class TestConfigDriveDataSource(MockerTestCase):
         cfg_ds = ds.DataSourceConfigDrive(settings.CFG_BUILTIN,
                                           None,
                                           helpers.Paths({}))
-        found = ds.read_config_drive_dir(self.tmp)
+        found = ds.read_config_drive(self.tmp)
         ec2_md = found['ec2-metadata']
         os_md = found['metadata']
         cfg_ds.ec2_metadata = ec2_md
@@ -165,13 +166,13 @@ class TestConfigDriveDataSource(MockerTestCase):
                 my_mock.replay()
                 device = cfg_ds.device_name_to_device(name)
                 self.assertEquals(dev_name, device)
-
+ 
     def test_dev_ec2_map(self):
         populate_dir(self.tmp, CFG_DRIVE_FILES_V2)
         cfg_ds = ds.DataSourceConfigDrive(settings.CFG_BUILTIN,
                                           None,
                                           helpers.Paths({}))
-        found = ds.read_config_drive_dir(self.tmp)
+        found = ds.read_config_drive(self.tmp)
         exists_mock = self.mocker.replace(os.path.exists,
                                           spec=False, passthrough=False)
         exists_mock(mocker.ARGS)
@@ -200,10 +201,11 @@ class TestConfigDriveDataSource(MockerTestCase):
 
         populate_dir(self.tmp, CFG_DRIVE_FILES_V2)
 
-        found = ds.read_config_drive_dir(self.tmp)
+        found = ds.read_config_drive(self.tmp)
 
         expected_md = copy(OSTACK_META)
         expected_md['instance-id'] = expected_md['uuid']
+        expected_md['local-hostname'] = expected_md['hostname']
 
         self.assertEqual(USER_DATA, found['userdata'])
         self.assertEqual(expected_md, found['metadata'])
@@ -219,10 +221,11 @@ class TestConfigDriveDataSource(MockerTestCase):
 
         populate_dir(self.tmp, data)
 
-        found = ds.read_config_drive_dir(self.tmp)
+        found = ds.read_config_drive(self.tmp)
 
         expected_md = copy(OSTACK_META)
         expected_md['instance-id'] = expected_md['uuid']
+        expected_md['local-hostname'] = expected_md['hostname']
 
         self.assertEqual(expected_md, found['metadata'])
 
@@ -235,8 +238,8 @@ class TestConfigDriveDataSource(MockerTestCase):
 
         populate_dir(self.tmp, data)
 
-        self.assertRaises(ds.BrokenConfigDriveDir,
-                          ds.read_config_drive_dir, self.tmp)
+        self.assertRaises(openstack.BrokenMetadata,
+                          ds.read_config_drive, self.tmp)
 
     def test_seed_dir_no_configdrive(self):
         """Verify that no metadata raises NonConfigDriveDir."""
@@ -247,14 +250,14 @@ class TestConfigDriveDataSource(MockerTestCase):
         data["openstack/latest/random-file.txt"] = "random-content"
         data["content/foo"] = "foocontent"
 
-        self.assertRaises(ds.NonConfigDriveDir,
-                          ds.read_config_drive_dir, my_d)
+        self.assertRaises(openstack.NonReadable,
+                          ds.read_config_drive, my_d)
 
     def test_seed_dir_missing(self):
         """Verify that missing seed_dir raises NonConfigDriveDir."""
         my_d = os.path.join(self.tmp, "nonexistantdirectory")
-        self.assertRaises(ds.NonConfigDriveDir,
-                          ds.read_config_drive_dir, my_d)
+        self.assertRaises(openstack.NonReadable,
+                          ds.read_config_drive, my_d)
 
     def test_find_candidates(self):
         devs_with_answers = {}
@@ -303,7 +306,7 @@ class TestConfigDriveDataSource(MockerTestCase):
 
 
 def cfg_ds_from_dir(seed_d):
-    found = ds.read_config_drive_dir(seed_d)
+    found = ds.read_config_drive(seed_d)
     cfg_ds = ds.DataSourceConfigDrive(settings.CFG_BUILTIN, None,
                                       helpers.Paths({}))
     populate_ds_from_read_config(cfg_ds, seed_d, found)
@@ -318,7 +321,7 @@ def populate_ds_from_read_config(cfg_ds, source, results):
     cfg_ds.metadata = results.get('metadata')
     cfg_ds.ec2_metadata = results.get('ec2-metadata')
     cfg_ds.userdata_raw = results.get('userdata')
-    cfg_ds.version = results.get('cfgdrive_ver')
+    cfg_ds.version = results.get('version')
 
 
 def populate_dir(seed_dir, files):
diff --git a/tests/unittests/test_ec2_util.py b/tests/unittests/test_ec2_util.py
index dd588aca..18d36d86 100644
--- a/tests/unittests/test_ec2_util.py
+++ b/tests/unittests/test_ec2_util.py
@@ -1,6 +1,7 @@
 from tests.unittests import helpers
 
 from cloudinit import ec2_utils as eu
+from cloudinit import url_helper as uh
 
 import httpretty as hp
 
@@ -40,11 +41,11 @@ class TestEc2Util(helpers.TestCase):
                         body="\n".join(['hostname',
                                         'instance-id',
                                         'ami-launch-index']))
-        hp.register_uri(hp.GET, eu.combine_url(base_url, 'hostname'),
+        hp.register_uri(hp.GET, uh.combine_url(base_url, 'hostname'),
                         status=200, body='ec2.fake.host.name.com')
-        hp.register_uri(hp.GET, eu.combine_url(base_url, 'instance-id'),
+        hp.register_uri(hp.GET, uh.combine_url(base_url, 'instance-id'),
                         status=200, body='123')
-        hp.register_uri(hp.GET, eu.combine_url(base_url, 'ami-launch-index'),
+        hp.register_uri(hp.GET, uh.combine_url(base_url, 'ami-launch-index'),
                         status=200, body='1')
         md = eu.get_instance_metadata(self.VERSION, retries=0)
         self.assertEquals(md['hostname'], 'ec2.fake.host.name.com')
@@ -58,14 +59,14 @@ class TestEc2Util(helpers.TestCase):
                         body="\n".join(['hostname',
                                         'instance-id',
                                         'public-keys/']))
-        hp.register_uri(hp.GET, eu.combine_url(base_url, 'hostname'),
+        hp.register_uri(hp.GET, uh.combine_url(base_url, 'hostname'),
                         status=200, body='ec2.fake.host.name.com')
-        hp.register_uri(hp.GET, eu.combine_url(base_url, 'instance-id'),
+        hp.register_uri(hp.GET, uh.combine_url(base_url, 'instance-id'),
                         status=200, body='123')
-        hp.register_uri(hp.GET, eu.combine_url(base_url, 'public-keys/'),
+        hp.register_uri(hp.GET, uh.combine_url(base_url, 'public-keys/'),
                         status=200, body='0=my-public-key')
         hp.register_uri(hp.GET,
-                        eu.combine_url(base_url, 'public-keys/0/openssh-key'),
+                        uh.combine_url(base_url, 'public-keys/0/openssh-key'),
                         status=200, body='ssh-rsa AAAA.....wZEf my-public-key')
         md = eu.get_instance_metadata(self.VERSION, retries=0, timeout=0.1)
         self.assertEquals(md['hostname'], 'ec2.fake.host.name.com')
@@ -79,18 +80,18 @@ class TestEc2Util(helpers.TestCase):
                         body="\n".join(['hostname',
                                         'instance-id',
                                         'public-keys/']))
-        hp.register_uri(hp.GET, eu.combine_url(base_url, 'hostname'),
+        hp.register_uri(hp.GET, uh.combine_url(base_url, 'hostname'),
                         status=200, body='ec2.fake.host.name.com')
-        hp.register_uri(hp.GET, eu.combine_url(base_url, 'instance-id'),
+        hp.register_uri(hp.GET, uh.combine_url(base_url, 'instance-id'),
                         status=200, body='123')
-        hp.register_uri(hp.GET, eu.combine_url(base_url, 'public-keys/'),
+        hp.register_uri(hp.GET, uh.combine_url(base_url, 'public-keys/'),
                         status=200,
                         body="\n".join(['0=my-public-key', '1=my-other-key']))
         hp.register_uri(hp.GET,
-                        eu.combine_url(base_url, 'public-keys/0/openssh-key'),
+                        uh.combine_url(base_url, 'public-keys/0/openssh-key'),
                         status=200, body='ssh-rsa AAAA.....wZEf my-public-key')
         hp.register_uri(hp.GET,
-                        eu.combine_url(base_url, 'public-keys/1/openssh-key'),
+                        uh.combine_url(base_url, 'public-keys/1/openssh-key'),
                         status=200, body='ssh-rsa AAAA.....wZEf my-other-key')
         md = eu.get_instance_metadata(self.VERSION, retries=0, timeout=0.1)
         self.assertEquals(md['hostname'], 'ec2.fake.host.name.com')
@@ -104,20 +105,20 @@ class TestEc2Util(helpers.TestCase):
                         body="\n".join(['hostname',
                                         'instance-id',
                                         'block-device-mapping/']))
-        hp.register_uri(hp.GET, eu.combine_url(base_url, 'hostname'),
+        hp.register_uri(hp.GET, uh.combine_url(base_url, 'hostname'),
                         status=200, body='ec2.fake.host.name.com')
-        hp.register_uri(hp.GET, eu.combine_url(base_url, 'instance-id'),
+        hp.register_uri(hp.GET, uh.combine_url(base_url, 'instance-id'),
                         status=200, body='123')
         hp.register_uri(hp.GET,
-                        eu.combine_url(base_url, 'block-device-mapping/'),
+                        uh.combine_url(base_url, 'block-device-mapping/'),
                         status=200,
                         body="\n".join(['ami', 'ephemeral0']))
         hp.register_uri(hp.GET,
-                        eu.combine_url(base_url, 'block-device-mapping/ami'),
+                        uh.combine_url(base_url, 'block-device-mapping/ami'),
                         status=200,
                         body="sdb")
         hp.register_uri(hp.GET,
-                        eu.combine_url(base_url,
+                        uh.combine_url(base_url,
                                        'block-device-mapping/ephemeral0'),
                         status=200,
                         body="sdc")