From b55a9606e9455056a4280b06ef3785964af6d3df Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Thu, 26 Sep 2013 08:18:29 -0600 Subject: Added support for formating the ephemeral disk on Windows Azure. --- cloudinit/sources/DataSourceAzure.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) (limited to 'cloudinit/sources/DataSourceAzure.py') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index a77c3d9a..3f96a0a5 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -44,8 +44,17 @@ BUILTIN_DS_CONFIG = { 'policy': True, 'command': BOUNCE_COMMAND, 'hostname_command': 'hostname', - } + }, + 'ephemeral_disk': '/dev/sdb1', + 'disk_setup': { + 'ephemeral0': {'table_type': 'mbr', + 'layout': True, + 'overwrite': False} + }, + 'fs_setup': [{'filesystem': 'ext4', 'device': '/dev/sdb', + 'partition': 'auto'}], } + DS_CFG_PATH = ['datasource', DS_NAME] @@ -111,11 +120,22 @@ class DataSourceAzureNet(sources.DataSource): if seed: self.metadata['random_seed'] = seed + # now update ds_cfg to reflect contents pass in config usercfg = util.get_cfg_by_path(self.cfg, DS_CFG_PATH, {}) self.ds_cfg = util.mergemanydict([usercfg, self.ds_cfg]) mycfg = self.ds_cfg + # Put the disk format elements into the config object + if 'disk_setup' in mycfg: + self.cfg['disk_setup'] = mycfg['disk_setup'] + + if 'fs_setup' in mycfg: + self.cfg['fs_setup'] = mycfg['fs_setup'] + + if 'ephemeral_disk' in mycfg: + self.cfg['ephemeral_disk'] = mycfg['ephemeral_disk'] + # walinux agent writes files world readable, but expects # the directory to be protected. write_files(mycfg['data_dir'], files, dirmode=0700) @@ -161,9 +181,12 @@ class DataSourceAzureNet(sources.DataSource): pubkeys = pubkeys_from_crt_files(fp_files) self.metadata['public-keys'] = pubkeys - return True + def device_name_to_device(self, name): + if 'ephemeral0' in name and 'ephemeral_disk' in self.cfg: + return self.cfg['ephemeral_disk'] + def get_config_obj(self): return self.cfg -- cgit v1.2.3 From 85bce58f1202dc2868e7d3f0abd7f3fcd96700f0 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 Sep 2013 10:56:23 -0400 Subject: fix pep8 --- cloudinit/sources/DataSourceAzure.py | 1 - 1 file changed, 1 deletion(-) (limited to 'cloudinit/sources/DataSourceAzure.py') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 3f96a0a5..ef3f073a 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -120,7 +120,6 @@ class DataSourceAzureNet(sources.DataSource): if seed: self.metadata['random_seed'] = seed - # now update ds_cfg to reflect contents pass in config usercfg = util.get_cfg_by_path(self.cfg, DS_CFG_PATH, {}) self.ds_cfg = util.mergemanydict([usercfg, self.ds_cfg]) -- cgit v1.2.3 From 4d13fa79b32636bfab27540c76f46fcfb7ce7f5c Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 Sep 2013 11:46:54 -0400 Subject: re-work 'ephemeral_disk' and location of builtin config Previously we had this 'ephemeral_disk' entry in the datasource config for Azure, and then we also copied some entries into the .cfg for that datasource from the datasource config. Ie, datasource['Azure']['disk_setup'] would be oddly copied into the .cfg object that was returned by 'get_config_obj' Now, instead, we have a BUILTIN_CLOUD_CONFIG, which has those same values in it. The other change here is that 'ephemeral_disk' now has no meaning. Instead, we add a populated-by-default entry 'disk_aliases' to the BUILTIN_DS_CFG, and then just return entries in it for 'device_name_to_device' --- cloudinit/sources/DataSourceAzure.py | 24 ++++++++---------------- cloudinit/sources/DataSourceSmartOS.py | 12 ++++++------ 2 files changed, 14 insertions(+), 22 deletions(-) (limited to 'cloudinit/sources/DataSourceAzure.py') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index ef3f073a..f7ca9eb6 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -45,7 +45,10 @@ BUILTIN_DS_CONFIG = { 'command': BOUNCE_COMMAND, 'hostname_command': 'hostname', }, - 'ephemeral_disk': '/dev/sdb1', + 'disk_aliases': {'ephemeral0': '/dev/sdb1'}, +} + +BUILTIN_CLOUD_CONFIG = { 'disk_setup': { 'ephemeral0': {'table_type': 'mbr', 'layout': True, @@ -103,7 +106,7 @@ class DataSourceAzureNet(sources.DataSource): (md, self.userdata_raw, cfg, files) = ret self.seed = cdev self.metadata = util.mergemanydict([md, DEFAULT_METADATA]) - self.cfg = cfg + self.cfg = util.mergemanydict([cfg, BUILTIN_CLOUD_CONFIG]) found = cdev LOG.debug("found datasource in %s", cdev) @@ -121,20 +124,10 @@ class DataSourceAzureNet(sources.DataSource): self.metadata['random_seed'] = seed # now update ds_cfg to reflect contents pass in config - usercfg = util.get_cfg_by_path(self.cfg, DS_CFG_PATH, {}) - self.ds_cfg = util.mergemanydict([usercfg, self.ds_cfg]) + user_ds_cfg = util.get_cfg_by_path(self.cfg, DS_CFG_PATH, {}) + self.ds_cfg = util.mergemanydict([user_ds_cfg, self.ds_cfg]) mycfg = self.ds_cfg - # Put the disk format elements into the config object - if 'disk_setup' in mycfg: - self.cfg['disk_setup'] = mycfg['disk_setup'] - - if 'fs_setup' in mycfg: - self.cfg['fs_setup'] = mycfg['fs_setup'] - - if 'ephemeral_disk' in mycfg: - self.cfg['ephemeral_disk'] = mycfg['ephemeral_disk'] - # walinux agent writes files world readable, but expects # the directory to be protected. write_files(mycfg['data_dir'], files, dirmode=0700) @@ -183,8 +176,7 @@ class DataSourceAzureNet(sources.DataSource): return True def device_name_to_device(self, name): - if 'ephemeral0' in name and 'ephemeral_disk' in self.cfg: - return self.cfg['ephemeral_disk'] + return self.ds_cfg['disk_aliases'].get(name) def get_config_obj(self): return self.cfg diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index da1eec79..050325fa 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -72,7 +72,10 @@ BUILTIN_DS_CONFIG = { 'iptables_disable'], 'base64_keys': [], 'base64_all': False, - 'ephemeral_disk': '/dev/vdb', + 'disk_aliases': {'ephemeral0': '/dev/vdb'}, +} + +BUILTIN_CLOUD_CONFIG = { 'disk_setup': { 'ephemeral0': {'table_type': 'mbr', 'layout': True, @@ -94,9 +97,7 @@ class DataSourceSmartOS(sources.DataSource): BUILTIN_DS_CONFIG]) self.metadata = {} - self.cfg = {} - self.cfg['disk_setup'] = self.ds_cfg.get('disk_setup') - self.cfg['fs_setup'] = self.ds_cfg.get('fs_setup') + self.cfg = BUILTIN_CLOUD_CONFIG self.seed = self.ds_cfg.get("serial_device") self.seed_timeout = self.ds_cfg.get("serial_timeout") @@ -154,8 +155,7 @@ class DataSourceSmartOS(sources.DataSource): return True def device_name_to_device(self, name): - if 'ephemeral0' in name: - return self.ds_cfg['ephemeral_disk'] + return self.ds_cfg['disk_aliases'].get(name) def get_config_obj(self): return self.cfg -- cgit v1.2.3 From 55490f1f040f7e00985375872938c80e41803f4e Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 Sep 2013 12:26:25 -0400 Subject: fix tests small other changes Also * cloudinit/sources/DataSourceAzure.py: invalid xml in a file called 'ovfenv.xml' should raise BrokenAzureDatasource rather than NonAzureDataSource * cloudinit/sources/DataSourceSmartOS.py: cloudinit/sources/DataSourceAzure.py use 'ephemeral0' as the device name in builtin fs_setup * tests/unittests/test_datasource/test_azure.py: * always patch 'list_possible_azure_ds_devs' as it calls find_devs_with which calls blkid, and dramatically was slowing down tests on my system. * test_user_cfg_set_agent_command_plain: fix this test to not depend on specific format of yaml.dumps(). * test_userdata_arrives: add a test that user-data makes it through --- cloudinit/sources/DataSourceAzure.py | 4 +- cloudinit/sources/DataSourceSmartOS.py | 2 +- tests/unittests/test_datasource/test_azure.py | 51 ++++++++++++++----------- tests/unittests/test_datasource/test_smartos.py | 28 +++++++------- 4 files changed, 44 insertions(+), 41 deletions(-) (limited to 'cloudinit/sources/DataSourceAzure.py') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index f7ca9eb6..a9680d79 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -54,7 +54,7 @@ BUILTIN_CLOUD_CONFIG = { 'layout': True, 'overwrite': False} }, - 'fs_setup': [{'filesystem': 'ext4', 'device': '/dev/sdb', + 'fs_setup': [{'filesystem': 'ext4', 'device': 'ephemeral0', 'partition': 'auto'}], } @@ -363,7 +363,7 @@ def read_azure_ovf(contents): try: dom = minidom.parseString(contents) except Exception as e: - raise NonAzureDataSource("invalid xml: %s" % e) + raise BrokenAzureDataSource("invalid xml: %s" % e) results = find_child(dom.documentElement, lambda n: n.localName == "ProvisioningSection") diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 050325fa..fa26633a 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -82,7 +82,7 @@ BUILTIN_CLOUD_CONFIG = { 'overwrite': False} }, 'fs_setup': [{'label': 'ephemeral0', 'filesystem': 'ext3', - 'device': '/dev/xvdb', 'partition': 'auto'}], + 'device': 'ephemeral0', 'partition': 'auto'}], } diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index aa37337c..3bcc9520 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -120,8 +120,7 @@ class TestAzureDataSource(MockerTestCase): mod = DataSourceAzure - if data.get('dsdevs'): - self.apply_patches([(mod, 'list_possible_azure_ds_devs', dsdevs)]) + self.apply_patches([(mod, 'list_possible_azure_ds_devs', dsdevs)]) self.apply_patches([(mod, 'invoke_agent', _invoke_agent), (mod, 'write_files', _write_files), @@ -154,9 +153,12 @@ class TestAzureDataSource(MockerTestCase): def test_user_cfg_set_agent_command_plain(self): # set dscfg in via plaintext - cfg = {'agent_command': "my_command"} + # we must have friendly-to-xml formatted plaintext in yaml_cfg + # not all plaintext is expected to work. + yaml_cfg = "{agent_command: my_command}\n" + cfg = yaml.safe_load(yaml_cfg) odata = {'HostName': "myhost", 'UserName': "myuser", - 'dscfg': {'text': yaml.dump(cfg), 'encoding': 'plain'}} + 'dscfg': {'text': yaml_cfg, 'encoding': 'plain'}} data = {'ovfcontent': construct_valid_ovf_env(data=odata)} dsrc = self._get_ds(data) @@ -308,38 +310,41 @@ class TestAzureDataSource(MockerTestCase): self.assertIsInstance(cfg['disk_setup'], dict) self.assertIsInstance(cfg['fs_setup'], list) - def test_overriden_ephemeral(self): - # Make sure that the merge happens correctly - dscfg = {'ephemeral_disk': '/dev/sdc', - 'disk_setup': {'/dev/sdc': {'something': '...'}, - 'ephemeral0': False, - }, - 'fs_setup': [{'label': 'something'}]} + def test_provide_disk_aliases(self): + # Make sure that user can affect disk aliases + dscfg = {'disk_aliases': {'ephemeral0': '/dev/sdc'}} odata = {'HostName': "myhost", 'UserName': "myuser", - 'dscfg': {'text': yaml.dump(dscfg), 'encoding': 'plain'}} - data = {'ovfcontent': construct_valid_ovf_env(data=odata), - 'sys_cfg': {}} + 'dscfg': {'text': base64.b64encode(yaml.dump(dscfg)), + 'encoding': 'base64'}} + usercfg = {'disk_setup': {'/dev/sdc': {'something': '...'}, + 'ephemeral0': False}} + userdata = '#cloud-config' + yaml.dump(usercfg) + "\n" + + ovfcontent = construct_valid_ovf_env(data=odata, userdata=userdata) + data = {'ovfcontent': ovfcontent, 'sys_cfg': {}} dsrc = self._get_ds(data) ret = dsrc.get_data() - cfg = dsrc.get_config_obj() self.assertTrue(ret) + cfg = dsrc.get_config_obj() self.assertTrue(cfg) self.assertEquals(dsrc.device_name_to_device("ephemeral0"), "/dev/sdc") - assert 'disk_setup' in cfg - assert 'fs_setup' in cfg - self.assertIsInstance(cfg['disk_setup'], dict) - self.assertIsInstance(cfg['fs_setup'], list) - assert 'ephemeral0' in cfg['disk_setup'] - assert '/dev/sdc' in cfg['disk_setup'] - self.assertFalse(cfg['disk_setup']['ephemeral0']) + + def test_userdata_arrives(self): + userdata = "This is my user-data" + xml = construct_valid_ovf_env(data={}, userdata=userdata) + data = {'ovfcontent': xml} + dsrc = self._get_ds(data) + dsrc.get_data() + + self.assertEqual(userdata, dsrc.userdata_raw) class TestReadAzureOvf(MockerTestCase): def test_invalid_xml_raises_non_azure_ds(self): invalid_xml = "" + construct_valid_ovf_env(data={}) - self.assertRaises(DataSourceAzure.NonAzureDataSource, + self.assertRaises(DataSourceAzure.BrokenAzureDataSource, DataSourceAzure.read_azure_ovf, invalid_xml) def test_load_with_pubkeys(self): diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 56fe811e..956767d8 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -79,7 +79,6 @@ class MockSerial(object): if self.last in self.mockdata: if not self.mocked_out: self.mocked_out = [x for x in self._format_out()] - print self.mocked_out if len(self.mocked_out) > self.count: self.count += 1 @@ -275,26 +274,25 @@ class TestSmartOSDataSource(MockerTestCase): self.assertIsInstance(cfg['disk_setup'], dict) self.assertIsInstance(cfg['fs_setup'], list) - def test_override_builtin_ds(self): + def test_override_disk_aliases(self): # Test to make sure that the built-in DS is overriden - data = {} - data['disk_setup'] = {'test_dev': {}} - data['fs_setup'] = [{'label': 'test_dev'}] - data['serial_device'] = '/dev/ttyS2' - dsrc = self._get_ds(ds_cfg=data) - cfg = dsrc.get_config_obj() + builtin = DataSourceSmartOS.BUILTIN_DS_CONFIG + + mydscfg = {'disk_aliases': {'FOO': '/dev/bar'}} + # expect that these values are in builtin, or this is pointless + for k in mydscfg: + self.assertIn(k, builtin) + + dsrc = self._get_ds(ds_cfg=mydscfg) ret = dsrc.get_data() self.assertTrue(ret) - assert 'disk_setup' in cfg - assert 'fs_setup' in cfg - self.assertIsInstance(cfg['disk_setup'], dict) - self.assertIsInstance(cfg['fs_setup'], list) - assert 'test_dev' in cfg['disk_setup'] - assert 'test_dev' in cfg['fs_setup'][0]['label'] + self.assertEqual(mydscfg['disk_aliases']['FOO'], + dsrc.ds_cfg['disk_aliases']['FOO']) - self.assertEquals(data['serial_device'], dsrc.seed) + self.assertEqual(dsrc.device_name_to_device('FOO'), + mydscfg['disk_aliases']['FOO']) def apply_patches(patches): -- cgit v1.2.3 From e192858b1d1e0212455b5a4bb017c8d7216fc12f Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 Sep 2013 13:00:35 -0400 Subject: azure data source 'ephemeral0' point to '/dev/sdb', not /dev/sdb1 in general block device mappings should be to block devices, not partitoins. --- cloudinit/sources/DataSourceAzure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/sources/DataSourceAzure.py') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index a9680d79..7ba6cea8 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -45,7 +45,7 @@ BUILTIN_DS_CONFIG = { 'command': BOUNCE_COMMAND, 'hostname_command': 'hostname', }, - 'disk_aliases': {'ephemeral0': '/dev/sdb1'}, + 'disk_aliases': {'ephemeral0': '/dev/sdb'}, } BUILTIN_CLOUD_CONFIG = { -- cgit v1.2.3