From b55a9606e9455056a4280b06ef3785964af6d3df Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Thu, 26 Sep 2013 08:18:29 -0600 Subject: Added support for formating the ephemeral disk on Windows Azure. --- cloudinit/sources/DataSourceAzure.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index a77c3d9a..3f96a0a5 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -44,8 +44,17 @@ BUILTIN_DS_CONFIG = { 'policy': True, 'command': BOUNCE_COMMAND, 'hostname_command': 'hostname', - } + }, + 'ephemeral_disk': '/dev/sdb1', + 'disk_setup': { + 'ephemeral0': {'table_type': 'mbr', + 'layout': True, + 'overwrite': False} + }, + 'fs_setup': [{'filesystem': 'ext4', 'device': '/dev/sdb', + 'partition': 'auto'}], } + DS_CFG_PATH = ['datasource', DS_NAME] @@ -111,11 +120,22 @@ class DataSourceAzureNet(sources.DataSource): if seed: self.metadata['random_seed'] = seed + # now update ds_cfg to reflect contents pass in config usercfg = util.get_cfg_by_path(self.cfg, DS_CFG_PATH, {}) self.ds_cfg = util.mergemanydict([usercfg, self.ds_cfg]) mycfg = self.ds_cfg + # Put the disk format elements into the config object + if 'disk_setup' in mycfg: + self.cfg['disk_setup'] = mycfg['disk_setup'] + + if 'fs_setup' in mycfg: + self.cfg['fs_setup'] = mycfg['fs_setup'] + + if 'ephemeral_disk' in mycfg: + self.cfg['ephemeral_disk'] = mycfg['ephemeral_disk'] + # walinux agent writes files world readable, but expects # the directory to be protected. write_files(mycfg['data_dir'], files, dirmode=0700) @@ -161,9 +181,12 @@ class DataSourceAzureNet(sources.DataSource): pubkeys = pubkeys_from_crt_files(fp_files) self.metadata['public-keys'] = pubkeys - return True + def device_name_to_device(self, name): + if 'ephemeral0' in name and 'ephemeral_disk' in self.cfg: + return self.cfg['ephemeral_disk'] + def get_config_obj(self): return self.cfg -- cgit v1.2.3 From 85bce58f1202dc2868e7d3f0abd7f3fcd96700f0 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 Sep 2013 10:56:23 -0400 Subject: fix pep8 --- cloudinit/sources/DataSourceAzure.py | 1 - 1 file changed, 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 3f96a0a5..ef3f073a 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -120,7 +120,6 @@ class DataSourceAzureNet(sources.DataSource): if seed: self.metadata['random_seed'] = seed - # now update ds_cfg to reflect contents pass in config usercfg = util.get_cfg_by_path(self.cfg, DS_CFG_PATH, {}) self.ds_cfg = util.mergemanydict([usercfg, self.ds_cfg]) -- cgit v1.2.3 From 4d13fa79b32636bfab27540c76f46fcfb7ce7f5c Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 Sep 2013 11:46:54 -0400 Subject: re-work 'ephemeral_disk' and location of builtin config Previously we had this 'ephemeral_disk' entry in the datasource config for Azure, and then we also copied some entries into the .cfg for that datasource from the datasource config. Ie, datasource['Azure']['disk_setup'] would be oddly copied into the .cfg object that was returned by 'get_config_obj' Now, instead, we have a BUILTIN_CLOUD_CONFIG, which has those same values in it. The other change here is that 'ephemeral_disk' now has no meaning. Instead, we add a populated-by-default entry 'disk_aliases' to the BUILTIN_DS_CFG, and then just return entries in it for 'device_name_to_device' --- cloudinit/sources/DataSourceAzure.py | 24 ++++++++---------------- cloudinit/sources/DataSourceSmartOS.py | 12 ++++++------ 2 files changed, 14 insertions(+), 22 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index ef3f073a..f7ca9eb6 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -45,7 +45,10 @@ BUILTIN_DS_CONFIG = { 'command': BOUNCE_COMMAND, 'hostname_command': 'hostname', }, - 'ephemeral_disk': '/dev/sdb1', + 'disk_aliases': {'ephemeral0': '/dev/sdb1'}, +} + +BUILTIN_CLOUD_CONFIG = { 'disk_setup': { 'ephemeral0': {'table_type': 'mbr', 'layout': True, @@ -103,7 +106,7 @@ class DataSourceAzureNet(sources.DataSource): (md, self.userdata_raw, cfg, files) = ret self.seed = cdev self.metadata = util.mergemanydict([md, DEFAULT_METADATA]) - self.cfg = cfg + self.cfg = util.mergemanydict([cfg, BUILTIN_CLOUD_CONFIG]) found = cdev LOG.debug("found datasource in %s", cdev) @@ -121,20 +124,10 @@ class DataSourceAzureNet(sources.DataSource): self.metadata['random_seed'] = seed # now update ds_cfg to reflect contents pass in config - usercfg = util.get_cfg_by_path(self.cfg, DS_CFG_PATH, {}) - self.ds_cfg = util.mergemanydict([usercfg, self.ds_cfg]) + user_ds_cfg = util.get_cfg_by_path(self.cfg, DS_CFG_PATH, {}) + self.ds_cfg = util.mergemanydict([user_ds_cfg, self.ds_cfg]) mycfg = self.ds_cfg - # Put the disk format elements into the config object - if 'disk_setup' in mycfg: - self.cfg['disk_setup'] = mycfg['disk_setup'] - - if 'fs_setup' in mycfg: - self.cfg['fs_setup'] = mycfg['fs_setup'] - - if 'ephemeral_disk' in mycfg: - self.cfg['ephemeral_disk'] = mycfg['ephemeral_disk'] - # walinux agent writes files world readable, but expects # the directory to be protected. write_files(mycfg['data_dir'], files, dirmode=0700) @@ -183,8 +176,7 @@ class DataSourceAzureNet(sources.DataSource): return True def device_name_to_device(self, name): - if 'ephemeral0' in name and 'ephemeral_disk' in self.cfg: - return self.cfg['ephemeral_disk'] + return self.ds_cfg['disk_aliases'].get(name) def get_config_obj(self): return self.cfg diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index da1eec79..050325fa 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -72,7 +72,10 @@ BUILTIN_DS_CONFIG = { 'iptables_disable'], 'base64_keys': [], 'base64_all': False, - 'ephemeral_disk': '/dev/vdb', + 'disk_aliases': {'ephemeral0': '/dev/vdb'}, +} + +BUILTIN_CLOUD_CONFIG = { 'disk_setup': { 'ephemeral0': {'table_type': 'mbr', 'layout': True, @@ -94,9 +97,7 @@ class DataSourceSmartOS(sources.DataSource): BUILTIN_DS_CONFIG]) self.metadata = {} - self.cfg = {} - self.cfg['disk_setup'] = self.ds_cfg.get('disk_setup') - self.cfg['fs_setup'] = self.ds_cfg.get('fs_setup') + self.cfg = BUILTIN_CLOUD_CONFIG self.seed = self.ds_cfg.get("serial_device") self.seed_timeout = self.ds_cfg.get("serial_timeout") @@ -154,8 +155,7 @@ class DataSourceSmartOS(sources.DataSource): return True def device_name_to_device(self, name): - if 'ephemeral0' in name: - return self.ds_cfg['ephemeral_disk'] + return self.ds_cfg['disk_aliases'].get(name) def get_config_obj(self): return self.cfg -- cgit v1.2.3 From 55490f1f040f7e00985375872938c80e41803f4e Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 Sep 2013 12:26:25 -0400 Subject: fix tests small other changes Also * cloudinit/sources/DataSourceAzure.py: invalid xml in a file called 'ovfenv.xml' should raise BrokenAzureDatasource rather than NonAzureDataSource * cloudinit/sources/DataSourceSmartOS.py: cloudinit/sources/DataSourceAzure.py use 'ephemeral0' as the device name in builtin fs_setup * tests/unittests/test_datasource/test_azure.py: * always patch 'list_possible_azure_ds_devs' as it calls find_devs_with which calls blkid, and dramatically was slowing down tests on my system. * test_user_cfg_set_agent_command_plain: fix this test to not depend on specific format of yaml.dumps(). * test_userdata_arrives: add a test that user-data makes it through --- cloudinit/sources/DataSourceAzure.py | 4 +- cloudinit/sources/DataSourceSmartOS.py | 2 +- tests/unittests/test_datasource/test_azure.py | 51 ++++++++++++++----------- tests/unittests/test_datasource/test_smartos.py | 28 +++++++------- 4 files changed, 44 insertions(+), 41 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index f7ca9eb6..a9680d79 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -54,7 +54,7 @@ BUILTIN_CLOUD_CONFIG = { 'layout': True, 'overwrite': False} }, - 'fs_setup': [{'filesystem': 'ext4', 'device': '/dev/sdb', + 'fs_setup': [{'filesystem': 'ext4', 'device': 'ephemeral0', 'partition': 'auto'}], } @@ -363,7 +363,7 @@ def read_azure_ovf(contents): try: dom = minidom.parseString(contents) except Exception as e: - raise NonAzureDataSource("invalid xml: %s" % e) + raise BrokenAzureDataSource("invalid xml: %s" % e) results = find_child(dom.documentElement, lambda n: n.localName == "ProvisioningSection") diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 050325fa..fa26633a 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -82,7 +82,7 @@ BUILTIN_CLOUD_CONFIG = { 'overwrite': False} }, 'fs_setup': [{'label': 'ephemeral0', 'filesystem': 'ext3', - 'device': '/dev/xvdb', 'partition': 'auto'}], + 'device': 'ephemeral0', 'partition': 'auto'}], } diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index aa37337c..3bcc9520 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -120,8 +120,7 @@ class TestAzureDataSource(MockerTestCase): mod = DataSourceAzure - if data.get('dsdevs'): - self.apply_patches([(mod, 'list_possible_azure_ds_devs', dsdevs)]) + self.apply_patches([(mod, 'list_possible_azure_ds_devs', dsdevs)]) self.apply_patches([(mod, 'invoke_agent', _invoke_agent), (mod, 'write_files', _write_files), @@ -154,9 +153,12 @@ class TestAzureDataSource(MockerTestCase): def test_user_cfg_set_agent_command_plain(self): # set dscfg in via plaintext - cfg = {'agent_command': "my_command"} + # we must have friendly-to-xml formatted plaintext in yaml_cfg + # not all plaintext is expected to work. + yaml_cfg = "{agent_command: my_command}\n" + cfg = yaml.safe_load(yaml_cfg) odata = {'HostName': "myhost", 'UserName': "myuser", - 'dscfg': {'text': yaml.dump(cfg), 'encoding': 'plain'}} + 'dscfg': {'text': yaml_cfg, 'encoding': 'plain'}} data = {'ovfcontent': construct_valid_ovf_env(data=odata)} dsrc = self._get_ds(data) @@ -308,38 +310,41 @@ class TestAzureDataSource(MockerTestCase): self.assertIsInstance(cfg['disk_setup'], dict) self.assertIsInstance(cfg['fs_setup'], list) - def test_overriden_ephemeral(self): - # Make sure that the merge happens correctly - dscfg = {'ephemeral_disk': '/dev/sdc', - 'disk_setup': {'/dev/sdc': {'something': '...'}, - 'ephemeral0': False, - }, - 'fs_setup': [{'label': 'something'}]} + def test_provide_disk_aliases(self): + # Make sure that user can affect disk aliases + dscfg = {'disk_aliases': {'ephemeral0': '/dev/sdc'}} odata = {'HostName': "myhost", 'UserName': "myuser", - 'dscfg': {'text': yaml.dump(dscfg), 'encoding': 'plain'}} - data = {'ovfcontent': construct_valid_ovf_env(data=odata), - 'sys_cfg': {}} + 'dscfg': {'text': base64.b64encode(yaml.dump(dscfg)), + 'encoding': 'base64'}} + usercfg = {'disk_setup': {'/dev/sdc': {'something': '...'}, + 'ephemeral0': False}} + userdata = '#cloud-config' + yaml.dump(usercfg) + "\n" + + ovfcontent = construct_valid_ovf_env(data=odata, userdata=userdata) + data = {'ovfcontent': ovfcontent, 'sys_cfg': {}} dsrc = self._get_ds(data) ret = dsrc.get_data() - cfg = dsrc.get_config_obj() self.assertTrue(ret) + cfg = dsrc.get_config_obj() self.assertTrue(cfg) self.assertEquals(dsrc.device_name_to_device("ephemeral0"), "/dev/sdc") - assert 'disk_setup' in cfg - assert 'fs_setup' in cfg - self.assertIsInstance(cfg['disk_setup'], dict) - self.assertIsInstance(cfg['fs_setup'], list) - assert 'ephemeral0' in cfg['disk_setup'] - assert '/dev/sdc' in cfg['disk_setup'] - self.assertFalse(cfg['disk_setup']['ephemeral0']) + + def test_userdata_arrives(self): + userdata = "This is my user-data" + xml = construct_valid_ovf_env(data={}, userdata=userdata) + data = {'ovfcontent': xml} + dsrc = self._get_ds(data) + dsrc.get_data() + + self.assertEqual(userdata, dsrc.userdata_raw) class TestReadAzureOvf(MockerTestCase): def test_invalid_xml_raises_non_azure_ds(self): invalid_xml = "" + construct_valid_ovf_env(data={}) - self.assertRaises(DataSourceAzure.NonAzureDataSource, + self.assertRaises(DataSourceAzure.BrokenAzureDataSource, DataSourceAzure.read_azure_ovf, invalid_xml) def test_load_with_pubkeys(self): diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 56fe811e..956767d8 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -79,7 +79,6 @@ class MockSerial(object): if self.last in self.mockdata: if not self.mocked_out: self.mocked_out = [x for x in self._format_out()] - print self.mocked_out if len(self.mocked_out) > self.count: self.count += 1 @@ -275,26 +274,25 @@ class TestSmartOSDataSource(MockerTestCase): self.assertIsInstance(cfg['disk_setup'], dict) self.assertIsInstance(cfg['fs_setup'], list) - def test_override_builtin_ds(self): + def test_override_disk_aliases(self): # Test to make sure that the built-in DS is overriden - data = {} - data['disk_setup'] = {'test_dev': {}} - data['fs_setup'] = [{'label': 'test_dev'}] - data['serial_device'] = '/dev/ttyS2' - dsrc = self._get_ds(ds_cfg=data) - cfg = dsrc.get_config_obj() + builtin = DataSourceSmartOS.BUILTIN_DS_CONFIG + + mydscfg = {'disk_aliases': {'FOO': '/dev/bar'}} + # expect that these values are in builtin, or this is pointless + for k in mydscfg: + self.assertIn(k, builtin) + + dsrc = self._get_ds(ds_cfg=mydscfg) ret = dsrc.get_data() self.assertTrue(ret) - assert 'disk_setup' in cfg - assert 'fs_setup' in cfg - self.assertIsInstance(cfg['disk_setup'], dict) - self.assertIsInstance(cfg['fs_setup'], list) - assert 'test_dev' in cfg['disk_setup'] - assert 'test_dev' in cfg['fs_setup'][0]['label'] + self.assertEqual(mydscfg['disk_aliases']['FOO'], + dsrc.ds_cfg['disk_aliases']['FOO']) - self.assertEquals(data['serial_device'], dsrc.seed) + self.assertEqual(dsrc.device_name_to_device('FOO'), + mydscfg['disk_aliases']['FOO']) def apply_patches(patches): -- cgit v1.2.3 From 447404c45887d4e7de54ab4ac5278b05f6324691 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 Sep 2013 12:56:01 -0400 Subject: fix probably debug code that explicitly set 'partition' to 'any'. --- cloudinit/config/cc_disk_setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index fb404c5d..8b7581a7 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -576,10 +576,9 @@ def mkfs(cloud, fs_cfg): When 'cmd' is provided then no other parameter is required. """ - fs_cfg['partition'] = 'any' label = fs_cfg.get('label') device = fs_cfg.get('device') - partition = str(fs_cfg.get('partition')) + partition = str(fs_cfg.get('partition'), 'any') fs_type = fs_cfg.get('filesystem') fs_cmd = fs_cfg.get('cmd', []) fs_opts = fs_cfg.get('extra_opts', []) -- cgit v1.2.3 From e192858b1d1e0212455b5a4bb017c8d7216fc12f Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 Sep 2013 13:00:35 -0400 Subject: azure data source 'ephemeral0' point to '/dev/sdb', not /dev/sdb1 in general block device mappings should be to block devices, not partitoins. --- cloudinit/sources/DataSourceAzure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index a9680d79..7ba6cea8 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -45,7 +45,7 @@ BUILTIN_DS_CONFIG = { 'command': BOUNCE_COMMAND, 'hostname_command': 'hostname', }, - 'disk_aliases': {'ephemeral0': '/dev/sdb1'}, + 'disk_aliases': {'ephemeral0': '/dev/sdb'}, } BUILTIN_CLOUD_CONFIG = { -- cgit v1.2.3 From 7c7ba39a8f40530172f7f9d91592b2cf1d402f87 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 Sep 2013 13:16:38 -0400 Subject: fix syntax error in last commit --- cloudinit/config/cc_disk_setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index 8b7581a7..a170f81f 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -578,7 +578,7 @@ def mkfs(cloud, fs_cfg): """ label = fs_cfg.get('label') device = fs_cfg.get('device') - partition = str(fs_cfg.get('partition'), 'any') + partition = str(fs_cfg.get('partition', 'any')) fs_type = fs_cfg.get('filesystem') fs_cmd = fs_cfg.get('cmd', []) fs_opts = fs_cfg.get('extra_opts', []) -- cgit v1.2.3 From 2c8c4c25adf2d75c6a769fa8b05151716ea4a98c Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Fri, 27 Sep 2013 11:35:36 -0600 Subject: Disable partitioning of ephemeral for SmartOS at request of Joyent --- cloudinit/sources/DataSourceSmartOS.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index fa26633a..93b8b50b 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -78,7 +78,7 @@ BUILTIN_DS_CONFIG = { BUILTIN_CLOUD_CONFIG = { 'disk_setup': { 'ephemeral0': {'table_type': 'mbr', - 'layout': True, + 'layout': False, 'overwrite': False} }, 'fs_setup': [{'label': 'ephemeral0', 'filesystem': 'ext3', -- cgit v1.2.3 From 1388890380ed4edf47c2ad0421cedfc117f44cdd Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 Sep 2013 15:45:56 -0400 Subject: update to get functional with any alias in a 'device' entry --- cloudinit/config/cc_disk_setup.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index a170f81f..85e184cb 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -513,9 +513,6 @@ def mkpart(device, cloud, definition): # Check if the default device is a partition or not LOG.debug("Checking against default devices") if _device and (_device != device): - if not is_device_valid(_device): - _device = _device[:-1] - if not is_device_valid(_device): raise Exception("Unable to find backing block device for %s" % \ device) @@ -586,7 +583,7 @@ def mkfs(cloud, fs_cfg): # This allows you to define the default ephemeral or swap LOG.debug("Checking %s against default devices" % device) - _device = is_default_device(label, cloud, fallback=device) + _device = is_default_device(device, cloud, fallback=device) if _device and (_device != device): if not is_device_valid(_device): raise Exception("Unable to find backing block device for %s" % \ -- cgit v1.2.3 From 1c79f38fb2a9d11a0ab7743cf5d2965b2c3cbd56 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 Sep 2013 15:58:56 -0400 Subject: adjust aliases early rather than late. this adds 2 functions update_disk_setup_devices update_fs_setup_devices Which update the appropriate datatype, and translate the names. Translating early means we don't have to deal with updating in the mkfs or mkpart calls explicitly. These are more easily unit tested as they just take a dictionary of the expected type and a 'transformer' that should return a new name or None. --- cloudinit/config/cc_disk_setup.py | 79 ++++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 39 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index 85e184cb..487a2582 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -41,6 +41,7 @@ def handle(_name, cfg, cloud, log, _args): """ disk_setup = cfg.get("disk_setup") if isinstance(disk_setup, dict): + update_disk_setup_devices(disk_setup, cloud.device_name_to_device) log.info("Partitioning disks.") for disk, definition in disk_setup.items(): if not isinstance(definition, dict): @@ -51,13 +52,13 @@ def handle(_name, cfg, cloud, log, _args): log.debug("Creating new partition table/disk") util.log_time(logfunc=LOG.debug, msg="Creating partition on %s" % disk, - func=mkpart, args=(disk, cloud, definition)) + func=mkpart, args=(disk, definition)) except Exception as e: util.logexc(LOG, "Failed partitioning operation\n%s" % e) fs_setup = cfg.get("fs_setup") if isinstance(fs_setup, list): - log.info("Creating file systems.") + update_fs_setup_devices(fs_setup, cloud.device_name_to_device) for definition in fs_setup: if not isinstance(definition, dict): log.warn("Invalid file system definition: %s" % definition) @@ -68,31 +69,48 @@ def handle(_name, cfg, cloud, log, _args): device = definition.get('device') util.log_time(logfunc=LOG.debug, msg="Creating fs for %s" % device, - func=mkfs, args=(cloud, definition)) + func=mkfs, args=(definition,)) except Exception as e: util.logexc(LOG, "Failed during filesystem operation\n%s" % e) -def is_default_device(name, cloud, fallback=None): - """ - Ask the cloud datasource if the 'name' maps to a default - device. If so, return that value, otherwise return 'name', or - fallback if so defined. - """ - - _dev = None - try: - _dev = cloud.device_name_to_device(name) - except Exception as e: - util.logexc(LOG, "Failed to find mapping for %s" % e) +def update_disk_setup_devices(disk_setup, tformer): + # update 'disk_setup' dictionary anywhere were a device may occur + # update it with the response from 'tformer' + for origname in disk_setup.keys(): + transformed = tformer(origname) + if transformed is None or transformed == origname: + continue + if transformed in disk_setup: + LOG.info("Replacing %s in disk_setup for translation of %s", + origname, transformed) + del disk_setup[transformed] + + disk_setup[transformed] = disk_setup[origname] + disk_setup[transformed]['_origname'] = origname + del disk_setup[origname] + LOG.debug("updated disk_setup device entry '%s' to '%s'", + origname, transformed) + + +def update_fs_setup_devices(disk_setup, tformer): + # update 'fs_setup' dictionary anywhere were a device may occur + # update it with the response from 'tformer' + for definition in disk_setup: + if not isinstance(definition, dict): + LOG.warn("entry in disk_setup not a dict: %s", definition) + continue - if _dev: - return _dev + origname = definition.get('device') + if origname is None: + continue - if fallback: - return fallback + transformed = tformer(origname) + if transformed is None or transformed == origname: + continue - return name + definition['_origname'] = origname + definition['device'] = transformed def value_splitter(values, start=None): @@ -488,12 +506,11 @@ def exec_mkpart(table_type, device, layout): return get_dyn_func("exec_mkpart_%s", table_type, device, layout) -def mkpart(device, cloud, definition): +def mkpart(device, definition): """ Creates the partition table. Parameters: - cloud: the cloud object definition: dictionary describing how to create the partition. The following are supported values in the dict: @@ -508,17 +525,9 @@ def mkpart(device, cloud, definition): overwrite = definition.get('overwrite', False) layout = definition.get('layout', False) table_type = definition.get('table_type', 'mbr') - _device = is_default_device(device, cloud) # Check if the default device is a partition or not LOG.debug("Checking against default devices") - if _device and (_device != device): - if not is_device_valid(_device): - raise Exception("Unable to find backing block device for %s" % \ - device) - else: - LOG.debug("Mapped %s to physical device %s" % (device, _device)) - device = _device if (isinstance(layout, bool) and not layout) or not layout: LOG.debug("Device is not to be partitioned, skipping") @@ -552,7 +561,7 @@ def mkpart(device, cloud, definition): LOG.debug("Partition table created for %s" % device) -def mkfs(cloud, fs_cfg): +def mkfs(fs_cfg): """ Create a file system on the device. @@ -583,14 +592,6 @@ def mkfs(cloud, fs_cfg): # This allows you to define the default ephemeral or swap LOG.debug("Checking %s against default devices" % device) - _device = is_default_device(device, cloud, fallback=device) - if _device and (_device != device): - if not is_device_valid(_device): - raise Exception("Unable to find backing block device for %s" % \ - device) - else: - LOG.debug("Mapped %s to physical device %s" % (device, _device)) - device = _device if not partition or partition.isdigit(): # Handle manual definition of partition -- cgit v1.2.3 From e2ad3a6634e8eb5f0fa93a83bc1577b67e335129 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 Sep 2013 16:17:58 -0400 Subject: demote .info to .debug, add another debug --- cloudinit/config/cc_disk_setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index 487a2582..e64176ba 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -42,7 +42,7 @@ def handle(_name, cfg, cloud, log, _args): disk_setup = cfg.get("disk_setup") if isinstance(disk_setup, dict): update_disk_setup_devices(disk_setup, cloud.device_name_to_device) - log.info("Partitioning disks.") + log.debug("Partitioning disks: %s", str(disk_setup)) for disk, definition in disk_setup.items(): if not isinstance(definition, dict): log.warn("Invalid disk definition for %s" % disk) @@ -58,6 +58,7 @@ def handle(_name, cfg, cloud, log, _args): fs_setup = cfg.get("fs_setup") if isinstance(fs_setup, list): + log.debug("setting up filesystems: %s", str(fs_setup)) update_fs_setup_devices(fs_setup, cloud.device_name_to_device) for definition in fs_setup: if not isinstance(definition, dict): -- cgit v1.2.3 From 5b0e1eb11f12772e73205657355752b8c64d20cd Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 Sep 2013 16:44:41 -0400 Subject: fix one bug --- cloudinit/config/cc_disk_setup.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index e64176ba..ebff6e62 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -650,13 +650,14 @@ def mkfs(fs_cfg): # Make sure the device is defined if not device: - LOG.critical("Device is not known: %s" % fs_cfg) + LOG.warn("Device is not known: %s", device) return # Check that we can create the FS - if not label or not fs_type: - LOG.debug("Command to create filesystem %s is bad. Skipping." % \ - label) + if not (fs_type or fs_cmd): + raise Exception("No way to create filesystem '%s'. fs_type or fs_cmd " + "must be set.", label) + # Create the commands if fs_cmd: -- cgit v1.2.3 From 327c023bffc4ab2e30bb9839d0062159be848147 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 Sep 2013 16:53:45 -0400 Subject: change LOG.debug('format' % var) to ('format', var) --- cloudinit/config/cc_disk_setup.py | 56 +++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 29 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index ebff6e62..74965899 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -238,8 +238,8 @@ def find_device_node(device, fs_type=None, label=None, valid_targets=None, for key, value in value_splitter(part): d[key.lower()] = value - if d['fstype'] == fs_type and \ - ((label_match and d['label'] == label) or not label_match): + if (d['fstype'] == fs_type and + ((label_match and d['label'] == label) or not label_match)): # If we find a matching device, we return that return ('/dev/%s' % d['name'], True) @@ -416,8 +416,8 @@ def get_partition_mbr_layout(size, layout): # Create a single partition return "0," - if (len(layout) == 0 and isinstance(layout, list)) or \ - not isinstance(layout, list): + if ((len(layout) == 0 and isinstance(layout, list)) or + not isinstance(layout, list)): raise Exception("Partition layout is invalid") last_part_num = len(layout) @@ -433,8 +433,7 @@ def get_partition_mbr_layout(size, layout): if isinstance(part, list): if len(part) != 2: - raise Exception("Partition was incorrectly defined: %s" % \ - part) + raise Exception("Partition was incorrectly defined: %s" % part) percent, part_type = part part_size = int((float(size) * (float(percent) / 100)) / 1024) @@ -535,9 +534,9 @@ def mkpart(device, definition): return # Device is not to be partitioned # This prevents you from overwriting the device - LOG.debug("Checking if device %s is a valid device" % device) + LOG.debug("Checking if device %s is a valid device", device) if not is_device_valid(device): - raise Exception("Device %s is not a disk device!" % device) + raise Exception("Device %s is not a disk device!", device) LOG.debug("Checking if device layout matches") if check_partition_layout(table_type, device, layout): @@ -556,10 +555,10 @@ def mkpart(device, definition): part_definition = get_partition_layout(table_type, device_size, layout) LOG.debug(" Layout is: %s" % part_definition) - LOG.debug("Creating partition table on %s" % device) + LOG.debug("Creating partition table on %s", device) exec_mkpart(table_type, device, part_definition) - LOG.debug("Partition table created for %s" % device) + LOG.debug("Partition table created for %s", device) def mkfs(fs_cfg): @@ -592,36 +591,36 @@ def mkfs(fs_cfg): overwrite = fs_cfg.get('overwrite', False) # This allows you to define the default ephemeral or swap - LOG.debug("Checking %s against default devices" % device) + LOG.debug("Checking %s against default devices", device) if not partition or partition.isdigit(): # Handle manual definition of partition if partition.isdigit(): device = "%s%s" % (device, partition) - LOG.debug("Manual request of partition %s for %s" % ( - partition, device)) + LOG.debug("Manual request of partition %s for %s", + partition, device) # Check to see if the fs already exists - LOG.debug("Checking device %s" % device) + LOG.debug("Checking device %s", device) check_label, check_fstype, _ = check_fs(device) - LOG.debug("Device %s has %s %s" % (device, check_label, check_fstype)) + LOG.debug("Device %s has %s %s", device, check_label, check_fstype) if check_label == label and check_fstype == fs_type: - LOG.debug("Existing file system found at %s" % device) + LOG.debug("Existing file system found at %s", device) if not overwrite: - LOG.warn("Device %s has required file system" % device) + LOG.debug("Device %s has required file system", device) return else: - LOG.warn("Destroying filesystem on %s" % device) + LOG.warn("Destroying filesystem on %s", device) else: - LOG.debug("Device %s is cleared for formating" % device) + LOG.debug("Device %s is cleared for formating", device) elif partition and str(partition).lower() in ('auto', 'any'): # For auto devices, we match if the filesystem does exist odevice = device - LOG.debug("Identifying device to create %s filesytem on" % label) + LOG.debug("Identifying device to create %s filesytem on", label) # any mean pick the first match on the device with matching fs_type label_match = True @@ -630,23 +629,22 @@ def mkfs(fs_cfg): device, reuse = find_device_node(device, fs_type=fs_type, label=label, label_match=label_match) - LOG.debug("Automatic device for %s identified as %s" % ( - odevice, device)) + LOG.debug("Automatic device for %s identified as %s", odevice, device) if reuse: LOG.debug("Found filesystem match, skipping formating.") return if not device: - LOG.debug("No device aviable that matches request.") - LOG.debug("Skipping fs creation for %s" % fs_cfg) + LOG.debug("No device aviable that matches request. " + "Skipping fs creation for %s", fs_cfg) return else: LOG.debug("Error in device identification handling.") return - LOG.debug("File system %s will be created on %s" % (label, device)) + LOG.debug("File system %s will be created on %s", label, device) # Make sure the device is defined if not device: @@ -658,7 +656,6 @@ def mkfs(fs_cfg): raise Exception("No way to create filesystem '%s'. fs_type or fs_cmd " "must be set.", label) - # Create the commands if fs_cmd: fs_cmd = fs_cfg['cmd'] % {'label': label, @@ -672,7 +669,8 @@ def mkfs(fs_cfg): mkfs_cmd = util.which("mk%s" % fs_type) if not mkfs_cmd: - LOG.critical("Unable to locate command to create filesystem.") + LOG.warn("Cannot create fstype '%s'. No mkfs.%s command", fs_type, + fs_type) return fs_cmd = [mkfs_cmd, device] @@ -684,8 +682,8 @@ def mkfs(fs_cfg): if fs_opts: fs_cmd.extend(fs_opts) - LOG.debug("Creating file system %s on %s" % (label, device)) - LOG.debug(" Using cmd: %s" % "".join(fs_cmd)) + LOG.debug("Creating file system %s on %s", label, device) + LOG.debug(" Using cmd: %s", "".join(fs_cmd)) try: util.subp(fs_cmd) except Exception as e: -- cgit v1.2.3 From 2cb3d6f52c9d1ee803f151c1aacda355974fb083 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 Sep 2013 17:08:40 -0400 Subject: find_device_node: treat label=None as label="" Since for a string there is no difference, we're just checking for this here. --- cloudinit/config/cc_disk_setup.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index 74965899..d274f81a 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -214,6 +214,10 @@ def find_device_node(device, fs_type=None, label=None, valid_targets=None, Note: This works with GPT partition tables! """ + # label of None is same as no label + if label is None: + label = "" + if not valid_targets: valid_targets = ['disk', 'part'] -- cgit v1.2.3