From ea88f3e1208ba501b50d1f41187b83cf11f15785 Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Wed, 2 Oct 2013 15:05:15 -0600 Subject: Added ability to define disks via 'ephemeralX.Y'. Modified cc_mounts to identify whether ephermalX is partitioned. Changed datasources for Azure and SmartOS to use 'ephemeralX.Y' format. Added disk remove functionally --- cloudinit/config/cc_mounts.py | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) (limited to 'cloudinit/config/cc_mounts.py') diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 390ba711..f4c2e3d8 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -20,6 +20,7 @@ from string import whitespace # pylint: disable=W0402 +import os.path import re from cloudinit import type_utils @@ -75,7 +76,9 @@ def handle(_name, cfg, cloud, log, _args): "name from ephemeral to ephemeral0"), (i + 1)) if is_mdname(startname): - newname = cloud.device_name_to_device(startname) + candidate_name = cloud.device_name_to_device(startname) + newname = disk_or_part(candidate_name) + if not newname: log.debug("Ignoring nonexistant named mount %s", startname) cfgmnt[i][1] = None @@ -119,7 +122,8 @@ def handle(_name, cfg, cloud, log, _args): # entry has the same device name for defmnt in defmnts: startname = defmnt[0] - devname = cloud.device_name_to_device(startname) + candidate_name = cloud.device_name_to_device(startname) + devname = disk_or_part(candidate_name) if devname is None: log.debug("Ignoring nonexistant named default mount %s", startname) continue @@ -198,3 +202,32 @@ def handle(_name, cfg, cloud, log, _args): util.subp(("mount", "-a")) except: util.logexc(log, "Activating mounts via 'mount -a' failed") + + +def disk_or_part(device): + """ + Find where the file system is on the disk, either on + the disk itself or on the first partition. We don't go + any deeper than partition 1 though. + """ + + if not device: + return None + + short_name = device.split('/')[-1] + sys_path = "/sys/block/%s" % short_name + + if not os.path.exists(sys_path): + LOG.warn("Device %s does not exist in sysfs" % device) + return None + + sys_long_path = sys_path + "/" + short_name + "%s" + valid_mappings = [ sys_long_path % "1", + sys_long_path % "p1", + sys_path ] + + for cdisk in valid_mappings: + if not os.path.exists(cdisk): + continue + return "/dev/%s" % cdisk.split('/')[-1] + return None -- cgit v1.2.3 From ecf2a600e41a9632ad305eb6a8cd5665908f31fb Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Thu, 3 Oct 2013 16:15:56 -0600 Subject: Moved ephemeralX.Y handling from Datasource into the cc_disk_setup, which makes it cloud agnostic. --- cloudinit/config/cc_disk_setup.py | 21 +++++++++++++++++++-- cloudinit/config/cc_mounts.py | 13 +++++++------ cloudinit/sources/DataSourceAzure.py | 4 +--- cloudinit/sources/DataSourceSmartOS.py | 4 +--- cloudinit/util.py | 11 +++++++---- 5 files changed, 35 insertions(+), 18 deletions(-) (limited to 'cloudinit/config/cc_mounts.py') diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index e744cd57..554dfdd3 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -104,13 +104,30 @@ def update_fs_setup_devices(disk_setup, tformer): continue origname = definition.get('device') + if origname is None: continue - transformed = tformer(origname) + transformed = None + if len(origname.split('.')) > 1: + # this maps ephemeralX.Y to a proper disk name. For example, + # if the origname is 'ephemeral0.1' and transformed is /dev/sdb + # then the returned device will be /dev/sdb1 _if_ /dev/sdb1 exists + # otherwise NONE + base_name = origname.split('.')[0] + tformed = tformer(base_name) + LOG.info("base device for %s is %s" % (origname, tformed)) + + transformed = util.map_device_alias(tformed, alias=origname) + LOG.info("%s is mapped to %s" % (origname, transformed)) + + else: + transformed = tformer(origname) + if transformed is None or transformed == origname: continue + LOG.info("Mapped %s to physical device %s" % (origname, transformed)) definition['_origname'] = origname definition['device'] = transformed @@ -497,7 +514,7 @@ def purge_disk(device): try: LOG.info("Purging filesystem on /dev/%s" % d['name']) util.subp(wipefs_cmd) - except Exception as e: + except Exception: raise Exception("Failed FS purge of /dev/%s" % d['name']) purge_disk_ptable(device) diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index f4c2e3d8..6b47d326 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -217,14 +217,15 @@ def disk_or_part(device): short_name = device.split('/')[-1] sys_path = "/sys/block/%s" % short_name - if not os.path.exists(sys_path): - LOG.warn("Device %s does not exist in sysfs" % device) - return None + # if the sys path does not exist but the device exists, + # then the device is a partition, no sense looking any further + if not os.path.exists(sys_path) and os.path.exists(device): + return device sys_long_path = sys_path + "/" + short_name + "%s" - valid_mappings = [ sys_long_path % "1", - sys_long_path % "p1", - sys_path ] + valid_mappings = [sys_long_path % "1", + sys_long_path % "p1", + sys_path] for cdisk in valid_mappings: if not os.path.exists(cdisk): diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index b7de0187..8321dee0 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -177,9 +177,7 @@ class DataSourceAzureNet(sources.DataSource): return True def device_name_to_device(self, name): - device = name.split('.')[0] - return util.map_device_alias(self.ds_cfg['disk_aliases'].get(device), - alias=name) + return self.ds_cfg['disk_aliases'].get(name) def get_config_obj(self): return self.cfg diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 9b3fdf1a..666129ec 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -156,9 +156,7 @@ class DataSourceSmartOS(sources.DataSource): return True def device_name_to_device(self, name): - device = name.split('.')[0] - return util.map_device_alias(self.ds_cfg['disk_aliases'].get(device), - alias=name) + return self.ds_cfg['disk_aliases'].get(name) def get_config_obj(self): return self.cfg diff --git a/cloudinit/util.py b/cloudinit/util.py index 14519586..053e1efc 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1869,7 +1869,7 @@ def map_device_alias(device, partition=None, alias=None): """ if not device: - return None + raise Exception("Device cannot be undefined!") if not partition and not alias: raise Exception("partition or alias is required") @@ -1877,9 +1877,9 @@ def map_device_alias(device, partition=None, alias=None): if alias: partition = map_partition(alias) - # if the partition doesn't map, return the device - if not partition: - return device + # if the partition doesn't map, return the device + if not partition: + return device short_name = device.split('/')[-1] sys_path = "/sys/block/%s" % short_name @@ -1898,6 +1898,9 @@ def map_device_alias(device, partition=None, alias=None): dev_path = "/dev/%s" % cdisk.split('/')[-1] if os.path.exists(dev_path): return dev_path + else: + LOG.warn("Specificed parition %s does not exist on %s" % ( + partition, device)) return None -- cgit v1.2.3 From 88dfa795a039834f3509cb12aed9f50b9f44d05b Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 4 Oct 2013 17:29:25 -0400 Subject: try to support ephemeral0.1 in cc_mounts --- cloudinit/config/cc_mounts.py | 128 +++++++++++++++++++++++++++--------------- cloudinit/util.py | 37 ------------ 2 files changed, 82 insertions(+), 83 deletions(-) (limited to 'cloudinit/config/cc_mounts.py') diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 6b47d326..1ec39e3c 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -45,6 +45,33 @@ def is_mdname(name): return False +def sanitize_devname(devname, transformer, log) + log.debug("Attempting to determine the real name of %s", startname) + + # workaround, allow user to specify 'ephemeral' + # rather than more ec2 correct 'ephemeral0' + devname = startname + if devname == "ephemeral": + devname = "ephemeral0" + log.debug("Adjusted mount option from ephemeral to ephemeral0") + + (blockdev, part) = util.expand_dotted_devname(devname) + + if is_mdname(blockdev): + orig = blockdev + blockdev = transformer(blockdev) + if not blockdev: + return None + if not blockdev.startswith("/"): + blockdev = "/dev/%s" % blockdev + log.debug("Mapped metadata name %s to %s", orig, blockdev) + else: + if SHORTNAME.match(startname): + blockdev = "/dev/%s" % blockdev + + return devnode_for_dev_part(blockdev, part) + + def handle(_name, cfg, cloud, log, _args): # fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno defvals = [None, None, "auto", "defaults,nobootwait", "0", "2"] @@ -65,34 +92,14 @@ def handle(_name, cfg, cloud, log, _args): (i + 1), type_utils.obj_name(cfgmnt[i])) continue - startname = str(cfgmnt[i][0]) - log.debug("Attempting to determine the real name of %s", startname) - - # workaround, allow user to specify 'ephemeral' - # rather than more ec2 correct 'ephemeral0' - if startname == "ephemeral": - cfgmnt[i][0] = "ephemeral0" - log.debug(("Adjusted mount option %s " - "name from ephemeral to ephemeral0"), (i + 1)) - - if is_mdname(startname): - candidate_name = cloud.device_name_to_device(startname) - newname = disk_or_part(candidate_name) + start = str(cfgmnt[i][0]) + sanitized = sanitize_devname(start, cloud.device_name_to_device, log) + if sanitized is None: + log.debug("Ignorming nonexistant named mount %s", start) - if not newname: - log.debug("Ignoring nonexistant named mount %s", startname) - cfgmnt[i][1] = None - else: - renamed = newname - if not newname.startswith("/"): - renamed = "/dev/%s" % newname - cfgmnt[i][0] = renamed - log.debug("Mapped metadata name %s to %s", startname, renamed) - else: - if SHORTNAME.match(startname): - renamed = "/dev/%s" % startname - log.debug("Mapped shortname name %s to %s", startname, renamed) - cfgmnt[i][0] = renamed + if sanitized != start: + log.debug("changed %s => %s" % (start, sanitized)) + cfgmnt[i][0] = sanitized # in case the user did not quote a field (likely fs-freq, fs_passno) # but do not convert None to 'None' (LP: #898365) @@ -121,12 +128,21 @@ def handle(_name, cfg, cloud, log, _args): # for each of the "default" mounts, add them only if no other # entry has the same device name for defmnt in defmnts: - startname = defmnt[0] - candidate_name = cloud.device_name_to_device(startname) - devname = disk_or_part(candidate_name) + start = defmnt[0] + sanitized = sanitize_devname(start, cloud.device_name_to_device, log) + if sanitized is None: + log.debug("Ignorming nonexistant named mount %s", start) + if sanitized != start: + log.debug("changed %s => %s" % (start, sanitized)) + defmnt[0] = sanitized + + devname = defmnt[0] + if candidate_name is not None: + dev = candidate_name if devname is None: log.debug("Ignoring nonexistant named default mount %s", startname) continue + devname = devnode_for_dev_part(devname, part) if devname.startswith("/"): defmnt[0] = devname else: @@ -204,31 +220,51 @@ def handle(_name, cfg, cloud, log, _args): util.logexc(log, "Activating mounts via 'mount -a' failed") -def disk_or_part(device): +def devnode_for_dev_part(device, partition): """ - Find where the file system is on the disk, either on - the disk itself or on the first partition. We don't go - any deeper than partition 1 though. + Find the name of the partition. While this might seem rather + straight forward, its not since some devices are '' + while others are 'p'. For example, /dev/xvda3 on EC2 + will present as /dev/xvda3p1 for the first partition since /dev/xvda3 is + a block device. """ - - if not device: + if not os.path.exists(device): return None - short_name = device.split('/')[-1] + if not partition: + return device + + short_name = os.path.basename(device) sys_path = "/sys/block/%s" % short_name - # if the sys path does not exist but the device exists, - # then the device is a partition, no sense looking any further - if not os.path.exists(sys_path) and os.path.exists(device): - return device + if not os.path.exists(sys_path): + LOG.debug("did not find entry for %s in /sys/block", short_name) + return None - sys_long_path = sys_path + "/" + short_name + "%s" - valid_mappings = [sys_long_path % "1", - sys_long_path % "p1", - sys_path] + sys_long_path = sys_path + "/" + short_name + + if partition is not None: + partition = str(partition) + + if partition is None: + valid_mappings = [sys_long_path + "1", + sys_long_path + "p1" % partition] + elif partition != "0": + valid_mappings = [sys_long_path + "%s" % partition, + sys_long_path + "p%s" % partition] + else: + valid_mappings = [] for cdisk in valid_mappings: if not os.path.exists(cdisk): continue - return "/dev/%s" % cdisk.split('/')[-1] + + dev_path = "/dev/%s" % os.path.basename(cdisk) + if os.path.exists(dev_path): + return dev_path + + if partition is None or partition == "0": + return device + + LOG.debug("Did not fine partition %s for device %s", partition, device) return None diff --git a/cloudinit/util.py b/cloudinit/util.py index f9957c67..9e6e0a73 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1835,40 +1835,3 @@ def expand_dotted_devname(dotted): return toks else: return (dotted, None) - - -def devnode_for_dev_part(device, partition): - """ - Find the name of the partition. While this might seem rather - straight forward, its not since some devices are '' - while others are 'p'. For example, /dev/xvda3 on EC2 - will present as /dev/xvda3p1 for the first partition since /dev/xvda3 is - a block device. - """ - if not os.path.exists(device): - return None - - if not partition: - return device - - short_name = os.path.basename(device) - sys_path = "/sys/block/%s" % short_name - - if not os.path.exists(sys_path): - LOG.debug("did not find entry for %s in /sys/block", short_name) - return None - - sys_long_path = sys_path + "/" + short_name - valid_mappings = [sys_long_path + "%s" % partition, - sys_long_path + "p%s" % partition] - - for cdisk in valid_mappings: - if not os.path.exists(cdisk): - continue - - dev_path = "/dev/%s" % os.path.basename(cdisk) - if os.path.exists(dev_path): - return dev_path - - LOG.debug("Did not fine partition %s for device %s", partition, device) - return None -- cgit v1.2.3 From 4caf39f2631643ac4facf2706716a7972d146128 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 4 Oct 2013 17:32:48 -0400 Subject: remove duplicate code --- cloudinit/config/cc_mounts.py | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) (limited to 'cloudinit/config/cc_mounts.py') diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 1ec39e3c..1a8ebaa3 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -131,25 +131,11 @@ def handle(_name, cfg, cloud, log, _args): start = defmnt[0] sanitized = sanitize_devname(start, cloud.device_name_to_device, log) if sanitized is None: - log.debug("Ignorming nonexistant named mount %s", start) + log.debug("Ignoring nonexistant default named mount %s", start) if sanitized != start: - log.debug("changed %s => %s" % (start, sanitized)) + log.debug("changed default device %s => %s" % (start, sanitized)) defmnt[0] = sanitized - devname = defmnt[0] - if candidate_name is not None: - dev = candidate_name - if devname is None: - log.debug("Ignoring nonexistant named default mount %s", startname) - continue - devname = devnode_for_dev_part(devname, part) - if devname.startswith("/"): - defmnt[0] = devname - else: - defmnt[0] = "/dev/%s" % devname - - log.debug("Mapped default device %s to %s", startname, defmnt[0]) - cfgmnt_has = False for cfgm in cfgmnt: if cfgm[0] == defmnt[0]: -- cgit v1.2.3 From ff820acaf999479f75a12414ac5fcb12d51f258b Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Fri, 4 Oct 2013 15:52:20 -0600 Subject: Fixed up issue with attempt to mount nonexistant swap --- cloudinit/config/cc_mounts.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'cloudinit/config/cc_mounts.py') diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 1a8ebaa3..ff7bf506 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -45,7 +45,7 @@ def is_mdname(name): return False -def sanitize_devname(devname, transformer, log) +def sanitize_devname(startname, transformer, log): log.debug("Attempting to determine the real name of %s", startname) # workaround, allow user to specify 'ephemeral' @@ -96,6 +96,7 @@ def handle(_name, cfg, cloud, log, _args): sanitized = sanitize_devname(start, cloud.device_name_to_device, log) if sanitized is None: log.debug("Ignorming nonexistant named mount %s", start) + continue if sanitized != start: log.debug("changed %s => %s" % (start, sanitized)) @@ -132,6 +133,7 @@ def handle(_name, cfg, cloud, log, _args): sanitized = sanitize_devname(start, cloud.device_name_to_device, log) if sanitized is None: log.debug("Ignoring nonexistant default named mount %s", start) + continue if sanitized != start: log.debug("changed default device %s => %s" % (start, sanitized)) defmnt[0] = sanitized -- cgit v1.2.3 From fb7b982da08619fad2c582f921c05cf982621c0b Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 4 Oct 2013 18:29:12 -0400 Subject: pep8 and pylint fixes --- cloudinit/config/cc_disk_setup.py | 5 +++-- cloudinit/config/cc_mounts.py | 7 +++++-- 2 files changed, 8 insertions(+), 4 deletions(-) (limited to 'cloudinit/config/cc_mounts.py') diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index e903dd4d..0b970e4e 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -484,12 +484,13 @@ def get_partition_mbr_layout(size, layout): def purge_disk_ptable(device): # wipe the first and last megabyte of a disk (or file) # gpt stores partition table both at front and at end. + null = '\0' # pylint: disable=W1401 start_len = 1024 * 1024 end_len = 1024 * 1024 with open(device, "rb+") as fp: - fp.write('\0' * (start_len)) + fp.write(null * (start_len)) fp.seek(-end_len, os.SEEK_END) - fp.write('\0' * end_len) + fp.write(null * end_len) fp.flush() read_parttbl(device) diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index ff7bf506..84ec928f 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -20,6 +20,7 @@ from string import whitespace # pylint: disable=W0402 +import logging import os.path import re @@ -32,6 +33,8 @@ SHORTNAME = re.compile(SHORTNAME_FILTER) WS = re.compile("[%s]+" % (whitespace)) FSTAB_PATH = "/etc/fstab" +LOG = logging.getLogger(__name__) + def is_mdname(name): # return true if this is a metadata service name @@ -146,7 +149,7 @@ def handle(_name, cfg, cloud, log, _args): if cfgmnt_has: log.debug(("Not including %s, already" - " previously included"), startname) + " previously included"), start) continue cfgmnt.append(defmnt) @@ -230,7 +233,7 @@ def devnode_for_dev_part(device, partition): return None sys_long_path = sys_path + "/" + short_name - + if partition is not None: partition = str(partition) -- cgit v1.2.3