From ea88f3e1208ba501b50d1f41187b83cf11f15785 Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Wed, 2 Oct 2013 15:05:15 -0600 Subject: Added ability to define disks via 'ephemeralX.Y'. Modified cc_mounts to identify whether ephermalX is partitioned. Changed datasources for Azure and SmartOS to use 'ephemeralX.Y' format. Added disk remove functionally --- cloudinit/util.py | 76 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) (limited to 'cloudinit/util.py') diff --git a/cloudinit/util.py b/cloudinit/util.py index 50ca7959..14519586 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -32,6 +32,7 @@ import grp import gzip import hashlib import os +import os.path import platform import pwd import random @@ -1826,3 +1827,78 @@ def log_time(logfunc, msg, func, args=None, kwargs=None, get_uptime=False): except: pass return ret + + +def map_partition(alias): + """ + Return partition number for devices like ephemeral0.0 or ephemeral0.1 + + Parameters: + alaias: the alias, i.e. ephemeral0 or swap0 + device: the actual device to markup + + Rules: + - anything after a . is a parittion + - device.0 is the same as device + """ + + if len(alias.split('.')) == 1: + return None + + suffix = alias.split('.')[-1] + try: + if int(suffix) == 0: + return None + return int(suffix) + except ValueError: + pass + + return None + + +def map_device_alias(device, partition=None, alias=None): + """ + Find the name of the partition. While this might seem rather + straight forward, its not since some devices are '' + while others are 'p'. For example, /dev/xvda3 on EC2 + will present as /dev/xvda3p1 for the first partition since /dev/xvda3 is + a block device. + + The primary use is to map 'ephemeral0.1' in the datasource to a + real device name + """ + + if not device: + return None + + if not partition and not alias: + raise Exception("partition or alias is required") + + if alias: + partition = map_partition(alias) + + # if the partition doesn't map, return the device + if not partition: + return device + + short_name = device.split('/')[-1] + sys_path = "/sys/block/%s" % short_name + + if not os.path.exists(sys_path): + return None + + sys_long_path = sys_path + "/" + short_name + valid_mappings = [sys_long_path + "%s" % partition, + sys_long_path + "p%s" % partition] + + for cdisk in valid_mappings: + if not os.path.exists(cdisk): + continue + + dev_path = "/dev/%s" % cdisk.split('/')[-1] + if os.path.exists(dev_path): + return dev_path + + return None + + -- cgit v1.2.3 From ecf2a600e41a9632ad305eb6a8cd5665908f31fb Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Thu, 3 Oct 2013 16:15:56 -0600 Subject: Moved ephemeralX.Y handling from Datasource into the cc_disk_setup, which makes it cloud agnostic. --- cloudinit/config/cc_disk_setup.py | 21 +++++++++++++++++++-- cloudinit/config/cc_mounts.py | 13 +++++++------ cloudinit/sources/DataSourceAzure.py | 4 +--- cloudinit/sources/DataSourceSmartOS.py | 4 +--- cloudinit/util.py | 11 +++++++---- 5 files changed, 35 insertions(+), 18 deletions(-) (limited to 'cloudinit/util.py') diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index e744cd57..554dfdd3 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -104,13 +104,30 @@ def update_fs_setup_devices(disk_setup, tformer): continue origname = definition.get('device') + if origname is None: continue - transformed = tformer(origname) + transformed = None + if len(origname.split('.')) > 1: + # this maps ephemeralX.Y to a proper disk name. For example, + # if the origname is 'ephemeral0.1' and transformed is /dev/sdb + # then the returned device will be /dev/sdb1 _if_ /dev/sdb1 exists + # otherwise NONE + base_name = origname.split('.')[0] + tformed = tformer(base_name) + LOG.info("base device for %s is %s" % (origname, tformed)) + + transformed = util.map_device_alias(tformed, alias=origname) + LOG.info("%s is mapped to %s" % (origname, transformed)) + + else: + transformed = tformer(origname) + if transformed is None or transformed == origname: continue + LOG.info("Mapped %s to physical device %s" % (origname, transformed)) definition['_origname'] = origname definition['device'] = transformed @@ -497,7 +514,7 @@ def purge_disk(device): try: LOG.info("Purging filesystem on /dev/%s" % d['name']) util.subp(wipefs_cmd) - except Exception as e: + except Exception: raise Exception("Failed FS purge of /dev/%s" % d['name']) purge_disk_ptable(device) diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index f4c2e3d8..6b47d326 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -217,14 +217,15 @@ def disk_or_part(device): short_name = device.split('/')[-1] sys_path = "/sys/block/%s" % short_name - if not os.path.exists(sys_path): - LOG.warn("Device %s does not exist in sysfs" % device) - return None + # if the sys path does not exist but the device exists, + # then the device is a partition, no sense looking any further + if not os.path.exists(sys_path) and os.path.exists(device): + return device sys_long_path = sys_path + "/" + short_name + "%s" - valid_mappings = [ sys_long_path % "1", - sys_long_path % "p1", - sys_path ] + valid_mappings = [sys_long_path % "1", + sys_long_path % "p1", + sys_path] for cdisk in valid_mappings: if not os.path.exists(cdisk): diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index b7de0187..8321dee0 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -177,9 +177,7 @@ class DataSourceAzureNet(sources.DataSource): return True def device_name_to_device(self, name): - device = name.split('.')[0] - return util.map_device_alias(self.ds_cfg['disk_aliases'].get(device), - alias=name) + return self.ds_cfg['disk_aliases'].get(name) def get_config_obj(self): return self.cfg diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 9b3fdf1a..666129ec 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -156,9 +156,7 @@ class DataSourceSmartOS(sources.DataSource): return True def device_name_to_device(self, name): - device = name.split('.')[0] - return util.map_device_alias(self.ds_cfg['disk_aliases'].get(device), - alias=name) + return self.ds_cfg['disk_aliases'].get(name) def get_config_obj(self): return self.cfg diff --git a/cloudinit/util.py b/cloudinit/util.py index 14519586..053e1efc 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1869,7 +1869,7 @@ def map_device_alias(device, partition=None, alias=None): """ if not device: - return None + raise Exception("Device cannot be undefined!") if not partition and not alias: raise Exception("partition or alias is required") @@ -1877,9 +1877,9 @@ def map_device_alias(device, partition=None, alias=None): if alias: partition = map_partition(alias) - # if the partition doesn't map, return the device - if not partition: - return device + # if the partition doesn't map, return the device + if not partition: + return device short_name = device.split('/')[-1] sys_path = "/sys/block/%s" % short_name @@ -1898,6 +1898,9 @@ def map_device_alias(device, partition=None, alias=None): dev_path = "/dev/%s" % cdisk.split('/')[-1] if os.path.exists(dev_path): return dev_path + else: + LOG.warn("Specificed parition %s does not exist on %s" % ( + partition, device)) return None -- cgit v1.2.3 From 193704c37b97a5687cab09416f9f38e632c76fa3 Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Thu, 3 Oct 2013 17:07:23 -0600 Subject: Make {pep8,pylint,test} pass commit. --- cloudinit/config/cc_disk_setup.py | 3 ++- cloudinit/util.py | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) (limited to 'cloudinit/util.py') diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index b9e1e9eb..127b5879 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -186,6 +186,7 @@ def enumerate_disk(device, nodeps=False): yield d + def device_type(device): """ Return the device type of the device by calling lsblk. @@ -193,7 +194,7 @@ def device_type(device): for d in enumerate_disk(device, nodeps=True): if "type" in d: - return d["type"].lower() + return d["type"].lower() return None diff --git a/cloudinit/util.py b/cloudinit/util.py index 053e1efc..34d111e7 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1903,5 +1903,3 @@ def map_device_alias(device, partition=None, alias=None): partition, device)) return None - - -- cgit v1.2.3 From 4846bff62e1a5e8f4edd216f900c27c55a319c5b Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 4 Oct 2013 16:16:12 -0400 Subject: simplifications and some function renames --- cloudinit/config/cc_disk_setup.py | 50 ++++++++-------------------------- cloudinit/util.py | 57 +++++++++------------------------------ 2 files changed, 24 insertions(+), 83 deletions(-) (limited to 'cloudinit/util.py') diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index 5faffefc..e903dd4d 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -94,15 +94,6 @@ def update_disk_setup_devices(disk_setup, tformer): LOG.debug("updated disk_setup device entry '%s' to '%s'", origname, transformed) -def reset_part_definition(definition, value): - if not value and 'partition' in definition: - definition['opartition'] = definition['partition'] - del definition['partition'] - - else: - definition['partition'] = value - - return definition def update_fs_setup_devices(disk_setup, tformer): # update 'fs_setup' dictionary anywhere were a device may occur @@ -117,38 +108,19 @@ def update_fs_setup_devices(disk_setup, tformer): if origname is None: continue - transformed = None - if len(origname.split('.')) == 2: - # this maps ephemeralX.Y to a proper disk name. For example, - # if the origname is 'ephemeral0.1' and transformed is /dev/sdb - # then the returned device will be /dev/sdb1 _if_ /dev/sdb1 exists - # otherwise NONE - base_name, partition = origname.split('.') - tformed = tformer(base_name) - LOG.info("base device for %s is %s" % (origname, tformed)) + (dev, part) = util.expand_dotted_devname(origname) - if partition == "0": - transformed = tformed - definition = reset_part_definition(definition, None) - - elif partition in ("auto", "any"): - definition = reset_part_definition(definition, partition) - transformed = tformed - - else: - definition = reset_part_definition(definition, None) - transformed = util.map_device_alias(tformed, alias=origname) - LOG.info("%s is mapped to %s" % (origname, transformed)) - - else: - transformed = tformer(origname) - - if transformed is None or transformed == origname: - continue + tformed = tformer(dev) + if tformed is not None: + dev = tformed + LOG.debug("%s is mapped to disk=%s part=%s", + origname, tformed, part) + definition['_origname'] = origname + definition['device'] = tformed - LOG.info("Mapped %s to physical device %s" % (origname, transformed)) - definition['_origname'] = origname - definition['device'] = transformed + if part and 'partition' in definition: + definition['_partition'] = definition['partition'] + definition['partition'] = part def value_splitter(values, start=None): diff --git a/cloudinit/util.py b/cloudinit/util.py index 34d111e7..f9957c67 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1829,62 +1829,33 @@ def log_time(logfunc, msg, func, args=None, kwargs=None, get_uptime=False): return ret -def map_partition(alias): - """ - Return partition number for devices like ephemeral0.0 or ephemeral0.1 - - Parameters: - alaias: the alias, i.e. ephemeral0 or swap0 - device: the actual device to markup - - Rules: - - anything after a . is a parittion - - device.0 is the same as device - """ - - if len(alias.split('.')) == 1: - return None - - suffix = alias.split('.')[-1] - try: - if int(suffix) == 0: - return None - return int(suffix) - except ValueError: - pass - - return None +def expand_dotted_devname(dotted): + toks = dotted.rsplit(".", 1) + if len(toks) > 1: + return toks + else: + return (dotted, None) -def map_device_alias(device, partition=None, alias=None): +def devnode_for_dev_part(device, partition): """ Find the name of the partition. While this might seem rather straight forward, its not since some devices are '' while others are 'p'. For example, /dev/xvda3 on EC2 will present as /dev/xvda3p1 for the first partition since /dev/xvda3 is a block device. - - The primary use is to map 'ephemeral0.1' in the datasource to a - real device name """ + if not os.path.exists(device): + return None - if not device: - raise Exception("Device cannot be undefined!") - - if not partition and not alias: - raise Exception("partition or alias is required") - - if alias: - partition = map_partition(alias) - - # if the partition doesn't map, return the device if not partition: return device - short_name = device.split('/')[-1] + short_name = os.path.basename(device) sys_path = "/sys/block/%s" % short_name if not os.path.exists(sys_path): + LOG.debug("did not find entry for %s in /sys/block", short_name) return None sys_long_path = sys_path + "/" + short_name @@ -1895,11 +1866,9 @@ def map_device_alias(device, partition=None, alias=None): if not os.path.exists(cdisk): continue - dev_path = "/dev/%s" % cdisk.split('/')[-1] + dev_path = "/dev/%s" % os.path.basename(cdisk) if os.path.exists(dev_path): return dev_path - else: - LOG.warn("Specificed parition %s does not exist on %s" % ( - partition, device)) + LOG.debug("Did not fine partition %s for device %s", partition, device) return None -- cgit v1.2.3 From 88dfa795a039834f3509cb12aed9f50b9f44d05b Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 4 Oct 2013 17:29:25 -0400 Subject: try to support ephemeral0.1 in cc_mounts --- cloudinit/config/cc_mounts.py | 128 +++++++++++++++++++++++++++--------------- cloudinit/util.py | 37 ------------ 2 files changed, 82 insertions(+), 83 deletions(-) (limited to 'cloudinit/util.py') diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 6b47d326..1ec39e3c 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -45,6 +45,33 @@ def is_mdname(name): return False +def sanitize_devname(devname, transformer, log) + log.debug("Attempting to determine the real name of %s", startname) + + # workaround, allow user to specify 'ephemeral' + # rather than more ec2 correct 'ephemeral0' + devname = startname + if devname == "ephemeral": + devname = "ephemeral0" + log.debug("Adjusted mount option from ephemeral to ephemeral0") + + (blockdev, part) = util.expand_dotted_devname(devname) + + if is_mdname(blockdev): + orig = blockdev + blockdev = transformer(blockdev) + if not blockdev: + return None + if not blockdev.startswith("/"): + blockdev = "/dev/%s" % blockdev + log.debug("Mapped metadata name %s to %s", orig, blockdev) + else: + if SHORTNAME.match(startname): + blockdev = "/dev/%s" % blockdev + + return devnode_for_dev_part(blockdev, part) + + def handle(_name, cfg, cloud, log, _args): # fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno defvals = [None, None, "auto", "defaults,nobootwait", "0", "2"] @@ -65,34 +92,14 @@ def handle(_name, cfg, cloud, log, _args): (i + 1), type_utils.obj_name(cfgmnt[i])) continue - startname = str(cfgmnt[i][0]) - log.debug("Attempting to determine the real name of %s", startname) - - # workaround, allow user to specify 'ephemeral' - # rather than more ec2 correct 'ephemeral0' - if startname == "ephemeral": - cfgmnt[i][0] = "ephemeral0" - log.debug(("Adjusted mount option %s " - "name from ephemeral to ephemeral0"), (i + 1)) - - if is_mdname(startname): - candidate_name = cloud.device_name_to_device(startname) - newname = disk_or_part(candidate_name) + start = str(cfgmnt[i][0]) + sanitized = sanitize_devname(start, cloud.device_name_to_device, log) + if sanitized is None: + log.debug("Ignorming nonexistant named mount %s", start) - if not newname: - log.debug("Ignoring nonexistant named mount %s", startname) - cfgmnt[i][1] = None - else: - renamed = newname - if not newname.startswith("/"): - renamed = "/dev/%s" % newname - cfgmnt[i][0] = renamed - log.debug("Mapped metadata name %s to %s", startname, renamed) - else: - if SHORTNAME.match(startname): - renamed = "/dev/%s" % startname - log.debug("Mapped shortname name %s to %s", startname, renamed) - cfgmnt[i][0] = renamed + if sanitized != start: + log.debug("changed %s => %s" % (start, sanitized)) + cfgmnt[i][0] = sanitized # in case the user did not quote a field (likely fs-freq, fs_passno) # but do not convert None to 'None' (LP: #898365) @@ -121,12 +128,21 @@ def handle(_name, cfg, cloud, log, _args): # for each of the "default" mounts, add them only if no other # entry has the same device name for defmnt in defmnts: - startname = defmnt[0] - candidate_name = cloud.device_name_to_device(startname) - devname = disk_or_part(candidate_name) + start = defmnt[0] + sanitized = sanitize_devname(start, cloud.device_name_to_device, log) + if sanitized is None: + log.debug("Ignorming nonexistant named mount %s", start) + if sanitized != start: + log.debug("changed %s => %s" % (start, sanitized)) + defmnt[0] = sanitized + + devname = defmnt[0] + if candidate_name is not None: + dev = candidate_name if devname is None: log.debug("Ignoring nonexistant named default mount %s", startname) continue + devname = devnode_for_dev_part(devname, part) if devname.startswith("/"): defmnt[0] = devname else: @@ -204,31 +220,51 @@ def handle(_name, cfg, cloud, log, _args): util.logexc(log, "Activating mounts via 'mount -a' failed") -def disk_or_part(device): +def devnode_for_dev_part(device, partition): """ - Find where the file system is on the disk, either on - the disk itself or on the first partition. We don't go - any deeper than partition 1 though. + Find the name of the partition. While this might seem rather + straight forward, its not since some devices are '' + while others are 'p'. For example, /dev/xvda3 on EC2 + will present as /dev/xvda3p1 for the first partition since /dev/xvda3 is + a block device. """ - - if not device: + if not os.path.exists(device): return None - short_name = device.split('/')[-1] + if not partition: + return device + + short_name = os.path.basename(device) sys_path = "/sys/block/%s" % short_name - # if the sys path does not exist but the device exists, - # then the device is a partition, no sense looking any further - if not os.path.exists(sys_path) and os.path.exists(device): - return device + if not os.path.exists(sys_path): + LOG.debug("did not find entry for %s in /sys/block", short_name) + return None - sys_long_path = sys_path + "/" + short_name + "%s" - valid_mappings = [sys_long_path % "1", - sys_long_path % "p1", - sys_path] + sys_long_path = sys_path + "/" + short_name + + if partition is not None: + partition = str(partition) + + if partition is None: + valid_mappings = [sys_long_path + "1", + sys_long_path + "p1" % partition] + elif partition != "0": + valid_mappings = [sys_long_path + "%s" % partition, + sys_long_path + "p%s" % partition] + else: + valid_mappings = [] for cdisk in valid_mappings: if not os.path.exists(cdisk): continue - return "/dev/%s" % cdisk.split('/')[-1] + + dev_path = "/dev/%s" % os.path.basename(cdisk) + if os.path.exists(dev_path): + return dev_path + + if partition is None or partition == "0": + return device + + LOG.debug("Did not fine partition %s for device %s", partition, device) return None diff --git a/cloudinit/util.py b/cloudinit/util.py index f9957c67..9e6e0a73 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1835,40 +1835,3 @@ def expand_dotted_devname(dotted): return toks else: return (dotted, None) - - -def devnode_for_dev_part(device, partition): - """ - Find the name of the partition. While this might seem rather - straight forward, its not since some devices are '' - while others are 'p'. For example, /dev/xvda3 on EC2 - will present as /dev/xvda3p1 for the first partition since /dev/xvda3 is - a block device. - """ - if not os.path.exists(device): - return None - - if not partition: - return device - - short_name = os.path.basename(device) - sys_path = "/sys/block/%s" % short_name - - if not os.path.exists(sys_path): - LOG.debug("did not find entry for %s in /sys/block", short_name) - return None - - sys_long_path = sys_path + "/" + short_name - valid_mappings = [sys_long_path + "%s" % partition, - sys_long_path + "p%s" % partition] - - for cdisk in valid_mappings: - if not os.path.exists(cdisk): - continue - - dev_path = "/dev/%s" % os.path.basename(cdisk) - if os.path.exists(dev_path): - return dev_path - - LOG.debug("Did not fine partition %s for device %s", partition, device) - return None -- cgit v1.2.3