From ea88f3e1208ba501b50d1f41187b83cf11f15785 Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Wed, 2 Oct 2013 15:05:15 -0600 Subject: Added ability to define disks via 'ephemeralX.Y'. Modified cc_mounts to identify whether ephermalX is partitioned. Changed datasources for Azure and SmartOS to use 'ephemeralX.Y' format. Added disk remove functionally --- cloudinit/config/cc_disk_setup.py | 129 ++++++++++++++++++++++---- cloudinit/config/cc_mounts.py | 37 +++++++- cloudinit/sources/DataSourceAzure.py | 9 +- cloudinit/sources/DataSourceSmartOS.py | 9 +- cloudinit/util.py | 76 +++++++++++++++ doc/examples/cloud-config-disk-setup.txt | 10 +- tests/unittests/test_datasource/test_azure.py | 2 - 7 files changed, 238 insertions(+), 34 deletions(-) diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index ade9c2ad..faf424ba 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -16,8 +16,8 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from cloudinit.settings import PER_INSTANCE from cloudinit import util +from cloudinit.settings import PER_INSTANCE import logging import shlex @@ -29,13 +29,13 @@ SFDISK_CMD = util.which("sfdisk") LSBLK_CMD = util.which("lsblk") BLKID_CMD = util.which("blkid") BLKDEV_CMD = util.which("blockdev") +WIPEFS_CMD = util.which("wipefs") LOG = logging.getLogger(__name__) def handle(_name, cfg, cloud, log, _args): """ - Call util.prep_disk for disk_setup cloud-config. See doc/examples/cloud-config_disk-setup.txt for documentation on the format. """ @@ -203,23 +203,11 @@ def is_filesystem(device): return fs_type -def find_device_node(device, fs_type=None, label=None, valid_targets=None, - label_match=True): +def enumerate_disk(device): """ - Find a device that is either matches the spec, or the first - - The return is value is (, ) where the device is the - device to use and the bool is whether the device matches the - fs_type and label. - - Note: This works with GPT partition tables! + Enumerate the elements of a child device. Return a dict of name, + type, fstype, and label """ - # label of None is same as no label - if label is None: - label = "" - - if not valid_targets: - valid_targets = ['disk', 'part'] lsblk_cmd = [LSBLK_CMD, '--pairs', '--out', 'NAME,TYPE,FSTYPE,LABEL', device] @@ -229,7 +217,6 @@ def find_device_node(device, fs_type=None, label=None, valid_targets=None, except Exception as e: raise Exception("Failed during disk check for %s\n%s" % (device, e)) - raw_device_used = False parts = [x for x in (info.strip()).splitlines() if len(x.split()) > 0] for part in parts: @@ -242,6 +229,35 @@ def find_device_node(device, fs_type=None, label=None, valid_targets=None, for key, value in value_splitter(part): d[key.lower()] = value + LOG.info(d) + yield d + + +def find_device_node(device, fs_type=None, label=None, valid_targets=None, + label_match=True, replace_fs=None): + """ + Find a device that is either matches the spec, or the first + + The return is value is (, ) where the device is the + device to use and the bool is whether the device matches the + fs_type and label. + + Note: This works with GPT partition tables! + """ + # label of None is same as no label + if label is None: + label = "" + + if not valid_targets: + valid_targets = ['disk', 'part'] + + raw_device_used = False + for d in enumerate_disk(device): + + if d['fstype'] == replace_fs and label_match == False: + # We found a device where we want to replace the FS + return ('/dev/%s' % d['name'], False) + if (d['fstype'] == fs_type and ((label_match and d['label'] == label) or not label_match)): # If we find a matching device, we return that @@ -454,6 +470,42 @@ def get_partition_mbr_layout(size, layout): return sfdisk_definition +def purge_disk(device): + """ + Remove parition table entries + """ + + # wipe any file systems first + for d in enumerate_disk(device): + LOG.info(d) + if d['type'] not in ["disk", "crypt"]: + wipefs_cmd = [WIPEFS_CMD, "--all", "/dev/%s" % d['name']] + try: + LOG.info("Purging filesystem on /dev/%s" % d['name']) + util.subp(wipefs_cmd) + LOG.info("Purged filesystem on /dev/%s" % d['name']) + except Exception as e: + raise Exception("Failed FS purge of /dev/%s" % d['name']) + + dd_cmd = util.which("dd") + last_seek = int(get_hdd_size(device) / 1024) - 2 + first_mb = [dd_cmd, "if=/dev/zero", "of=%s" % device, "bs=1M", "count=1"] + last_mb = [dd_cmd, "if=/dev/zero", "of=%s" % device, "bs=1M", "seek=%s" % last_seek] + try: + util.subp(first_mb) + LOG.info("Purged MBR/Partition table from %s" % device) + util.subp(last_mb, rcs=[0,1]) + LOG.info("Purged any chance of GPT table from %s" % device) + + # Wipe it for good measure + wipefs_cmd = [WIPEFS_CMD, "--all", device] + util.subp(wipefs_cmd) + except Exception as e: + LOG.critical(e) + raise Exception("Failed to remove MBR/Part from %s" % device) + + read_parttbl(device) + def get_partition_layout(table_type, size, layout): """ @@ -542,6 +594,12 @@ def mkpart(device, definition): if not is_device_valid(device): raise Exception("Device %s is not a disk device!", device) + # Remove the partition table entries + if isinstance(layout, str) and layout.lower() == "remove": + LOG.debug("Instructed to remove partition table entries") + purge_disk(device) + return + LOG.debug("Checking if device layout matches") if check_partition_layout(table_type, device, layout): LOG.debug("Device partitioning layout matches") @@ -565,6 +623,26 @@ def mkpart(device, definition): LOG.debug("Partition table created for %s", device) +def lookup_force_flag(fs): + """ + A force flag might be -F or -F, this look it up + """ + flags = {'ext': '-F', + 'btrfs': '-f', + 'xfs': '-f', + 'reiserfs': '-f', + } + + if 'ext' in fs.lower(): + fs = 'ext' + + if fs.lower() in flags: + return flags[fs] + + LOG.warn("Force flag for %s is unknown." % fs) + return '' + + def mkfs(fs_cfg): """ Create a file system on the device. @@ -592,6 +670,7 @@ def mkfs(fs_cfg): fs_type = fs_cfg.get('filesystem') fs_cmd = fs_cfg.get('cmd', []) fs_opts = fs_cfg.get('extra_opts', []) + fs_replace = fs_cfg.get('replace_fs', False) overwrite = fs_cfg.get('overwrite', False) # This allows you to define the default ephemeral or swap @@ -632,17 +711,23 @@ def mkfs(fs_cfg): label_match = False device, reuse = find_device_node(device, fs_type=fs_type, label=label, - label_match=label_match) + label_match=label_match, + replace_fs=fs_replace) LOG.debug("Automatic device for %s identified as %s", odevice, device) if reuse: LOG.debug("Found filesystem match, skipping formating.") return + if not reuse and fs_replace and device: + LOG.debug("Replacing file system on %s as instructed." % device) + if not device: LOG.debug("No device aviable that matches request. " "Skipping fs creation for %s", fs_cfg) return + elif not partition or str(partition).lower() == 'none': + LOG.debug("Using the raw device to place filesystem %s on" % label) else: LOG.debug("Error in device identification handling.") @@ -682,12 +767,16 @@ def mkfs(fs_cfg): if label: fs_cmd.extend(["-L", label]) + # File systems that support the -F flag + if not fs_cmd and (overwrite or device_type(device) == "disk"): + fs_cmd.append(lookup_force_flag(fs_type)) + # Add the extends FS options if fs_opts: fs_cmd.extend(fs_opts) LOG.debug("Creating file system %s on %s", label, device) - LOG.debug(" Using cmd: %s", "".join(fs_cmd)) + LOG.debug(" Using cmd: %s", " ".join(fs_cmd)) try: util.subp(fs_cmd) except Exception as e: diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 390ba711..f4c2e3d8 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -20,6 +20,7 @@ from string import whitespace # pylint: disable=W0402 +import os.path import re from cloudinit import type_utils @@ -75,7 +76,9 @@ def handle(_name, cfg, cloud, log, _args): "name from ephemeral to ephemeral0"), (i + 1)) if is_mdname(startname): - newname = cloud.device_name_to_device(startname) + candidate_name = cloud.device_name_to_device(startname) + newname = disk_or_part(candidate_name) + if not newname: log.debug("Ignoring nonexistant named mount %s", startname) cfgmnt[i][1] = None @@ -119,7 +122,8 @@ def handle(_name, cfg, cloud, log, _args): # entry has the same device name for defmnt in defmnts: startname = defmnt[0] - devname = cloud.device_name_to_device(startname) + candidate_name = cloud.device_name_to_device(startname) + devname = disk_or_part(candidate_name) if devname is None: log.debug("Ignoring nonexistant named default mount %s", startname) continue @@ -198,3 +202,32 @@ def handle(_name, cfg, cloud, log, _args): util.subp(("mount", "-a")) except: util.logexc(log, "Activating mounts via 'mount -a' failed") + + +def disk_or_part(device): + """ + Find where the file system is on the disk, either on + the disk itself or on the first partition. We don't go + any deeper than partition 1 though. + """ + + if not device: + return None + + short_name = device.split('/')[-1] + sys_path = "/sys/block/%s" % short_name + + if not os.path.exists(sys_path): + LOG.warn("Device %s does not exist in sysfs" % device) + return None + + sys_long_path = sys_path + "/" + short_name + "%s" + valid_mappings = [ sys_long_path % "1", + sys_long_path % "p1", + sys_path ] + + for cdisk in valid_mappings: + if not os.path.exists(cdisk): + continue + return "/dev/%s" % cdisk.split('/')[-1] + return None diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 7ba6cea8..b7de0187 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -54,8 +54,9 @@ BUILTIN_CLOUD_CONFIG = { 'layout': True, 'overwrite': False} }, - 'fs_setup': [{'filesystem': 'ext4', 'device': 'ephemeral0', - 'partition': 'auto'}], + 'fs_setup': [{'filesystem': 'ext4', + 'device': 'ephemeral0.1', + 'replace_fs': 'ntfs'}] } DS_CFG_PATH = ['datasource', DS_NAME] @@ -176,7 +177,9 @@ class DataSourceAzureNet(sources.DataSource): return True def device_name_to_device(self, name): - return self.ds_cfg['disk_aliases'].get(name) + device = name.split('.')[0] + return util.map_device_alias(self.ds_cfg['disk_aliases'].get(device), + alias=name) def get_config_obj(self): return self.cfg diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 93b8b50b..9b3fdf1a 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -81,8 +81,9 @@ BUILTIN_CLOUD_CONFIG = { 'layout': False, 'overwrite': False} }, - 'fs_setup': [{'label': 'ephemeral0', 'filesystem': 'ext3', - 'device': 'ephemeral0', 'partition': 'auto'}], + 'fs_setup': [{'label': 'ephemeral0', + 'filesystem': 'ext3', + 'device': 'ephemeral0'}], } @@ -155,7 +156,9 @@ class DataSourceSmartOS(sources.DataSource): return True def device_name_to_device(self, name): - return self.ds_cfg['disk_aliases'].get(name) + device = name.split('.')[0] + return util.map_device_alias(self.ds_cfg['disk_aliases'].get(device), + alias=name) def get_config_obj(self): return self.cfg diff --git a/cloudinit/util.py b/cloudinit/util.py index 50ca7959..14519586 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -32,6 +32,7 @@ import grp import gzip import hashlib import os +import os.path import platform import pwd import random @@ -1826,3 +1827,78 @@ def log_time(logfunc, msg, func, args=None, kwargs=None, get_uptime=False): except: pass return ret + + +def map_partition(alias): + """ + Return partition number for devices like ephemeral0.0 or ephemeral0.1 + + Parameters: + alaias: the alias, i.e. ephemeral0 or swap0 + device: the actual device to markup + + Rules: + - anything after a . is a parittion + - device.0 is the same as device + """ + + if len(alias.split('.')) == 1: + return None + + suffix = alias.split('.')[-1] + try: + if int(suffix) == 0: + return None + return int(suffix) + except ValueError: + pass + + return None + + +def map_device_alias(device, partition=None, alias=None): + """ + Find the name of the partition. While this might seem rather + straight forward, its not since some devices are '' + while others are 'p'. For example, /dev/xvda3 on EC2 + will present as /dev/xvda3p1 for the first partition since /dev/xvda3 is + a block device. + + The primary use is to map 'ephemeral0.1' in the datasource to a + real device name + """ + + if not device: + return None + + if not partition and not alias: + raise Exception("partition or alias is required") + + if alias: + partition = map_partition(alias) + + # if the partition doesn't map, return the device + if not partition: + return device + + short_name = device.split('/')[-1] + sys_path = "/sys/block/%s" % short_name + + if not os.path.exists(sys_path): + return None + + sys_long_path = sys_path + "/" + short_name + valid_mappings = [sys_long_path + "%s" % partition, + sys_long_path + "p%s" % partition] + + for cdisk in valid_mappings: + if not os.path.exists(cdisk): + continue + + dev_path = "/dev/%s" % cdisk.split('/')[-1] + if os.path.exists(dev_path): + return dev_path + + return None + + diff --git a/doc/examples/cloud-config-disk-setup.txt b/doc/examples/cloud-config-disk-setup.txt index 3fc47699..bc6e1923 100644 --- a/doc/examples/cloud-config-disk-setup.txt +++ b/doc/examples/cloud-config-disk-setup.txt @@ -30,8 +30,8 @@ disk_setup: fs_setup: - label: ephemeral0 filesystem: ext4 - device: ephemeral0 - partition: auto + device: ephemeral0.1 + replace_fs: ntfs Default disk definitions for SmartOS @@ -47,8 +47,7 @@ disk_setup: fs_setup: - label: ephemeral0 filesystem: ext3 - device: ephemeral0 - partition: auto + device: ephemeral0.0 Cavaut for SmartOS: if ephemeral disk is not defined, then the disk will not be automatically added to the mounts. @@ -187,6 +186,9 @@ Where: label as 'ephemeralX' otherwise there may be issues with the mounting of the ephemeral storage layer. + If you define the device as 'ephemeralX.Y' then Y will be interpetted + as a partition value. However, ephermalX.0 is the _same_ as ephemeralX. + : The valid options are: "auto|any": tell cloud-init not to care whether there is a partition or not. Auto will use the first partition that does not contain a diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index df7e52d0..aad84206 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -328,8 +328,6 @@ class TestAzureDataSource(MockerTestCase): self.assertTrue(ret) cfg = dsrc.get_config_obj() self.assertTrue(cfg) - self.assertEquals(dsrc.device_name_to_device("ephemeral0"), - "/dev/sdc") def test_userdata_arrives(self): userdata = "This is my user-data" -- cgit v1.2.3 From 9f8822edfa5c1737933b83db54a3aa3143aaa280 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 3 Oct 2013 07:47:05 -0400 Subject: fix pep8 --- cloudinit/config/cc_disk_setup.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index faf424ba..c9885d09 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -16,8 +16,8 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from cloudinit import util from cloudinit.settings import PER_INSTANCE +from cloudinit import util import logging import shlex @@ -254,7 +254,7 @@ def find_device_node(device, fs_type=None, label=None, valid_targets=None, raw_device_used = False for d in enumerate_disk(device): - if d['fstype'] == replace_fs and label_match == False: + if d['fstype'] == replace_fs and label_match is False: # We found a device where we want to replace the FS return ('/dev/%s' % d['name'], False) @@ -470,6 +470,7 @@ def get_partition_mbr_layout(size, layout): return sfdisk_definition + def purge_disk(device): """ Remove parition table entries @@ -494,7 +495,7 @@ def purge_disk(device): try: util.subp(first_mb) LOG.info("Purged MBR/Partition table from %s" % device) - util.subp(last_mb, rcs=[0,1]) + util.subp(last_mb, rcs=[0, 1]) LOG.info("Purged any chance of GPT table from %s" % device) # Wipe it for good measure @@ -634,7 +635,7 @@ def lookup_force_flag(fs): } if 'ext' in fs.lower(): - fs = 'ext' + fs = 'ext' if fs.lower() in flags: return flags[fs] -- cgit v1.2.3 From 5bce3c2160e3b1b367e8bf3e33254847f31363b0 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 3 Oct 2013 08:36:33 -0400 Subject: use native python code for wiping partition table I'm pretty sure the previous code wasn't seeking correctly and probably writing near the end, but not to the end. This is simpler and probably faster. --- cloudinit/config/cc_disk_setup.py | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index c9885d09..4abaf13d 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -19,6 +19,7 @@ from cloudinit.settings import PER_INSTANCE from cloudinit import util import logging +import os import shlex frequency = PER_INSTANCE @@ -471,6 +472,20 @@ def get_partition_mbr_layout(size, layout): return sfdisk_definition +def purge_disk_ptable(device): + # wipe the first and last megabyte of a disk (or file) + # gpt stores partition table both at front and at end. + start_len = 1024 * 1024 + end_len = 1024 * 1024 + with open(device, "rb+") as fp: + fp.write('\0' * (start_len)) + fp.seek(-end_len, os.SEEK_END) + fp.write('\0' * end_len) + fp.flush() + + read_parttbl(device) + + def purge_disk(device): """ Remove parition table entries @@ -488,24 +503,7 @@ def purge_disk(device): except Exception as e: raise Exception("Failed FS purge of /dev/%s" % d['name']) - dd_cmd = util.which("dd") - last_seek = int(get_hdd_size(device) / 1024) - 2 - first_mb = [dd_cmd, "if=/dev/zero", "of=%s" % device, "bs=1M", "count=1"] - last_mb = [dd_cmd, "if=/dev/zero", "of=%s" % device, "bs=1M", "seek=%s" % last_seek] - try: - util.subp(first_mb) - LOG.info("Purged MBR/Partition table from %s" % device) - util.subp(last_mb, rcs=[0, 1]) - LOG.info("Purged any chance of GPT table from %s" % device) - - # Wipe it for good measure - wipefs_cmd = [WIPEFS_CMD, "--all", device] - util.subp(wipefs_cmd) - except Exception as e: - LOG.critical(e) - raise Exception("Failed to remove MBR/Part from %s" % device) - - read_parttbl(device) + purge_disk_ptable(device) def get_partition_layout(table_type, size, layout): -- cgit v1.2.3 From c3daa2fa160c930084509f4ef5abfbb562b7059d Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 3 Oct 2013 08:46:12 -0400 Subject: remove verbosity of log.info --- cloudinit/config/cc_disk_setup.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index 4abaf13d..e744cd57 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -230,7 +230,6 @@ def enumerate_disk(device): for key, value in value_splitter(part): d[key.lower()] = value - LOG.info(d) yield d @@ -493,13 +492,11 @@ def purge_disk(device): # wipe any file systems first for d in enumerate_disk(device): - LOG.info(d) if d['type'] not in ["disk", "crypt"]: wipefs_cmd = [WIPEFS_CMD, "--all", "/dev/%s" % d['name']] try: LOG.info("Purging filesystem on /dev/%s" % d['name']) util.subp(wipefs_cmd) - LOG.info("Purged filesystem on /dev/%s" % d['name']) except Exception as e: raise Exception("Failed FS purge of /dev/%s" % d['name']) -- cgit v1.2.3 From ecf2a600e41a9632ad305eb6a8cd5665908f31fb Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Thu, 3 Oct 2013 16:15:56 -0600 Subject: Moved ephemeralX.Y handling from Datasource into the cc_disk_setup, which makes it cloud agnostic. --- cloudinit/config/cc_disk_setup.py | 21 +++++++++++++++++++-- cloudinit/config/cc_mounts.py | 13 +++++++------ cloudinit/sources/DataSourceAzure.py | 4 +--- cloudinit/sources/DataSourceSmartOS.py | 4 +--- cloudinit/util.py | 11 +++++++---- 5 files changed, 35 insertions(+), 18 deletions(-) diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index e744cd57..554dfdd3 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -104,13 +104,30 @@ def update_fs_setup_devices(disk_setup, tformer): continue origname = definition.get('device') + if origname is None: continue - transformed = tformer(origname) + transformed = None + if len(origname.split('.')) > 1: + # this maps ephemeralX.Y to a proper disk name. For example, + # if the origname is 'ephemeral0.1' and transformed is /dev/sdb + # then the returned device will be /dev/sdb1 _if_ /dev/sdb1 exists + # otherwise NONE + base_name = origname.split('.')[0] + tformed = tformer(base_name) + LOG.info("base device for %s is %s" % (origname, tformed)) + + transformed = util.map_device_alias(tformed, alias=origname) + LOG.info("%s is mapped to %s" % (origname, transformed)) + + else: + transformed = tformer(origname) + if transformed is None or transformed == origname: continue + LOG.info("Mapped %s to physical device %s" % (origname, transformed)) definition['_origname'] = origname definition['device'] = transformed @@ -497,7 +514,7 @@ def purge_disk(device): try: LOG.info("Purging filesystem on /dev/%s" % d['name']) util.subp(wipefs_cmd) - except Exception as e: + except Exception: raise Exception("Failed FS purge of /dev/%s" % d['name']) purge_disk_ptable(device) diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index f4c2e3d8..6b47d326 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -217,14 +217,15 @@ def disk_or_part(device): short_name = device.split('/')[-1] sys_path = "/sys/block/%s" % short_name - if not os.path.exists(sys_path): - LOG.warn("Device %s does not exist in sysfs" % device) - return None + # if the sys path does not exist but the device exists, + # then the device is a partition, no sense looking any further + if not os.path.exists(sys_path) and os.path.exists(device): + return device sys_long_path = sys_path + "/" + short_name + "%s" - valid_mappings = [ sys_long_path % "1", - sys_long_path % "p1", - sys_path ] + valid_mappings = [sys_long_path % "1", + sys_long_path % "p1", + sys_path] for cdisk in valid_mappings: if not os.path.exists(cdisk): diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index b7de0187..8321dee0 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -177,9 +177,7 @@ class DataSourceAzureNet(sources.DataSource): return True def device_name_to_device(self, name): - device = name.split('.')[0] - return util.map_device_alias(self.ds_cfg['disk_aliases'].get(device), - alias=name) + return self.ds_cfg['disk_aliases'].get(name) def get_config_obj(self): return self.cfg diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 9b3fdf1a..666129ec 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -156,9 +156,7 @@ class DataSourceSmartOS(sources.DataSource): return True def device_name_to_device(self, name): - device = name.split('.')[0] - return util.map_device_alias(self.ds_cfg['disk_aliases'].get(device), - alias=name) + return self.ds_cfg['disk_aliases'].get(name) def get_config_obj(self): return self.cfg diff --git a/cloudinit/util.py b/cloudinit/util.py index 14519586..053e1efc 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1869,7 +1869,7 @@ def map_device_alias(device, partition=None, alias=None): """ if not device: - return None + raise Exception("Device cannot be undefined!") if not partition and not alias: raise Exception("partition or alias is required") @@ -1877,9 +1877,9 @@ def map_device_alias(device, partition=None, alias=None): if alias: partition = map_partition(alias) - # if the partition doesn't map, return the device - if not partition: - return device + # if the partition doesn't map, return the device + if not partition: + return device short_name = device.split('/')[-1] sys_path = "/sys/block/%s" % short_name @@ -1898,6 +1898,9 @@ def map_device_alias(device, partition=None, alias=None): dev_path = "/dev/%s" % cdisk.split('/')[-1] if os.path.exists(dev_path): return dev_path + else: + LOG.warn("Specificed parition %s does not exist on %s" % ( + partition, device)) return None -- cgit v1.2.3 From 1f107c05948a34cfe53dc26b2dcb863ee4a0f304 Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Thu, 3 Oct 2013 16:41:32 -0600 Subject: Configure SmartOS Datasource to be region aware --- cloudinit/sources/DataSourceSmartOS.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 666129ec..2813ffb3 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -46,6 +46,7 @@ SMARTOS_ATTRIB_MAP = { 'user-data': ('user-data', False), 'iptables_disable': ('iptables_disable', True), 'motd_sys_info': ('motd_sys_info', True), + 'availability_zone': ('region', True), } DS_NAME = 'SmartOS' @@ -175,6 +176,13 @@ class DataSourceSmartOS(sources.DataSource): seed_timeout=self.seed_timeout, default=default, b64=b64) + @property + def availability_zone(self): + try: + return self.metadata['availability-zone'] + except KeyError: + return None + def get_serial(seed_device, seed_timeout): """This is replaced in unit testing, allowing us to replace -- cgit v1.2.3 From 6bbe48d8551919960b3f4acce6299b0a6a53e9d5 Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Thu, 3 Oct 2013 17:04:22 -0600 Subject: Wrapped use of 'lsblk' to a generator function; removed other fcalls to 'lsblk' --- cloudinit/config/cc_disk_setup.py | 95 ++++++++++++++++++++------------------- 1 file changed, 48 insertions(+), 47 deletions(-) diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index 554dfdd3..b9e1e9eb 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -145,23 +145,55 @@ def value_splitter(values, start=None): yield key, value -def device_type(device): +def enumerate_disk(device, nodeps=False): """ - Return the device type of the device by calling lsblk. + Enumerate the elements of a child device. + + Parameters: + device: the kernel device name + nodeps : don't enumerate children devices + + Return a dict describing the disk: + type: the entry type, i.e disk or part + fstype: the filesystem type, if it exists + label: file system label, if it exists + name: the device name, i.e. sda """ - lsblk_cmd = [LSBLK_CMD, '--pairs', '--nodeps', '--out', 'NAME,TYPE', + lsblk_cmd = [LSBLK_CMD, '--pairs', '--out', 'NAME,TYPE,FSTYPE,LABEL', device] + + if nodeps: + lsblk_cmd.append('--nodeps') + info = None try: info, _err = util.subp(lsblk_cmd) except Exception as e: raise Exception("Failed during disk check for %s\n%s" % (device, e)) - for key, value in value_splitter(info): - if key.lower() == "type": - return value.lower() + parts = [x for x in (info.strip()).splitlines() if len(x.split()) > 0] + for part in parts: + d = {'name': None, + 'type': None, + 'fstype': None, + 'label': None, + } + + for key, value in value_splitter(part): + d[key.lower()] = value + + yield d + +def device_type(device): + """ + Return the device type of the device by calling lsblk. + """ + + for d in enumerate_disk(device, nodeps=True): + if "type" in d: + return d["type"].lower() return None @@ -221,35 +253,6 @@ def is_filesystem(device): return fs_type -def enumerate_disk(device): - """ - Enumerate the elements of a child device. Return a dict of name, - type, fstype, and label - """ - - lsblk_cmd = [LSBLK_CMD, '--pairs', '--out', 'NAME,TYPE,FSTYPE,LABEL', - device] - info = None - try: - info, _err = util.subp(lsblk_cmd) - except Exception as e: - raise Exception("Failed during disk check for %s\n%s" % (device, e)) - - parts = [x for x in (info.strip()).splitlines() if len(x.split()) > 0] - - for part in parts: - d = {'name': None, - 'type': None, - 'fstype': None, - 'label': None, - } - - for key, value in value_splitter(part): - d[key.lower()] = value - - yield d - - def find_device_node(device, fs_type=None, label=None, valid_targets=None, label_match=True, replace_fs=None): """ @@ -301,22 +304,20 @@ def find_device_node(device, fs_type=None, label=None, valid_targets=None, def is_disk_used(device): """ - Check if the device is currently used. Returns false if there + Check if the device is currently used. Returns true if the device + has either a file system or a partition entry is no filesystem found on the disk. """ - lsblk_cmd = [LSBLK_CMD, '--pairs', '--out', 'NAME,TYPE', - device] - info = None - try: - info, _err = util.subp(lsblk_cmd) - except Exception as e: - # if we error out, we can't use the device - util.logexc(LOG, - "Error checking for filesystem on %s\n%s" % (device, e)) + + # If the child count is higher 1, then there are child nodes + # such as partition or device mapper nodes + use_count = [x for x in enumerate_disk(device)] + if len(use_count.splitlines()) > 1: return True - # If there is any output, then the device has something - if len(info.splitlines()) > 1: + # If we see a file system, then its used + _, check_fstype, _ = check_fs(device) + if check_fstype: return True return False -- cgit v1.2.3 From 193704c37b97a5687cab09416f9f38e632c76fa3 Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Thu, 3 Oct 2013 17:07:23 -0600 Subject: Make {pep8,pylint,test} pass commit. --- cloudinit/config/cc_disk_setup.py | 3 ++- cloudinit/util.py | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index b9e1e9eb..127b5879 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -186,6 +186,7 @@ def enumerate_disk(device, nodeps=False): yield d + def device_type(device): """ Return the device type of the device by calling lsblk. @@ -193,7 +194,7 @@ def device_type(device): for d in enumerate_disk(device, nodeps=True): if "type" in d: - return d["type"].lower() + return d["type"].lower() return None diff --git a/cloudinit/util.py b/cloudinit/util.py index 053e1efc..34d111e7 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1903,5 +1903,3 @@ def map_device_alias(device, partition=None, alias=None): partition, device)) return None - - -- cgit v1.2.3 From 7b5480e46e14c6e31492e0a1e9ef4eb05a9d746d Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Thu, 3 Oct 2013 17:30:57 -0600 Subject: Added support for 'ephmeral0.' for device mappings in disk formating support. --- cloudinit/config/cc_disk_setup.py | 27 +++++++++++++++++++++++---- doc/examples/cloud-config-disk-setup.txt | 11 ++++++++++- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index 127b5879..5faffefc 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -94,6 +94,15 @@ def update_disk_setup_devices(disk_setup, tformer): LOG.debug("updated disk_setup device entry '%s' to '%s'", origname, transformed) +def reset_part_definition(definition, value): + if not value and 'partition' in definition: + definition['opartition'] = definition['partition'] + del definition['partition'] + + else: + definition['partition'] = value + + return definition def update_fs_setup_devices(disk_setup, tformer): # update 'fs_setup' dictionary anywhere were a device may occur @@ -109,17 +118,27 @@ def update_fs_setup_devices(disk_setup, tformer): continue transformed = None - if len(origname.split('.')) > 1: + if len(origname.split('.')) == 2: # this maps ephemeralX.Y to a proper disk name. For example, # if the origname is 'ephemeral0.1' and transformed is /dev/sdb # then the returned device will be /dev/sdb1 _if_ /dev/sdb1 exists # otherwise NONE - base_name = origname.split('.')[0] + base_name, partition = origname.split('.') tformed = tformer(base_name) LOG.info("base device for %s is %s" % (origname, tformed)) - transformed = util.map_device_alias(tformed, alias=origname) - LOG.info("%s is mapped to %s" % (origname, transformed)) + if partition == "0": + transformed = tformed + definition = reset_part_definition(definition, None) + + elif partition in ("auto", "any"): + definition = reset_part_definition(definition, partition) + transformed = tformed + + else: + definition = reset_part_definition(definition, None) + transformed = util.map_device_alias(tformed, alias=origname) + LOG.info("%s is mapped to %s" % (origname, transformed)) else: transformed = tformer(origname) diff --git a/doc/examples/cloud-config-disk-setup.txt b/doc/examples/cloud-config-disk-setup.txt index bc6e1923..6ad61c33 100644 --- a/doc/examples/cloud-config-disk-setup.txt +++ b/doc/examples/cloud-config-disk-setup.txt @@ -170,6 +170,7 @@ The general format is: device: partition: overwrite: + replace_fs: Where: