Added ability to define disks via 'ephemeralX.Y'.

Modified cc_mounts to identify whether ephermalX is partitioned.
Changed datasources for Azure and SmartOS to use 'ephemeralX.Y' format.
Added disk remove functionally

7 files changed, 238 insertions, 34 deletions

diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py
index ade9c2ad..faf424ba 100644
--- a/cloudinit/config/cc_disk_setup.py
+++ b/cloudinit/config/cc_disk_setup.py
@@ -16,8 +16,8 @@
 #
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-from cloudinit.settings import PER_INSTANCE
 from cloudinit import util
+from cloudinit.settings import PER_INSTANCE
 import logging
 import shlex
 
@@ -29,13 +29,13 @@ SFDISK_CMD = util.which("sfdisk")
 LSBLK_CMD = util.which("lsblk")
 BLKID_CMD = util.which("blkid")
 BLKDEV_CMD = util.which("blockdev")
+WIPEFS_CMD = util.which("wipefs")
 
 LOG = logging.getLogger(__name__)
 
 
 def handle(_name, cfg, cloud, log, _args):
     """
-    Call util.prep_disk for disk_setup cloud-config.
     See doc/examples/cloud-config_disk-setup.txt for documentation on the
     format.
     """
@@ -203,23 +203,11 @@ def is_filesystem(device):
     return fs_type
 
 
-def find_device_node(device, fs_type=None, label=None, valid_targets=None,
-                     label_match=True):
+def enumerate_disk(device):
     """
-    Find a device that is either matches the spec, or the first
-
-    The return is value is (<device>, <bool>) where the device is the
-    device to use and the bool is whether the device matches the
-    fs_type and label.
-
-    Note: This works with GPT partition tables!
+    Enumerate the elements of a child device. Return a dict of name,
+        type, fstype, and label
     """
-    # label of None is same as no label
-    if label is None:
-        label = ""
-
-    if not valid_targets:
-        valid_targets = ['disk', 'part']
 
     lsblk_cmd = [LSBLK_CMD, '--pairs', '--out', 'NAME,TYPE,FSTYPE,LABEL',
                  device]
@@ -229,7 +217,6 @@ def find_device_node(device, fs_type=None, label=None, valid_targets=None,
     except Exception as e:
         raise Exception("Failed during disk check for %s\n%s" % (device, e))
 
-    raw_device_used = False
     parts = [x for x in (info.strip()).splitlines() if len(x.split()) > 0]
 
     for part in parts:
@@ -242,6 +229,35 @@ def find_device_node(device, fs_type=None, label=None, valid_targets=None,
         for key, value in value_splitter(part):
             d[key.lower()] = value
 
+        LOG.info(d)
+        yield d
+
+
+def find_device_node(device, fs_type=None, label=None, valid_targets=None,
+                     label_match=True, replace_fs=None):
+    """
+    Find a device that is either matches the spec, or the first
+
+    The return is value is (<device>, <bool>) where the device is the
+    device to use and the bool is whether the device matches the
+    fs_type and label.
+
+    Note: This works with GPT partition tables!
+    """
+    # label of None is same as no label
+    if label is None:
+        label = ""
+
+    if not valid_targets:
+        valid_targets = ['disk', 'part']
+
+    raw_device_used = False
+    for d in enumerate_disk(device):
+
+        if d['fstype'] == replace_fs and label_match == False:
+            # We found a device where we want to replace the FS
+            return ('/dev/%s' % d['name'], False)
+
         if (d['fstype'] == fs_type and
             ((label_match and d['label'] == label) or not label_match)):
             # If we find a matching device, we return that
@@ -454,6 +470,42 @@ def get_partition_mbr_layout(size, layout):
 
     return sfdisk_definition
 
+def purge_disk(device):
+    """
+    Remove parition table entries
+    """
+
+    # wipe any file systems first
+    for d in enumerate_disk(device):
+        LOG.info(d)
+        if d['type'] not in ["disk", "crypt"]:
+            wipefs_cmd = [WIPEFS_CMD, "--all", "/dev/%s" % d['name']]
+            try:
+                LOG.info("Purging filesystem on /dev/%s" % d['name'])
+                util.subp(wipefs_cmd)
+                LOG.info("Purged filesystem on /dev/%s" % d['name'])
+            except Exception as e:
+                raise Exception("Failed FS purge of /dev/%s" % d['name'])
+
+    dd_cmd = util.which("dd")
+    last_seek = int(get_hdd_size(device) / 1024) - 2
+    first_mb = [dd_cmd, "if=/dev/zero", "of=%s" % device, "bs=1M", "count=1"]
+    last_mb = [dd_cmd, "if=/dev/zero", "of=%s" % device, "bs=1M", "seek=%s" % last_seek]
+    try:
+        util.subp(first_mb)
+        LOG.info("Purged MBR/Partition table from %s" % device)
+        util.subp(last_mb, rcs=[0,1])
+        LOG.info("Purged any chance of GPT table from %s" % device)
+
+        # Wipe it for good measure
+        wipefs_cmd = [WIPEFS_CMD, "--all", device]
+        util.subp(wipefs_cmd)
+    except Exception as e:
+        LOG.critical(e)
+        raise Exception("Failed to remove MBR/Part from %s" % device)
+
+    read_parttbl(device)
+
 
 def get_partition_layout(table_type, size, layout):
     """
@@ -542,6 +594,12 @@ def mkpart(device, definition):
     if not is_device_valid(device):
         raise Exception("Device %s is not a disk device!", device)
 
+    # Remove the partition table entries
+    if isinstance(layout, str) and layout.lower() == "remove":
+        LOG.debug("Instructed to remove partition table entries")
+        purge_disk(device)
+        return
+
     LOG.debug("Checking if device layout matches")
     if check_partition_layout(table_type, device, layout):
         LOG.debug("Device partitioning layout matches")
@@ -565,6 +623,26 @@ def mkpart(device, definition):
     LOG.debug("Partition table created for %s", device)
 
 
+def lookup_force_flag(fs):
+    """
+    A force flag might be -F or -F, this look it up
+    """
+    flags = {'ext': '-F',
+             'btrfs': '-f',
+             'xfs': '-f',
+             'reiserfs': '-f',
+            }
+
+    if 'ext' in fs.lower():
+       fs = 'ext'
+
+    if fs.lower() in flags:
+        return flags[fs]
+
+    LOG.warn("Force flag for %s is unknown." % fs)
+    return ''
+
+
 def mkfs(fs_cfg):
     """
     Create a file system on the device.
@@ -592,6 +670,7 @@ def mkfs(fs_cfg):
     fs_type = fs_cfg.get('filesystem')
     fs_cmd = fs_cfg.get('cmd', [])
     fs_opts = fs_cfg.get('extra_opts', [])
+    fs_replace = fs_cfg.get('replace_fs', False)
     overwrite = fs_cfg.get('overwrite', False)
 
     # This allows you to define the default ephemeral or swap
@@ -632,17 +711,23 @@ def mkfs(fs_cfg):
             label_match = False
 
         device, reuse = find_device_node(device, fs_type=fs_type, label=label,
-                                         label_match=label_match)
+                                         label_match=label_match,
+                                         replace_fs=fs_replace)
         LOG.debug("Automatic device for %s identified as %s", odevice, device)
 
         if reuse:
             LOG.debug("Found filesystem match, skipping formating.")
             return
 
+        if not reuse and fs_replace and device:
+            LOG.debug("Replacing file system on %s as instructed." % device)
+
         if not device:
             LOG.debug("No device aviable that matches request. "
                       "Skipping fs creation for %s", fs_cfg)
             return
+    elif not partition or str(partition).lower() == 'none':
+        LOG.debug("Using the raw device to place filesystem %s on" % label)
 
     else:
         LOG.debug("Error in device identification handling.")
@@ -682,12 +767,16 @@ def mkfs(fs_cfg):
         if label:
             fs_cmd.extend(["-L", label])
 
+    # File systems that support the -F flag
+    if not fs_cmd and (overwrite or device_type(device) == "disk"):
+        fs_cmd.append(lookup_force_flag(fs_type))
+
     # Add the extends FS options
     if fs_opts:
         fs_cmd.extend(fs_opts)
 
     LOG.debug("Creating file system %s on %s", label, device)
-    LOG.debug("     Using cmd: %s", "".join(fs_cmd))
+    LOG.debug("     Using cmd: %s", " ".join(fs_cmd))
     try:
         util.subp(fs_cmd)
     except Exception as e:
diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py
index 390ba711..f4c2e3d8 100644
--- a/cloudinit/config/cc_mounts.py
+++ b/cloudinit/config/cc_mounts.py
@@ -20,6 +20,7 @@
 
 from string import whitespace  # pylint: disable=W0402
 
+import os.path
 import re
 
 from cloudinit import type_utils
@@ -75,7 +76,9 @@ def handle(_name, cfg, cloud, log, _args):
                        "name from ephemeral to ephemeral0"), (i + 1))
 
         if is_mdname(startname):
-            newname = cloud.device_name_to_device(startname)
+            candidate_name = cloud.device_name_to_device(startname)
+            newname = disk_or_part(candidate_name)
+
             if not newname:
                 log.debug("Ignoring nonexistant named mount %s", startname)
                 cfgmnt[i][1] = None
@@ -119,7 +122,8 @@ def handle(_name, cfg, cloud, log, _args):
     # entry has the same device name
     for defmnt in defmnts:
         startname = defmnt[0]
-        devname = cloud.device_name_to_device(startname)
+        candidate_name = cloud.device_name_to_device(startname)
+        devname = disk_or_part(candidate_name)
         if devname is None:
             log.debug("Ignoring nonexistant named default mount %s", startname)
             continue
@@ -198,3 +202,32 @@ def handle(_name, cfg, cloud, log, _args):
         util.subp(("mount", "-a"))
     except:
         util.logexc(log, "Activating mounts via 'mount -a' failed")
+
+
+def disk_or_part(device):
+    """
+    Find where the file system is on the disk, either on
+    the disk itself or on the first partition. We don't go
+    any deeper than partition 1 though.
+    """
+
+    if not device:
+        return None
+
+    short_name = device.split('/')[-1]
+    sys_path = "/sys/block/%s" % short_name
+
+    if not os.path.exists(sys_path):
+        LOG.warn("Device %s does not exist in sysfs" % device)
+        return None
+
+    sys_long_path = sys_path + "/" + short_name + "%s"
+    valid_mappings = [ sys_long_path % "1",
+                       sys_long_path % "p1",
+                       sys_path ]
+
+    for cdisk in valid_mappings:
+        if not os.path.exists(cdisk):
+            continue
+        return "/dev/%s" % cdisk.split('/')[-1]
+    return None
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index 7ba6cea8..b7de0187 100644
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -54,8 +54,9 @@ BUILTIN_CLOUD_CONFIG = {
                        'layout': True,
                        'overwrite': False}
          },
-    'fs_setup': [{'filesystem': 'ext4', 'device': 'ephemeral0',
-                  'partition': 'auto'}],
+    'fs_setup': [{'filesystem': 'ext4',
+                  'device': 'ephemeral0.1',
+                  'replace_fs': 'ntfs'}]
 }
 
 DS_CFG_PATH = ['datasource', DS_NAME]
@@ -176,7 +177,9 @@ class DataSourceAzureNet(sources.DataSource):
         return True
 
     def device_name_to_device(self, name):
-        return self.ds_cfg['disk_aliases'].get(name)
+        device = name.split('.')[0]
+        return util.map_device_alias(self.ds_cfg['disk_aliases'].get(device),
+                                     alias=name)
 
     def get_config_obj(self):
         return self.cfg
diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index 93b8b50b..9b3fdf1a 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -81,8 +81,9 @@ BUILTIN_CLOUD_CONFIG = {
                        'layout': False,
                        'overwrite': False}
          },
-    'fs_setup': [{'label': 'ephemeral0', 'filesystem': 'ext3',
-                  'device': 'ephemeral0', 'partition': 'auto'}],
+    'fs_setup': [{'label': 'ephemeral0',
+                  'filesystem': 'ext3',
+                  'device': 'ephemeral0'}],
 }
 
 
@@ -155,7 +156,9 @@ class DataSourceSmartOS(sources.DataSource):
         return True
 
     def device_name_to_device(self, name):
-        return self.ds_cfg['disk_aliases'].get(name)
+        device = name.split('.')[0]
+        return util.map_device_alias(self.ds_cfg['disk_aliases'].get(device),
+                                     alias=name)
 
     def get_config_obj(self):
         return self.cfg
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 50ca7959..14519586 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -32,6 +32,7 @@ import grp
 import gzip
 import hashlib
 import os
+import os.path
 import platform
 import pwd
 import random
@@ -1826,3 +1827,78 @@ def log_time(logfunc, msg, func, args=None, kwargs=None, get_uptime=False):
         except:
             pass
     return ret
+
+
+def map_partition(alias):
+    """
+    Return partition number for devices like ephemeral0.0 or ephemeral0.1
+
+    Parameters:
+        alaias: the alias, i.e. ephemeral0 or swap0
+        device: the actual device to markup
+
+    Rules:
+        - anything after a . is a parittion
+        - device.0 is the same as device
+    """
+
+    if len(alias.split('.')) == 1:
+        return None
+
+    suffix = alias.split('.')[-1]
+    try:
+        if int(suffix) == 0:
+            return None
+        return int(suffix)
+    except ValueError:
+        pass
+
+    return None
+
+
+def map_device_alias(device, partition=None, alias=None):
+    """
+    Find the name of the partition. While this might seem rather
+    straight forward, its not since some devices are '<device><partition>'
+    while others are '<device>p<partition>'. For example, /dev/xvda3 on EC2
+    will present as /dev/xvda3p1 for the first partition since /dev/xvda3 is
+    a block device.
+
+    The primary use is to map 'ephemeral0.1' in the datasource to a
+    real device name
+    """
+
+    if not device:
+        return None
+
+    if not partition and not alias:
+        raise Exception("partition or alias is required")
+
+    if alias:
+        partition = map_partition(alias)
+
+        # if the partition doesn't map, return the device
+        if not partition:
+            return device
+
+    short_name = device.split('/')[-1]
+    sys_path = "/sys/block/%s" % short_name
+
+    if not os.path.exists(sys_path):
+        return None
+
+    sys_long_path = sys_path + "/" + short_name
+    valid_mappings = [sys_long_path + "%s" % partition,
+                      sys_long_path + "p%s" % partition]
+
+    for cdisk in valid_mappings:
+        if not os.path.exists(cdisk):
+            continue
+
+        dev_path = "/dev/%s" % cdisk.split('/')[-1]
+        if os.path.exists(dev_path):
+            return dev_path
+
+    return None
+
+
diff --git a/doc/examples/cloud-config-disk-setup.txt b/doc/examples/cloud-config-disk-setup.txt
index 3fc47699..bc6e1923 100644
--- a/doc/examples/cloud-config-disk-setup.txt
+++ b/doc/examples/cloud-config-disk-setup.txt
@@ -30,8 +30,8 @@ disk_setup:
 fs_setup:
     - label: ephemeral0
       filesystem: ext4
-      device: ephemeral0
-      partition: auto
+      device: ephemeral0.1
+      replace_fs: ntfs
 
 
 Default disk definitions for SmartOS
@@ -47,8 +47,7 @@ disk_setup:
 fs_setup:
     - label: ephemeral0
       filesystem: ext3
-      device: ephemeral0
-      partition: auto
+      device: ephemeral0.0
 
 Cavaut for SmartOS: if ephemeral disk is not defined, then the disk will
     not be automatically added to the mounts.
@@ -187,6 +186,9 @@ Where:
         label as 'ephemeralX' otherwise there may be issues with the mounting
         of the ephemeral storage layer.
 
+        If you define the device as 'ephemeralX.Y' then Y will be interpetted
+        as a partition value. However, ephermalX.0 is the _same_ as ephemeralX.
+
     <PART_VALUE>: The valid options are:
         "auto|any": tell cloud-init not to care whether there is a partition
             or not. Auto will use the first partition that does not contain a
diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
index df7e52d0..aad84206 100644
--- a/tests/unittests/test_datasource/test_azure.py
+++ b/tests/unittests/test_datasource/test_azure.py
@@ -328,8 +328,6 @@ class TestAzureDataSource(MockerTestCase):
         self.assertTrue(ret)
         cfg = dsrc.get_config_obj()
         self.assertTrue(cfg)
-        self.assertEquals(dsrc.device_name_to_device("ephemeral0"),
-                          "/dev/sdc")
 
     def test_userdata_arrives(self):
         userdata = "This is my user-data"