fix failure to create disks on azure, create new 'dev.part' notation

This fixes azure partition creation, and introduces 'ephemeral0.1'
notation to indicate "partition 1 on whatever device is ephemeral0".

LP: #1233698

7 files changed, 278 insertions, 93 deletions

diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py
index ade9c2ad..0b970e4e 100644
--- a/cloudinit/config/cc_disk_setup.py
+++ b/cloudinit/config/cc_disk_setup.py
@@ -19,6 +19,7 @@
 from cloudinit.settings import PER_INSTANCE
 from cloudinit import util
 import logging
+import os
 import shlex
 
 frequency = PER_INSTANCE
@@ -29,13 +30,13 @@ SFDISK_CMD = util.which("sfdisk")
 LSBLK_CMD = util.which("lsblk")
 BLKID_CMD = util.which("blkid")
 BLKDEV_CMD = util.which("blockdev")
+WIPEFS_CMD = util.which("wipefs")
 
 LOG = logging.getLogger(__name__)
 
 
 def handle(_name, cfg, cloud, log, _args):
     """
-    Call util.prep_disk for disk_setup cloud-config.
     See doc/examples/cloud-config_disk-setup.txt for documentation on the
     format.
     """
@@ -103,15 +104,23 @@ def update_fs_setup_devices(disk_setup, tformer):
             continue
 
         origname = definition.get('device')
+
         if origname is None:
             continue
 
-        transformed = tformer(origname)
-        if transformed is None or transformed == origname:
-            continue
+        (dev, part) = util.expand_dotted_devname(origname)
+
+        tformed = tformer(dev)
+        if tformed is not None:
+            dev = tformed
+            LOG.debug("%s is mapped to disk=%s part=%s",
+                      origname, tformed, part)
+            definition['_origname'] = origname
+            definition['device'] = tformed
 
-        definition['_origname'] = origname
-        definition['device'] = transformed
+        if part and 'partition' in definition:
+            definition['_partition'] = definition['partition']
+        definition['partition'] = part
 
 
 def value_splitter(values, start=None):
@@ -127,23 +136,56 @@ def value_splitter(values, start=None):
         yield key, value
 
 
-def device_type(device):
+def enumerate_disk(device, nodeps=False):
     """
-    Return the device type of the device by calling lsblk.
+    Enumerate the elements of a child device.
+
+    Parameters:
+        device: the kernel device name
+        nodeps <BOOL>: don't enumerate children devices
+
+    Return a dict describing the disk:
+        type: the entry type, i.e disk or part
+        fstype: the filesystem type, if it exists
+        label: file system label, if it exists
+        name: the device name, i.e. sda
     """
 
-    lsblk_cmd = [LSBLK_CMD, '--pairs', '--nodeps', '--out', 'NAME,TYPE',
+    lsblk_cmd = [LSBLK_CMD, '--pairs', '--out', 'NAME,TYPE,FSTYPE,LABEL',
                  device]
+
+    if nodeps:
+        lsblk_cmd.append('--nodeps')
+
     info = None
     try:
         info, _err = util.subp(lsblk_cmd)
     except Exception as e:
         raise Exception("Failed during disk check for %s\n%s" % (device, e))
 
-    for key, value in value_splitter(info):
-        if key.lower() == "type":
-            return value.lower()
+    parts = [x for x in (info.strip()).splitlines() if len(x.split()) > 0]
 
+    for part in parts:
+        d = {'name': None,
+             'type': None,
+             'fstype': None,
+             'label': None,
+            }
+
+        for key, value in value_splitter(part):
+            d[key.lower()] = value
+
+        yield d
+
+
+def device_type(device):
+    """
+    Return the device type of the device by calling lsblk.
+    """
+
+    for d in enumerate_disk(device, nodeps=True):
+        if "type" in d:
+            return d["type"].lower()
     return None
 
 
@@ -204,7 +246,7 @@ def is_filesystem(device):
 
 
 def find_device_node(device, fs_type=None, label=None, valid_targets=None,
-                     label_match=True):
+                     label_match=True, replace_fs=None):
     """
     Find a device that is either matches the spec, or the first
 
@@ -221,26 +263,12 @@ def find_device_node(device, fs_type=None, label=None, valid_targets=None,
     if not valid_targets:
         valid_targets = ['disk', 'part']
 
-    lsblk_cmd = [LSBLK_CMD, '--pairs', '--out', 'NAME,TYPE,FSTYPE,LABEL',
-                 device]
-    info = None
-    try:
-        info, _err = util.subp(lsblk_cmd)
-    except Exception as e:
-        raise Exception("Failed during disk check for %s\n%s" % (device, e))
-
     raw_device_used = False
-    parts = [x for x in (info.strip()).splitlines() if len(x.split()) > 0]
-
-    for part in parts:
-        d = {'name': None,
-             'type': None,
-             'fstype': None,
-             'label': None,
-            }
+    for d in enumerate_disk(device):
 
-        for key, value in value_splitter(part):
-            d[key.lower()] = value
+        if d['fstype'] == replace_fs and label_match is False:
+            # We found a device where we want to replace the FS
+            return ('/dev/%s' % d['name'], False)
 
         if (d['fstype'] == fs_type and
             ((label_match and d['label'] == label) or not label_match)):
@@ -268,22 +296,20 @@ def find_device_node(device, fs_type=None, label=None, valid_targets=None,
 
 def is_disk_used(device):
     """
-    Check if the device is currently used. Returns false if there
+    Check if the device is currently used. Returns true if the device
+    has either a file system or a partition entry
     is no filesystem found on the disk.
     """
-    lsblk_cmd = [LSBLK_CMD, '--pairs', '--out', 'NAME,TYPE',
-                 device]
-    info = None
-    try:
-        info, _err = util.subp(lsblk_cmd)
-    except Exception as e:
-        # if we error out, we can't use the device
-        util.logexc(LOG,
-                    "Error checking for filesystem on %s\n%s" % (device, e))
+
+    # If the child count is higher 1, then there are child nodes
+    # such as partition or device mapper nodes
+    use_count = [x for x in enumerate_disk(device)]
+    if len(use_count.splitlines()) > 1:
         return True
 
-    # If there is any output, then the device has something
-    if len(info.splitlines()) > 1:
+    # If we see a file system, then its used
+    _, check_fstype, _ = check_fs(device)
+    if check_fstype:
         return True
 
     return False
@@ -455,6 +481,39 @@ def get_partition_mbr_layout(size, layout):
     return sfdisk_definition
 
 
+def purge_disk_ptable(device):
+    # wipe the first and last megabyte of a disk (or file)
+    # gpt stores partition table both at front and at end.
+    null = '\0'  # pylint: disable=W1401
+    start_len = 1024 * 1024
+    end_len = 1024 * 1024
+    with open(device, "rb+") as fp:
+        fp.write(null * (start_len))
+        fp.seek(-end_len, os.SEEK_END)
+        fp.write(null * end_len)
+        fp.flush()
+
+    read_parttbl(device)
+
+
+def purge_disk(device):
+    """
+    Remove parition table entries
+    """
+
+    # wipe any file systems first
+    for d in enumerate_disk(device):
+        if d['type'] not in ["disk", "crypt"]:
+            wipefs_cmd = [WIPEFS_CMD, "--all", "/dev/%s" % d['name']]
+            try:
+                LOG.info("Purging filesystem on /dev/%s" % d['name'])
+                util.subp(wipefs_cmd)
+            except Exception:
+                raise Exception("Failed FS purge of /dev/%s" % d['name'])
+
+    purge_disk_ptable(device)
+
+
 def get_partition_layout(table_type, size, layout):
     """
     Call the appropriate function for creating the table
@@ -542,6 +601,12 @@ def mkpart(device, definition):
     if not is_device_valid(device):
         raise Exception("Device %s is not a disk device!", device)
 
+    # Remove the partition table entries
+    if isinstance(layout, str) and layout.lower() == "remove":
+        LOG.debug("Instructed to remove partition table entries")
+        purge_disk(device)
+        return
+
     LOG.debug("Checking if device layout matches")
     if check_partition_layout(table_type, device, layout):
         LOG.debug("Device partitioning layout matches")
@@ -565,6 +630,26 @@ def mkpart(device, definition):
     LOG.debug("Partition table created for %s", device)
 
 
+def lookup_force_flag(fs):
+    """
+    A force flag might be -F or -F, this look it up
+    """
+    flags = {'ext': '-F',
+             'btrfs': '-f',
+             'xfs': '-f',
+             'reiserfs': '-f',
+            }
+
+    if 'ext' in fs.lower():
+        fs = 'ext'
+
+    if fs.lower() in flags:
+        return flags[fs]
+
+    LOG.warn("Force flag for %s is unknown." % fs)
+    return ''
+
+
 def mkfs(fs_cfg):
     """
     Create a file system on the device.
@@ -592,6 +677,7 @@ def mkfs(fs_cfg):
     fs_type = fs_cfg.get('filesystem')
     fs_cmd = fs_cfg.get('cmd', [])
     fs_opts = fs_cfg.get('extra_opts', [])
+    fs_replace = fs_cfg.get('replace_fs', False)
     overwrite = fs_cfg.get('overwrite', False)
 
     # This allows you to define the default ephemeral or swap
@@ -632,17 +718,23 @@ def mkfs(fs_cfg):
             label_match = False
 
         device, reuse = find_device_node(device, fs_type=fs_type, label=label,
-                                         label_match=label_match)
+                                         label_match=label_match,
+                                         replace_fs=fs_replace)
         LOG.debug("Automatic device for %s identified as %s", odevice, device)
 
         if reuse:
             LOG.debug("Found filesystem match, skipping formating.")
             return
 
+        if not reuse and fs_replace and device:
+            LOG.debug("Replacing file system on %s as instructed." % device)
+
         if not device:
             LOG.debug("No device aviable that matches request. "
                       "Skipping fs creation for %s", fs_cfg)
             return
+    elif not partition or str(partition).lower() == 'none':
+        LOG.debug("Using the raw device to place filesystem %s on" % label)
 
     else:
         LOG.debug("Error in device identification handling.")
@@ -682,12 +774,16 @@ def mkfs(fs_cfg):
         if label:
             fs_cmd.extend(["-L", label])
 
+    # File systems that support the -F flag
+    if not fs_cmd and (overwrite or device_type(device) == "disk"):
+        fs_cmd.append(lookup_force_flag(fs_type))
+
     # Add the extends FS options
     if fs_opts:
         fs_cmd.extend(fs_opts)
 
     LOG.debug("Creating file system %s on %s", label, device)
-    LOG.debug("     Using cmd: %s", "".join(fs_cmd))
+    LOG.debug("     Using cmd: %s", " ".join(fs_cmd))
     try:
         util.subp(fs_cmd)
     except Exception as e:
diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py
index 390ba711..84ec928f 100644
--- a/cloudinit/config/cc_mounts.py
+++ b/cloudinit/config/cc_mounts.py
@@ -20,6 +20,8 @@
 
 from string import whitespace  # pylint: disable=W0402
 
+import logging
+import os.path
 import re
 
 from cloudinit import type_utils
@@ -31,6 +33,8 @@ SHORTNAME = re.compile(SHORTNAME_FILTER)
 WS = re.compile("[%s]+" % (whitespace))
 FSTAB_PATH = "/etc/fstab"
 
+LOG = logging.getLogger(__name__)
+
 
 def is_mdname(name):
     # return true if this is a metadata service name
@@ -44,6 +48,33 @@ def is_mdname(name):
     return False
 
 
+def sanitize_devname(startname, transformer, log):
+    log.debug("Attempting to determine the real name of %s", startname)
+
+    # workaround, allow user to specify 'ephemeral'
+    # rather than more ec2 correct 'ephemeral0'
+    devname = startname
+    if devname == "ephemeral":
+        devname = "ephemeral0"
+        log.debug("Adjusted mount option from ephemeral to ephemeral0")
+
+    (blockdev, part) = util.expand_dotted_devname(devname)
+
+    if is_mdname(blockdev):
+        orig = blockdev
+        blockdev = transformer(blockdev)
+        if not blockdev:
+            return None
+        if not blockdev.startswith("/"):
+            blockdev = "/dev/%s" % blockdev
+        log.debug("Mapped metadata name %s to %s", orig, blockdev)
+    else:
+        if SHORTNAME.match(startname):
+            blockdev = "/dev/%s" % blockdev
+
+    return devnode_for_dev_part(blockdev, part)
+
+
 def handle(_name, cfg, cloud, log, _args):
     # fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno
     defvals = [None, None, "auto", "defaults,nobootwait", "0", "2"]
@@ -64,32 +95,15 @@ def handle(_name, cfg, cloud, log, _args):
                      (i + 1), type_utils.obj_name(cfgmnt[i]))
             continue
 
-        startname = str(cfgmnt[i][0])
-        log.debug("Attempting to determine the real name of %s", startname)
-
-        # workaround, allow user to specify 'ephemeral'
-        # rather than more ec2 correct 'ephemeral0'
-        if startname == "ephemeral":
-            cfgmnt[i][0] = "ephemeral0"
-            log.debug(("Adjusted mount option %s "
-                       "name from ephemeral to ephemeral0"), (i + 1))
-
-        if is_mdname(startname):
-            newname = cloud.device_name_to_device(startname)
-            if not newname:
-                log.debug("Ignoring nonexistant named mount %s", startname)
-                cfgmnt[i][1] = None
-            else:
-                renamed = newname
-                if not newname.startswith("/"):
-                    renamed = "/dev/%s" % newname
-                cfgmnt[i][0] = renamed
-                log.debug("Mapped metadata name %s to %s", startname, renamed)
-        else:
-            if SHORTNAME.match(startname):
-                renamed = "/dev/%s" % startname
-                log.debug("Mapped shortname name %s to %s", startname, renamed)
-                cfgmnt[i][0] = renamed
+        start = str(cfgmnt[i][0])
+        sanitized = sanitize_devname(start, cloud.device_name_to_device, log)
+        if sanitized is None:
+            log.debug("Ignorming nonexistant named mount %s", start)
+            continue
+
+        if sanitized != start:
+            log.debug("changed %s => %s" % (start, sanitized))
+        cfgmnt[i][0] = sanitized
 
         # in case the user did not quote a field (likely fs-freq, fs_passno)
         # but do not convert None to 'None' (LP: #898365)
@@ -118,17 +132,14 @@ def handle(_name, cfg, cloud, log, _args):
     # for each of the "default" mounts, add them only if no other
     # entry has the same device name
     for defmnt in defmnts:
-        startname = defmnt[0]
-        devname = cloud.device_name_to_device(startname)
-        if devname is None:
-            log.debug("Ignoring nonexistant named default mount %s", startname)
+        start = defmnt[0]
+        sanitized = sanitize_devname(start, cloud.device_name_to_device, log)
+        if sanitized is None:
+            log.debug("Ignoring nonexistant default named mount %s", start)
             continue
-        if devname.startswith("/"):
-            defmnt[0] = devname
-        else:
-            defmnt[0] = "/dev/%s" % devname
-
-        log.debug("Mapped default device %s to %s", startname, defmnt[0])
+        if sanitized != start:
+            log.debug("changed default device %s => %s" % (start, sanitized))
+        defmnt[0] = sanitized
 
         cfgmnt_has = False
         for cfgm in cfgmnt:
@@ -138,7 +149,7 @@ def handle(_name, cfg, cloud, log, _args):
 
         if cfgmnt_has:
             log.debug(("Not including %s, already"
-                       " previously included"), startname)
+                       " previously included"), start)
             continue
         cfgmnt.append(defmnt)
 
@@ -198,3 +209,53 @@ def handle(_name, cfg, cloud, log, _args):
         util.subp(("mount", "-a"))
     except:
         util.logexc(log, "Activating mounts via 'mount -a' failed")
+
+
+def devnode_for_dev_part(device, partition):
+    """
+    Find the name of the partition. While this might seem rather
+    straight forward, its not since some devices are '<device><partition>'
+    while others are '<device>p<partition>'. For example, /dev/xvda3 on EC2
+    will present as /dev/xvda3p1 for the first partition since /dev/xvda3 is
+    a block device.
+    """
+    if not os.path.exists(device):
+        return None
+
+    if not partition:
+        return device
+
+    short_name = os.path.basename(device)
+    sys_path = "/sys/block/%s" % short_name
+
+    if not os.path.exists(sys_path):
+        LOG.debug("did not find entry for %s in /sys/block", short_name)
+        return None
+
+    sys_long_path = sys_path + "/" + short_name
+
+    if partition is not None:
+        partition = str(partition)
+
+    if partition is None:
+        valid_mappings = [sys_long_path + "1",
+                          sys_long_path + "p1" % partition]
+    elif partition != "0":
+        valid_mappings = [sys_long_path + "%s" % partition,
+                          sys_long_path + "p%s" % partition]
+    else:
+        valid_mappings = []
+
+    for cdisk in valid_mappings:
+        if not os.path.exists(cdisk):
+            continue
+
+        dev_path = "/dev/%s" % os.path.basename(cdisk)
+        if os.path.exists(dev_path):
+            return dev_path
+
+    if partition is None or partition == "0":
+        return device
+
+    LOG.debug("Did not fine partition %s for device %s", partition, device)
+    return None
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index 7ba6cea8..8321dee0 100644
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -54,8 +54,9 @@ BUILTIN_CLOUD_CONFIG = {
                        'layout': True,
                        'overwrite': False}
          },
-    'fs_setup': [{'filesystem': 'ext4', 'device': 'ephemeral0',
-                  'partition': 'auto'}],
+    'fs_setup': [{'filesystem': 'ext4',
+                  'device': 'ephemeral0.1',
+                  'replace_fs': 'ntfs'}]
 }
 
 DS_CFG_PATH = ['datasource', DS_NAME]
diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index 93b8b50b..2813ffb3 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -46,6 +46,7 @@ SMARTOS_ATTRIB_MAP = {
     'user-data': ('user-data', False),
     'iptables_disable': ('iptables_disable', True),
     'motd_sys_info': ('motd_sys_info', True),
+    'availability_zone': ('region', True),
 }
 
 DS_NAME = 'SmartOS'
@@ -81,8 +82,9 @@ BUILTIN_CLOUD_CONFIG = {
                        'layout': False,
                        'overwrite': False}
          },
-    'fs_setup': [{'label': 'ephemeral0', 'filesystem': 'ext3',
-                  'device': 'ephemeral0', 'partition': 'auto'}],
+    'fs_setup': [{'label': 'ephemeral0',
+                  'filesystem': 'ext3',
+                  'device': 'ephemeral0'}],
 }
 
 
@@ -174,6 +176,13 @@ class DataSourceSmartOS(sources.DataSource):
                           seed_timeout=self.seed_timeout, default=default,
                           b64=b64)
 
+    @property
+    def availability_zone(self):
+        try:
+            return self.metadata['availability-zone']
+        except KeyError:
+            return None
+
 
 def get_serial(seed_device, seed_timeout):
     """This is replaced in unit testing, allowing us to replace
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 50ca7959..9e6e0a73 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -32,6 +32,7 @@ import grp
 import gzip
 import hashlib
 import os
+import os.path
 import platform
 import pwd
 import random
@@ -1826,3 +1827,11 @@ def log_time(logfunc, msg, func, args=None, kwargs=None, get_uptime=False):
         except:
             pass
     return ret
+
+
+def expand_dotted_devname(dotted):
+    toks = dotted.rsplit(".", 1)
+    if len(toks) > 1:
+        return toks
+    else:
+        return (dotted, None)
diff --git a/doc/examples/cloud-config-disk-setup.txt b/doc/examples/cloud-config-disk-setup.txt
index 3fc47699..6ad61c33 100644
--- a/doc/examples/cloud-config-disk-setup.txt
+++ b/doc/examples/cloud-config-disk-setup.txt
@@ -30,8 +30,8 @@ disk_setup:
 fs_setup:
     - label: ephemeral0
       filesystem: ext4
-      device: ephemeral0
-      partition: auto
+      device: ephemeral0.1
+      replace_fs: ntfs
 
 
 Default disk definitions for SmartOS
@@ -47,8 +47,7 @@ disk_setup:
 fs_setup:
     - label: ephemeral0
       filesystem: ext3
-      device: ephemeral0
-      partition: auto
+      device: ephemeral0.0
 
 Cavaut for SmartOS: if ephemeral disk is not defined, then the disk will
     not be automatically added to the mounts.
@@ -171,6 +170,7 @@ The general format is:
           device: <DEVICE>
           partition: <PART_VALUE>
           overwrite: <OVERWRITE>
+          replace_fs: <FS_TYPE>
 
 Where:
     <LABEL>: The file system label to be used. If set to None, no label is
@@ -187,7 +187,13 @@ Where:
         label as 'ephemeralX' otherwise there may be issues with the mounting
         of the ephemeral storage layer.
 
-    <PART_VALUE>: The valid options are:
+        If you define the device as 'ephemeralX.Y' then Y will be interpetted
+        as a partition value. However, ephermalX.0 is the _same_ as ephemeralX.
+
+    <PART_VALUE>:
+        Partition definitions are overwriten if you use the '<DEVICE>.Y' notation.
+
+        The valid options are:
         "auto|any": tell cloud-init not to care whether there is a partition
             or not. Auto will use the first partition that does not contain a
             file system already. In the absence of a partition table, it will
@@ -236,5 +242,10 @@ Where:
 
         "false": If an existing file system exists, skip the creation.
 
+    <REPLACE_FS>: This is a special directive, used for Windows Azure that
+        instructs cloud-init to replace a file system of <FS_TYPE>. NOTE:
+        unless you define a label, this requires the use of the 'any' partition
+        directive.
+
 Behavior Caveat: The default behavior is to _check_ if the file system exists.
     If a file system matches the specification, then the operation is a no-op.
diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
index df7e52d0..aad84206 100644
--- a/tests/unittests/test_datasource/test_azure.py
+++ b/tests/unittests/test_datasource/test_azure.py
@@ -328,8 +328,6 @@ class TestAzureDataSource(MockerTestCase):
         self.assertTrue(ret)
         cfg = dsrc.get_config_obj()
         self.assertTrue(cfg)
-        self.assertEquals(dsrc.device_name_to_device("ephemeral0"),
-                          "/dev/sdc")
 
     def test_userdata_arrives(self):
         userdata = "This is my user-data"