summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--cloudinit/config/cc_disk_setup.py186
-rw-r--r--cloudinit/config/cc_mounts.py135
-rw-r--r--cloudinit/sources/DataSourceAzure.py5
-rw-r--r--cloudinit/sources/DataSourceSmartOS.py13
-rw-r--r--cloudinit/util.py9
-rw-r--r--doc/examples/cloud-config-disk-setup.txt21
-rw-r--r--tests/unittests/test_datasource/test_azure.py2
7 files changed, 278 insertions, 93 deletions
diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py
index ade9c2ad..0b970e4e 100644
--- a/cloudinit/config/cc_disk_setup.py
+++ b/cloudinit/config/cc_disk_setup.py
@@ -19,6 +19,7 @@
from cloudinit.settings import PER_INSTANCE
from cloudinit import util
import logging
+import os
import shlex
frequency = PER_INSTANCE
@@ -29,13 +30,13 @@ SFDISK_CMD = util.which("sfdisk")
LSBLK_CMD = util.which("lsblk")
BLKID_CMD = util.which("blkid")
BLKDEV_CMD = util.which("blockdev")
+WIPEFS_CMD = util.which("wipefs")
LOG = logging.getLogger(__name__)
def handle(_name, cfg, cloud, log, _args):
"""
- Call util.prep_disk for disk_setup cloud-config.
See doc/examples/cloud-config_disk-setup.txt for documentation on the
format.
"""
@@ -103,15 +104,23 @@ def update_fs_setup_devices(disk_setup, tformer):
continue
origname = definition.get('device')
+
if origname is None:
continue
- transformed = tformer(origname)
- if transformed is None or transformed == origname:
- continue
+ (dev, part) = util.expand_dotted_devname(origname)
+
+ tformed = tformer(dev)
+ if tformed is not None:
+ dev = tformed
+ LOG.debug("%s is mapped to disk=%s part=%s",
+ origname, tformed, part)
+ definition['_origname'] = origname
+ definition['device'] = tformed
- definition['_origname'] = origname
- definition['device'] = transformed
+ if part and 'partition' in definition:
+ definition['_partition'] = definition['partition']
+ definition['partition'] = part
def value_splitter(values, start=None):
@@ -127,23 +136,56 @@ def value_splitter(values, start=None):
yield key, value
-def device_type(device):
+def enumerate_disk(device, nodeps=False):
"""
- Return the device type of the device by calling lsblk.
+ Enumerate the elements of a child device.
+
+ Parameters:
+ device: the kernel device name
+ nodeps <BOOL>: don't enumerate children devices
+
+ Return a dict describing the disk:
+ type: the entry type, i.e disk or part
+ fstype: the filesystem type, if it exists
+ label: file system label, if it exists
+ name: the device name, i.e. sda
"""
- lsblk_cmd = [LSBLK_CMD, '--pairs', '--nodeps', '--out', 'NAME,TYPE',
+ lsblk_cmd = [LSBLK_CMD, '--pairs', '--out', 'NAME,TYPE,FSTYPE,LABEL',
device]
+
+ if nodeps:
+ lsblk_cmd.append('--nodeps')
+
info = None
try:
info, _err = util.subp(lsblk_cmd)
except Exception as e:
raise Exception("Failed during disk check for %s\n%s" % (device, e))
- for key, value in value_splitter(info):
- if key.lower() == "type":
- return value.lower()
+ parts = [x for x in (info.strip()).splitlines() if len(x.split()) > 0]
+ for part in parts:
+ d = {'name': None,
+ 'type': None,
+ 'fstype': None,
+ 'label': None,
+ }
+
+ for key, value in value_splitter(part):
+ d[key.lower()] = value
+
+ yield d
+
+
+def device_type(device):
+ """
+ Return the device type of the device by calling lsblk.
+ """
+
+ for d in enumerate_disk(device, nodeps=True):
+ if "type" in d:
+ return d["type"].lower()
return None
@@ -204,7 +246,7 @@ def is_filesystem(device):
def find_device_node(device, fs_type=None, label=None, valid_targets=None,
- label_match=True):
+ label_match=True, replace_fs=None):
"""
Find a device that is either matches the spec, or the first
@@ -221,26 +263,12 @@ def find_device_node(device, fs_type=None, label=None, valid_targets=None,
if not valid_targets:
valid_targets = ['disk', 'part']
- lsblk_cmd = [LSBLK_CMD, '--pairs', '--out', 'NAME,TYPE,FSTYPE,LABEL',
- device]
- info = None
- try:
- info, _err = util.subp(lsblk_cmd)
- except Exception as e:
- raise Exception("Failed during disk check for %s\n%s" % (device, e))
-
raw_device_used = False
- parts = [x for x in (info.strip()).splitlines() if len(x.split()) > 0]
-
- for part in parts:
- d = {'name': None,
- 'type': None,
- 'fstype': None,
- 'label': None,
- }
+ for d in enumerate_disk(device):
- for key, value in value_splitter(part):
- d[key.lower()] = value
+ if d['fstype'] == replace_fs and label_match is False:
+ # We found a device where we want to replace the FS
+ return ('/dev/%s' % d['name'], False)
if (d['fstype'] == fs_type and
((label_match and d['label'] == label) or not label_match)):
@@ -268,22 +296,20 @@ def find_device_node(device, fs_type=None, label=None, valid_targets=None,
def is_disk_used(device):
"""
- Check if the device is currently used. Returns false if there
+ Check if the device is currently used. Returns true if the device
+ has either a file system or a partition entry
is no filesystem found on the disk.
"""
- lsblk_cmd = [LSBLK_CMD, '--pairs', '--out', 'NAME,TYPE',
- device]
- info = None
- try:
- info, _err = util.subp(lsblk_cmd)
- except Exception as e:
- # if we error out, we can't use the device
- util.logexc(LOG,
- "Error checking for filesystem on %s\n%s" % (device, e))
+
+ # If the child count is higher 1, then there are child nodes
+ # such as partition or device mapper nodes
+ use_count = [x for x in enumerate_disk(device)]
+ if len(use_count.splitlines()) > 1:
return True
- # If there is any output, then the device has something
- if len(info.splitlines()) > 1:
+ # If we see a file system, then its used
+ _, check_fstype, _ = check_fs(device)
+ if check_fstype:
return True
return False
@@ -455,6 +481,39 @@ def get_partition_mbr_layout(size, layout):
return sfdisk_definition
+def purge_disk_ptable(device):
+ # wipe the first and last megabyte of a disk (or file)
+ # gpt stores partition table both at front and at end.
+ null = '\0' # pylint: disable=W1401
+ start_len = 1024 * 1024
+ end_len = 1024 * 1024
+ with open(device, "rb+") as fp:
+ fp.write(null * (start_len))
+ fp.seek(-end_len, os.SEEK_END)
+ fp.write(null * end_len)
+ fp.flush()
+
+ read_parttbl(device)
+
+
+def purge_disk(device):
+ """
+ Remove parition table entries
+ """
+
+ # wipe any file systems first
+ for d in enumerate_disk(device):
+ if d['type'] not in ["disk", "crypt"]:
+ wipefs_cmd = [WIPEFS_CMD, "--all", "/dev/%s" % d['name']]
+ try:
+ LOG.info("Purging filesystem on /dev/%s" % d['name'])
+ util.subp(wipefs_cmd)
+ except Exception:
+ raise Exception("Failed FS purge of /dev/%s" % d['name'])
+
+ purge_disk_ptable(device)
+
+
def get_partition_layout(table_type, size, layout):
"""
Call the appropriate function for creating the table
@@ -542,6 +601,12 @@ def mkpart(device, definition):
if not is_device_valid(device):
raise Exception("Device %s is not a disk device!", device)
+ # Remove the partition table entries
+ if isinstance(layout, str) and layout.lower() == "remove":
+ LOG.debug("Instructed to remove partition table entries")
+ purge_disk(device)
+ return
+
LOG.debug("Checking if device layout matches")
if check_partition_layout(table_type, device, layout):
LOG.debug("Device partitioning layout matches")
@@ -565,6 +630,26 @@ def mkpart(device, definition):
LOG.debug("Partition table created for %s", device)
+def lookup_force_flag(fs):
+ """
+ A force flag might be -F or -F, this look it up
+ """
+ flags = {'ext': '-F',
+ 'btrfs': '-f',
+ 'xfs': '-f',
+ 'reiserfs': '-f',
+ }
+
+ if 'ext' in fs.lower():
+ fs = 'ext'
+
+ if fs.lower() in flags:
+ return flags[fs]
+
+ LOG.warn("Force flag for %s is unknown." % fs)
+ return ''
+
+
def mkfs(fs_cfg):
"""
Create a file system on the device.
@@ -592,6 +677,7 @@ def mkfs(fs_cfg):
fs_type = fs_cfg.get('filesystem')
fs_cmd = fs_cfg.get('cmd', [])
fs_opts = fs_cfg.get('extra_opts', [])
+ fs_replace = fs_cfg.get('replace_fs', False)
overwrite = fs_cfg.get('overwrite', False)
# This allows you to define the default ephemeral or swap
@@ -632,17 +718,23 @@ def mkfs(fs_cfg):
label_match = False
device, reuse = find_device_node(device, fs_type=fs_type, label=label,
- label_match=label_match)
+ label_match=label_match,
+ replace_fs=fs_replace)
LOG.debug("Automatic device for %s identified as %s", odevice, device)
if reuse:
LOG.debug("Found filesystem match, skipping formating.")
return
+ if not reuse and fs_replace and device:
+ LOG.debug("Replacing file system on %s as instructed." % device)
+
if not device:
LOG.debug("No device aviable that matches request. "
"Skipping fs creation for %s", fs_cfg)
return
+ elif not partition or str(partition).lower() == 'none':
+ LOG.debug("Using the raw device to place filesystem %s on" % label)
else:
LOG.debug("Error in device identification handling.")
@@ -682,12 +774,16 @@ def mkfs(fs_cfg):
if label:
fs_cmd.extend(["-L", label])
+ # File systems that support the -F flag
+ if not fs_cmd and (overwrite or device_type(device) == "disk"):
+ fs_cmd.append(lookup_force_flag(fs_type))
+
# Add the extends FS options
if fs_opts:
fs_cmd.extend(fs_opts)
LOG.debug("Creating file system %s on %s", label, device)
- LOG.debug(" Using cmd: %s", "".join(fs_cmd))
+ LOG.debug(" Using cmd: %s", " ".join(fs_cmd))
try:
util.subp(fs_cmd)
except Exception as e:
diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py
index 390ba711..84ec928f 100644
--- a/cloudinit/config/cc_mounts.py
+++ b/cloudinit/config/cc_mounts.py
@@ -20,6 +20,8 @@
from string import whitespace # pylint: disable=W0402
+import logging
+import os.path
import re
from cloudinit import type_utils
@@ -31,6 +33,8 @@ SHORTNAME = re.compile(SHORTNAME_FILTER)
WS = re.compile("[%s]+" % (whitespace))
FSTAB_PATH = "/etc/fstab"
+LOG = logging.getLogger(__name__)
+
def is_mdname(name):
# return true if this is a metadata service name
@@ -44,6 +48,33 @@ def is_mdname(name):
return False
+def sanitize_devname(startname, transformer, log):
+ log.debug("Attempting to determine the real name of %s", startname)
+
+ # workaround, allow user to specify 'ephemeral'
+ # rather than more ec2 correct 'ephemeral0'
+ devname = startname
+ if devname == "ephemeral":
+ devname = "ephemeral0"
+ log.debug("Adjusted mount option from ephemeral to ephemeral0")
+
+ (blockdev, part) = util.expand_dotted_devname(devname)
+
+ if is_mdname(blockdev):
+ orig = blockdev
+ blockdev = transformer(blockdev)
+ if not blockdev:
+ return None
+ if not blockdev.startswith("/"):
+ blockdev = "/dev/%s" % blockdev
+ log.debug("Mapped metadata name %s to %s", orig, blockdev)
+ else:
+ if SHORTNAME.match(startname):
+ blockdev = "/dev/%s" % blockdev
+
+ return devnode_for_dev_part(blockdev, part)
+
+
def handle(_name, cfg, cloud, log, _args):
# fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno
defvals = [None, None, "auto", "defaults,nobootwait", "0", "2"]
@@ -64,32 +95,15 @@ def handle(_name, cfg, cloud, log, _args):
(i + 1), type_utils.obj_name(cfgmnt[i]))
continue
- startname = str(cfgmnt[i][0])
- log.debug("Attempting to determine the real name of %s", startname)
-
- # workaround, allow user to specify 'ephemeral'
- # rather than more ec2 correct 'ephemeral0'
- if startname == "ephemeral":
- cfgmnt[i][0] = "ephemeral0"
- log.debug(("Adjusted mount option %s "
- "name from ephemeral to ephemeral0"), (i + 1))
-
- if is_mdname(startname):
- newname = cloud.device_name_to_device(startname)
- if not newname:
- log.debug("Ignoring nonexistant named mount %s", startname)
- cfgmnt[i][1] = None
- else:
- renamed = newname
- if not newname.startswith("/"):
- renamed = "/dev/%s" % newname
- cfgmnt[i][0] = renamed
- log.debug("Mapped metadata name %s to %s", startname, renamed)
- else:
- if SHORTNAME.match(startname):
- renamed = "/dev/%s" % startname
- log.debug("Mapped shortname name %s to %s", startname, renamed)
- cfgmnt[i][0] = renamed
+ start = str(cfgmnt[i][0])
+ sanitized = sanitize_devname(start, cloud.device_name_to_device, log)
+ if sanitized is None:
+ log.debug("Ignorming nonexistant named mount %s", start)
+ continue
+
+ if sanitized != start:
+ log.debug("changed %s => %s" % (start, sanitized))
+ cfgmnt[i][0] = sanitized
# in case the user did not quote a field (likely fs-freq, fs_passno)
# but do not convert None to 'None' (LP: #898365)
@@ -118,17 +132,14 @@ def handle(_name, cfg, cloud, log, _args):
# for each of the "default" mounts, add them only if no other
# entry has the same device name
for defmnt in defmnts:
- startname = defmnt[0]
- devname = cloud.device_name_to_device(startname)
- if devname is None:
- log.debug("Ignoring nonexistant named default mount %s", startname)
+ start = defmnt[0]
+ sanitized = sanitize_devname(start, cloud.device_name_to_device, log)
+ if sanitized is None:
+ log.debug("Ignoring nonexistant default named mount %s", start)
continue
- if devname.startswith("/"):
- defmnt[0] = devname
- else:
- defmnt[0] = "/dev/%s" % devname
-
- log.debug("Mapped default device %s to %s", startname, defmnt[0])
+ if sanitized != start:
+ log.debug("changed default device %s => %s" % (start, sanitized))
+ defmnt[0] = sanitized
cfgmnt_has = False
for cfgm in cfgmnt:
@@ -138,7 +149,7 @@ def handle(_name, cfg, cloud, log, _args):
if cfgmnt_has:
log.debug(("Not including %s, already"
- " previously included"), startname)
+ " previously included"), start)
continue
cfgmnt.append(defmnt)
@@ -198,3 +209,53 @@ def handle(_name, cfg, cloud, log, _args):
util.subp(("mount", "-a"))
except:
util.logexc(log, "Activating mounts via 'mount -a' failed")
+
+
+def devnode_for_dev_part(device, partition):
+ """
+ Find the name of the partition. While this might seem rather
+ straight forward, its not since some devices are '<device><partition>'
+ while others are '<device>p<partition>'. For example, /dev/xvda3 on EC2
+ will present as /dev/xvda3p1 for the first partition since /dev/xvda3 is
+ a block device.
+ """
+ if not os.path.exists(device):
+ return None
+
+ if not partition:
+ return device
+
+ short_name = os.path.basename(device)
+ sys_path = "/sys/block/%s" % short_name
+
+ if not os.path.exists(sys_path):
+ LOG.debug("did not find entry for %s in /sys/block", short_name)
+ return None
+
+ sys_long_path = sys_path + "/" + short_name
+
+ if partition is not None:
+ partition = str(partition)
+
+ if partition is None:
+ valid_mappings = [sys_long_path + "1",
+ sys_long_path + "p1" % partition]
+ elif partition != "0":
+ valid_mappings = [sys_long_path + "%s" % partition,
+ sys_long_path + "p%s" % partition]
+ else:
+ valid_mappings = []
+
+ for cdisk in valid_mappings:
+ if not os.path.exists(cdisk):
+ continue
+
+ dev_path = "/dev/%s" % os.path.basename(cdisk)
+ if os.path.exists(dev_path):
+ return dev_path
+
+ if partition is None or partition == "0":
+ return device
+
+ LOG.debug("Did not fine partition %s for device %s", partition, device)
+ return None
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index 7ba6cea8..8321dee0 100644
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -54,8 +54,9 @@ BUILTIN_CLOUD_CONFIG = {
'layout': True,
'overwrite': False}
},
- 'fs_setup': [{'filesystem': 'ext4', 'device': 'ephemeral0',
- 'partition': 'auto'}],
+ 'fs_setup': [{'filesystem': 'ext4',
+ 'device': 'ephemeral0.1',
+ 'replace_fs': 'ntfs'}]
}
DS_CFG_PATH = ['datasource', DS_NAME]
diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index 93b8b50b..2813ffb3 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -46,6 +46,7 @@ SMARTOS_ATTRIB_MAP = {
'user-data': ('user-data', False),
'iptables_disable': ('iptables_disable', True),
'motd_sys_info': ('motd_sys_info', True),
+ 'availability_zone': ('region', True),
}
DS_NAME = 'SmartOS'
@@ -81,8 +82,9 @@ BUILTIN_CLOUD_CONFIG = {
'layout': False,
'overwrite': False}
},
- 'fs_setup': [{'label': 'ephemeral0', 'filesystem': 'ext3',
- 'device': 'ephemeral0', 'partition': 'auto'}],
+ 'fs_setup': [{'label': 'ephemeral0',
+ 'filesystem': 'ext3',
+ 'device': 'ephemeral0'}],
}
@@ -174,6 +176,13 @@ class DataSourceSmartOS(sources.DataSource):
seed_timeout=self.seed_timeout, default=default,
b64=b64)
+ @property
+ def availability_zone(self):
+ try:
+ return self.metadata['availability-zone']
+ except KeyError:
+ return None
+
def get_serial(seed_device, seed_timeout):
"""This is replaced in unit testing, allowing us to replace
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 50ca7959..9e6e0a73 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -32,6 +32,7 @@ import grp
import gzip
import hashlib
import os
+import os.path
import platform
import pwd
import random
@@ -1826,3 +1827,11 @@ def log_time(logfunc, msg, func, args=None, kwargs=None, get_uptime=False):
except:
pass
return ret
+
+
+def expand_dotted_devname(dotted):
+ toks = dotted.rsplit(".", 1)
+ if len(toks) > 1:
+ return toks
+ else:
+ return (dotted, None)
diff --git a/doc/examples/cloud-config-disk-setup.txt b/doc/examples/cloud-config-disk-setup.txt
index 3fc47699..6ad61c33 100644
--- a/doc/examples/cloud-config-disk-setup.txt
+++ b/doc/examples/cloud-config-disk-setup.txt
@@ -30,8 +30,8 @@ disk_setup:
fs_setup:
- label: ephemeral0
filesystem: ext4
- device: ephemeral0
- partition: auto
+ device: ephemeral0.1
+ replace_fs: ntfs
Default disk definitions for SmartOS
@@ -47,8 +47,7 @@ disk_setup:
fs_setup:
- label: ephemeral0
filesystem: ext3
- device: ephemeral0
- partition: auto
+ device: ephemeral0.0
Cavaut for SmartOS: if ephemeral disk is not defined, then the disk will
not be automatically added to the mounts.
@@ -171,6 +170,7 @@ The general format is:
device: <DEVICE>
partition: <PART_VALUE>
overwrite: <OVERWRITE>
+ replace_fs: <FS_TYPE>
Where:
<LABEL>: The file system label to be used. If set to None, no label is
@@ -187,7 +187,13 @@ Where:
label as 'ephemeralX' otherwise there may be issues with the mounting
of the ephemeral storage layer.
- <PART_VALUE>: The valid options are:
+ If you define the device as 'ephemeralX.Y' then Y will be interpetted
+ as a partition value. However, ephermalX.0 is the _same_ as ephemeralX.
+
+ <PART_VALUE>:
+ Partition definitions are overwriten if you use the '<DEVICE>.Y' notation.
+
+ The valid options are:
"auto|any": tell cloud-init not to care whether there is a partition
or not. Auto will use the first partition that does not contain a
file system already. In the absence of a partition table, it will
@@ -236,5 +242,10 @@ Where:
"false": If an existing file system exists, skip the creation.
+ <REPLACE_FS>: This is a special directive, used for Windows Azure that
+ instructs cloud-init to replace a file system of <FS_TYPE>. NOTE:
+ unless you define a label, this requires the use of the 'any' partition
+ directive.
+
Behavior Caveat: The default behavior is to _check_ if the file system exists.
If a file system matches the specification, then the operation is a no-op.
diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
index df7e52d0..aad84206 100644
--- a/tests/unittests/test_datasource/test_azure.py
+++ b/tests/unittests/test_datasource/test_azure.py
@@ -328,8 +328,6 @@ class TestAzureDataSource(MockerTestCase):
self.assertTrue(ret)
cfg = dsrc.get_config_obj()
self.assertTrue(cfg)
- self.assertEquals(dsrc.device_name_to_device("ephemeral0"),
- "/dev/sdc")
def test_userdata_arrives(self):
userdata = "This is my user-data"
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-18 13:35:20 +0000