diff options
| -rw-r--r-- | ChangeLog | 7 | ||||
| -rw-r--r-- | cloudinit/config/cc_debug.py | 79 | ||||
| -rw-r--r-- | cloudinit/distros/debian.py | 28 | ||||
| -rw-r--r-- | cloudinit/importer.py | 6 | ||||
| -rw-r--r-- | cloudinit/util.py | 2 | ||||
| -rw-r--r-- | doc/examples/cloud-config-datasources.txt | 1 | ||||
| -rw-r--r-- | doc/examples/cloud-config-disk-setup.txt | 426 | ||||
| -rw-r--r-- | doc/examples/cloud-config.txt | 11 | ||||
| -rwxr-xr-x | packages/bddeb | 2 | ||||
| -rw-r--r-- | packages/debian/control.in | 1 |
10 files changed, 340 insertions, 223 deletions
@@ -1,5 +1,12 @@ 0.7.5: - open 0.7.5 + - Add a debug log message around import failures + - add a 'debug' module for easily printing out some information about + datasource and cloud-init [Shraddha Pandhe] + - support running apt with 'eatmydata' via configuration token + apt_get_wrapper (LP: #1236531). + - convert paths provided in config-drive 'files' to string before writing + (LP: #1260072). 0.7.4: - fix issue mounting 'ephemeral0' if ephemeral0 was an alias for a partitioned block device with target filesystem on ephemeral0.1. diff --git a/cloudinit/config/cc_debug.py b/cloudinit/config/cc_debug.py new file mode 100644 index 00000000..cfd31fa1 --- /dev/null +++ b/cloudinit/config/cc_debug.py @@ -0,0 +1,79 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2013 Yahoo! Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +from StringIO import StringIO +from cloudinit import util +from cloudinit import type_utils +import copy + + +def _make_header(text): + header = StringIO() + header.write("-" * 80) + header.write("\n") + header.write(text.center(80, ' ')) + header.write("\n") + header.write("-" * 80) + header.write("\n") + return header.getvalue() + + +def handle(name, cfg, cloud, log, args): + verbose = util.get_cfg_by_path(cfg, ('debug', 'verbose'), default=True) + if args: + # if args are provided (from cmdline) then explicitly set verbose + out_file = args[0] + verbose = True + else: + out_file = util.get_cfg_by_path(cfg, ('debug', 'output')) + + if not verbose: + log.debug(("Skipping module named %s," + " verbose printing disabled"), name) + return + # Clean out some keys that we just don't care about showing... + dump_cfg = copy.deepcopy(cfg) + for k in ['log_cfgs']: + dump_cfg.pop(k, None) + all_keys = list(dump_cfg.keys()) + for k in all_keys: + if k.startswith("_"): + dump_cfg.pop(k, None) + # Now dump it... + to_print = StringIO() + to_print.write(_make_header("Config")) + to_print.write(util.yaml_dumps(dump_cfg)) + to_print.write("\n") + to_print.write(_make_header("MetaData")) + to_print.write(util.yaml_dumps(cloud.datasource.metadata)) + to_print.write("\n") + to_print.write(_make_header("Misc")) + to_print.write("Datasource: %s\n" % + (type_utils.obj_name(cloud.datasource))) + to_print.write("Distro: %s\n" % (type_utils.obj_name(cloud.distro))) + to_print.write("Hostname: %s\n" % (cloud.get_hostname(True))) + to_print.write("Instance ID: %s\n" % (cloud.get_instance_id())) + to_print.write("Locale: %s\n" % (cloud.get_locale())) + to_print.write("Launch IDX: %s\n" % (cloud.launch_index)) + contents = to_print.getvalue() + content_to_file = [] + for line in contents.splitlines(): + line = "ci-info: %s\n" % (line) + content_to_file.append(line) + if out_file: + util.write_file(out_file, "".join(content_to_file), 0644, "w") + else: + util.multi_log("".join(content_to_file), console=True, stderr=False) diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 8fe49cbe..1ae232fd 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -36,6 +36,10 @@ LOG = logging.getLogger(__name__) APT_GET_COMMAND = ('apt-get', '--option=Dpkg::Options::=--force-confold', '--option=Dpkg::options::=--force-unsafe-io', '--assume-yes', '--quiet') +APT_GET_WRAPPER = { + 'command': 'eatmydata', + 'enabled': 'auto', +} class Distro(distros.Distro): @@ -148,7 +152,13 @@ class Distro(distros.Distro): # See: http://tiny.cc/kg91fw # Or: http://tiny.cc/mh91fw e['DEBIAN_FRONTEND'] = 'noninteractive' - cmd = list(self.get_option("apt_get_command", APT_GET_COMMAND)) + + wcfg = self.get_option("apt_get_wrapper", APT_GET_WRAPPER) + cmd = _get_wrapper_prefix( + wcfg.get('command', APT_GET_WRAPPER['command']), + wcfg.get('enabled', APT_GET_WRAPPER['enabled'])) + + cmd.extend(list(self.get_option("apt_get_command", APT_GET_COMMAND))) if args and isinstance(args, str): cmd.append(args) @@ -166,7 +176,9 @@ class Distro(distros.Distro): cmd.extend(pkglist) # Allow the output of this to flow outwards (ie not be captured) - util.subp(cmd, env=e, capture=False) + util.log_time(logfunc=LOG.debug, + msg="apt-%s [%s]" % (command, ' '.join(cmd)), func=util.subp, + args=(cmd,), kwargs={'env': e, 'capture': False}) def update_package_sources(self): self._runner.run("update-sources", self.package_command, @@ -175,3 +187,15 @@ class Distro(distros.Distro): def get_primary_arch(self): (arch, _err) = util.subp(['dpkg', '--print-architecture']) return str(arch).strip() + + +def _get_wrapper_prefix(cmd, mode): + if isinstance(cmd, str): + cmd = [str(cmd)] + + if (util.is_true(mode) or + (str(mode).lower() == "auto" and cmd[0] and + util.which(cmd[0]))): + return cmd + else: + return [] diff --git a/cloudinit/importer.py b/cloudinit/importer.py index 71cf2726..a094141a 100644 --- a/cloudinit/importer.py +++ b/cloudinit/importer.py @@ -36,6 +36,7 @@ def find_module(base_name, search_paths, required_attrs=None): found_places = [] if not required_attrs: required_attrs = [] + # NOTE(harlowja): translate the search paths to include the base name. real_paths = [] for path in search_paths: real_path = [] @@ -50,8 +51,9 @@ def find_module(base_name, search_paths, required_attrs=None): mod = None try: mod = import_module(full_path) - except ImportError: - pass + except ImportError as e: + LOG.debug("Failed at attempted import of '%s' due to: %s", + full_path, e) if not mod: continue found_attrs = 0 diff --git a/cloudinit/util.py b/cloudinit/util.py index b69e2bb0..6b30af5e 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -170,6 +170,8 @@ class SeLinuxGuard(object): def __exit__(self, excp_type, excp_value, excp_traceback): if self.selinux and self.selinux.is_selinux_enabled(): path = os.path.realpath(os.path.expanduser(self.path)) + # path should be a string, not unicode + path = str(path) do_restore = False try: # See if even worth restoring?? diff --git a/doc/examples/cloud-config-datasources.txt b/doc/examples/cloud-config-datasources.txt index 65a3cdf5..3bde4aac 100644 --- a/doc/examples/cloud-config-datasources.txt +++ b/doc/examples/cloud-config-datasources.txt @@ -49,7 +49,6 @@ datasource: hostname_bounce: interface: eth0 policy: on # [can be 'on', 'off' or 'force'] - } SmartOS: # Smart OS datasource works over a serial console interacting with diff --git a/doc/examples/cloud-config-disk-setup.txt b/doc/examples/cloud-config-disk-setup.txt index 6ad61c33..0dfef8e8 100644 --- a/doc/examples/cloud-config-disk-setup.txt +++ b/doc/examples/cloud-config-disk-setup.txt @@ -1,24 +1,24 @@ -Cloud-init supports the creation of simple partition tables and file systems -on devices. +# Cloud-init supports the creation of simple partition tables and file systems +# on devices. -Default disk definitions for AWS --------------------------------- -(Not implemented yet, but provided for future documentation) +# Default disk definitions for AWS +# -------------------------------- +# (Not implemented yet, but provided for future documentation) - disk_setup: - ephmeral0: - type: 'mbr' - layout: True - overwrite: False +disk_setup: + ephmeral0: + type: 'mbr' + layout: True + overwrite: False - fs_setup: - - label: None, - filesystem: ext3 - device: ephemeral0 - partition: auto +fs_setup: + - label: None, + filesystem: ext3 + device: ephemeral0 + partition: auto -Default disk definitions for Windows Azure ------------------------------------------- +# Default disk definitions for Windows Azure +# ------------------------------------------ device_aliases: {'ephemeral0': '/dev/sdb'} disk_setup: @@ -34,8 +34,8 @@ fs_setup: replace_fs: ntfs -Default disk definitions for SmartOS ------------------------------------- +# Default disk definitions for SmartOS +# ------------------------------------ device_aliases: {'ephemeral0': '/dev/sdb'} disk_setup: @@ -49,203 +49,203 @@ fs_setup: filesystem: ext3 device: ephemeral0.0 -Cavaut for SmartOS: if ephemeral disk is not defined, then the disk will - not be automatically added to the mounts. - - -The default definition is used to make sure that the ephemeral storage is -setup properly. - -"disk_setup": disk partitioning --------------------------------- - -The disk_setup directive instructs Cloud-init to partition a disk. The format is: - - disk_setup: - ephmeral0: - type: 'mbr' - layout: 'auto' - /dev/xvdh: - type: 'mbr' - layout: - - 33 - - [33, 82] - - 33 - overwrite: True - -The format is a list of dicts of dicts. The first value is the name of the -device and the subsequent values define how to create and layout the partition. - -The general format is: - disk_setup: - <DEVICE>: - type: 'mbr' - layout: <LAYOUT|BOOL> - overwrite: <BOOL> - -Where: - <DEVICE>: The name of the device. 'ephemeralX' and 'swap' are special - values which are specific to the cloud. For these devices - Cloud-init will look up what the real devices is and then - use it. - - For other devices, the kernel device name is used. At this - time only simply kernel devices are supported, meaning - that device mapper and other targets may not work. - - Note: At this time, there is no handling or setup of - device mapper targets. +# Cavaut for SmartOS: if ephemeral disk is not defined, then the disk will +# not be automatically added to the mounts. - type=<TYPE>: Currently the following are supported: - 'mbr': default and setups a MS-DOS partition table - Note: At this time only 'mbr' partition tables are allowed. - It is anticipated in the future that we'll have GPT as - option in the future, or even "RAID" to create a mdadm - RAID. +# The default definition is used to make sure that the ephemeral storage is +# setup properly. - layout={...}: The device layout. This is a list of values, with the - percentage of disk that partition will take. - Valid options are: - [<SIZE>, [<SIZE>, <PART_TYPE]] +# "disk_setup": disk partitioning +# -------------------------------- - Where <SIZE> is the _percentage_ of the disk to use, while - <PART_TYPE> is the numerical value of the partition type. +# The disk_setup directive instructs Cloud-init to partition a disk. The format is: - The following setups two partitions, with the first - partition having a swap label, taking 1/3 of the disk space - and the remainder being used as the second partition. - /dev/xvdh': - type: 'mbr' - layout: - - [33,82] - - 66 - overwrite: True - - When layout is "true" it means single partition the entire - device. - - When layout is "false" it means don't partition or ignore - existing partitioning. - - If layout is set to "true" and overwrite is set to "false", - it will skip partitioning the device without a failure. - - overwrite=<BOOL>: This describes whether to ride with saftey's on and - everything holstered. - - 'false' is the default, which means that: - 1. The device will be checked for a partition table - 2. The device will be checked for a file system - 3. If either a partition of file system is found, then - the operation will be _skipped_. - - 'true' is cowboy mode. There are no checks and things are - done blindly. USE with caution, you can do things you - really, really don't want to do. - - -fs_setup: Setup the file system -------------------------------- - -fs_setup describes the how the file systems are supposed to look. +disk_setup: + ephmeral0: + type: 'mbr' + layout: 'auto' + /dev/xvdh: + type: 'mbr' + layout: + - 33 + - [33, 82] + - 33 + overwrite: True + +# The format is a list of dicts of dicts. The first value is the name of the +# device and the subsequent values define how to create and layout the +# partition. +# The general format is: +# disk_setup: +# <DEVICE>: +# type: 'mbr' +# layout: <LAYOUT|BOOL> +# overwrite: <BOOL> +# +# Where: +# <DEVICE>: The name of the device. 'ephemeralX' and 'swap' are special +# values which are specific to the cloud. For these devices +# Cloud-init will look up what the real devices is and then +# use it. +# +# For other devices, the kernel device name is used. At this +# time only simply kernel devices are supported, meaning +# that device mapper and other targets may not work. +# +# Note: At this time, there is no handling or setup of +# device mapper targets. +# +# type=<TYPE>: Currently the following are supported: +# 'mbr': default and setups a MS-DOS partition table +# +# Note: At this time only 'mbr' partition tables are allowed. +# It is anticipated in the future that we'll have GPT as +# option in the future, or even "RAID" to create a mdadm +# RAID. +# +# layout={...}: The device layout. This is a list of values, with the +# percentage of disk that partition will take. +# Valid options are: +# [<SIZE>, [<SIZE>, <PART_TYPE]] +# +# Where <SIZE> is the _percentage_ of the disk to use, while +# <PART_TYPE> is the numerical value of the partition type. +# +# The following setups two partitions, with the first +# partition having a swap label, taking 1/3 of the disk space +# and the remainder being used as the second partition. +# /dev/xvdh': +# type: 'mbr' +# layout: +# - [33,82] +# - 66 +# overwrite: True +# +# When layout is "true" it means single partition the entire +# device. +# +# When layout is "false" it means don't partition or ignore +# existing partitioning. +# +# If layout is set to "true" and overwrite is set to "false", +# it will skip partitioning the device without a failure. +# +# overwrite=<BOOL>: This describes whether to ride with saftey's on and +# everything holstered. +# +# 'false' is the default, which means that: +# 1. The device will be checked for a partition table +# 2. The device will be checked for a file system +# 3. If either a partition of file system is found, then +# the operation will be _skipped_. +# +# 'true' is cowboy mode. There are no checks and things are +# done blindly. USE with caution, you can do things you +# really, really don't want to do. +# +# +# fs_setup: Setup the file system +# ------------------------------- +# +# fs_setup describes the how the file systems are supposed to look. - fs_setup: - - label: ephemeral0 - filesystem: 'ext3' - device: 'ephemeral0' - partition: 'auto' - - label: mylabl2 - filesystem: 'ext4' - device: '/dev/xvda1' - - special: - cmd: mkfs -t %(FILESYSTEM)s -L %(LABEL)s %(DEVICE)s - filesystem: 'btrfs' - device: '/dev/xvdh' - -The general format is: - fs_setup: - - label: <LABEL> - filesystem: <FS_TYPE> - device: <DEVICE> - partition: <PART_VALUE> - overwrite: <OVERWRITE> - replace_fs: <FS_TYPE> - -Where: - <LABEL>: The file system label to be used. If set to None, no label is - used. - - <FS_TYPE>: The file system type. It is assumed that the there - will be a "mkfs.<FS_TYPE>" that behaves likes "mkfs". On a standard - Ubuntu Cloud Image, this means that you have the option of ext{2,3,4}, - and vfat by default. - - <DEVICE>: The device name. Special names of 'ephemeralX' or 'swap' - are allowed and the actual device is acquired from the cloud datasource. - When using 'ephemeralX' (i.e. ephemeral0), make sure to leave the - label as 'ephemeralX' otherwise there may be issues with the mounting - of the ephemeral storage layer. - - If you define the device as 'ephemeralX.Y' then Y will be interpetted - as a partition value. However, ephermalX.0 is the _same_ as ephemeralX. - - <PART_VALUE>: - Partition definitions are overwriten if you use the '<DEVICE>.Y' notation. - - The valid options are: - "auto|any": tell cloud-init not to care whether there is a partition - or not. Auto will use the first partition that does not contain a - file system already. In the absence of a partition table, it will - put it directly on the disk. - - "auto": If a file system that matches the specification in terms of - label, type and device, then cloud-init will skip the creation of - the file system. - - "any": If a file system that matches the file system type and device, - then cloud-init will skip the creation of the file system. - - Devices are selected based on first-detected, starting with partitions - and then the raw disk. Consider the following: - NAME FSTYPE LABEL - xvdb - |-xvdb1 ext4 - |-xvdb2 - |-xvdb3 btrfs test - \-xvdb4 ext4 test - - If you ask for 'auto', label of 'test, and file system of 'ext4' - then cloud-init will select the 2nd partition, even though there - is a partition match at the 4th partition. - - If you ask for 'any' and a label of 'test', then cloud-init will - select the 1st partition. - - If you ask for 'auto' and don't define label, then cloud-init will - select the 1st partition. - - In general, if you have a specific partition configuration in mind, - you should define either the device or the partition number. 'auto' - and 'any' are specifically intended for formating ephemeral storage or - for simple schemes. - - "none": Put the file system directly on the device. - - <NUM>: where NUM is the actual partition number. - - <OVERWRITE>: Defines whether or not to overwrite any existing - filesystem. - - "true": Indiscriminately destroy any pre-existing file system. Use at - your own peril. - - "false": If an existing file system exists, skip the creation. - - <REPLACE_FS>: This is a special directive, used for Windows Azure that - instructs cloud-init to replace a file system of <FS_TYPE>. NOTE: - unless you define a label, this requires the use of the 'any' partition - directive. - -Behavior Caveat: The default behavior is to _check_ if the file system exists. - If a file system matches the specification, then the operation is a no-op. +fs_setup: + - label: ephemeral0 + filesystem: 'ext3' + device: 'ephemeral0' + partition: 'auto' + - label: mylabl2 + filesystem: 'ext4' + device: '/dev/xvda1' + - special: + cmd: mkfs -t %(FILESYSTEM)s -L %(LABEL)s %(DEVICE)s + filesystem: 'btrfs' + device: '/dev/xvdh' + +# The general format is: +# fs_setup: +# - label: <LABEL> +# filesystem: <FS_TYPE> +# device: <DEVICE> +# partition: <PART_VALUE> +# overwrite: <OVERWRITE> +# replace_fs: <FS_TYPE> +# +# Where: +# <LABEL>: The file system label to be used. If set to None, no label is +# used. +# +# <FS_TYPE>: The file system type. It is assumed that the there +# will be a "mkfs.<FS_TYPE>" that behaves likes "mkfs". On a standard +# Ubuntu Cloud Image, this means that you have the option of ext{2,3,4}, +# and vfat by default. +# +# <DEVICE>: The device name. Special names of 'ephemeralX' or 'swap' +# are allowed and the actual device is acquired from the cloud datasource. +# When using 'ephemeralX' (i.e. ephemeral0), make sure to leave the +# label as 'ephemeralX' otherwise there may be issues with the mounting +# of the ephemeral storage layer. +# +# If you define the device as 'ephemeralX.Y' then Y will be interpetted +# as a partition value. However, ephermalX.0 is the _same_ as ephemeralX. +# +# <PART_VALUE>: +# Partition definitions are overwriten if you use the '<DEVICE>.Y' notation. +# +# The valid options are: +# "auto|any": tell cloud-init not to care whether there is a partition +# or not. Auto will use the first partition that does not contain a +# file system already. In the absence of a partition table, it will +# put it directly on the disk. +# +# "auto": If a file system that matches the specification in terms of +# label, type and device, then cloud-init will skip the creation of +# the file system. +# +# "any": If a file system that matches the file system type and device, +# then cloud-init will skip the creation of the file system. +# +# Devices are selected based on first-detected, starting with partitions +# and then the raw disk. Consider the following: +# NAME FSTYPE LABEL +# xvdb +# |-xvdb1 ext4 +# |-xvdb2 +# |-xvdb3 btrfs test +# \-xvdb4 ext4 test +# +# If you ask for 'auto', label of 'test, and file system of 'ext4' +# then cloud-init will select the 2nd partition, even though there +# is a partition match at the 4th partition. +# +# If you ask for 'any' and a label of 'test', then cloud-init will +# select the 1st partition. +# +# If you ask for 'auto' and don't define label, then cloud-init will +# select the 1st partition. +# +# In general, if you have a specific partition configuration in mind, +# you should define either the device or the partition number. 'auto' +# and 'any' are specifically intended for formating ephemeral storage or +# for simple schemes. +# +# "none": Put the file system directly on the device. +# +# <NUM>: where NUM is the actual partition number. +# +# <OVERWRITE>: Defines whether or not to overwrite any existing +# filesystem. +# +# "true": Indiscriminately destroy any pre-existing file system. Use at +# your own peril. +# +# "false": If an existing file system exists, skip the creation. +# +# <REPLACE_FS>: This is a special directive, used for Windows Azure that +# instructs cloud-init to replace a file system of <FS_TYPE>. NOTE: +# unless you define a label, this requires the use of the 'any' partition +# directive. +# +# Behavior Caveat: The default behavior is to _check_ if the file system exists. +# If a file system matches the specification, then the operation is a no-op. diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index b35b084d..61fa6065 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -74,7 +74,7 @@ apt_preserve_sources_list: true # 'source' entries in apt-sources that match this python regex # expression will be passed to add-apt-repository -add_apt_repo_match = "^[\w-]+:\w" +add_apt_repo_match: '^[\w-]+:\w' apt_sources: - source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main" @@ -147,8 +147,13 @@ apt_sources: # '--option=Dpkg::options::=--force-unsafe-io', '--assume-yes', '--quiet'] # # apt_get_upgrade_subcommand: -# Specify a different subcommand for 'upgrade. The default is 'dist-upgrade'. -# This is the subcommand that is invoked if package_upgrade is set to true above. +# Specify a different subcommand for 'upgrade. The default is 'dist-upgrade'. +# This is the subcommand that is invoked if package_upgrade is set to true above. +# +# apt_get_wrapper: +# command: eatmydata +# enabled: [True, False, "auto"] +# # Install additional packages on first boot # diff --git a/packages/bddeb b/packages/bddeb index 8de4d466..f52eb55f 100755 --- a/packages/bddeb +++ b/packages/bddeb @@ -32,7 +32,7 @@ PKG_MP = { 'boto': 'python-boto', 'cheetah': 'python-cheetah', 'configobj': 'python-configobj', - 'jsonpatch': 'python-json-patch', + 'jsonpatch': 'python-jsonpatch | python-json-patch', 'oauth': 'python-oauth', 'prettytable': 'python-prettytable', 'pyserial': 'python-serial', diff --git a/packages/debian/control.in b/packages/debian/control.in index b9352f5b..fd9f3ffd 100644 --- a/packages/debian/control.in +++ b/packages/debian/control.in @@ -25,7 +25,6 @@ Depends: procps, #end for python-software-properties | software-properties-common, \${misc:Depends}, - \${python:Depends} XB-Python-Version: \${python:Versions} Description: Init scripts for cloud instances Cloud instances need special scripts to run during initialisation |