From 9a922bfa70306d89535574c6580296c9b02ec43e Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Thu, 12 Dec 2013 14:22:14 -0500
Subject: fix 'make yaml'.

---
 doc/examples/cloud-config-datasources.txt |   1 -
 doc/examples/cloud-config-disk-setup.txt  | 426 +++++++++++++++---------------
 doc/examples/cloud-config.txt             |   2 +-
 3 files changed, 214 insertions(+), 215 deletions(-)

(limited to 'doc')

diff --git a/doc/examples/cloud-config-datasources.txt b/doc/examples/cloud-config-datasources.txt
index 65a3cdf5..3bde4aac 100644
--- a/doc/examples/cloud-config-datasources.txt
+++ b/doc/examples/cloud-config-datasources.txt
@@ -49,7 +49,6 @@ datasource:
     hostname_bounce:
       interface: eth0
       policy: on # [can be 'on', 'off' or 'force']
-    }
 
   SmartOS:
     # Smart OS datasource works over a serial console interacting with
diff --git a/doc/examples/cloud-config-disk-setup.txt b/doc/examples/cloud-config-disk-setup.txt
index 6ad61c33..0dfef8e8 100644
--- a/doc/examples/cloud-config-disk-setup.txt
+++ b/doc/examples/cloud-config-disk-setup.txt
@@ -1,24 +1,24 @@
-Cloud-init supports the creation of simple partition tables and file systems
-on devices.
+# Cloud-init supports the creation of simple partition tables and file systems
+# on devices.
 
-Default disk definitions for AWS
---------------------------------
-(Not implemented yet, but provided for future documentation)
+# Default disk definitions for AWS
+# --------------------------------
+# (Not implemented yet, but provided for future documentation)
 
- disk_setup:
-    ephmeral0:
-        type: 'mbr'
-        layout: True
-        overwrite: False
+disk_setup:
+   ephmeral0:
+       type: 'mbr'
+       layout: True
+       overwrite: False
 
- fs_setup:
-    - label: None,
-      filesystem: ext3
-      device: ephemeral0
-      partition: auto
+fs_setup:
+   - label: None,
+     filesystem: ext3
+     device: ephemeral0
+     partition: auto
 
-Default disk definitions for Windows Azure
-------------------------------------------
+# Default disk definitions for Windows Azure
+# ------------------------------------------
 
 device_aliases: {'ephemeral0': '/dev/sdb'}
 disk_setup:
@@ -34,8 +34,8 @@ fs_setup:
       replace_fs: ntfs
 
 
-Default disk definitions for SmartOS
-------------------------------------
+# Default disk definitions for SmartOS
+# ------------------------------------
 
 device_aliases: {'ephemeral0': '/dev/sdb'}
 disk_setup:
@@ -49,203 +49,203 @@ fs_setup:
       filesystem: ext3
       device: ephemeral0.0
 
-Cavaut for SmartOS: if ephemeral disk is not defined, then the disk will
-    not be automatically added to the mounts.
-
-
-The default definition is used to make sure that the ephemeral storage is
-setup properly.
-
-"disk_setup": disk partitioning
---------------------------------
-
-The disk_setup directive instructs Cloud-init to partition a disk. The format is:
-
- disk_setup:
-    ephmeral0:
-        type: 'mbr'
-        layout: 'auto'
-    /dev/xvdh:
-        type: 'mbr'
-        layout:
-            - 33
-            - [33, 82]
-            - 33
-        overwrite: True
-
-The format is a list of dicts of dicts. The first value is the name of the
-device and the subsequent values define how to create and layout the partition.
-
-The general format is:
-    disk_setup:
-        <DEVICE>:
-            type: 'mbr'
-            layout: <LAYOUT|BOOL>
-            overwrite: <BOOL>
-
-Where:
-    <DEVICE>: The name of the device. 'ephemeralX' and 'swap' are special
-                values which are specific to the cloud. For these devices
-                Cloud-init will look up what the real devices is and then
-                use it.
-
-                For other devices, the kernel device name is used. At this
-                time only simply kernel devices are supported, meaning
-                that device mapper and other targets may not work.
-
-                Note: At this time, there is no handling or setup of
-                device mapper targets.
+# Cavaut for SmartOS: if ephemeral disk is not defined, then the disk will
+#    not be automatically added to the mounts.
 
-    type=<TYPE>: Currently the following are supported:
-                    'mbr': default and setups a MS-DOS partition table
 
-                Note: At this time only 'mbr' partition tables are allowed.
-                    It is anticipated in the future that we'll have GPT as
-                    option in the future, or even "RAID" to create a mdadm
-                    RAID.
+# The default definition is used to make sure that the ephemeral storage is
+# setup properly.
 
-    layout={...}: The device layout. This is a list of values, with the
-                percentage of disk that partition will take.
-                Valid options are:
-                    [<SIZE>, [<SIZE>, <PART_TYPE]]
+# "disk_setup": disk partitioning
+# --------------------------------
 
-                Where <SIZE> is the _percentage_ of the disk to use, while
-                <PART_TYPE> is the numerical value of the partition type.
+# The disk_setup directive instructs Cloud-init to partition a disk. The format is:
 
-                The following setups two partitions, with the first
-                partition having a swap label, taking 1/3 of the disk space
-                and the remainder being used as the second partition.
-                    /dev/xvdh':
-                        type: 'mbr'
-                        layout:
-                            - [33,82]
-                            - 66
-                        overwrite: True
-
-                When layout is "true" it means single partition the entire
-                device.
-
-                When layout is "false" it means don't partition or ignore
-                existing partitioning.
-
-                If layout is set to "true" and overwrite is set to "false",
-                it will skip partitioning the device without a failure.
-
-    overwrite=<BOOL>: This describes whether to ride with saftey's on and
-                everything holstered.
-
-                'false' is the default, which means that:
-                    1. The device will be checked for a partition table
-                    2. The device will be checked for a file system
-                    3. If either a partition of file system is found, then
-                        the operation will be _skipped_.
-
-                'true' is cowboy mode. There are no checks and things are
-                    done blindly. USE with caution, you can do things you
-                    really, really don't want to do.
-
-
-fs_setup: Setup the file system
--------------------------------
-
-fs_setup describes the how the file systems are supposed to look.
+disk_setup:
+   ephmeral0:
+       type: 'mbr'
+       layout: 'auto'
+   /dev/xvdh:
+       type: 'mbr'
+       layout:
+           - 33
+           - [33, 82]
+           - 33
+       overwrite: True
+
+# The format is a list of dicts of dicts. The first value is the name of the
+# device and the subsequent values define how to create and layout the
+# partition.
+# The general format is:
+#    disk_setup:
+#        <DEVICE>:
+#            type: 'mbr'
+#            layout: <LAYOUT|BOOL>
+#            overwrite: <BOOL>
+#
+# Where:
+#    <DEVICE>: The name of the device. 'ephemeralX' and 'swap' are special
+#                values which are specific to the cloud. For these devices
+#                Cloud-init will look up what the real devices is and then
+#                use it.
+#
+#                For other devices, the kernel device name is used. At this
+#                time only simply kernel devices are supported, meaning
+#                that device mapper and other targets may not work.
+#
+#                Note: At this time, there is no handling or setup of
+#                device mapper targets.
+#
+#    type=<TYPE>: Currently the following are supported:
+#                    'mbr': default and setups a MS-DOS partition table
+#
+#                Note: At this time only 'mbr' partition tables are allowed.
+#                    It is anticipated in the future that we'll have GPT as
+#                    option in the future, or even "RAID" to create a mdadm
+#                    RAID.
+#
+#    layout={...}: The device layout. This is a list of values, with the
+#                percentage of disk that partition will take.
+#                Valid options are:
+#                    [<SIZE>, [<SIZE>, <PART_TYPE]]
+#
+#                Where <SIZE> is the _percentage_ of the disk to use, while
+#                <PART_TYPE> is the numerical value of the partition type.
+#
+#                The following setups two partitions, with the first
+#                partition having a swap label, taking 1/3 of the disk space
+#                and the remainder being used as the second partition.
+#                    /dev/xvdh':
+#                        type: 'mbr'
+#                        layout:
+#                            - [33,82]
+#                            - 66
+#                        overwrite: True
+#
+#                When layout is "true" it means single partition the entire
+#                device.
+#
+#                When layout is "false" it means don't partition or ignore
+#                existing partitioning.
+#
+#                If layout is set to "true" and overwrite is set to "false",
+#                it will skip partitioning the device without a failure.
+#
+#    overwrite=<BOOL>: This describes whether to ride with saftey's on and
+#                everything holstered.
+#
+#                'false' is the default, which means that:
+#                    1. The device will be checked for a partition table
+#                    2. The device will be checked for a file system
+#                    3. If either a partition of file system is found, then
+#                        the operation will be _skipped_.
+#
+#                'true' is cowboy mode. There are no checks and things are
+#                    done blindly. USE with caution, you can do things you
+#                    really, really don't want to do.
+#
+#
+# fs_setup: Setup the file system
+# -------------------------------
+#
+# fs_setup describes the how the file systems are supposed to look.
 
- fs_setup:
-    - label: ephemeral0
-      filesystem: 'ext3'
-      device: 'ephemeral0'
-      partition: 'auto'
-    - label:  mylabl2
-      filesystem: 'ext4'
-      device: '/dev/xvda1'
-    - special:
-      cmd: mkfs -t %(FILESYSTEM)s -L %(LABEL)s %(DEVICE)s
-      filesystem: 'btrfs'
-      device: '/dev/xvdh'
-
-The general format is:
-    fs_setup:
-        - label: <LABEL>
-          filesystem: <FS_TYPE>
-          device: <DEVICE>
-          partition: <PART_VALUE>
-          overwrite: <OVERWRITE>
-          replace_fs: <FS_TYPE>
-
-Where:
-    <LABEL>: The file system label to be used. If set to None, no label is
-        used.
-
-    <FS_TYPE>: The file system type. It is assumed that the there
-        will be a "mkfs.<FS_TYPE>" that behaves likes "mkfs". On a standard
-        Ubuntu Cloud Image, this means that you have the option of ext{2,3,4},
-        and vfat by default.
-
-    <DEVICE>: The device name. Special names of 'ephemeralX' or 'swap'
-        are allowed and the actual device is acquired from the cloud datasource.
-        When using 'ephemeralX' (i.e. ephemeral0), make sure to leave the
-        label as 'ephemeralX' otherwise there may be issues with the mounting
-        of the ephemeral storage layer.
-
-        If you define the device as 'ephemeralX.Y' then Y will be interpetted
-        as a partition value. However, ephermalX.0 is the _same_ as ephemeralX.
-
-    <PART_VALUE>:
-        Partition definitions are overwriten if you use the '<DEVICE>.Y' notation.
-
-        The valid options are:
-        "auto|any": tell cloud-init not to care whether there is a partition
-            or not. Auto will use the first partition that does not contain a
-            file system already. In the absence of a partition table, it will
-            put it directly on the disk.
-
-            "auto": If a file system that matches the specification in terms of
-            label, type and device, then cloud-init will skip the creation of
-            the file system.
-
-            "any": If a file system that matches the file system type and device,
-            then cloud-init will skip the creation of the file system.
-
-            Devices are selected based on first-detected, starting with partitions
-            and then the raw disk. Consider the following:
-                NAME     FSTYPE LABEL
-                xvdb
-                |-xvdb1  ext4
-                |-xvdb2
-                |-xvdb3  btrfs  test
-                \-xvdb4  ext4   test
-
-            If you ask for 'auto', label of 'test, and file system of 'ext4'
-            then cloud-init will select the 2nd partition, even though there
-            is a partition match at the 4th partition.
-
-            If you ask for 'any' and a label of 'test', then cloud-init will
-            select the 1st partition.
-
-            If you ask for 'auto' and don't define label, then cloud-init will
-            select the 1st partition.
-
-            In general, if you have a specific partition configuration in mind,
-            you should define either the device or the partition number. 'auto'
-            and 'any' are specifically intended for formating ephemeral storage or
-            for simple schemes.
-
-        "none": Put the file system directly on the device.
-
-        <NUM>: where NUM is the actual partition number.
-
-    <OVERWRITE>: Defines whether or not to overwrite any existing
-        filesystem.
-
-        "true": Indiscriminately destroy any pre-existing file system. Use at
-            your own peril.
-
-        "false": If an existing file system exists, skip the creation.
-
-    <REPLACE_FS>: This is a special directive, used for Windows Azure that
-        instructs cloud-init to replace a file system of <FS_TYPE>. NOTE:
-        unless you define a label, this requires the use of the 'any' partition
-        directive.
-
-Behavior Caveat: The default behavior is to _check_ if the file system exists.
-    If a file system matches the specification, then the operation is a no-op.
+fs_setup:
+   - label: ephemeral0
+     filesystem: 'ext3'
+     device: 'ephemeral0'
+     partition: 'auto'
+   - label:  mylabl2
+     filesystem: 'ext4'
+     device: '/dev/xvda1'
+   - special:
+     cmd: mkfs -t %(FILESYSTEM)s -L %(LABEL)s %(DEVICE)s
+     filesystem: 'btrfs'
+     device: '/dev/xvdh'
+
+# The general format is:
+#    fs_setup:
+#        - label: <LABEL>
+#          filesystem: <FS_TYPE>
+#          device: <DEVICE>
+#          partition: <PART_VALUE>
+#          overwrite: <OVERWRITE>
+#          replace_fs: <FS_TYPE>
+#
+# Where:
+#     <LABEL>: The file system label to be used. If set to None, no label is
+#        used.
+#
+#    <FS_TYPE>: The file system type. It is assumed that the there
+#        will be a "mkfs.<FS_TYPE>" that behaves likes "mkfs". On a standard
+#        Ubuntu Cloud Image, this means that you have the option of ext{2,3,4},
+#        and vfat by default.
+#
+#    <DEVICE>: The device name. Special names of 'ephemeralX' or 'swap'
+#        are allowed and the actual device is acquired from the cloud datasource.
+#        When using 'ephemeralX' (i.e. ephemeral0), make sure to leave the
+#        label as 'ephemeralX' otherwise there may be issues with the mounting
+#        of the ephemeral storage layer.
+#
+#        If you define the device as 'ephemeralX.Y' then Y will be interpetted
+#        as a partition value. However, ephermalX.0 is the _same_ as ephemeralX.
+#
+#    <PART_VALUE>:
+#        Partition definitions are overwriten if you use the '<DEVICE>.Y' notation.
+#
+#        The valid options are:
+#        "auto|any": tell cloud-init not to care whether there is a partition
+#            or not. Auto will use the first partition that does not contain a
+#            file system already. In the absence of a partition table, it will
+#            put it directly on the disk.
+#
+#            "auto": If a file system that matches the specification in terms of
+#            label, type and device, then cloud-init will skip the creation of
+#            the file system.
+#
+#            "any": If a file system that matches the file system type and device,
+#            then cloud-init will skip the creation of the file system.
+#
+#            Devices are selected based on first-detected, starting with partitions
+#            and then the raw disk. Consider the following:
+#                NAME     FSTYPE LABEL
+#                xvdb
+#                |-xvdb1  ext4
+#                |-xvdb2
+#                |-xvdb3  btrfs  test
+#                \-xvdb4  ext4   test
+#
+#            If you ask for 'auto', label of 'test, and file system of 'ext4'
+#            then cloud-init will select the 2nd partition, even though there
+#            is a partition match at the 4th partition.
+#
+#            If you ask for 'any' and a label of 'test', then cloud-init will
+#            select the 1st partition.
+#
+#            If you ask for 'auto' and don't define label, then cloud-init will
+#            select the 1st partition.
+#
+#            In general, if you have a specific partition configuration in mind,
+#            you should define either the device or the partition number. 'auto'
+#            and 'any' are specifically intended for formating ephemeral storage or
+#            for simple schemes.
+#
+#        "none": Put the file system directly on the device.
+#
+#        <NUM>: where NUM is the actual partition number.
+#
+#    <OVERWRITE>: Defines whether or not to overwrite any existing
+#        filesystem.
+#
+#        "true": Indiscriminately destroy any pre-existing file system. Use at
+#            your own peril.
+#
+#        "false": If an existing file system exists, skip the creation.
+#
+#    <REPLACE_FS>: This is a special directive, used for Windows Azure that
+#        instructs cloud-init to replace a file system of <FS_TYPE>. NOTE:
+#        unless you define a label, this requires the use of the 'any' partition
+#        directive.
+#
+# Behavior Caveat: The default behavior is to _check_ if the file system exists.
+#    If a file system matches the specification, then the operation is a no-op.
diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt
index b35b084d..8d756c61 100644
--- a/doc/examples/cloud-config.txt
+++ b/doc/examples/cloud-config.txt
@@ -74,7 +74,7 @@ apt_preserve_sources_list: true
 
 # 'source' entries in apt-sources that match this python regex
 # expression will be passed to add-apt-repository
-add_apt_repo_match = "^[\w-]+:\w"
+add_apt_repo_match: '^[\w-]+:\w'
 
 apt_sources:
  - source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main"
-- 
cgit v1.2.3


From 923a7d0efc3cf3224b891dc783321018d59ce33e Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Sat, 14 Dec 2013 14:17:31 -0500
Subject: support calling apt with eatmydata, enable by default if available.

This allows a general config option to prefix apt-get commands via
'apt_get_wrapper'.  By default, the command is set to 'eatmydata', and the
mode set to 'auto'.  That means if eatmydata is available (via which), it
will use it.

The 'command' can be either a array or a string.


LP: #1236531
---
 ChangeLog                     |  2 ++
 cloudinit/distros/debian.py   | 28 ++++++++++++++++++++++++++--
 doc/examples/cloud-config.txt |  9 +++++++--
 3 files changed, 35 insertions(+), 4 deletions(-)

(limited to 'doc')

diff --git a/ChangeLog b/ChangeLog
index 1f4ca198..1b7948f7 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -3,6 +3,8 @@
  - Add a debug log message around import failures
  - add a 'debug' module for easily printing out some information about
    datasource and cloud-init [Shraddha Pandhe]
+ - support running apt with 'eatmydata' via configuration token
+   apt_get_wrapper (LP: #1236531).
 0.7.4:
  - fix issue mounting 'ephemeral0' if ephemeral0 was an alias for a
    partitioned block device with target filesystem on ephemeral0.1.
diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py
index 8fe49cbe..1ae232fd 100644
--- a/cloudinit/distros/debian.py
+++ b/cloudinit/distros/debian.py
@@ -36,6 +36,10 @@ LOG = logging.getLogger(__name__)
 APT_GET_COMMAND = ('apt-get', '--option=Dpkg::Options::=--force-confold',
                    '--option=Dpkg::options::=--force-unsafe-io',
                    '--assume-yes', '--quiet')
+APT_GET_WRAPPER = {
+    'command': 'eatmydata',
+    'enabled': 'auto',
+}
 
 
 class Distro(distros.Distro):
@@ -148,7 +152,13 @@ class Distro(distros.Distro):
         # See: http://tiny.cc/kg91fw
         # Or: http://tiny.cc/mh91fw
         e['DEBIAN_FRONTEND'] = 'noninteractive'
-        cmd = list(self.get_option("apt_get_command", APT_GET_COMMAND))
+
+        wcfg = self.get_option("apt_get_wrapper", APT_GET_WRAPPER)
+        cmd = _get_wrapper_prefix(
+            wcfg.get('command', APT_GET_WRAPPER['command']),
+            wcfg.get('enabled', APT_GET_WRAPPER['enabled']))
+
+        cmd.extend(list(self.get_option("apt_get_command", APT_GET_COMMAND)))
 
         if args and isinstance(args, str):
             cmd.append(args)
@@ -166,7 +176,9 @@ class Distro(distros.Distro):
         cmd.extend(pkglist)
 
         # Allow the output of this to flow outwards (ie not be captured)
-        util.subp(cmd, env=e, capture=False)
+        util.log_time(logfunc=LOG.debug,
+            msg="apt-%s [%s]" % (command, ' '.join(cmd)), func=util.subp,
+            args=(cmd,), kwargs={'env': e, 'capture': False})
 
     def update_package_sources(self):
         self._runner.run("update-sources", self.package_command,
@@ -175,3 +187,15 @@ class Distro(distros.Distro):
     def get_primary_arch(self):
         (arch, _err) = util.subp(['dpkg', '--print-architecture'])
         return str(arch).strip()
+
+
+def _get_wrapper_prefix(cmd, mode):
+    if isinstance(cmd, str):
+        cmd = [str(cmd)]
+
+    if (util.is_true(mode) or
+        (str(mode).lower() == "auto" and cmd[0] and
+         util.which(cmd[0]))):
+        return cmd
+    else:
+        return []
diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt
index 8d756c61..61fa6065 100644
--- a/doc/examples/cloud-config.txt
+++ b/doc/examples/cloud-config.txt
@@ -147,8 +147,13 @@ apt_sources:
 #     '--option=Dpkg::options::=--force-unsafe-io', '--assume-yes', '--quiet']
 #
 # apt_get_upgrade_subcommand:
-# Specify a different subcommand for 'upgrade. The default is 'dist-upgrade'.
-# This is the subcommand that is invoked if package_upgrade is set to true above.
+#  Specify a different subcommand for 'upgrade. The default is 'dist-upgrade'.
+#  This is the subcommand that is invoked if package_upgrade is set to true above.
+#
+# apt_get_wrapper:
+#   command: eatmydata
+#   enabled: [True, False, "auto"]
+#  
 
 # Install additional packages on first boot
 #
-- 
cgit v1.2.3