merge from master at 18.3

10 files changed, 304 insertions, 18 deletions

diff --git a/doc/examples/cloud-config-disk-setup.txt b/doc/examples/cloud-config-disk-setup.txt
index dd91477d..43a62a26 100644
--- a/doc/examples/cloud-config-disk-setup.txt
+++ b/doc/examples/cloud-config-disk-setup.txt
@@ -37,7 +37,7 @@ fs_setup:
 # Default disk definitions for SmartOS
 # ------------------------------------
 
-device_aliases: {'ephemeral0': '/dev/sdb'}
+device_aliases: {'ephemeral0': '/dev/vdb'}
 disk_setup:
     ephemeral0:
          table_type: mbr
@@ -46,7 +46,7 @@ disk_setup:
 
 fs_setup:
     - label: ephemeral0
-      filesystem: ext3
+      filesystem: ext4
       device: ephemeral0.0
 
 # Cavaut for SmartOS: if ephemeral disk is not defined, then the disk will
diff --git a/doc/examples/cloud-config-user-groups.txt b/doc/examples/cloud-config-user-groups.txt
index 7bca24a3..01ecad7b 100644
--- a/doc/examples/cloud-config-user-groups.txt
+++ b/doc/examples/cloud-config-user-groups.txt
@@ -30,6 +30,11 @@ users:
     gecos: Magic Cloud App Daemon User
     inactive: true
     system: true
+  - name: fizzbuzz
+    sudo: False
+    ssh_authorized_keys:
+      - <ssh pub key 1>
+      - <ssh pub key 2>
   - snapuser: joe@joeuser.io
 
 # Valid Values:
@@ -71,13 +76,21 @@ users:
 #   no_log_init: When set to true, do not initialize lastlog and faillog database.
 #   ssh_import_id: Optional. Import SSH ids
 #   ssh_authorized_keys: Optional. [list] Add keys to user's authorized keys file
-#   sudo: Defaults to none. Set to the sudo string you want to use, i.e.
-#           ALL=(ALL) NOPASSWD:ALL. To add multiple rules, use the following
-#           format.
-#               sudo:
-#                   - ALL=(ALL) NOPASSWD:/bin/mysql
-#                   - ALL=(ALL) ALL
-#           Note: Please double check your syntax and make sure it is valid.
+#   sudo: Defaults to none. Accepts a sudo rule string, a list of sudo rule
+#         strings or False to explicitly deny sudo usage. Examples:
+#
+#         Allow a user unrestricted sudo access.
+#             sudo:  ALL=(ALL) NOPASSWD:ALL
+#
+#         Adding multiple sudo rule strings.
+#             sudo:
+#               - ALL=(ALL) NOPASSWD:/bin/mysql
+#               - ALL=(ALL) ALL
+#
+#         Prevent sudo access for a user.
+#             sudo: False
+#
+#         Note: Please double check your syntax and make sure it is valid.
 #               cloud-init does not parse/check the syntax of the sudo
 #               directive.
 #   system: Create the user as a system user. This means no home directory.
diff --git a/doc/rtd/topics/datasources.rst b/doc/rtd/topics/datasources.rst
index 7e2854de..30e57d85 100644
--- a/doc/rtd/topics/datasources.rst
+++ b/doc/rtd/topics/datasources.rst
@@ -17,6 +17,103 @@ own way) internally a datasource abstract class was created to allow for a
 single way to access the different cloud systems methods to provide this data
 through the typical usage of subclasses.
 
+
+instance-data
+-------------
+For reference, cloud-init stores all the metadata, vendordata and userdata
+provided by a cloud in a json blob at ``/run/cloud-init/instance-data.json``.
+While the json contains datasource-specific keys and names, cloud-init will
+maintain a minimal set of standardized keys that will remain stable on any
+cloud. Standardized instance-data keys will be present under a "v1" key.
+Any datasource metadata cloud-init consumes will all be present under the
+"ds" key.
+
+Below is an instance-data.json example from an OpenStack instance:
+
+.. sourcecode:: json
+
+  {
+   "base64-encoded-keys": [
+    "ds/meta-data/random_seed",
+    "ds/user-data"
+   ],
+   "ds": {
+    "ec2_metadata": {
+     "ami-id": "ami-0000032f",
+     "ami-launch-index": "0",
+     "ami-manifest-path": "FIXME",
+     "block-device-mapping": {
+      "ami": "vda",
+      "ephemeral0": "/dev/vdb",
+      "root": "/dev/vda"
+     },
+     "hostname": "xenial-test.novalocal",
+     "instance-action": "none",
+     "instance-id": "i-0006e030",
+     "instance-type": "m1.small",
+     "local-hostname": "xenial-test.novalocal",
+     "local-ipv4": "10.5.0.6",
+     "placement": {
+      "availability-zone": "None"
+     },
+     "public-hostname": "xenial-test.novalocal",
+     "public-ipv4": "10.245.162.145",
+     "reservation-id": "r-fxm623oa",
+     "security-groups": "default"
+    },
+    "meta-data": {
+     "availability_zone": null,
+     "devices": [],
+     "hostname": "xenial-test.novalocal",
+     "instance-id": "3e39d278-0644-4728-9479-678f9212d8f0",
+     "launch_index": 0,
+     "local-hostname": "xenial-test.novalocal",
+     "name": "xenial-test",
+     "project_id": "e0eb2d2538814...",
+     "random_seed": "A6yPN...",
+     "uuid": "3e39d278-0644-4728-9479-678f92..."
+    },
+    "network_json": {
+     "links": [
+      {
+       "ethernet_mac_address": "fa:16:3e:7d:74:9b",
+       "id": "tap9ca524d5-6e",
+       "mtu": 8958,
+       "type": "ovs",
+       "vif_id": "9ca524d5-6e5a-4809-936a-6901..."
+      }
+     ],
+     "networks": [
+      {
+       "id": "network0",
+       "link": "tap9ca524d5-6e",
+       "network_id": "c6adfc18-9753-42eb-b3ea-18b57e6b837f",
+       "type": "ipv4_dhcp"
+      }
+     ],
+     "services": [
+      {
+       "address": "10.10.160.2",
+       "type": "dns"
+      }
+     ]
+    },
+    "user-data": "I2Nsb3VkLWNvbmZpZ...",
+    "vendor-data": null
+   },
+   "v1": {
+    "availability-zone": null,
+    "cloud-name": "openstack",
+    "instance-id": "3e39d278-0644-4728-9479-678f9212d8f0",
+    "local-hostname": "xenial-test",
+    "region": null
+   }
+  }
+
+
+
+Datasource API
+--------------
 The current interface that a datasource object must provide is the following:
 
 .. sourcecode:: python
@@ -80,6 +177,7 @@ Follow for more information.
 .. toctree::
    :maxdepth: 2
 
+   datasources/aliyun.rst
    datasources/altcloud.rst
    datasources/azure.rst
    datasources/cloudsigma.rst
diff --git a/doc/rtd/topics/datasources/aliyun.rst b/doc/rtd/topics/datasources/aliyun.rst
new file mode 100644
index 00000000..3f4f40ca
--- /dev/null
+++ b/doc/rtd/topics/datasources/aliyun.rst
@@ -0,0 +1,74 @@
+.. _datasource_aliyun:
+
+Alibaba Cloud (AliYun)
+======================
+The ``AliYun`` datasource reads data from Alibaba Cloud ECS.  Support is
+present in cloud-init since 0.7.9.
+
+Metadata Service
+----------------
+The Alibaba Cloud metadata service is available at the well known url
+``http://100.100.100.200/``. For more information see
+Alibaba Cloud ECS on `metadata
+<https://www.alibabacloud.com/help/zh/faq-detail/49122.htm>`__.
+
+Versions
+^^^^^^^^
+Like the EC2 metadata service, Alibaba Cloud's metadata service provides
+versioned data under specific paths.  As of April 2018, there are only
+``2016-01-01`` and ``latest`` versions.
+
+It is expected that the dated version will maintain a stable interface but
+``latest`` may change content at a future date.
+
+Cloud-init uses the ``2016-01-01`` version.
+
+You can list the versions available to your instance with:
+
+.. code-block:: shell-session
+
+    $ curl http://100.100.100.200/
+    2016-01-01
+    latest
+
+Metadata
+^^^^^^^^
+Instance metadata can be queried at
+``http://100.100.100.200/2016-01-01/meta-data``
+
+.. code-block:: shell-session
+
+    $ curl http://100.100.100.200/2016-01-01/meta-data
+    dns-conf/
+    eipv4
+    hostname
+    image-id
+    instance-id
+    instance/
+    mac
+    network-type
+    network/
+    ntp-conf/
+    owner-account-id
+    private-ipv4
+    public-keys/
+    region-id
+    serial-number
+    source-address
+    sub-private-ipv4-list
+    vpc-cidr-block
+    vpc-id
+
+Userdata
+^^^^^^^^
+If provided, user-data will appear at
+``http://100.100.100.200/2016-01-01/user-data``.
+If no user-data is provided, this will return a 404.
+
+.. code-block:: shell-session
+
+    $ curl http://100.100.100.200/2016-01-01/user-data
+    #!/bin/sh
+    echo "Hello World."
+
+.. vi: textwidth=78
diff --git a/doc/rtd/topics/datasources/cloudstack.rst b/doc/rtd/topics/datasources/cloudstack.rst
index 225093a1..a3101ed7 100644
--- a/doc/rtd/topics/datasources/cloudstack.rst
+++ b/doc/rtd/topics/datasources/cloudstack.rst
@@ -4,7 +4,9 @@ CloudStack
 ==========
 
 `Apache CloudStack`_ expose user-data, meta-data, user password and account
-sshkey thru the Virtual-Router. For more details on meta-data and user-data,
+sshkey thru the Virtual-Router. The datasource obtains the VR address via
+dhcp lease information given to the instance.
+For more details on meta-data and user-data,
 refer the `CloudStack Administrator Guide`_. 
 
 URLs to access user-data and meta-data from the Virtual Machine. Here 10.1.1.1
@@ -18,14 +20,26 @@ is the Virtual Router IP:
 
 Configuration
 -------------
+The following configuration can be set for the datasource in system
+configuration (in `/etc/cloud/cloud.cfg` or `/etc/cloud/cloud.cfg.d/`).
 
-Apache CloudStack datasource can be configured as follows:
+The settings that may be configured are:
 
-.. code:: yaml
+ * **max_wait**:  the maximum amount of clock time in seconds that should be
+   spent searching metadata_urls.  A value less than zero will result in only
+   one request being made, to the first in the list. (default: 120)
+ * **timeout**: the timeout value provided to urlopen for each individual http
+   request.  This is used both when selecting a metadata_url and when crawling
+   the metadata service. (default: 50)
 
-    datasource:
-      CloudStack: {}
-      None: {}
+An example configuration with the default values is provided below:
+
+.. sourcecode:: yaml
+
+  datasource:
+   CloudStack:
+    max_wait: 120
+    timeout: 50
     datasource_list:
       - CloudStack
 
diff --git a/doc/rtd/topics/datasources/ec2.rst b/doc/rtd/topics/datasources/ec2.rst
index 3bc66e17..64c325d8 100644
--- a/doc/rtd/topics/datasources/ec2.rst
+++ b/doc/rtd/topics/datasources/ec2.rst
@@ -60,4 +60,34 @@ To see which versions are supported from your cloud provider use the following U
     ...
     latest
 
+
+
+Configuration
+-------------
+The following configuration can be set for the datasource in system
+configuration (in `/etc/cloud/cloud.cfg` or `/etc/cloud/cloud.cfg.d/`).
+
+The settings that may be configured are:
+
+ * **metadata_urls**: This list of urls will be searched for an Ec2
+   metadata service. The first entry that successfully returns a 200 response
+   for <url>/<version>/meta-data/instance-id will be selected.
+   (default: ['http://169.254.169.254', 'http://instance-data:8773']).
+ * **max_wait**:  the maximum amount of clock time in seconds that should be
+   spent searching metadata_urls.  A value less than zero will result in only
+   one request being made, to the first in the list. (default: 120)
+ * **timeout**: the timeout value provided to urlopen for each individual http
+   request.  This is used both when selecting a metadata_url and when crawling
+   the metadata service. (default: 50)
+
+An example configuration with the default values is provided below:
+
+.. sourcecode:: yaml
+
+  datasource:
+   Ec2:
+    metadata_urls: ["http://169.254.169.254:80", "http://instance-data:8773"]
+    max_wait: 120
+    timeout: 50
+
 .. vi: textwidth=78
diff --git a/doc/rtd/topics/datasources/openstack.rst b/doc/rtd/topics/datasources/openstack.rst
index 43592dec..421da08f 100644
--- a/doc/rtd/topics/datasources/openstack.rst
+++ b/doc/rtd/topics/datasources/openstack.rst
@@ -7,6 +7,21 @@ This datasource supports reading data from the
 `OpenStack Metadata Service
 <https://docs.openstack.org/nova/latest/admin/networking-nova.html#metadata-service>`_.
 
+Discovery
+-------------
+To determine whether a platform looks like it may be OpenStack, cloud-init
+checks the following environment attributes as a potential OpenStack platform:
+
+ * Maybe OpenStack if
+
+   * **non-x86 cpu architecture**: because DMI data is buggy on some arches
+ * Is OpenStack **if x86 architecture and ANY** of the following
+
+   * **/proc/1/environ**: Nova-lxd contains *product_name=OpenStack Nova*
+   * **DMI product_name**: Either *Openstack Nova* or *OpenStack Compute*
+   * **DMI chassis_asset_tag** is *OpenTelekomCloud*
+
+
 Configuration
 -------------
 The following configuration can be set for the datasource in system
@@ -25,18 +40,22 @@ The settings that may be configured are:
    the metadata service. (default: 10)
  * **retries**: The number of retries that should be done for an http request.
    This value is used only after metadata_url is selected. (default: 5)
+ * **apply_network_config**: A boolean specifying whether to configure the
+   network for the instance based on network_data.json provided by the
+   metadata service. When False, only configure dhcp on the primary nic for
+   this instances. (default: True)
 
-An example configuration with the default values is provided as example below:
+An example configuration with the default values is provided below:
 
 .. sourcecode:: yaml
 
-  #cloud-config
   datasource:
    OpenStack:
     metadata_urls: ["http://169.254.169.254"]
     max_wait: -1
     timeout: 10
     retries: 5
+    apply_network_config: True
 
 
 Vendor Data
diff --git a/doc/rtd/topics/network-config-format-v1.rst b/doc/rtd/topics/network-config-format-v1.rst
index 2f8ab54c..3b0148ca 100644
--- a/doc/rtd/topics/network-config-format-v1.rst
+++ b/doc/rtd/topics/network-config-format-v1.rst
@@ -130,6 +130,18 @@ the bond interfaces.
 The ``bond_interfaces`` key accepts a list of network device ``name`` values
 from the configuration.  This list may be empty.
 
+**mtu**: *<MTU SizeBytes>*
+
+The MTU key represents a device's Maximum Transmission Unit, the largest size
+packet or frame, specified in octets (eight-bit bytes), that can be sent in a
+packet- or frame-based network.  Specifying ``mtu`` is optional.
+
+.. note::
+
+  The possible supported values of a device's MTU is not available at
+  configuration time.  It's possible to specify a value too large or to
+  small for a device and may be ignored by the device.
+
 **params**:  *<Dictionary of key: value bonding parameter pairs>*
 
 The ``params`` key in a bond holds a dictionary of bonding parameters.
@@ -268,6 +280,21 @@ Type ``vlan`` requires the following keys:
 - ``vlan_link``: Specify the underlying link via its ``name``.
 - ``vlan_id``: Specify the VLAN numeric id.
 
+The following optional keys are supported:
+
+**mtu**: *<MTU SizeBytes>*
+
+The MTU key represents a device's Maximum Transmission Unit, the largest size
+packet or frame, specified in octets (eight-bit bytes), that can be sent in a
+packet- or frame-based network.  Specifying ``mtu`` is optional.
+
+.. note::
+
+  The possible supported values of a device's MTU is not available at
+  configuration time.  It's possible to specify a value too large or to
+  small for a device and may be ignored by the device.
+
+
 **VLAN Example**::
 
    network:
diff --git a/doc/rtd/topics/network-config-format-v2.rst b/doc/rtd/topics/network-config-format-v2.rst
index 335d236a..ea370ef5 100644
--- a/doc/rtd/topics/network-config-format-v2.rst
+++ b/doc/rtd/topics/network-config-format-v2.rst
@@ -174,6 +174,12 @@ recognized by ``inet_pton(3)``
 Example for IPv4: ``gateway4: 172.16.0.1``
 Example for IPv6: ``gateway6: 2001:4::1``
 
+**mtu**: *<MTU SizeBytes>*
+
+The MTU key represents a device's Maximum Transmission Unit, the largest size
+packet or frame, specified in octets (eight-bit bytes), that can be sent in a
+packet- or frame-based network.  Specifying ``mtu`` is optional.
+
 **nameservers**: *<(mapping)>*
 
 Set DNS servers and search domains, for manual address configuration. There
diff --git a/doc/rtd/topics/tests.rst b/doc/rtd/topics/tests.rst
index cac4a6e4..b83bd899 100644
--- a/doc/rtd/topics/tests.rst
+++ b/doc/rtd/topics/tests.rst
@@ -58,7 +58,8 @@ explaining how to run one or the other independently.
     $ tox -e citest -- run --verbose \
         --os-name stretch --os-name xenial \
         --deb cloud-init_0.7.8~my_patch_all.deb \
-        --preserve-data --data-dir ~/collection
+        --preserve-data --data-dir ~/collection \
+        --preserve-instance
 
 The above command will do the following:
 
@@ -76,6 +77,10 @@ The above command will do the following:
 * ``--preserve-data`` always preserve collected data, do not remove data
   after successful test run
 
+* ``--preserve-instance`` do not destroy the instance after test to allow
+  for debugging the stopped instance during integration test development. By
+  default, test instances are destroyed after the test completes.
+
 * ``--data-dir ~/collection`` write collected data into `~/collection`,
   rather than using a temporary directory