merge from master at 0.7.9-233-ge586fe35

6 files changed, 411 insertions, 36 deletions

diff --git a/cloudinit/sources/DataSourceAliYun.py b/cloudinit/sources/DataSourceAliYun.py
index 9debe947..380e27cb 100644
--- a/cloudinit/sources/DataSourceAliYun.py
+++ b/cloudinit/sources/DataSourceAliYun.py
@@ -4,8 +4,10 @@ import os
 
 from cloudinit import sources
 from cloudinit.sources import DataSourceEc2 as EC2
+from cloudinit import util
 
 DEF_MD_VERSION = "2016-01-01"
+ALIYUN_PRODUCT = "Alibaba Cloud ECS"
 
 
 class DataSourceAliYun(EC2.DataSourceEc2):
@@ -24,7 +26,17 @@ class DataSourceAliYun(EC2.DataSourceEc2):
 
     @property
     def cloud_platform(self):
-        return EC2.Platforms.ALIYUN
+        if self._cloud_platform is None:
+            if _is_aliyun():
+                self._cloud_platform = EC2.Platforms.ALIYUN
+            else:
+                self._cloud_platform = EC2.Platforms.NO_EC2_METADATA
+
+        return self._cloud_platform
+
+
+def _is_aliyun():
+    return util.read_dmi_data('system-product-name') == ALIYUN_PRODUCT
 
 
 def parse_public_keys(public_keys):
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index b9458ffa..b5a95a1f 100644
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -16,6 +16,7 @@ from xml.dom import minidom
 import xml.etree.ElementTree as ET
 
 from cloudinit import log as logging
+from cloudinit import net
 from cloudinit import sources
 from cloudinit.sources.helpers.azure import get_metadata_from_fabric
 from cloudinit import util
@@ -36,6 +37,8 @@ RESOURCE_DISK_PATH = '/dev/disk/cloud/azure_resource'
 DEFAULT_PRIMARY_NIC = 'eth0'
 LEASE_FILE = '/var/lib/dhcp/dhclient.eth0.leases'
 DEFAULT_FS = 'ext4'
+# DMI chassis-asset-tag is set static for all azure instances
+AZURE_CHASSIS_ASSET_TAG = '7783-7084-3265-9085-8269-3286-77'
 
 
 def find_storvscid_from_sysctl_pnpinfo(sysctl_out, deviceid):
@@ -99,7 +102,7 @@ def get_dev_storvsc_sysctl():
         sysctl_out, err = util.subp(['sysctl', 'dev.storvsc'])
     except util.ProcessExecutionError:
         LOG.debug("Fail to execute sysctl dev.storvsc")
-        return None
+        sysctl_out = ""
     return sysctl_out
 
 
@@ -175,6 +178,11 @@ if util.is_FreeBSD():
         RESOURCE_DISK_PATH = "/dev/" + res_disk
     else:
         LOG.debug("resource disk is None")
+    BOUNCE_COMMAND = [
+        'sh', '-xc',
+        ("i=$interface; x=0; ifconfig down $i || x=$?; "
+         "ifconfig up $i || x=$?; exit $x")
+    ]
 
 BUILTIN_DS_CONFIG = {
     'agent_command': AGENT_START_BUILTIN,
@@ -238,7 +246,9 @@ def temporary_hostname(temp_hostname, cfg, hostname_command='hostname'):
         set_hostname(previous_hostname, hostname_command)
 
 
-class DataSourceAzureNet(sources.DataSource):
+class DataSourceAzure(sources.DataSource):
+    _negotiated = False
+
     def __init__(self, sys_cfg, distro, paths):
         sources.DataSource.__init__(self, sys_cfg, distro, paths)
         self.seed_dir = os.path.join(paths.seed_dir, 'azure')
@@ -248,6 +258,7 @@ class DataSourceAzureNet(sources.DataSource):
             util.get_cfg_by_path(sys_cfg, DS_CFG_PATH, {}),
             BUILTIN_DS_CONFIG])
         self.dhclient_lease_file = self.ds_cfg.get('dhclient_lease_file')
+        self._network_config = None
 
     def __str__(self):
         root = sources.DataSource.__str__(self)
@@ -320,6 +331,11 @@ class DataSourceAzureNet(sources.DataSource):
         # azure removes/ejects the cdrom containing the ovf-env.xml
         # file on reboot.  So, in order to successfully reboot we
         # need to look in the datadir and consider that valid
+        asset_tag = util.read_dmi_data('chassis-asset-tag')
+        if asset_tag != AZURE_CHASSIS_ASSET_TAG:
+            LOG.debug("Non-Azure DMI asset tag '%s' discovered.", asset_tag)
+            return False
+
         ddir = self.ds_cfg['data_dir']
 
         candidates = [self.seed_dir]
@@ -364,13 +380,14 @@ class DataSourceAzureNet(sources.DataSource):
             LOG.debug("using files cached in %s", ddir)
 
         # azure / hyper-v provides random data here
+        # TODO. find the seed on FreeBSD platform
+        # now update ds_cfg to reflect contents pass in config
         if not util.is_FreeBSD():
             seed = util.load_file("/sys/firmware/acpi/tables/OEM0",
                                   quiet=True, decode=False)
             if seed:
                 self.metadata['random_seed'] = seed
-        # TODO. find the seed on FreeBSD platform
-        # now update ds_cfg to reflect contents pass in config
+
         user_ds_cfg = util.get_cfg_by_path(self.cfg, DS_CFG_PATH, {})
         self.ds_cfg = util.mergemanydict([user_ds_cfg, self.ds_cfg])
 
@@ -378,6 +395,40 @@ class DataSourceAzureNet(sources.DataSource):
         # the directory to be protected.
         write_files(ddir, files, dirmode=0o700)
 
+        self.metadata['instance-id'] = util.read_dmi_data('system-uuid')
+
+        return True
+
+    def device_name_to_device(self, name):
+        return self.ds_cfg['disk_aliases'].get(name)
+
+    def get_config_obj(self):
+        return self.cfg
+
+    def check_instance_id(self, sys_cfg):
+        # quickly (local check only) if self.instance_id is still valid
+        return sources.instance_id_matches_system_uuid(self.get_instance_id())
+
+    def setup(self, is_new_instance):
+        if self._negotiated is False:
+            LOG.debug("negotiating for %s (new_instance=%s)",
+                      self.get_instance_id(), is_new_instance)
+            fabric_data = self._negotiate()
+            LOG.debug("negotiating returned %s", fabric_data)
+            if fabric_data:
+                self.metadata.update(fabric_data)
+            self._negotiated = True
+        else:
+            LOG.debug("negotiating already done for %s",
+                      self.get_instance_id())
+
+    def _negotiate(self):
+        """Negotiate with fabric and return data from it.
+
+           On success, returns a dictionary including 'public_keys'.
+           On failure, returns False.
+        """
+
         if self.ds_cfg['agent_command'] == AGENT_START_BUILTIN:
             self.bounce_network_with_azure_hostname()
 
@@ -387,31 +438,64 @@ class DataSourceAzureNet(sources.DataSource):
         else:
             metadata_func = self.get_metadata_from_agent
 
+        LOG.debug("negotiating with fabric via agent command %s",
+                  self.ds_cfg['agent_command'])
         try:
             fabric_data = metadata_func()
         except Exception as exc:
-            LOG.info("Error communicating with Azure fabric; assume we aren't"
-                     " on Azure.", exc_info=True)
+            LOG.warning(
+                "Error communicating with Azure fabric; You may experience."
+                "connectivity issues.", exc_info=True)
             return False
-        self.metadata['instance-id'] = util.read_dmi_data('system-uuid')
-        self.metadata.update(fabric_data)
-
-        return True
-
-    def device_name_to_device(self, name):
-        return self.ds_cfg['disk_aliases'].get(name)
 
-    def get_config_obj(self):
-        return self.cfg
-
-    def check_instance_id(self, sys_cfg):
-        # quickly (local check only) if self.instance_id is still valid
-        return sources.instance_id_matches_system_uuid(self.get_instance_id())
+        return fabric_data
 
     def activate(self, cfg, is_new_instance):
         address_ephemeral_resize(is_new_instance=is_new_instance)
         return
 
+    @property
+    def network_config(self):
+        """Generate a network config like net.generate_fallback_network() with
+           the following execptions.
+
+           1. Probe the drivers of the net-devices present and inject them in
+              the network configuration under params: driver: <driver> value
+           2. If the driver value is 'mlx4_core', the control mode should be
+              set to manual.  The device will be later used to build a bond,
+              for now we want to ensure the device gets named but does not
+              break any network configuration
+        """
+        blacklist = ['mlx4_core']
+        if not self._network_config:
+            LOG.debug('Azure: generating fallback configuration')
+            # generate a network config, blacklist picking any mlx4_core devs
+            netconfig = net.generate_fallback_config(
+                blacklist_drivers=blacklist, config_driver=True)
+
+            # if we have any blacklisted devices, update the network_config to
+            # include the device, mac, and driver values, but with no ip
+            # config; this ensures udev rules are generated but won't affect
+            # ip configuration
+            bl_found = 0
+            for bl_dev in [dev for dev in net.get_devicelist()
+                           if net.device_driver(dev) in blacklist]:
+                bl_found += 1
+                cfg = {
+                    'type': 'physical',
+                    'name': 'vf%d' % bl_found,
+                    'mac_address': net.get_interface_mac(bl_dev),
+                    'params': {
+                        'driver': net.device_driver(bl_dev),
+                        'device_id': net.device_devid(bl_dev),
+                    },
+                }
+                netconfig['config'].append(cfg)
+
+            self._network_config = netconfig
+
+        return self._network_config
+
 
 def _partitions_on_device(devpath, maxnum=16):
     # return a list of tuples (ptnum, path) for each part on devpath
@@ -694,7 +778,7 @@ def read_azure_ovf(contents):
     try:
         dom = minidom.parseString(contents)
     except Exception as e:
-        raise BrokenAzureDataSource("invalid xml: %s" % e)
+        raise BrokenAzureDataSource("Invalid ovf-env.xml: %s" % e)
 
     results = find_child(dom.documentElement,
                          lambda n: n.localName == "ProvisioningSection")
@@ -792,19 +876,23 @@ def encrypt_pass(password, salt_id="$6$"):
     return crypt.crypt(password, salt_id + util.rand_str(strlen=16))
 
 
+def _check_freebsd_cdrom(cdrom_dev):
+    """Return boolean indicating path to cdrom device has content."""
+    try:
+        with open(cdrom_dev) as fp:
+            fp.read(1024)
+            return True
+    except IOError:
+        LOG.debug("cdrom (%s) is not configured", cdrom_dev)
+    return False
+
+
 def list_possible_azure_ds_devs():
-    # return a sorted list of devices that might have a azure datasource
     devlist = []
     if util.is_FreeBSD():
         cdrom_dev = "/dev/cd0"
-        try:
-            util.subp(["mount", "-o", "ro", "-t", "udf", cdrom_dev,
-                       "/mnt/cdrom/secure"])
-        except util.ProcessExecutionError:
-            LOG.debug("Fail to mount cd")
-            return devlist
-        util.subp(["umount", "/mnt/cdrom/secure"])
-        devlist.append(cdrom_dev)
+        if _check_freebsd_cdrom(cdrom_dev):
+            return [cdrom_dev]
     else:
         for fstype in ("iso9660", "udf"):
             devlist.extend(util.find_devs_with("TYPE=%s" % fstype))
@@ -834,9 +922,12 @@ class NonAzureDataSource(Exception):
     pass
 
 
+# Legacy: Must be present in case we load an old pkl object
+DataSourceAzureNet = DataSourceAzure
+
 # Used to match classes to dependencies
 datasources = [
-    (DataSourceAzureNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
+    (DataSourceAzure, (sources.DEP_FILESYSTEM, )),
 ]
 
 
diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
index 2f9c7edf..4ec9592f 100644
--- a/cloudinit/sources/DataSourceEc2.py
+++ b/cloudinit/sources/DataSourceEc2.py
@@ -32,7 +32,12 @@ class Platforms(object):
     AWS = "AWS"
     BRIGHTBOX = "Brightbox"
     SEEDED = "Seeded"
+    # UNKNOWN indicates no positive id.  If strict_id is 'warn' or 'false',
+    # then an attempt at the Ec2 Metadata service will be made.
     UNKNOWN = "Unknown"
+    # NO_EC2_METADATA indicates this platform does not have a Ec2 metadata
+    # service available. No attempt at the Ec2 Metadata service will be made.
+    NO_EC2_METADATA = "No-EC2-Metadata"
 
 
 class DataSourceEc2(sources.DataSource):
@@ -65,6 +70,8 @@ class DataSourceEc2(sources.DataSource):
                   strict_mode, self.cloud_platform)
         if strict_mode == "true" and self.cloud_platform == Platforms.UNKNOWN:
             return False
+        elif self.cloud_platform == Platforms.NO_EC2_METADATA:
+            return False
 
         try:
             if not self.wait_for_metadata_service():
@@ -309,10 +316,16 @@ def identify_platform():
 
 
 def _collect_platform_data():
-    # returns a dictionary with all lower case values:
-    #   uuid: system-uuid from dmi or /sys/hypervisor
-    #   uuid_source: 'hypervisor' (/sys/hypervisor/uuid) or 'dmi'
-    #   serial: dmi 'system-serial-number' (/sys/.../product_serial)
+    """Returns a dictionary of platform info from dmi or /sys/hypervisor.
+
+    Keys in the dictionary are as follows:
+       uuid: system-uuid from dmi or /sys/hypervisor
+       uuid_source: 'hypervisor' (/sys/hypervisor/uuid) or 'dmi'
+       serial: dmi 'system-serial-number' (/sys/.../product_serial)
+
+    On Ec2 instances experimentation is that product_serial is upper case,
+    and product_uuid is lower case.  This returns lower case values for both.
+    """
     data = {}
     try:
         uuid = util.load_file("/sys/hypervisor/uuid").strip()
diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py
index c68f6b8c..e641244d 100644
--- a/cloudinit/sources/DataSourceNoCloud.py
+++ b/cloudinit/sources/DataSourceNoCloud.py
@@ -43,6 +43,18 @@ class DataSourceNoCloud(sources.DataSource):
                   'network-config': None}
 
         try:
+            # Parse the system serial label from dmi. If not empty, try parsing
+            # like the commandline
+            md = {}
+            serial = util.read_dmi_data('system-serial-number')
+            if serial and load_cmdline_data(md, serial):
+                found.append("dmi")
+                mydata = _merge_new_seed(mydata, {'meta-data': md})
+        except Exception:
+            util.logexc(LOG, "Unable to parse dmi data")
+            return False
+
+        try:
             # Parse the kernel command line, getting data passed in
             md = {}
             if load_cmdline_data(md):
diff --git a/cloudinit/sources/DataSourceScaleway.py b/cloudinit/sources/DataSourceScaleway.py
new file mode 100644
index 00000000..3a8a8e8f
--- /dev/null
+++ b/cloudinit/sources/DataSourceScaleway.py
@@ -0,0 +1,234 @@
+# Author: Julien Castets <castets.j@gmail.com>
+#
+# This file is part of cloud-init. See LICENSE file for license information.
+
+# Scaleway API:
+# https://developer.scaleway.com/#metadata
+
+import json
+import os
+import socket
+import time
+
+import requests
+
+# pylint fails to import the two modules below.
+# These are imported via requests.packages rather than urllib3 because:
+#  a.) the provider of the requests package should ensure that urllib3
+#      contained in it is consistent/correct.
+#  b.) cloud-init does not specifically have a dependency on urllib3
+#
+# For future reference, see:
+#   https://github.com/kennethreitz/requests/pull/2375
+#   https://github.com/requests/requests/issues/4104
+# pylint: disable=E0401
+from requests.packages.urllib3.connection import HTTPConnection
+from requests.packages.urllib3.poolmanager import PoolManager
+
+from cloudinit import log as logging
+from cloudinit import sources
+from cloudinit import url_helper
+from cloudinit import util
+
+
+LOG = logging.getLogger(__name__)
+
+DS_BASE_URL = 'http://169.254.42.42'
+
+BUILTIN_DS_CONFIG = {
+    'metadata_url': DS_BASE_URL + '/conf?format=json',
+    'userdata_url': DS_BASE_URL + '/user_data/cloud-init',
+    'vendordata_url': DS_BASE_URL + '/vendor_data/cloud-init'
+}
+
+DEF_MD_RETRIES = 5
+DEF_MD_TIMEOUT = 10
+
+
+def on_scaleway():
+    """
+    There are three ways to detect if you are on Scaleway:
+
+    * check DMI data: not yet implemented by Scaleway, but the check is made to
+      be future-proof.
+    * the initrd created the file /var/run/scaleway.
+    * "scaleway" is in the kernel cmdline.
+    """
+    vendor_name = util.read_dmi_data('system-manufacturer')
+    if vendor_name == 'Scaleway':
+        return True
+
+    if os.path.exists('/var/run/scaleway'):
+        return True
+
+    cmdline = util.get_cmdline()
+    if 'scaleway' in cmdline:
+        return True
+
+    return False
+
+
+class SourceAddressAdapter(requests.adapters.HTTPAdapter):
+    """
+    Adapter for requests to choose the local address to bind to.
+    """
+    def __init__(self, source_address, **kwargs):
+        self.source_address = source_address
+        super(SourceAddressAdapter, self).__init__(**kwargs)
+
+    def init_poolmanager(self, connections, maxsize, block=False):
+        socket_options = HTTPConnection.default_socket_options + [
+            (socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
+        ]
+        self.poolmanager = PoolManager(num_pools=connections,
+                                       maxsize=maxsize,
+                                       block=block,
+                                       source_address=self.source_address,
+                                       socket_options=socket_options)
+
+
+def query_data_api_once(api_address, timeout, requests_session):
+    """
+    Retrieve user data or vendor data.
+
+    Scaleway user/vendor data API returns HTTP/404 if user/vendor data is not
+    set.
+
+    This function calls `url_helper.readurl` but instead of considering
+    HTTP/404 as an error that requires a retry, it considers it as empty
+    user/vendor data.
+
+    Also, be aware the user data/vendor API requires the source port to be
+    below 1024 to ensure the client is root (since non-root users can't bind
+    ports below 1024). If requests raises ConnectionError (EADDRINUSE), the
+    caller should retry to call this function on an other port.
+    """
+    try:
+        resp = url_helper.readurl(
+            api_address,
+            data=None,
+            timeout=timeout,
+            # It's the caller's responsability to recall this function in case
+            # of exception. Don't let url_helper.readurl() retry by itself.
+            retries=0,
+            session=requests_session,
+            # If the error is a HTTP/404 or a ConnectionError, go into raise
+            # block below.
+            exception_cb=lambda _, exc: exc.code == 404 or (
+                isinstance(exc.cause, requests.exceptions.ConnectionError)
+            )
+        )
+        return util.decode_binary(resp.contents)
+    except url_helper.UrlError as exc:
+        # Empty user data.
+        if exc.code == 404:
+            return None
+        raise
+
+
+def query_data_api(api_type, api_address, retries, timeout):
+    """Get user or vendor data.
+
+    Handle the retrying logic in case the source port is used.
+
+    Scaleway metadata service requires the source port of the client to
+    be a privileged port (<1024).  This is done to ensure that only a
+    privileged user on the system can access the metadata service.
+    """
+    # Query user/vendor data. Try to make a request on the first privileged
+    # port available.
+    for port in range(1, max(retries, 2)):
+        try:
+            LOG.debug(
+                'Trying to get %s data (bind on port %d)...',
+                api_type, port
+            )
+            requests_session = requests.Session()
+            requests_session.mount(
+                'http://',
+                SourceAddressAdapter(source_address=('0.0.0.0', port))
+            )
+            data = query_data_api_once(
+                api_address,
+                timeout=timeout,
+                requests_session=requests_session
+            )
+            LOG.debug('%s-data downloaded', api_type)
+            return data
+
+        except url_helper.UrlError as exc:
+            # Local port already in use or HTTP/429.
+            LOG.warning('Error while trying to get %s data: %s', api_type, exc)
+            time.sleep(5)
+            last_exc = exc
+            continue
+
+    # Max number of retries reached.
+    raise last_exc
+
+
+class DataSourceScaleway(sources.DataSource):
+
+    def __init__(self, sys_cfg, distro, paths):
+        super(DataSourceScaleway, self).__init__(sys_cfg, distro, paths)
+
+        self.ds_cfg = util.mergemanydict([
+            util.get_cfg_by_path(sys_cfg, ["datasource", "Scaleway"], {}),
+            BUILTIN_DS_CONFIG
+        ])
+
+        self.metadata_address = self.ds_cfg['metadata_url']
+        self.userdata_address = self.ds_cfg['userdata_url']
+        self.vendordata_address = self.ds_cfg['vendordata_url']
+
+        self.retries = int(self.ds_cfg.get('retries', DEF_MD_RETRIES))
+        self.timeout = int(self.ds_cfg.get('timeout', DEF_MD_TIMEOUT))
+
+    def get_data(self):
+        if not on_scaleway():
+            return False
+
+        resp = url_helper.readurl(self.metadata_address,
+                                  timeout=self.timeout,
+                                  retries=self.retries)
+        self.metadata = json.loads(util.decode_binary(resp.contents))
+
+        self.userdata_raw = query_data_api(
+            'user-data', self.userdata_address,
+            self.retries, self.timeout
+        )
+        self.vendordata_raw = query_data_api(
+            'vendor-data', self.vendordata_address,
+            self.retries, self.timeout
+        )
+        return True
+
+    @property
+    def launch_index(self):
+        return None
+
+    def get_instance_id(self):
+        return self.metadata['id']
+
+    def get_public_ssh_keys(self):
+        return [key['key'] for key in self.metadata['ssh_public_keys']]
+
+    def get_hostname(self, fqdn=False, resolve_ip=False):
+        return self.metadata['hostname']
+
+    @property
+    def availability_zone(self):
+        return None
+
+    @property
+    def region(self):
+        return None
+
+
+datasources = [
+    (DataSourceScaleway, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
+]
+
+
+def get_datasource_list(depends):
+    return sources.list_from_depends(depends, datasources)
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index c3ce36d6..952caf35 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -251,10 +251,23 @@ class DataSource(object):
     def first_instance_boot(self):
         return
 
+    def setup(self, is_new_instance):
+        """setup(is_new_instance)
+
+        This is called before user-data and vendor-data have been processed.
+
+        Unless the datasource has set mode to 'local', then networking
+        per 'fallback' or per 'network_config' will have been written and
+        brought up the OS at this point.
+        """
+        return
+
     def activate(self, cfg, is_new_instance):
         """activate(cfg, is_new_instance)
 
-        This is called before the init_modules will be called.
+        This is called before the init_modules will be called but after
+        the user-data and vendor-data have been fully processed.
+
         The cfg is fully up to date config, it contains a merged view of
            system config, datasource config, user config, vendor config.
         It should be used rather than the sys_cfg passed to __init__.