Scaleway: add datasource with user and vendor data for Scaleway.

Here we add and enable by default a datasource for Scaleway cloud.
The datasource quickly exits unless one of three things:
 a.) 'Scaleway' found as the system vendor
 b.) 'scaleway' found on the kernel command line.
 c.) the directory /var/run/scaleway exists (this is currently created
     by the scaleway initramfs module).

One interesting bit of this particular datasource is that it requires
the source port of the http request to be < 1024.

6 files changed, 524 insertions, 3 deletions

diff --git a/cloudinit/settings.py b/cloudinit/settings.py
index 0abd8a4a..c120498f 100644
--- a/cloudinit/settings.py
+++ b/cloudinit/settings.py
@@ -35,6 +35,7 @@ CFG_BUILTIN = {
         'CloudStack',
         'SmartOS',
         'Bigstep',
+        'Scaleway',
         # At the end to act as a 'catch' when none of the above work...
         'None',
     ],
diff --git a/cloudinit/sources/DataSourceScaleway.py b/cloudinit/sources/DataSourceScaleway.py
new file mode 100644
index 00000000..3a8a8e8f
--- /dev/null
+++ b/cloudinit/sources/DataSourceScaleway.py
@@ -0,0 +1,234 @@
+# Author: Julien Castets <castets.j@gmail.com>
+#
+# This file is part of cloud-init. See LICENSE file for license information.
+
+# Scaleway API:
+# https://developer.scaleway.com/#metadata
+
+import json
+import os
+import socket
+import time
+
+import requests
+
+# pylint fails to import the two modules below.
+# These are imported via requests.packages rather than urllib3 because:
+#  a.) the provider of the requests package should ensure that urllib3
+#      contained in it is consistent/correct.
+#  b.) cloud-init does not specifically have a dependency on urllib3
+#
+# For future reference, see:
+#   https://github.com/kennethreitz/requests/pull/2375
+#   https://github.com/requests/requests/issues/4104
+# pylint: disable=E0401
+from requests.packages.urllib3.connection import HTTPConnection
+from requests.packages.urllib3.poolmanager import PoolManager
+
+from cloudinit import log as logging
+from cloudinit import sources
+from cloudinit import url_helper
+from cloudinit import util
+
+
+LOG = logging.getLogger(__name__)
+
+DS_BASE_URL = 'http://169.254.42.42'
+
+BUILTIN_DS_CONFIG = {
+    'metadata_url': DS_BASE_URL + '/conf?format=json',
+    'userdata_url': DS_BASE_URL + '/user_data/cloud-init',
+    'vendordata_url': DS_BASE_URL + '/vendor_data/cloud-init'
+}
+
+DEF_MD_RETRIES = 5
+DEF_MD_TIMEOUT = 10
+
+
+def on_scaleway():
+    """
+    There are three ways to detect if you are on Scaleway:
+
+    * check DMI data: not yet implemented by Scaleway, but the check is made to
+      be future-proof.
+    * the initrd created the file /var/run/scaleway.
+    * "scaleway" is in the kernel cmdline.
+    """
+    vendor_name = util.read_dmi_data('system-manufacturer')
+    if vendor_name == 'Scaleway':
+        return True
+
+    if os.path.exists('/var/run/scaleway'):
+        return True
+
+    cmdline = util.get_cmdline()
+    if 'scaleway' in cmdline:
+        return True
+
+    return False
+
+
+class SourceAddressAdapter(requests.adapters.HTTPAdapter):
+    """
+    Adapter for requests to choose the local address to bind to.
+    """
+    def __init__(self, source_address, **kwargs):
+        self.source_address = source_address
+        super(SourceAddressAdapter, self).__init__(**kwargs)
+
+    def init_poolmanager(self, connections, maxsize, block=False):
+        socket_options = HTTPConnection.default_socket_options + [
+            (socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
+        ]
+        self.poolmanager = PoolManager(num_pools=connections,
+                                       maxsize=maxsize,
+                                       block=block,
+                                       source_address=self.source_address,
+                                       socket_options=socket_options)
+
+
+def query_data_api_once(api_address, timeout, requests_session):
+    """
+    Retrieve user data or vendor data.
+
+    Scaleway user/vendor data API returns HTTP/404 if user/vendor data is not
+    set.
+
+    This function calls `url_helper.readurl` but instead of considering
+    HTTP/404 as an error that requires a retry, it considers it as empty
+    user/vendor data.
+
+    Also, be aware the user data/vendor API requires the source port to be
+    below 1024 to ensure the client is root (since non-root users can't bind
+    ports below 1024). If requests raises ConnectionError (EADDRINUSE), the
+    caller should retry to call this function on an other port.
+    """
+    try:
+        resp = url_helper.readurl(
+            api_address,
+            data=None,
+            timeout=timeout,
+            # It's the caller's responsability to recall this function in case
+            # of exception. Don't let url_helper.readurl() retry by itself.
+            retries=0,
+            session=requests_session,
+            # If the error is a HTTP/404 or a ConnectionError, go into raise
+            # block below.
+            exception_cb=lambda _, exc: exc.code == 404 or (
+                isinstance(exc.cause, requests.exceptions.ConnectionError)
+            )
+        )
+        return util.decode_binary(resp.contents)
+    except url_helper.UrlError as exc:
+        # Empty user data.
+        if exc.code == 404:
+            return None
+        raise
+
+
+def query_data_api(api_type, api_address, retries, timeout):
+    """Get user or vendor data.
+
+    Handle the retrying logic in case the source port is used.
+
+    Scaleway metadata service requires the source port of the client to
+    be a privileged port (<1024).  This is done to ensure that only a
+    privileged user on the system can access the metadata service.
+    """
+    # Query user/vendor data. Try to make a request on the first privileged
+    # port available.
+    for port in range(1, max(retries, 2)):
+        try:
+            LOG.debug(
+                'Trying to get %s data (bind on port %d)...',
+                api_type, port
+            )
+            requests_session = requests.Session()
+            requests_session.mount(
+                'http://',
+                SourceAddressAdapter(source_address=('0.0.0.0', port))
+            )
+            data = query_data_api_once(
+                api_address,
+                timeout=timeout,
+                requests_session=requests_session
+            )
+            LOG.debug('%s-data downloaded', api_type)
+            return data
+
+        except url_helper.UrlError as exc:
+            # Local port already in use or HTTP/429.
+            LOG.warning('Error while trying to get %s data: %s', api_type, exc)
+            time.sleep(5)
+            last_exc = exc
+            continue
+
+    # Max number of retries reached.
+    raise last_exc
+
+
+class DataSourceScaleway(sources.DataSource):
+
+    def __init__(self, sys_cfg, distro, paths):
+        super(DataSourceScaleway, self).__init__(sys_cfg, distro, paths)
+
+        self.ds_cfg = util.mergemanydict([
+            util.get_cfg_by_path(sys_cfg, ["datasource", "Scaleway"], {}),
+            BUILTIN_DS_CONFIG
+        ])
+
+        self.metadata_address = self.ds_cfg['metadata_url']
+        self.userdata_address = self.ds_cfg['userdata_url']
+        self.vendordata_address = self.ds_cfg['vendordata_url']
+
+        self.retries = int(self.ds_cfg.get('retries', DEF_MD_RETRIES))
+        self.timeout = int(self.ds_cfg.get('timeout', DEF_MD_TIMEOUT))
+
+    def get_data(self):
+        if not on_scaleway():
+            return False
+
+        resp = url_helper.readurl(self.metadata_address,
+                                  timeout=self.timeout,
+                                  retries=self.retries)
+        self.metadata = json.loads(util.decode_binary(resp.contents))
+
+        self.userdata_raw = query_data_api(
+            'user-data', self.userdata_address,
+            self.retries, self.timeout
+        )
+        self.vendordata_raw = query_data_api(
+            'vendor-data', self.vendordata_address,
+            self.retries, self.timeout
+        )
+        return True
+
+    @property
+    def launch_index(self):
+        return None
+
+    def get_instance_id(self):
+        return self.metadata['id']
+
+    def get_public_ssh_keys(self):
+        return [key['key'] for key in self.metadata['ssh_public_keys']]
+
+    def get_hostname(self, fqdn=False, resolve_ip=False):
+        return self.metadata['hostname']
+
+    @property
+    def availability_zone(self):
+        return None
+
+    @property
+    def region(self):
+        return None
+
+
+datasources = [
+    (DataSourceScaleway, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
+]
+
+
+def get_datasource_list(depends):
+    return sources.list_from_depends(depends, datasources)
diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py
index d2b92e6a..7cf76aae 100644
--- a/cloudinit/url_helper.py
+++ b/cloudinit/url_helper.py
@@ -172,7 +172,8 @@ def _get_ssl_args(url, ssl_details):
 
 def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
             headers=None, headers_cb=None, ssl_details=None,
-            check_status=True, allow_redirects=True, exception_cb=None):
+            check_status=True, allow_redirects=True, exception_cb=None,
+            session=None):
     url = _cleanurl(url)
     req_args = {
         'url': url,
@@ -231,7 +232,12 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
             LOG.debug("[%s/%s] open '%s' with %s configuration", i,
                       manual_tries, url, filtered_req_args)
 
-            r = requests.request(**req_args)
+            if session is None:
+                session = requests.Session()
+
+            with session as sess:
+                r = sess.request(**req_args)
+
             if check_status:
                 r.raise_for_status()
             LOG.debug("Read from %s (%s, %sb) after %s attempts", url,
diff --git a/tests/unittests/test_datasource/test_common.py b/tests/unittests/test_datasource/test_common.py
index 2ff1d9df..413e87ac 100644
--- a/tests/unittests/test_datasource/test_common.py
+++ b/tests/unittests/test_datasource/test_common.py
@@ -19,6 +19,7 @@ from cloudinit.sources import (
     DataSourceOpenNebula as OpenNebula,
     DataSourceOpenStack as OpenStack,
     DataSourceOVF as OVF,
+    DataSourceScaleway as Scaleway,
     DataSourceSmartOS as SmartOS,
 )
 from cloudinit.sources import DataSourceNone as DSNone
@@ -48,6 +49,7 @@ DEFAULT_NETWORK = [
     NoCloud.DataSourceNoCloudNet,
     OpenStack.DataSourceOpenStack,
     OVF.DataSourceOVFNet,
+    Scaleway.DataSourceScaleway,
 ]
 
 
diff --git a/tests/unittests/test_datasource/test_scaleway.py b/tests/unittests/test_datasource/test_scaleway.py
new file mode 100644
index 00000000..65d83ad7
--- /dev/null
+++ b/tests/unittests/test_datasource/test_scaleway.py
@@ -0,0 +1,262 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+import json
+
+import httpretty
+import requests
+
+from cloudinit import helpers
+from cloudinit import settings
+from cloudinit.sources import DataSourceScaleway
+
+from ..helpers import mock, HttprettyTestCase, TestCase
+
+
+class DataResponses(object):
+    """
+    Possible responses of the API endpoint
+    169.254.42.42/user_data/cloud-init and
+    169.254.42.42/vendor_data/cloud-init.
+    """
+
+    FAKE_USER_DATA = '#!/bin/bash\necho "user-data"'
+
+    @staticmethod
+    def rate_limited(method, uri, headers):
+        return 429, headers, ''
+
+    @staticmethod
+    def api_error(method, uri, headers):
+        return 500, headers, ''
+
+    @classmethod
+    def get_ok(cls, method, uri, headers):
+        return 200, headers, cls.FAKE_USER_DATA
+
+    @staticmethod
+    def empty(method, uri, headers):
+        """
+        No user data for this server.
+        """
+        return 404, headers, ''
+
+
+class MetadataResponses(object):
+    """
+    Possible responses of the metadata API.
+    """
+
+    FAKE_METADATA = {
+        'id': '00000000-0000-0000-0000-000000000000',
+        'hostname': 'scaleway.host',
+        'ssh_public_keys': [{
+            'key': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABA',
+            'fingerprint': '2048 06:ae:...  login (RSA)'
+        }, {
+            'key': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABCCCCC',
+            'fingerprint': '2048 06:ff:...  login2 (RSA)'
+        }]
+    }
+
+    @classmethod
+    def get_ok(cls, method, uri, headers):
+        return 200, headers, json.dumps(cls.FAKE_METADATA)
+
+
+class TestOnScaleway(TestCase):
+
+    def install_mocks(self, fake_dmi, fake_file_exists, fake_cmdline):
+        mock, faked = fake_dmi
+        mock.return_value = 'Scaleway' if faked else 'Whatever'
+
+        mock, faked = fake_file_exists
+        mock.return_value = faked
+
+        mock, faked = fake_cmdline
+        mock.return_value = \
+            'initrd=initrd showopts scaleway nousb' if faked \
+            else 'BOOT_IMAGE=/vmlinuz-3.11.0-26-generic'
+
+    @mock.patch('cloudinit.util.get_cmdline')
+    @mock.patch('os.path.exists')
+    @mock.patch('cloudinit.util.read_dmi_data')
+    def test_not_on_scaleway(self, m_read_dmi_data, m_file_exists,
+                             m_get_cmdline):
+        self.install_mocks(
+            fake_dmi=(m_read_dmi_data, False),
+            fake_file_exists=(m_file_exists, False),
+            fake_cmdline=(m_get_cmdline, False)
+        )
+        self.assertFalse(DataSourceScaleway.on_scaleway())
+
+        # When not on Scaleway, get_data() returns False.
+        datasource = DataSourceScaleway.DataSourceScaleway(
+            settings.CFG_BUILTIN, None, helpers.Paths({})
+        )
+        self.assertFalse(datasource.get_data())
+
+    @mock.patch('cloudinit.util.get_cmdline')
+    @mock.patch('os.path.exists')
+    @mock.patch('cloudinit.util.read_dmi_data')
+    def test_on_scaleway_dmi(self, m_read_dmi_data, m_file_exists,
+                             m_get_cmdline):
+        """
+        dmidecode returns "Scaleway".
+        """
+        # dmidecode returns "Scaleway"
+        self.install_mocks(
+            fake_dmi=(m_read_dmi_data, True),
+            fake_file_exists=(m_file_exists, False),
+            fake_cmdline=(m_get_cmdline, False)
+        )
+        self.assertTrue(DataSourceScaleway.on_scaleway())
+
+    @mock.patch('cloudinit.util.get_cmdline')
+    @mock.patch('os.path.exists')
+    @mock.patch('cloudinit.util.read_dmi_data')
+    def test_on_scaleway_var_run_scaleway(self, m_read_dmi_data, m_file_exists,
+                                          m_get_cmdline):
+        """
+        /var/run/scaleway exists.
+        """
+        self.install_mocks(
+            fake_dmi=(m_read_dmi_data, False),
+            fake_file_exists=(m_file_exists, True),
+            fake_cmdline=(m_get_cmdline, False)
+        )
+        self.assertTrue(DataSourceScaleway.on_scaleway())
+
+    @mock.patch('cloudinit.util.get_cmdline')
+    @mock.patch('os.path.exists')
+    @mock.patch('cloudinit.util.read_dmi_data')
+    def test_on_scaleway_cmdline(self, m_read_dmi_data, m_file_exists,
+                                 m_get_cmdline):
+        """
+        "scaleway" in /proc/cmdline.
+        """
+        self.install_mocks(
+            fake_dmi=(m_read_dmi_data, False),
+            fake_file_exists=(m_file_exists, False),
+            fake_cmdline=(m_get_cmdline, True)
+        )
+        self.assertTrue(DataSourceScaleway.on_scaleway())
+
+
+def get_source_address_adapter(*args, **kwargs):
+    """
+    Scaleway user/vendor data API requires to be called with a privileged port.
+
+    If the unittests are run as non-root, the user doesn't have the permission
+    to bind on ports below 1024.
+
+    This function removes the bind on a privileged address, since anyway the
+    HTTP call is mocked by httpretty.
+    """
+    kwargs.pop('source_address')
+    return requests.adapters.HTTPAdapter(*args, **kwargs)
+
+
+class TestDataSourceScaleway(HttprettyTestCase):
+
+    def setUp(self):
+        self.datasource = DataSourceScaleway.DataSourceScaleway(
+            settings.CFG_BUILTIN, None, helpers.Paths({})
+        )
+        super(TestDataSourceScaleway, self).setUp()
+
+        self.metadata_url = \
+            DataSourceScaleway.BUILTIN_DS_CONFIG['metadata_url']
+        self.userdata_url = \
+            DataSourceScaleway.BUILTIN_DS_CONFIG['userdata_url']
+        self.vendordata_url = \
+            DataSourceScaleway.BUILTIN_DS_CONFIG['vendordata_url']
+
+    @httpretty.activate
+    @mock.patch('cloudinit.sources.DataSourceScaleway.SourceAddressAdapter',
+                get_source_address_adapter)
+    @mock.patch('cloudinit.util.get_cmdline')
+    @mock.patch('time.sleep', return_value=None)
+    def test_metadata_ok(self, sleep, m_get_cmdline):
+        """
+        get_data() returns metadata, user data and vendor data.
+        """
+        m_get_cmdline.return_value = 'scaleway'
+
+        # Make user data API return a valid response
+        httpretty.register_uri(httpretty.GET, self.metadata_url,
+                               body=MetadataResponses.get_ok)
+        httpretty.register_uri(httpretty.GET, self.userdata_url,
+                               body=DataResponses.get_ok)
+        httpretty.register_uri(httpretty.GET, self.vendordata_url,
+                               body=DataResponses.get_ok)
+        self.datasource.get_data()
+
+        self.assertEqual(self.datasource.get_instance_id(),
+                         MetadataResponses.FAKE_METADATA['id'])
+        self.assertEqual(self.datasource.get_public_ssh_keys(), [
+            elem['key'] for elem in
+            MetadataResponses.FAKE_METADATA['ssh_public_keys']
+        ])
+        self.assertEqual(self.datasource.get_hostname(),
+                         MetadataResponses.FAKE_METADATA['hostname'])
+        self.assertEqual(self.datasource.get_userdata_raw(),
+                         DataResponses.FAKE_USER_DATA)
+        self.assertEqual(self.datasource.get_vendordata_raw(),
+                         DataResponses.FAKE_USER_DATA)
+        self.assertIsNone(self.datasource.availability_zone)
+        self.assertIsNone(self.datasource.region)
+        self.assertEqual(sleep.call_count, 0)
+
+    @httpretty.activate
+    @mock.patch('cloudinit.sources.DataSourceScaleway.SourceAddressAdapter',
+                get_source_address_adapter)
+    @mock.patch('cloudinit.util.get_cmdline')
+    @mock.patch('time.sleep', return_value=None)
+    def test_metadata_404(self, sleep, m_get_cmdline):
+        """
+        get_data() returns metadata, but no user data nor vendor data.
+        """
+        m_get_cmdline.return_value = 'scaleway'
+
+        # Make user and vendor data APIs return HTTP/404, which means there is
+        # no user / vendor data for the server.
+        httpretty.register_uri(httpretty.GET, self.metadata_url,
+                               body=MetadataResponses.get_ok)
+        httpretty.register_uri(httpretty.GET, self.userdata_url,
+                               body=DataResponses.empty)
+        httpretty.register_uri(httpretty.GET, self.vendordata_url,
+                               body=DataResponses.empty)
+        self.datasource.get_data()
+        self.assertIsNone(self.datasource.get_userdata_raw())
+        self.assertIsNone(self.datasource.get_vendordata_raw())
+        self.assertEqual(sleep.call_count, 0)
+
+    @httpretty.activate
+    @mock.patch('cloudinit.sources.DataSourceScaleway.SourceAddressAdapter',
+                get_source_address_adapter)
+    @mock.patch('cloudinit.util.get_cmdline')
+    @mock.patch('time.sleep', return_value=None)
+    def test_metadata_rate_limit(self, sleep, m_get_cmdline):
+        """
+        get_data() is rate limited two times by the metadata API when fetching
+        user data.
+        """
+        m_get_cmdline.return_value = 'scaleway'
+
+        httpretty.register_uri(httpretty.GET, self.metadata_url,
+                               body=MetadataResponses.get_ok)
+        httpretty.register_uri(httpretty.GET, self.vendordata_url,
+                               body=DataResponses.empty)
+
+        httpretty.register_uri(
+            httpretty.GET, self.userdata_url,
+            responses=[
+                httpretty.Response(body=DataResponses.rate_limited),
+                httpretty.Response(body=DataResponses.rate_limited),
+                httpretty.Response(body=DataResponses.get_ok),
+            ]
+        )
+        self.datasource.get_data()
+        self.assertEqual(self.datasource.get_userdata_raw(),
+                         DataResponses.FAKE_USER_DATA)
+        self.assertEqual(sleep.call_count, 2)
diff --git a/tools/ds-identify b/tools/ds-identify
index 7c8b144b..33bd2991 100755
--- a/tools/ds-identify
+++ b/tools/ds-identify
@@ -112,7 +112,7 @@ DI_DSNAME=""
 # be searched if there is no setting found in config.
 DI_DSLIST_DEFAULT="MAAS ConfigDrive NoCloud AltCloud Azure Bigstep \
 CloudSigma CloudStack DigitalOcean AliYun Ec2 GCE OpenNebula OpenStack \
-OVF SmartOS"
+OVF SmartOS Scaleway"
 DI_DSLIST=""
 DI_MODE=""
 DI_ON_FOUND=""
@@ -896,6 +896,22 @@ dscheck_None() {
     return ${DS_NOT_FOUND}
 }
 
+dscheck_Scaleway() {
+    if [ "${DI_DMI_SYS_VENDOR}" = "Scaleway" ]; then
+        return $DS_FOUND
+    fi
+
+    case " ${DI_KERNEL_CMDLINE} " in
+        *\ scaleway\ *) return ${DS_FOUND};;
+    esac
+
+    if [ -f ${PATH_ROOT}/var/run/scaleway ]; then
+        return ${DS_FOUND}
+    fi
+
+    return ${DS_NOT_FOUND}
+}
+
 collect_info() {
     read_virt
     read_pid1_product_name