Scaleway: Add network configuration to the DataSource

DEP_NETWORK is removed since the network_config must
run at each boot. New EventType.BOOT event is used
for that.

Network is brought up early to fetch the metadata which
is required to configure the network (ipv4 and/or v6).

Adds unittests for the following and fixes test_common for
LOCAL and NETWORK sets.

3 files changed, 124 insertions, 11 deletions

diff --git a/cloudinit/sources/DataSourceScaleway.py b/cloudinit/sources/DataSourceScaleway.py
index e2502b02..9dc4ab23 100644
--- a/cloudinit/sources/DataSourceScaleway.py
+++ b/cloudinit/sources/DataSourceScaleway.py
@@ -29,7 +29,9 @@ from cloudinit import log as logging
 from cloudinit import sources
 from cloudinit import url_helper
 from cloudinit import util
-
+from cloudinit import net
+from cloudinit.net.dhcp import EphemeralDHCPv4, NoDHCPLeaseError
+from cloudinit.event import EventType
 
 LOG = logging.getLogger(__name__)
 
@@ -168,8 +170,8 @@ def query_data_api(api_type, api_address, retries, timeout):
 
 
 class DataSourceScaleway(sources.DataSource):
-
     dsname = "Scaleway"
+    update_events = {'network': [EventType.BOOT_NEW_INSTANCE, EventType.BOOT]}
 
     def __init__(self, sys_cfg, distro, paths):
         super(DataSourceScaleway, self).__init__(sys_cfg, distro, paths)
@@ -185,11 +187,10 @@ class DataSourceScaleway(sources.DataSource):
 
         self.retries = int(self.ds_cfg.get('retries', DEF_MD_RETRIES))
         self.timeout = int(self.ds_cfg.get('timeout', DEF_MD_TIMEOUT))
+        self._fallback_interface = None
+        self._network_config = None
 
-    def _get_data(self):
-        if not on_scaleway():
-            return False
-
+    def _crawl_metadata(self):
         resp = url_helper.readurl(self.metadata_address,
                                   timeout=self.timeout,
                                   retries=self.retries)
@@ -203,9 +204,48 @@ class DataSourceScaleway(sources.DataSource):
             'vendor-data', self.vendordata_address,
             self.retries, self.timeout
         )
+
+    def _get_data(self):
+        if not on_scaleway():
+            return False
+
+        if self._fallback_interface is None:
+            self._fallback_interface = net.find_fallback_nic()
+        try:
+            with EphemeralDHCPv4(self._fallback_interface):
+                util.log_time(
+                    logfunc=LOG.debug, msg='Crawl of metadata service',
+                    func=self._crawl_metadata)
+        except (NoDHCPLeaseError) as e:
+            util.logexc(LOG, str(e))
+            return False
         return True
 
     @property
+    def network_config(self):
+        """
+        Configure networking according to data received from the
+        metadata API.
+        """
+        if self._network_config:
+            return self._network_config
+
+        if self._fallback_interface is None:
+            self._fallback_interface = net.find_fallback_nic()
+
+        netcfg = {'type': 'physical', 'name': '%s' % self._fallback_interface}
+        subnets = [{'type': 'dhcp4'}]
+        if self.metadata['ipv6']:
+            subnets += [{'type': 'static',
+                         'address': '%s' % self.metadata['ipv6']['address'],
+                         'gateway': '%s' % self.metadata['ipv6']['gateway'],
+                         'netmask': '%s' % self.metadata['ipv6']['netmask'],
+                         }]
+        netcfg['subnets'] = subnets
+        self._network_config = {'version': 1, 'config': [netcfg]}
+        return self._network_config
+
+    @property
     def launch_index(self):
         return None
 
@@ -228,7 +268,7 @@ class DataSourceScaleway(sources.DataSource):
 
 
 datasources = [
-    (DataSourceScaleway, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
+    (DataSourceScaleway, (sources.DEP_FILESYSTEM,)),
 ]
 
 
diff --git a/tests/unittests/test_datasource/test_common.py b/tests/unittests/test_datasource/test_common.py
index 0d35dc29..1a5a3db2 100644
--- a/tests/unittests/test_datasource/test_common.py
+++ b/tests/unittests/test_datasource/test_common.py
@@ -41,6 +41,7 @@ DEFAULT_LOCAL = [
     SmartOS.DataSourceSmartOS,
     Ec2.DataSourceEc2Local,
     OpenStack.DataSourceOpenStackLocal,
+    Scaleway.DataSourceScaleway,
 ]
 
 DEFAULT_NETWORK = [
@@ -55,7 +56,6 @@ DEFAULT_NETWORK = [
     NoCloud.DataSourceNoCloudNet,
     OpenStack.DataSourceOpenStack,
     OVF.DataSourceOVFNet,
-    Scaleway.DataSourceScaleway,
 ]
 
 
diff --git a/tests/unittests/test_datasource/test_scaleway.py b/tests/unittests/test_datasource/test_scaleway.py
index e4e9bb20..c2bc7a00 100644
--- a/tests/unittests/test_datasource/test_scaleway.py
+++ b/tests/unittests/test_datasource/test_scaleway.py
@@ -176,11 +176,18 @@ class TestDataSourceScaleway(HttprettyTestCase):
         self.vendordata_url = \
             DataSourceScaleway.BUILTIN_DS_CONFIG['vendordata_url']
 
+        self.add_patch('cloudinit.sources.DataSourceScaleway.on_scaleway',
+                       '_m_on_scaleway', return_value=True)
+        self.add_patch(
+            'cloudinit.sources.DataSourceScaleway.net.find_fallback_nic',
+            '_m_find_fallback_nic', return_value='scalewaynic0')
+
+    @mock.patch('cloudinit.sources.DataSourceScaleway.EphemeralDHCPv4')
     @mock.patch('cloudinit.sources.DataSourceScaleway.SourceAddressAdapter',
                 get_source_address_adapter)
     @mock.patch('cloudinit.util.get_cmdline')
     @mock.patch('time.sleep', return_value=None)
-    def test_metadata_ok(self, sleep, m_get_cmdline):
+    def test_metadata_ok(self, sleep, m_get_cmdline, dhcpv4):
         """
         get_data() returns metadata, user data and vendor data.
         """
@@ -211,11 +218,12 @@ class TestDataSourceScaleway(HttprettyTestCase):
         self.assertIsNone(self.datasource.region)
         self.assertEqual(sleep.call_count, 0)
 
+    @mock.patch('cloudinit.sources.DataSourceScaleway.EphemeralDHCPv4')
     @mock.patch('cloudinit.sources.DataSourceScaleway.SourceAddressAdapter',
                 get_source_address_adapter)
     @mock.patch('cloudinit.util.get_cmdline')
     @mock.patch('time.sleep', return_value=None)
-    def test_metadata_404(self, sleep, m_get_cmdline):
+    def test_metadata_404(self, sleep, m_get_cmdline, dhcpv4):
         """
         get_data() returns metadata, but no user data nor vendor data.
         """
@@ -234,11 +242,12 @@ class TestDataSourceScaleway(HttprettyTestCase):
         self.assertIsNone(self.datasource.get_vendordata_raw())
         self.assertEqual(sleep.call_count, 0)
 
+    @mock.patch('cloudinit.sources.DataSourceScaleway.EphemeralDHCPv4')
     @mock.patch('cloudinit.sources.DataSourceScaleway.SourceAddressAdapter',
                 get_source_address_adapter)
     @mock.patch('cloudinit.util.get_cmdline')
     @mock.patch('time.sleep', return_value=None)
-    def test_metadata_rate_limit(self, sleep, m_get_cmdline):
+    def test_metadata_rate_limit(self, sleep, m_get_cmdline, dhcpv4):
         """
         get_data() is rate limited two times by the metadata API when fetching
         user data.
@@ -262,3 +271,67 @@ class TestDataSourceScaleway(HttprettyTestCase):
         self.assertEqual(self.datasource.get_userdata_raw(),
                          DataResponses.FAKE_USER_DATA)
         self.assertEqual(sleep.call_count, 2)
+
+    @mock.patch('cloudinit.sources.DataSourceScaleway.net.find_fallback_nic')
+    @mock.patch('cloudinit.util.get_cmdline')
+    def test_network_config_ok(self, m_get_cmdline, fallback_nic):
+        """
+        network_config will only generate IPv4 config if no ipv6 data is
+        available in the metadata
+        """
+        m_get_cmdline.return_value = 'scaleway'
+        fallback_nic.return_value = 'ens2'
+        self.datasource.metadata['ipv6'] = None
+
+        netcfg = self.datasource.network_config
+        resp = {'version': 1,
+                'config': [{
+                     'type': 'physical',
+                     'name': 'ens2',
+                     'subnets': [{'type': 'dhcp4'}]}]
+                }
+        self.assertEqual(netcfg, resp)
+
+    @mock.patch('cloudinit.sources.DataSourceScaleway.net.find_fallback_nic')
+    @mock.patch('cloudinit.util.get_cmdline')
+    def test_network_config_ipv6_ok(self, m_get_cmdline, fallback_nic):
+        """
+        network_config will only generate IPv4/v6 configs if ipv6 data is
+        available in the metadata
+        """
+        m_get_cmdline.return_value = 'scaleway'
+        fallback_nic.return_value = 'ens2'
+        self.datasource.metadata['ipv6'] = {
+                'address': '2000:abc:4444:9876::42:999',
+                'gateway': '2000:abc:4444:9876::42:000',
+                'netmask': '127',
+                }
+
+        netcfg = self.datasource.network_config
+        resp = {'version': 1,
+                'config': [{
+                     'type': 'physical',
+                     'name': 'ens2',
+                     'subnets': [{'type': 'dhcp4'},
+                                 {'type': 'static',
+                                  'address': '2000:abc:4444:9876::42:999',
+                                  'gateway': '2000:abc:4444:9876::42:000',
+                                  'netmask': '127', }
+                                 ]
+
+                     }]
+                }
+        self.assertEqual(netcfg, resp)
+
+    @mock.patch('cloudinit.sources.DataSourceScaleway.net.find_fallback_nic')
+    @mock.patch('cloudinit.util.get_cmdline')
+    def test_network_config_existing(self, m_get_cmdline, fallback_nic):
+        """
+        network_config() should return the same data if a network config
+        already exists
+        """
+        m_get_cmdline.return_value = 'scaleway'
+        self.datasource._network_config = '0xdeadbeef'
+
+        netcfg = self.datasource.network_config
+        self.assertEqual(netcfg, '0xdeadbeef')