improve network configuration

This branch accomplishes several things:
 - centrally handle 'dsmode' to be 'local' or 'net.
   This allows local data sources to run before networking
   but still have user-data read by default when networking is available.

 - support networking information being read on dreamcompute
   dreamcompute's openstack declares networking via the
   /etc/network/interfaces style 'network_config' format.
 
 - support reading and applying networking information on SmartOS

 - improve reading networking from openstack network_data.json (LP: #1577982)
   add support for mtu and routes and many miscellaneous fixes.

 - support for renaming devices in a container (LP: #1579130).
   Also rename network devices as instructed by the host on
   every boot where cloud-init networking is enabled.  This is required
   because a.) containers do not get systemd.link files applied
   as they do not have udev. b.) if the initramfs is out of date
   then we need to apply them.

 - remove blocking of udev rules (LP: #1577844, LP: #1571761)

LP: #1577982, #1579130, #1577844, #1571761

2 files changed, 201 insertions, 173 deletions

diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py
index 195b8207..83d75f9c 100644
--- a/tests/unittests/test_datasource/test_configdrive.py
+++ b/tests/unittests/test_datasource/test_configdrive.py
@@ -15,6 +15,7 @@ except ImportError:
     from contextlib2 import ExitStack
 
 from cloudinit import helpers
+from cloudinit import net
 from cloudinit import settings
 from cloudinit.sources import DataSourceConfigDrive as ds
 from cloudinit.sources.helpers import openstack
@@ -73,7 +74,7 @@ NETWORK_DATA = {
          'type': 'ovs', 'mtu': None, 'id': 'tap2f88d109-5b'},
         {'vif_id': '1a5382f8-04c5-4d75-ab98-d666c1ef52cc',
          'ethernet_mac_address': 'fa:16:3e:05:30:fe',
-         'type': 'ovs', 'mtu': None, 'id': 'tap1a5382f8-04'}
+         'type': 'ovs', 'mtu': None, 'id': 'tap1a5382f8-04', 'name': 'nic0'}
     ],
     'networks': [
         {'link': 'tap2ecc7709-b3', 'type': 'ipv4_dhcp',
@@ -88,6 +89,34 @@ NETWORK_DATA = {
     ]
 }
 
+NETWORK_DATA_2 = {
+    "services": [
+        {"type": "dns", "address": "1.1.1.191"},
+        {"type": "dns", "address": "1.1.1.4"}],
+    "networks": [
+        {"network_id": "d94bbe94-7abc-48d4-9c82-4628ea26164a", "type": "ipv4",
+         "netmask": "255.255.255.248", "link": "eth0",
+         "routes": [{"netmask": "0.0.0.0", "network": "0.0.0.0",
+                     "gateway": "2.2.2.9"}],
+         "ip_address": "2.2.2.10", "id": "network0-ipv4"},
+        {"network_id": "ca447c83-6409-499b-aaef-6ad1ae995348", "type": "ipv4",
+         "netmask": "255.255.255.224", "link": "eth1",
+         "routes": [], "ip_address": "3.3.3.24", "id": "network1-ipv4"}],
+    "links": [
+        {"ethernet_mac_address": "fa:16:3e:dd:50:9a", "mtu": 1500,
+         "type": "vif", "id": "eth0", "vif_id": "vif-foo1"},
+        {"ethernet_mac_address": "fa:16:3e:a8:14:69", "mtu": 1500,
+         "type": "vif", "id": "eth1", "vif_id": "vif-foo2"}]
+}
+
+
+KNOWN_MACS = {
+    'fa:16:3e:69:b0:58': 'enp0s1',
+    'fa:16:3e:d4:57:ad': 'enp0s2',
+    'fa:16:3e:dd:50:9a': 'foo1',
+    'fa:16:3e:a8:14:69': 'foo2',
+}
+
 CFG_DRIVE_FILES_V2 = {
     'ec2/2009-04-04/meta-data.json': json.dumps(EC2_META),
     'ec2/2009-04-04/user-data': USER_DATA,
@@ -365,10 +394,54 @@ class TestConfigDriveDataSource(TestCase):
         """Verify that network_data is converted and present on ds object."""
         populate_dir(self.tmp, CFG_DRIVE_FILES_V2)
         myds = cfg_ds_from_dir(self.tmp)
-        network_config = ds.convert_network_data(NETWORK_DATA)
+        network_config = ds.convert_network_data(NETWORK_DATA,
+                                                 known_macs=KNOWN_MACS)
         self.assertEqual(myds.network_config, network_config)
 
 
+class TestConvertNetworkData(TestCase):
+    def _getnames_in_config(self, ncfg):
+        return set([n['name'] for n in ncfg['config']
+                    if n['type'] == 'physical'])
+
+    def test_conversion_fills_names(self):
+        ncfg = ds.convert_network_data(NETWORK_DATA, known_macs=KNOWN_MACS)
+        expected = set(['nic0', 'enp0s1', 'enp0s2'])
+        found = self._getnames_in_config(ncfg)
+        self.assertEqual(found, expected)
+
+    @mock.patch('cloudinit.net.get_interfaces_by_mac')
+    def test_convert_reads_system_prefers_name(self, get_interfaces_by_mac):
+        macs = KNOWN_MACS.copy()
+        macs.update({'fa:16:3e:05:30:fe': 'foonic1',
+                     'fa:16:3e:69:b0:58': 'ens1'})
+        get_interfaces_by_mac.return_value = macs
+
+        ncfg = ds.convert_network_data(NETWORK_DATA)
+        expected = set(['nic0', 'ens1', 'enp0s2'])
+        found = self._getnames_in_config(ncfg)
+        self.assertEqual(found, expected)
+
+    def test_convert_raises_value_error_on_missing_name(self):
+        macs = {'aa:aa:aa:aa:aa:00': 'ens1'}
+        self.assertRaises(ValueError, ds.convert_network_data,
+                          NETWORK_DATA, known_macs=macs)
+
+    def test_conversion_with_route(self):
+        ncfg = ds.convert_network_data(NETWORK_DATA_2, known_macs=KNOWN_MACS)
+        # not the best test, but see that we get a route in the
+        # network config and that it gets rendered to an ENI file
+        routes = []
+        for n in ncfg['config']:
+            for s in n.get('subnets', []):
+                routes.extend(s.get('routes', []))
+        self.assertIn(
+            {'network': '0.0.0.0', 'netmask': '0.0.0.0', 'gateway': '2.2.2.9'},
+            routes)
+        eni = net.render_interfaces(net.parse_net_config_data(ncfg))
+        self.assertIn("route add default gw 2.2.2.9", eni)
+
+
 def cfg_ds_from_dir(seed_d):
     found = ds.read_config_drive(seed_d)
     cfg_ds = ds.DataSourceConfigDrive(settings.CFG_BUILTIN, None,
@@ -387,7 +460,8 @@ def populate_ds_from_read_config(cfg_ds, source, results):
     cfg_ds.userdata_raw = results.get('userdata')
     cfg_ds.version = results.get('version')
     cfg_ds.network_json = results.get('networkdata')
-    cfg_ds._network_config = ds.convert_network_data(cfg_ds.network_json)
+    cfg_ds._network_config = ds.convert_network_data(
+        cfg_ds.network_json, known_macs=KNOWN_MACS)
 
 
 def populate_dir(seed_dir, files):
diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py
index cdd04a72..c809117b 100644
--- a/tests/unittests/test_datasource/test_smartos.py
+++ b/tests/unittests/test_datasource/test_smartos.py
@@ -25,6 +25,7 @@
 from __future__ import print_function
 
 from binascii import crc32
+import json
 import os
 import os.path
 import re
@@ -40,12 +41,49 @@ from cloudinit import helpers as c_helpers
 from cloudinit.sources import DataSourceSmartOS
 from cloudinit.util import b64e
 
-from .. import helpers
-
-try:
-    from unittest import mock
-except ImportError:
-    import mock
+from ..helpers import mock, FilesystemMockingTestCase, TestCase
+
+SDC_NICS = json.loads("""
+[
+    {
+        "nic_tag": "external",
+        "primary": true,
+        "mtu": 1500,
+        "model": "virtio",
+        "gateway": "8.12.42.1",
+        "netmask": "255.255.255.0",
+        "ip": "8.12.42.102",
+        "network_uuid": "992fc7ce-6aac-4b74-aed6-7b9d2c6c0bfe",
+        "gateways": [
+            "8.12.42.1"
+        ],
+        "vlan_id": 324,
+        "mac": "90:b8:d0:f5:e4:f5",
+        "interface": "net0",
+        "ips": [
+            "8.12.42.102/24"
+        ]
+    },
+    {
+        "nic_tag": "sdc_overlay/16187209",
+        "gateway": "192.168.128.1",
+        "model": "virtio",
+        "mac": "90:b8:d0:a5:ff:cd",
+        "netmask": "255.255.252.0",
+        "ip": "192.168.128.93",
+        "network_uuid": "4cad71da-09bc-452b-986d-03562a03a0a9",
+        "gateways": [
+            "192.168.128.1"
+        ],
+        "vlan_id": 2,
+        "mtu": 8500,
+        "interface": "net1",
+        "ips": [
+            "192.168.128.93/22"
+        ]
+    }
+]
+""")
 
 MOCK_RETURNS = {
     'hostname': 'test-host',
@@ -60,79 +98,66 @@ MOCK_RETURNS = {
     'sdc:vendor-data': '\n'.join(['VENDOR_DATA', '']),
     'user-data': '\n'.join(['something', '']),
     'user-script': '\n'.join(['/bin/true', '']),
+    'sdc:nics': json.dumps(SDC_NICS),
 }
 
 DMI_DATA_RETURN = 'smartdc'
 
 
-def get_mock_client(mockdata):
-    class MockMetadataClient(object):
+class PsuedoJoyentClient(object):
+    def __init__(self, data=None):
+        if data is None:
+            data = MOCK_RETURNS.copy()
+        self.data = data
+        return
+
+    def get(self, key, default=None, strip=False):
+        if key in self.data:
+            r = self.data[key]
+            if strip:
+                r = r.strip()
+        else:
+            r = default
+        return r
 
-        def __init__(self, serial):
-            pass
+    def get_json(self, key, default=None):
+        result = self.get(key, default=default)
+        if result is None:
+            return default
+        return json.loads(result)
 
-        def get_metadata(self, metadata_key):
-            return mockdata.get(metadata_key)
-    return MockMetadataClient
+    def exists(self):
+        return True
 
 
-class TestSmartOSDataSource(helpers.FilesystemMockingTestCase):
+class TestSmartOSDataSource(FilesystemMockingTestCase):
     def setUp(self):
         super(TestSmartOSDataSource, self).setUp()
 
+        dsmos = 'cloudinit.sources.DataSourceSmartOS'
+        patcher = mock.patch(dsmos + ".jmc_client_factory")
+        self.jmc_cfact = patcher.start()
+        self.addCleanup(patcher.stop)
+        patcher = mock.patch(dsmos + ".get_smartos_environ")
+        self.get_smartos_environ = patcher.start()
+        self.addCleanup(patcher.stop)
+
         self.tmp = tempfile.mkdtemp()
         self.addCleanup(shutil.rmtree, self.tmp)
-        self.legacy_user_d = tempfile.mkdtemp()
-        self.addCleanup(shutil.rmtree, self.legacy_user_d)
-
-        # If you should want to watch the logs...
-        self._log = None
-        self._log_file = None
-        self._log_handler = None
-
-        # patch cloud_dir, so our 'seed_dir' is guaranteed empty
         self.paths = c_helpers.Paths({'cloud_dir': self.tmp})
 
-        self.unapply = []
-        super(TestSmartOSDataSource, self).setUp()
+        self.legacy_user_d = tempfile.mkdtemp()
+        self.orig_lud = DataSourceSmartOS.LEGACY_USER_D
+        DataSourceSmartOS.LEGACY_USER_D = self.legacy_user_d
 
     def tearDown(self):
-        helpers.FilesystemMockingTestCase.tearDown(self)
-        if self._log_handler and self._log:
-            self._log.removeHandler(self._log_handler)
-        apply_patches([i for i in reversed(self.unapply)])
+        DataSourceSmartOS.LEGACY_USER_D = self.orig_lud
         super(TestSmartOSDataSource, self).tearDown()
 
-    def _patchIn(self, root):
-        self.restore()
-        self.patchOS(root)
-        self.patchUtils(root)
-
-    def apply_patches(self, patches):
-        ret = apply_patches(patches)
-        self.unapply += ret
-
-    def _get_ds(self, sys_cfg=None, ds_cfg=None, mockdata=None, dmi_data=None,
-                is_lxbrand=False):
-        mod = DataSourceSmartOS
-
-        if mockdata is None:
-            mockdata = MOCK_RETURNS
-
-        if dmi_data is None:
-            dmi_data = DMI_DATA_RETURN
-
-        def _dmi_data():
-            return dmi_data
-
-        def _os_uname():
-            if not is_lxbrand:
-                # LP: #1243287. tests assume this runs, but running test on
-                # arm would cause them all to fail.
-                return ('LINUX', 'NODENAME', 'RELEASE', 'VERSION', 'x86_64')
-            else:
-                return ('LINUX', 'NODENAME', 'RELEASE', 'BRANDZ VIRTUAL LINUX',
-                        'X86_64')
+    def _get_ds(self, mockdata=None, mode=DataSourceSmartOS.SMARTOS_ENV_KVM,
+                sys_cfg=None, ds_cfg=None):
+        self.jmc_cfact.return_value = PsuedoJoyentClient(mockdata)
+        self.get_smartos_environ.return_value = mode
 
         if sys_cfg is None:
             sys_cfg = {}
@@ -141,44 +166,8 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase):
             sys_cfg['datasource'] = sys_cfg.get('datasource', {})
             sys_cfg['datasource']['SmartOS'] = ds_cfg
 
-        self.apply_patches([(mod, 'LEGACY_USER_D', self.legacy_user_d)])
-        self.apply_patches([
-            (mod, 'JoyentMetadataClient', get_mock_client(mockdata))])
-        self.apply_patches([(mod, 'dmi_data', _dmi_data)])
-        self.apply_patches([(os, 'uname', _os_uname)])
-        self.apply_patches([(mod, 'device_exists', lambda d: True)])
-        dsrc = mod.DataSourceSmartOS(sys_cfg, distro=None,
-                                     paths=self.paths)
-        self.apply_patches([(dsrc, '_get_seed_file_object', mock.MagicMock())])
-        return dsrc
-
-    def test_seed(self):
-        # default seed should be /dev/ttyS1
-        dsrc = self._get_ds()
-        ret = dsrc.get_data()
-        self.assertTrue(ret)
-        self.assertEqual('kvm', dsrc.smartos_type)
-        self.assertEqual('/dev/ttyS1', dsrc.seed)
-
-    def test_seed_lxbrand(self):
-        # default seed should be /dev/ttyS1
-        dsrc = self._get_ds(is_lxbrand=True)
-        ret = dsrc.get_data()
-        self.assertTrue(ret)
-        self.assertEqual('lx-brand', dsrc.smartos_type)
-        self.assertEqual('/native/.zonecontrol/metadata.sock', dsrc.seed)
-
-    def test_issmartdc(self):
-        dsrc = self._get_ds()
-        ret = dsrc.get_data()
-        self.assertTrue(ret)
-        self.assertTrue(dsrc.is_smartdc)
-
-    def test_issmartdc_lxbrand(self):
-        dsrc = self._get_ds(is_lxbrand=True)
-        ret = dsrc.get_data()
-        self.assertTrue(ret)
-        self.assertTrue(dsrc.is_smartdc)
+        return DataSourceSmartOS.DataSourceSmartOS(
+            sys_cfg, distro=None, paths=self.paths)
 
     def test_no_base64(self):
         ds_cfg = {'no_base64_decode': ['test_var1'], 'all_base': True}
@@ -214,58 +203,6 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase):
         self.assertEqual(MOCK_RETURNS['hostname'],
                          dsrc.metadata['local-hostname'])
 
-    def test_base64_all(self):
-        # metadata provided base64_all of true
-        my_returns = MOCK_RETURNS.copy()
-        my_returns['base64_all'] = "true"
-        for k in ('hostname', 'cloud-init:user-data'):
-            my_returns[k] = b64e(my_returns[k])
-
-        dsrc = self._get_ds(mockdata=my_returns)
-        ret = dsrc.get_data()
-        self.assertTrue(ret)
-        self.assertEqual(MOCK_RETURNS['hostname'],
-                         dsrc.metadata['local-hostname'])
-        self.assertEqual(MOCK_RETURNS['cloud-init:user-data'],
-                         dsrc.userdata_raw)
-        self.assertEqual(MOCK_RETURNS['root_authorized_keys'],
-                         dsrc.metadata['public-keys'])
-        self.assertEqual(MOCK_RETURNS['disable_iptables_flag'],
-                         dsrc.metadata['iptables_disable'])
-        self.assertEqual(MOCK_RETURNS['enable_motd_sys_info'],
-                         dsrc.metadata['motd_sys_info'])
-
-    def test_b64_userdata(self):
-        my_returns = MOCK_RETURNS.copy()
-        my_returns['b64-cloud-init:user-data'] = "true"
-        my_returns['b64-hostname'] = "true"
-        for k in ('hostname', 'cloud-init:user-data'):
-            my_returns[k] = b64e(my_returns[k])
-
-        dsrc = self._get_ds(mockdata=my_returns)
-        ret = dsrc.get_data()
-        self.assertTrue(ret)
-        self.assertEqual(MOCK_RETURNS['hostname'],
-                         dsrc.metadata['local-hostname'])
-        self.assertEqual(MOCK_RETURNS['cloud-init:user-data'],
-                         dsrc.userdata_raw)
-        self.assertEqual(MOCK_RETURNS['root_authorized_keys'],
-                         dsrc.metadata['public-keys'])
-
-    def test_b64_keys(self):
-        my_returns = MOCK_RETURNS.copy()
-        my_returns['base64_keys'] = 'hostname,ignored'
-        for k in ('hostname',):
-            my_returns[k] = b64e(my_returns[k])
-
-        dsrc = self._get_ds(mockdata=my_returns)
-        ret = dsrc.get_data()
-        self.assertTrue(ret)
-        self.assertEqual(MOCK_RETURNS['hostname'],
-                         dsrc.metadata['local-hostname'])
-        self.assertEqual(MOCK_RETURNS['cloud-init:user-data'],
-                         dsrc.userdata_raw)
-
     def test_userdata(self):
         dsrc = self._get_ds(mockdata=MOCK_RETURNS)
         ret = dsrc.get_data()
@@ -275,6 +212,13 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase):
         self.assertEqual(MOCK_RETURNS['cloud-init:user-data'],
                          dsrc.userdata_raw)
 
+    def test_sdc_nics(self):
+        dsrc = self._get_ds(mockdata=MOCK_RETURNS)
+        ret = dsrc.get_data()
+        self.assertTrue(ret)
+        self.assertEqual(json.loads(MOCK_RETURNS['sdc:nics']),
+                         dsrc.metadata['network-data'])
+
     def test_sdc_scripts(self):
         dsrc = self._get_ds(mockdata=MOCK_RETURNS)
         ret = dsrc.get_data()
@@ -430,18 +374,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase):
                          mydscfg['disk_aliases']['FOO'])
 
 
-def apply_patches(patches):
-    ret = []
-    for (ref, name, replace) in patches:
-        if replace is None:
-            continue
-        orig = getattr(ref, name)
-        setattr(ref, name, replace)
-        ret.append((ref, name, orig))
-    return ret
-
-
-class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase):
+class TestJoyentMetadataClient(FilesystemMockingTestCase):
 
     def setUp(self):
         super(TestJoyentMetadataClient, self).setUp()
@@ -481,7 +414,8 @@ class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase):
                        mock.Mock(return_value=self.request_id)))
 
     def _get_client(self):
-        return DataSourceSmartOS.JoyentMetadataClient(self.serial)
+        return DataSourceSmartOS.JoyentMetadataClient(
+            fp=self.serial, smartos_type=DataSourceSmartOS.SMARTOS_ENV_KVM)
 
     def assertEndsWith(self, haystack, prefix):
         self.assertTrue(haystack.endswith(prefix),
@@ -495,7 +429,7 @@ class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase):
 
     def test_get_metadata_writes_a_single_line(self):
         client = self._get_client()
-        client.get_metadata('some_key')
+        client.get('some_key')
         self.assertEqual(1, self.serial.write.call_count)
         written_line = self.serial.write.call_args[0][0]
         print(type(written_line))
@@ -505,7 +439,7 @@ class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase):
 
     def _get_written_line(self, key='some_key'):
         client = self._get_client()
-        client.get_metadata(key)
+        client.get(key)
         return self.serial.write.call_args[0][0]
 
     def test_get_metadata_writes_bytes(self):
@@ -549,32 +483,32 @@ class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase):
 
     def test_get_metadata_reads_a_line(self):
         client = self._get_client()
-        client.get_metadata('some_key')
+        client.get('some_key')
         self.assertEqual(self.metasource_data_len, self.serial.read.call_count)
 
     def test_get_metadata_returns_valid_value(self):
         client = self._get_client()
-        value = client.get_metadata('some_key')
+        value = client.get('some_key')
         self.assertEqual(self.metadata_value, value)
 
     def test_get_metadata_throws_exception_for_incorrect_length(self):
         self.response_parts['length'] = 0
         client = self._get_client()
         self.assertRaises(DataSourceSmartOS.JoyentMetadataFetchException,
-                          client.get_metadata, 'some_key')
+                          client.get, 'some_key')
 
     def test_get_metadata_throws_exception_for_incorrect_crc(self):
         self.response_parts['crc'] = 'deadbeef'
         client = self._get_client()
         self.assertRaises(DataSourceSmartOS.JoyentMetadataFetchException,
-                          client.get_metadata, 'some_key')
+                          client.get, 'some_key')
 
     def test_get_metadata_throws_exception_for_request_id_mismatch(self):
         self.response_parts['request_id'] = 'deadbeef'
         client = self._get_client()
         client._checksum = lambda _: self.response_parts['crc']
         self.assertRaises(DataSourceSmartOS.JoyentMetadataFetchException,
-                          client.get_metadata, 'some_key')
+                          client.get, 'some_key')
 
     def test_get_metadata_returns_None_if_value_not_found(self):
         self.response_parts['payload'] = ''
@@ -582,4 +516,24 @@ class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase):
         self.response_parts['length'] = 17
         client = self._get_client()
         client._checksum = lambda _: self.response_parts['crc']
-        self.assertIsNone(client.get_metadata('some_key'))
+        self.assertIsNone(client.get('some_key'))
+
+
+class TestNetworkConversion(TestCase):
+
+    def test_convert_simple(self):
+        expected = {
+            'version': 1,
+            'config': [
+                {'name': 'net0', 'type': 'physical',
+                 'subnets': [{'type': 'static', 'gateway': '8.12.42.1',
+                              'netmask': '255.255.255.0',
+                              'address': '8.12.42.102/24'}],
+                 'mtu': 1500, 'mac_address': '90:b8:d0:f5:e4:f5'},
+                {'name': 'net1', 'type': 'physical',
+                 'subnets': [{'type': 'static', 'gateway': '192.168.128.1',
+                              'netmask': '255.255.252.0',
+                              'address': '192.168.128.93/22'}],
+                 'mtu': 8500, 'mac_address': '90:b8:d0:a5:ff:cd'}]}
+        found = DataSourceSmartOS.convert_smartos_network_data(SDC_NICS)
+        self.assertEqual(expected, found)