support network configuration in cloud-init --local

this allows 'cloud-init --local' to fully run before networking comes up.
By doing so, we can now cleanly apply networking to the new system.

This adds support for reading ConfigDrive network configuration
and also from NoCloud.  The support is only present for ubuntu/debian
at the current time.  Other distros will follow.

Also ability to specify network configuration on kernel command line
via either ip= or network-config=<base64>.

3 files changed, 180 insertions, 3 deletions

diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py
index bfd787d1..89b15f54 100644
--- a/tests/unittests/test_datasource/test_configdrive.py
+++ b/tests/unittests/test_datasource/test_configdrive.py
@@ -59,6 +59,34 @@ OSTACK_META = {
 
 CONTENT_0 = b'This is contents of /etc/foo.cfg\n'
 CONTENT_1 = b'# this is /etc/bar/bar.cfg\n'
+NETWORK_DATA = {
+    'services': [
+        {'type': 'dns', 'address': '199.204.44.24'},
+        {'type': 'dns', 'address': '199.204.47.54'}
+    ],
+    'links': [
+        {'vif_id': '2ecc7709-b3f7-4448-9580-e1ec32d75bbd',
+         'ethernet_mac_address': 'fa:16:3e:69:b0:58',
+         'type': 'ovs', 'mtu': None, 'id': 'tap2ecc7709-b3'},
+        {'vif_id': '2f88d109-5b57-40e6-af32-2472df09dc33',
+         'ethernet_mac_address': 'fa:16:3e:d4:57:ad',
+         'type': 'ovs', 'mtu': None, 'id': 'tap2f88d109-5b'},
+        {'vif_id': '1a5382f8-04c5-4d75-ab98-d666c1ef52cc',
+         'ethernet_mac_address': 'fa:16:3e:05:30:fe',
+         'type': 'ovs', 'mtu': None, 'id': 'tap1a5382f8-04'}
+    ],
+    'networks': [
+        {'link': 'tap2ecc7709-b3', 'type': 'ipv4_dhcp',
+         'network_id': '6d6357ac-0f70-4afa-8bd7-c274cc4ea235',
+         'id': 'network0'},
+        {'link': 'tap2f88d109-5b', 'type': 'ipv4_dhcp',
+         'network_id': 'd227a9b3-6960-4d94-8976-ee5788b44f54',
+         'id': 'network1'},
+        {'link': 'tap1a5382f8-04', 'type': 'ipv4_dhcp',
+         'network_id': 'dab2ba57-cae2-4311-a5ed-010b263891f5',
+         'id': 'network2'}
+    ]
+}
 
 CFG_DRIVE_FILES_V2 = {
     'ec2/2009-04-04/meta-data.json': json.dumps(EC2_META),
@@ -70,7 +98,11 @@ CFG_DRIVE_FILES_V2 = {
     'openstack/content/0000': CONTENT_0,
     'openstack/content/0001': CONTENT_1,
     'openstack/latest/meta_data.json': json.dumps(OSTACK_META),
-    'openstack/latest/user_data': USER_DATA}
+    'openstack/latest/user_data': USER_DATA,
+    'openstack/latest/network_data.json': json.dumps(NETWORK_DATA),
+    'openstack/2015-10-15/meta_data.json': json.dumps(OSTACK_META),
+    'openstack/2015-10-15/user_data': USER_DATA,
+    'openstack/2015-10-15/network_data.json': json.dumps(NETWORK_DATA)}
 
 
 class TestConfigDriveDataSource(TestCase):
@@ -225,6 +257,7 @@ class TestConfigDriveDataSource(TestCase):
 
         self.assertEqual(USER_DATA, found['userdata'])
         self.assertEqual(expected_md, found['metadata'])
+        self.assertEqual(NETWORK_DATA, found['networkdata'])
         self.assertEqual(found['files']['/etc/foo.cfg'], CONTENT_0)
         self.assertEqual(found['files']['/etc/bar/bar.cfg'], CONTENT_1)
 
@@ -250,6 +283,7 @@ class TestConfigDriveDataSource(TestCase):
         data = copy(CFG_DRIVE_FILES_V2)
 
         data["openstack/2012-08-10/meta_data.json"] = "non-json garbage {}"
+        data["openstack/2015-10-15/meta_data.json"] = "non-json garbage {}"
         data["openstack/latest/meta_data.json"] = "non-json garbage {}"
 
         populate_dir(self.tmp, data)
@@ -321,6 +355,19 @@ class TestConfigDriveDataSource(TestCase):
         self.assertEqual(myds.get_public_ssh_keys(),
                          [OSTACK_META['public_keys']['mykey']])
 
+    def test_network_data_is_found(self):
+        """Verify that network_data is present in ds in config-drive-v2."""
+        populate_dir(self.tmp, CFG_DRIVE_FILES_V2)
+        myds = cfg_ds_from_dir(self.tmp)
+        self.assertEqual(myds.network_json, NETWORK_DATA)
+
+    def test_network_config_is_converted(self):
+        """Verify that network_data is converted and present on ds object."""
+        populate_dir(self.tmp, CFG_DRIVE_FILES_V2)
+        myds = cfg_ds_from_dir(self.tmp)
+        network_config = ds.convert_network_data(NETWORK_DATA)
+        self.assertEqual(myds.network_config, network_config)
+
 
 def cfg_ds_from_dir(seed_d):
     found = ds.read_config_drive(seed_d)
@@ -339,6 +386,8 @@ def populate_ds_from_read_config(cfg_ds, source, results):
     cfg_ds.ec2_metadata = results.get('ec2-metadata')
     cfg_ds.userdata_raw = results.get('userdata')
     cfg_ds.version = results.get('version')
+    cfg_ds.network_json = results.get('networkdata')
+    cfg_ds._network_config = ds.convert_network_data(cfg_ds.network_json)
 
 
 def populate_dir(seed_dir, files):
diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py
index 6d30c5b8..2c2a424d 100644
--- a/tests/unittests/test_distros/test_netconfig.py
+++ b/tests/unittests/test_distros/test_netconfig.py
@@ -109,8 +109,9 @@ class TestNetCfgDistro(TestCase):
             ub_distro.apply_network(BASE_NET_CFG, False)
 
             self.assertEquals(len(write_bufs), 1)
-            self.assertIn('/etc/network/interfaces', write_bufs)
-            write_buf = write_bufs['/etc/network/interfaces']
+            eni_name = '/etc/network/interfaces.d/50-cloud-init.cfg'
+            self.assertIn(eni_name, write_bufs)
+            write_buf = write_bufs[eni_name]
             self.assertEquals(str(write_buf).strip(), BASE_NET_CFG.strip())
             self.assertEquals(write_buf.mode, 0o644)
 
diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py
new file mode 100644
index 00000000..dfb31710
--- /dev/null
+++ b/tests/unittests/test_net.py
@@ -0,0 +1,127 @@
+from cloudinit import util
+from cloudinit import net
+from .helpers import TestCase
+
+import base64
+import copy
+import io
+import gzip
+import json
+import os
+
+DHCP_CONTENT_1 = """
+DEVICE='eth0'
+PROTO='dhcp'
+IPV4ADDR='192.168.122.89'
+IPV4BROADCAST='192.168.122.255'
+IPV4NETMASK='255.255.255.0'
+IPV4GATEWAY='192.168.122.1'
+IPV4DNS0='192.168.122.1'
+IPV4DNS1='0.0.0.0'
+HOSTNAME='foohost'
+DNSDOMAIN=''
+NISDOMAIN=''
+ROOTSERVER='192.168.122.1'
+ROOTPATH=''
+filename=''
+UPTIME='21'
+DHCPLEASETIME='3600'
+DOMAINSEARCH='foo.com'
+"""
+
+DHCP_EXPECTED_1 = {
+    'name': 'eth0',
+    'type': 'physical',
+    'subnets': [{'broadcast': '192.168.122.255',
+                 'gateway': '192.168.122.1',
+                 'dns_search': ['foo.com'],
+                 'type': 'dhcp',
+                 'netmask': '255.255.255.0',
+                 'dns_nameservers': ['192.168.122.1']}],
+}
+
+
+STATIC_CONTENT_1 = """
+DEVICE='eth1'
+PROTO='static'
+IPV4ADDR='10.0.0.2'
+IPV4BROADCAST='10.0.0.255'
+IPV4NETMASK='255.255.255.0'
+IPV4GATEWAY='10.0.0.1'
+IPV4DNS0='10.0.1.1'
+IPV4DNS1='0.0.0.0'
+HOSTNAME='foohost'
+UPTIME='21'
+DHCPLEASETIME='3600'
+DOMAINSEARCH='foo.com'
+"""
+
+STATIC_EXPECTED_1 = {
+    'name': 'eth1',
+    'type': 'physical',
+    'subnets': [{'broadcast': '10.0.0.255', 'gateway': '10.0.0.1',
+                 'dns_search': ['foo.com'], 'type': 'static',
+                 'netmask': '255.255.255.0',
+                 'dns_nameservers': ['10.0.1.1']}],
+}
+
+
+class TestNetConfigParsing(TestCase):
+    simple_cfg = {
+        'config': [{"type": "physical", "name": "eth0",
+                    "mac_address": "c0:d6:9f:2c:e8:80",
+                    "subnets": [{"type": "dhcp"}]}]}
+
+    def test_klibc_convert_dhcp(self):
+        found = net._klibc_to_config_entry(DHCP_CONTENT_1)
+        self.assertEqual(found, ('eth0', DHCP_EXPECTED_1))
+
+    def test_klibc_convert_static(self):
+        found = net._klibc_to_config_entry(STATIC_CONTENT_1)
+        self.assertEqual(found, ('eth1', STATIC_EXPECTED_1))
+
+    def test_config_from_klibc_net_cfg(self):
+        files = []
+        pairs = (('net-eth0.cfg', DHCP_CONTENT_1),
+                 ('net-eth1.cfg', STATIC_CONTENT_1))
+
+        macs = {'eth1': 'b8:ae:ed:75:ff:2b',
+                'eth0': 'b8:ae:ed:75:ff:2a'}
+
+        dhcp = copy.deepcopy(DHCP_EXPECTED_1)
+        dhcp['mac_address'] = macs['eth0']
+
+        static = copy.deepcopy(STATIC_EXPECTED_1)
+        static['mac_address'] = macs['eth1']
+
+        expected = {'version': 1, 'config': [dhcp, static]}
+        with util.tempdir() as tmpd:
+            for fname, content in pairs:
+                fp = os.path.join(tmpd, fname)
+                files.append(fp)
+                util.write_file(fp, content)
+
+            found = net.config_from_klibc_net_cfg(files=files, mac_addrs=macs)
+            self.assertEqual(found, expected)
+
+    def test_cmdline_with_b64(self):
+        data = base64.b64encode(json.dumps(self.simple_cfg).encode())
+        encoded_text = data.decode()
+        cmdline = 'ro network-config=' + encoded_text + ' root=foo'
+        found = net.read_kernel_cmdline_config(cmdline=cmdline)
+        self.assertEqual(found, self.simple_cfg)
+
+    def test_cmdline_with_b64_gz(self):
+        data = _gzip_data(json.dumps(self.simple_cfg).encode())
+        encoded_text = base64.b64encode(data).decode()
+        cmdline = 'ro network-config=' + encoded_text + ' root=foo'
+        found = net.read_kernel_cmdline_config(cmdline=cmdline)
+        self.assertEqual(found, self.simple_cfg)
+
+
+def _gzip_data(data):
+    with io.BytesIO() as iobuf:
+        gzfp = gzip.GzipFile(mode="wb", fileobj=iobuf)
+        gzfp.write(data)
+        gzfp.close()
+        return iobuf.getvalue()