From 657fd40f9ee692a817ec4614cd0d6cb0539ffabf Mon Sep 17 00:00:00 2001 From: Lars Kellogg-Stedman Date: Sat, 4 Mar 2017 17:07:16 -0500 Subject: net: do not raise exception for > 3 nameservers log a warning rather than raising ValueError if we see more than three nameserver addresses. LP: #1670052 --- tests/unittests/test_distros/test_resolv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tests') diff --git a/tests/unittests/test_distros/test_resolv.py b/tests/unittests/test_distros/test_resolv.py index 6b535a95..c9d03475 100644 --- a/tests/unittests/test_distros/test_resolv.py +++ b/tests/unittests/test_distros/test_resolv.py @@ -46,7 +46,7 @@ class TestResolvHelper(TestCase): self.assertNotIn('10.3', rp.nameservers) self.assertEqual(len(rp.nameservers), 3) rp.add_nameserver('10.2') - self.assertRaises(ValueError, rp.add_nameserver, '10.3') + rp.add_nameserver('10.3') self.assertNotIn('10.3', rp.nameservers) def test_search_domains(self): -- cgit v1.2.3 From 1d751a6f46f044e3c3827f3cef0e4a2e71d50fe7 Mon Sep 17 00:00:00 2001 From: Lars Kellogg-Stedman Date: Thu, 2 Mar 2017 11:08:26 -0500 Subject: net: support both ipv4 and ipv6 gateways in sysconfig. Previously, cloud-init would throw an exception if an interface had both ipv4 and ipv6 addresses and a default gateway for each address family. This change allows cloud-init to correctly configure interfaces in this situation. LP: #1669504 --- cloudinit/net/sysconfig.py | 33 +++++++++++----- tests/unittests/test_net.py | 94 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+), 9 deletions(-) (limited to 'tests') diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index 6e7739fb..06de660f 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -87,7 +87,8 @@ class Route(ConfigMap): def __init__(self, route_name, base_sysconf_dir): super(Route, self).__init__() self.last_idx = 1 - self.has_set_default = False + self.has_set_default_ipv4 = False + self.has_set_default_ipv6 = False self._route_name = route_name self._base_sysconf_dir = base_sysconf_dir @@ -95,7 +96,8 @@ class Route(ConfigMap): r = Route(self._route_name, self._base_sysconf_dir) r._conf = self._conf.copy() r.last_idx = self.last_idx - r.has_set_default = self.has_set_default + r.has_set_default_ipv4 = self.has_set_default_ipv4 + r.has_set_default_ipv6 = self.has_set_default_ipv6 return r @property @@ -119,10 +121,10 @@ class NetInterface(ConfigMap): super(NetInterface, self).__init__() self.children = [] self.routes = Route(iface_name, base_sysconf_dir) - self._kind = kind + self.kind = kind + self._iface_name = iface_name self._conf['DEVICE'] = iface_name - self._conf['TYPE'] = self.iface_types[kind] self._base_sysconf_dir = base_sysconf_dir @property @@ -140,6 +142,8 @@ class NetInterface(ConfigMap): @kind.setter def kind(self, kind): + if kind not in self.iface_types: + raise ValueError(kind) self._kind = kind self._conf['TYPE'] = self.iface_types[kind] @@ -173,7 +177,7 @@ class Renderer(renderer.Renderer): ('BOOTPROTO', 'none'), ]) - # If these keys exist, then there values will be used to form + # If these keys exist, then their values will be used to form # a BONDING_OPTS grouping; otherwise no grouping will be set. bond_tpl_opts = tuple([ ('bond_mode', "mode=%s"), @@ -199,6 +203,7 @@ class Renderer(renderer.Renderer): def _render_iface_shared(cls, iface, iface_cfg): for k, v in cls.iface_defaults: iface_cfg[k] = v + for (old_key, new_key) in [('mac_address', 'HWADDR'), ('mtu', 'MTU')]: old_value = iface.get(old_key) if old_value is not None: @@ -227,10 +232,20 @@ class Renderer(renderer.Renderer): if 'netmask' in subnet: iface_cfg['NETMASK'] = subnet['netmask'] for route in subnet.get('routes', []): + if subnet.get('ipv6'): + gw_cfg = 'IPV6_DEFAULTGW' + else: + gw_cfg = 'GATEWAY' + if _is_default_route(route): - if route_cfg.has_set_default: - raise ValueError("Duplicate declaration of default" - " route found for interface '%s'" + if ( + (subnet.get('ipv4') and + route_cfg.has_set_default_ipv4) or + (subnet.get('ipv6') and + route_cfg.has_set_default_ipv6) + ): + raise ValueError("Duplicate declaration of default " + "route found for interface '%s'" % (iface_cfg.name)) # NOTE(harlowja): ipv6 and ipv4 default gateways gw_key = 'GATEWAY0' @@ -242,7 +257,7 @@ class Renderer(renderer.Renderer): # also provided the default route? iface_cfg['DEFROUTE'] = True if 'gateway' in route: - iface_cfg['GATEWAY'] = route['gateway'] + iface_cfg[gw_cfg] = route['gateway'] route_cfg.has_set_default = True else: gw_key = 'GATEWAY%s' % route_cfg.last_idx diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 4b03ff72..8d253109 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -244,6 +244,100 @@ USERCTL=no ; Created by cloud-init on instance boot automatically, do not edit. ; nameserver 172.19.0.12 +""".lstrip()), + ('etc/udev/rules.d/70-persistent-net.rules', + "".join(['SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ', + 'ATTR{address}=="fa:16:3e:ed:9a:59", NAME="eth0"\n']))] + }, + { + 'in_data': { + "services": [{"type": "dns", "address": "172.19.0.12"}], + "networks": [{ + "network_id": "public-ipv4", + "type": "ipv4", "netmask": "255.255.252.0", + "link": "tap1a81968a-79", + "routes": [{ + "netmask": "0.0.0.0", + "network": "0.0.0.0", + "gateway": "172.19.3.254", + }], + "ip_address": "172.19.1.34", "id": "network0" + }, { + "network_id": "public-ipv6", + "type": "ipv6", "netmask": "", + "link": "tap1a81968a-79", + "routes": [ + { + "gateway": "2001:DB8::1", + "netmask": "::", + "network": "::" + } + ], + "ip_address": "2001:DB8::10", "id": "network1" + }], + "links": [ + { + "ethernet_mac_address": "fa:16:3e:ed:9a:59", + "mtu": None, "type": "bridge", "id": + "tap1a81968a-79", + "vif_id": "1a81968a-797a-400f-8a80-567f997eb93f" + }, + ], + }, + 'in_macs': { + 'fa:16:3e:ed:9a:59': 'eth0', + }, + 'out_sysconfig': [ + ('etc/sysconfig/network-scripts/ifcfg-eth0', + """ +# Created by cloud-init on instance boot automatically, do not edit. +# +BOOTPROTO=none +DEVICE=eth0 +HWADDR=fa:16:3e:ed:9a:59 +NM_CONTROLLED=no +ONBOOT=yes +TYPE=Ethernet +USERCTL=no +""".lstrip()), + ('etc/sysconfig/network-scripts/ifcfg-eth0:0', + """ +# Created by cloud-init on instance boot automatically, do not edit. +# +BOOTPROTO=static +DEFROUTE=yes +DEVICE=eth0:0 +GATEWAY=172.19.3.254 +HWADDR=fa:16:3e:ed:9a:59 +IPADDR=172.19.1.34 +NETMASK=255.255.252.0 +NM_CONTROLLED=no +ONBOOT=yes +TYPE=Ethernet +USERCTL=no +""".lstrip()), + ('etc/sysconfig/network-scripts/ifcfg-eth0:1', + """ +# Created by cloud-init on instance boot automatically, do not edit. +# +BOOTPROTO=static +DEFROUTE=yes +DEVICE=eth0:1 +HWADDR=fa:16:3e:ed:9a:59 +IPV6ADDR=2001:DB8::10 +IPV6INIT=yes +IPV6_DEFAULTGW=2001:DB8::1 +NETMASK= +NM_CONTROLLED=no +ONBOOT=yes +TYPE=Ethernet +USERCTL=no +""".lstrip()), + ('etc/resolv.conf', + """ +; Created by cloud-init on instance boot automatically, do not edit. +; +nameserver 172.19.0.12 """.lstrip()), ('etc/udev/rules.d/70-persistent-net.rules', "".join(['SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ', -- cgit v1.2.3 From 66dfb0915d4abf3f61948f281c4aff8394494ed3 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 13 Mar 2017 14:35:29 -0400 Subject: test: avoid differences in 'date' output due to daylight savings. When testing for timezone we were testing that 'date' output would contain 'HDT' for the current time. But after a 'spring forward', the current time started to have 'HST'. Instead of asking 'date' for the timezone that applies now, ask it for a static date. --- tests/cloud_tests/configs/modules/timezone.yaml | 4 +++- tests/cloud_tests/testcases/modules/timezone.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'tests') diff --git a/tests/cloud_tests/configs/modules/timezone.yaml b/tests/cloud_tests/configs/modules/timezone.yaml index 6a05aba1..8c96ed47 100644 --- a/tests/cloud_tests/configs/modules/timezone.yaml +++ b/tests/cloud_tests/configs/modules/timezone.yaml @@ -7,6 +7,8 @@ cloud_config: | collect_scripts: timezone: | #!/bin/bash - date +%Z + # date will convert this to system's configured time zone. + # use a static date to avoid dealing with daylight savings. + date "+%Z" --date="Thu, 03 Nov 2016 00:47:00 -0400" # vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/timezone.py b/tests/cloud_tests/testcases/modules/timezone.py index 272c266f..bf91d490 100644 --- a/tests/cloud_tests/testcases/modules/timezone.py +++ b/tests/cloud_tests/testcases/modules/timezone.py @@ -10,6 +10,6 @@ class TestTimezone(base.CloudTestCase): def test_timezone(self): """Test date prints correct timezone""" out = self.get_data_file('timezone') - self.assertIn('HST', out) + self.assertEqual('HDT', out.rstrip()) # vi: ts=4 expandtab -- cgit v1.2.3 From 3ec116a1bb3646c2ff932bed24b4bf087b51cb8c Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Fri, 17 Mar 2017 12:26:59 -0500 Subject: Integration Testing: improve testcase subclassing Use inspect.getmro(mod) rather than mod.__bases__ to test if a potential testcase class inherits from CloudTestCase. This allows testcases to be based on CloudTestCase indirectly, adding greater flexibility to the structure of test classes. --- tests/cloud_tests/testcases/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tests') diff --git a/tests/cloud_tests/testcases/__init__.py b/tests/cloud_tests/testcases/__init__.py index 182c090a..a1d86d45 100644 --- a/tests/cloud_tests/testcases/__init__.py +++ b/tests/cloud_tests/testcases/__init__.py @@ -21,7 +21,7 @@ def discover_tests(test_name): raise ValueError('no test verifier found at: {}'.format(testmod_name)) return [mod for name, mod in inspect.getmembers(testmod) - if inspect.isclass(mod) and base_test in mod.__bases__ and + if inspect.isclass(mod) and base_test in inspect.getmro(mod) and getattr(mod, '__test__', True)] -- cgit v1.2.3 From 1a2ca7530518d819cbab7287b12f942743427e38 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 15 Mar 2017 12:06:40 -0400 Subject: support 'loopback' as a device type. As reported in bug 1671927, sysconfig had an issue with rendering a loopback device. The problem was that some as yet unknown issue was causing the openstack config drive to parse the provided ENI file rather than reading the network_data.json. Parsing an ENI file would add a a 'lo' device of type 'physical', and sysconfig was failing to render that. The change here is: a.) add a 'loopback' type rather than 'physical' for network config. {'name': 'lo', 'type': 'loopback', 'subnets': ['type': 'loopback']} b.) support skipping that type in the eni and sysconfig renderers. c.) make network_state just piggy back on 'physical' renderer for loopback (this was what was happening before). Tests are added for eni and sysconfig renderer. --- cloudinit/net/eni.py | 16 +++++++++------ cloudinit/net/network_state.py | 4 ++++ cloudinit/net/sysconfig.py | 2 ++ tests/unittests/test_net.py | 44 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 6 deletions(-) (limited to 'tests') diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index 5b249f1f..69ecbb5d 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -273,8 +273,11 @@ def _ifaces_to_net_config_data(ifaces): # devname is 'eth0' for name='eth0:1' devname = name.partition(":")[0] if devname not in devs: - devs[devname] = {'type': 'physical', 'name': devname, - 'subnets': []} + if devname == "lo": + dtype = "loopback" + else: + dtype = "physical" + devs[devname] = {'type': dtype, 'name': devname, 'subnets': []} # this isnt strictly correct, but some might specify # hwaddress on a nic for matching / declaring name. if 'hwaddress' in data: @@ -423,10 +426,11 @@ class Renderer(renderer.Renderer): bonding ''' order = { - 'physical': 0, - 'bond': 1, - 'bridge': 2, - 'vlan': 3, + 'loopback': 0, + 'physical': 1, + 'bond': 2, + 'bridge': 3, + 'vlan': 4, } sections = [] diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 11ef585b..90b2835a 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -211,6 +211,10 @@ class NetworkStateInterpreter(object): exc_info=True) LOG.debug(self.dump_network_state()) + @ensure_command_keys(['name']) + def handle_loopback(self, command): + return self.handle_physical(command) + @ensure_command_keys(['name']) def handle_physical(self, command): ''' diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index 06de660f..7f52db4a 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -368,6 +368,8 @@ class Renderer(renderer.Renderer): '''Given state, return /etc/sysconfig files + contents''' iface_contents = {} for iface in network_state.iter_interfaces(): + if iface['type'] == "loopback": + continue iface_name = iface['name'] iface_cfg = NetInterface(iface_name, base_sysconf_dir) cls._render_iface_shared(iface, iface_cfg) diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 8d253109..c6535e1c 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -637,6 +637,14 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true } } +CONFIG_V1_EXPLICIT_LOOPBACK = { + 'version': 1, + 'config': [{'name': 'eth0', 'type': 'physical', + 'subnets': [{'control': 'auto', 'type': 'dhcp'}]}, + {'name': 'lo', 'type': 'loopback', + 'subnets': [{'control': 'auto', 'type': 'loopback'}]}, + ]} + def _setup_test(tmp_dir, mock_get_devicelist, mock_read_sys_net, mock_sys_dev_path): @@ -722,6 +730,27 @@ USERCTL=no with open(os.path.join(render_dir, fn)) as fh: self.assertEqual(expected_content, fh.read()) + def test_config_with_explicit_loopback(self): + ns = network_state.parse_net_config_data(CONFIG_V1_EXPLICIT_LOOPBACK) + render_dir = self.tmp_path("render") + os.makedirs(render_dir) + renderer = sysconfig.Renderer() + renderer.render_network_state(render_dir, ns) + found = dir2dict(render_dir) + nspath = '/etc/sysconfig/network-scripts/' + self.assertNotIn(nspath + 'ifcfg-lo', found.keys()) + expected = """\ +# Created by cloud-init on instance boot automatically, do not edit. +# +BOOTPROTO=dhcp +DEVICE=eth0 +NM_CONTROLLED=no +ONBOOT=yes +TYPE=Ethernet +USERCTL=no +""" + self.assertEqual(expected, found[nspath + 'ifcfg-eth0']) + class TestEniNetRendering(CiTestCase): @@ -762,6 +791,21 @@ iface eth1000 inet dhcp """ self.assertEqual(expected.lstrip(), contents.lstrip()) + def test_config_with_explicit_loopback(self): + tmp_dir = self.tmp_dir() + ns = network_state.parse_net_config_data(CONFIG_V1_EXPLICIT_LOOPBACK) + renderer = eni.Renderer() + renderer.render_network_state(tmp_dir, ns) + expected = """\ +auto lo +iface lo inet loopback + +auto eth0 +iface eth0 inet dhcp +""" + self.assertEqual( + expected, dir2dict(tmp_dir)['/etc/network/interfaces']) + class TestEniNetworkStateToEni(CiTestCase): mycfg = { -- cgit v1.2.3 From a33447344eed897010603b3e8ea1fd122052de76 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 16 Mar 2017 14:30:15 -0400 Subject: render_network_state: switch arguments around, do not require target render_network_state should default to rendering on /. The changes here just make it so render_network_state does not require a target, but defaults to None, and uses target_path to handle that. --- cloudinit/net/eni.py | 8 ++++---- cloudinit/net/sysconfig.py | 8 ++++---- tests/unittests/test_datasource/test_configdrive.py | 7 ++++--- tests/unittests/test_net.py | 12 ++++++------ 4 files changed, 18 insertions(+), 17 deletions(-) (limited to 'tests') diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index 69ecbb5d..efa034bf 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -448,14 +448,14 @@ class Renderer(renderer.Renderer): return '\n\n'.join(['\n'.join(s) for s in sections]) + "\n" - def render_network_state(self, target, network_state): - fpeni = os.path.join(target, self.eni_path) + def render_network_state(self, network_state, target=None): + fpeni = util.target_path(target, self.eni_path) util.ensure_dir(os.path.dirname(fpeni)) header = self.eni_header if self.eni_header else "" util.write_file(fpeni, header + self._render_interfaces(network_state)) if self.netrules_path: - netrules = os.path.join(target, self.netrules_path) + netrules = util.target_path(target, self.netrules_path) util.ensure_dir(os.path.dirname(netrules)) util.write_file(netrules, self._render_persistent_net(network_state)) @@ -465,7 +465,7 @@ class Renderer(renderer.Renderer): links_prefix=self.links_path_prefix) def _render_systemd_links(self, target, network_state, links_prefix): - fp_prefix = os.path.join(target, links_prefix) + fp_prefix = util.target_path(target, links_prefix) for f in glob.glob(fp_prefix + "*"): os.unlink(f) for iface in network_state.iter_interfaces(): diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index 7f52db4a..4eeaaa8a 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -389,19 +389,19 @@ class Renderer(renderer.Renderer): contents[iface_cfg.routes.path] = iface_cfg.routes.to_string() return contents - def render_network_state(self, target, network_state): - base_sysconf_dir = os.path.join(target, self.sysconf_dir) + def render_network_state(self, network_state, target=None): + base_sysconf_dir = util.target_path(target, self.sysconf_dir) for path, data in self._render_sysconfig(base_sysconf_dir, network_state).items(): util.write_file(path, data) if self.dns_path: - dns_path = os.path.join(target, self.dns_path) + dns_path = util.target_path(target, self.dns_path) resolv_content = self._render_dns(network_state, existing_dns_path=dns_path) util.write_file(dns_path, resolv_content) if self.netrules_path: netrules_content = self._render_persistent_net(network_state) - netrules_path = os.path.join(target, self.netrules_path) + netrules_path = util.target_path(target, self.netrules_path) util.write_file(netrules_path, netrules_content) # vi: ts=4 expandtab diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index 55153357..337be667 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -645,7 +645,7 @@ class TestConvertNetworkData(TestCase): routes) eni_renderer = eni.Renderer() eni_renderer.render_network_state( - self.tmp, network_state.parse_net_config_data(ncfg)) + network_state.parse_net_config_data(ncfg), self.tmp) with open(os.path.join(self.tmp, "etc", "network", "interfaces"), 'r') as f: eni_rendering = f.read() @@ -665,8 +665,9 @@ class TestConvertNetworkData(TestCase): ncfg = openstack.convert_net_json(NETWORK_DATA_BOND, known_macs=KNOWN_MACS) eni_renderer = eni.Renderer() + eni_renderer.render_network_state( - self.tmp, network_state.parse_net_config_data(ncfg)) + network_state.parse_net_config_data(ncfg), self.tmp) with open(os.path.join(self.tmp, "etc", "network", "interfaces"), 'r') as f: eni_rendering = f.read() @@ -697,7 +698,7 @@ class TestConvertNetworkData(TestCase): known_macs=KNOWN_MACS) eni_renderer = eni.Renderer() eni_renderer.render_network_state( - self.tmp, network_state.parse_net_config_data(ncfg)) + network_state.parse_net_config_data(ncfg), self.tmp) with open(os.path.join(self.tmp, "etc", "network", "interfaces"), 'r') as f: eni_rendering = f.read() diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index c6535e1c..dca44b37 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -697,7 +697,7 @@ class TestSysConfigRendering(CiTestCase): os.makedirs(render_dir) renderer = sysconfig.Renderer() - renderer.render_network_state(render_dir, ns) + renderer.render_network_state(ns, render_dir) render_file = 'etc/sysconfig/network-scripts/ifcfg-eth1000' with open(os.path.join(render_dir, render_file)) as fh: @@ -725,7 +725,7 @@ USERCTL=no ns = network_state.parse_net_config_data(network_cfg, skip_broken=False) renderer = sysconfig.Renderer() - renderer.render_network_state(render_dir, ns) + renderer.render_network_state(ns, render_dir) for fn, expected_content in os_sample.get('out_sysconfig', []): with open(os.path.join(render_dir, fn)) as fh: self.assertEqual(expected_content, fh.read()) @@ -735,7 +735,7 @@ USERCTL=no render_dir = self.tmp_path("render") os.makedirs(render_dir) renderer = sysconfig.Renderer() - renderer.render_network_state(render_dir, ns) + renderer.render_network_state(ns, render_dir) found = dir2dict(render_dir) nspath = '/etc/sysconfig/network-scripts/' self.assertNotIn(nspath + 'ifcfg-lo', found.keys()) @@ -775,7 +775,7 @@ class TestEniNetRendering(CiTestCase): {'links_path_prefix': None, 'eni_path': 'interfaces', 'netrules_path': None, }) - renderer.render_network_state(render_dir, ns) + renderer.render_network_state(ns, render_dir) self.assertTrue(os.path.exists(os.path.join(render_dir, 'interfaces'))) @@ -795,7 +795,7 @@ iface eth1000 inet dhcp tmp_dir = self.tmp_dir() ns = network_state.parse_net_config_data(CONFIG_V1_EXPLICIT_LOOPBACK) renderer = eni.Renderer() - renderer.render_network_state(tmp_dir, ns) + renderer.render_network_state(ns, tmp_dir) expected = """\ auto lo iface lo inet loopback @@ -972,7 +972,7 @@ class TestEniRoundTrip(CiTestCase): config={'eni_path': eni_path, 'links_path_prefix': links_prefix, 'netrules_path': netrules_path}) - renderer.render_network_state(dir, ns) + renderer.render_network_state(ns, dir) return dir2dict(dir) def testsimple_convert_and_render(self): -- cgit v1.2.3 From 41950e902f5dd6cb3118280d3d27409812702e41 Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Tue, 14 Mar 2017 15:28:08 -0600 Subject: test: Adding integratiron test for password as list This adds an integration test for setting passwords when given as a list rather than a string. This also updates the docs and tests so that Random is now RANDOM as is correct. --- cloudinit/config/cc_set_passwords.py | 4 +- .../configs/modules/set_password_list.yaml | 20 +++++---- .../configs/modules/set_password_list_string.yaml | 37 +++++++++++++++ tests/cloud_tests/testcases/base.py | 52 ++++++++++++++++++++++ .../testcases/modules/set_password_list.py | 20 ++------- .../testcases/modules/set_password_list_string.py | 11 +++++ 6 files changed, 117 insertions(+), 27 deletions(-) create mode 100644 tests/cloud_tests/configs/modules/set_password_list_string.yaml create mode 100644 tests/cloud_tests/testcases/modules/set_password_list_string.py (limited to 'tests') diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index 16117048..8440e593 100755 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -47,7 +47,7 @@ enabled, disabled, or left to system defaults using ``ssh_pwauth``. chpasswd: list: | user1:password1 - user2:Random + user2:RANDOM user3:password3 user4:R @@ -57,7 +57,7 @@ enabled, disabled, or left to system defaults using ``ssh_pwauth``. chpasswd: list: - user1:password1 - - user2:Random + - user2:RANDOM - user3:password3 - user4:R """ diff --git a/tests/cloud_tests/configs/modules/set_password_list.yaml b/tests/cloud_tests/configs/modules/set_password_list.yaml index 36129047..a1eadd75 100644 --- a/tests/cloud_tests/configs/modules/set_password_list.yaml +++ b/tests/cloud_tests/configs/modules/set_password_list.yaml @@ -6,22 +6,26 @@ cloud_config: | ssh_pwauth: yes users: - name: tom - password: $1$xyz$sPMsLNmf66Ohl.ol6JvzE. + # md5 gotomgo + passwd: "$1$S7$tT1BEDIYrczeryDQJfdPe0" lock_passwd: false - name: dick - password: $1$xyz$sPMsLNmf66Ohl.ol6JvzE. + # md5 gocubsgo + passwd: "$1$ssisyfpf$YqvuJLfrrW6Cg/l53Pi1n1" lock_passwd: false - name: harry - password: $1$xyz$sPMsLNmf66Ohl.ol6JvzE. + # sha512 goharrygo + passwd: "$6$LF$9Z2p6rWK6TNC1DC6393ec0As.18KRAvKDbfsGJEdWN3sRQRwpdfoh37EQ3yUh69tP4GSrGW5XKHxMLiKowJgm/" lock_passwd: false - name: jane - password: $1$xyz$sPMsLNmf66Ohl.ol6JvzE. + # sha256 gojanego + passwd: "$5$iW$XsxmWCdpwIW8Yhv.Jn/R3uk6A4UaicfW5Xp7C9p9pg." lock_passwd: false chpasswd: - list: | - tom:mypassword123! - dick:R - harry:Random + list: + - tom:mypassword123! + - dick:RANDOM + - harry:RANDOM collect_scripts: shadow: | #!/bin/bash diff --git a/tests/cloud_tests/configs/modules/set_password_list_string.yaml b/tests/cloud_tests/configs/modules/set_password_list_string.yaml new file mode 100644 index 00000000..cbb71bee --- /dev/null +++ b/tests/cloud_tests/configs/modules/set_password_list_string.yaml @@ -0,0 +1,37 @@ +# +# Set password of list of users as a string +# +cloud_config: | + #cloud-config + ssh_pwauth: yes + users: + - name: tom + # md5 gotomgo + passwd: "$1$S7$tT1BEDIYrczeryDQJfdPe0" + lock_passwd: false + - name: dick + # md5 gocubsgo + passwd: "$1$ssisyfpf$YqvuJLfrrW6Cg/l53Pi1n1" + lock_passwd: false + - name: harry + # sha512 goharrygo + passwd: "$6$LF$9Z2p6rWK6TNC1DC6393ec0As.18KRAvKDbfsGJEdWN3sRQRwpdfoh37EQ3yUh69tP4GSrGW5XKHxMLiKowJgm/" + lock_passwd: false + - name: jane + # sha256 gojanego + passwd: "$5$iW$XsxmWCdpwIW8Yhv.Jn/R3uk6A4UaicfW5Xp7C9p9pg." + lock_passwd: false + chpasswd: + list: | + tom:mypassword123! + dick:RANDOM + harry:RANDOM +collect_scripts: + shadow: | + #!/bin/bash + cat /etc/shadow + sshd_config: | + #!/bin/bash + grep '^PasswordAuth' /etc/ssh/sshd_config + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py index 5395b9a3..51ce2b41 100644 --- a/tests/cloud_tests/testcases/base.py +++ b/tests/cloud_tests/testcases/base.py @@ -2,6 +2,7 @@ from cloudinit import util as c_util +import crypt import json import unittest @@ -14,6 +15,9 @@ class CloudTestCase(unittest.TestCase): conf = None _cloud_config = None + def shortDescription(self): + return None + @property def cloud_config(self): """ @@ -78,4 +82,52 @@ class CloudTestCase(unittest.TestCase): result = self.get_status_data(self.get_data_file('result.json')) self.assertEqual(len(result['errors']), 0) + +class PasswordListTest(CloudTestCase): + def test_shadow_passwords(self): + shadow = self.get_data_file('shadow') + users = {} + dupes = [] + for line in shadow.splitlines(): + user, encpw = line.split(":")[0:2] + if user in users: + dupes.append(user) + users[user] = encpw + + jane_enc = "$5$iW$XsxmWCdpwIW8Yhv.Jn/R3uk6A4UaicfW5Xp7C9p9pg." + self.assertEqual([], dupes) + self.assertEqual(jane_enc, users['jane']) + + # shadow entry is $N$salt$, so we encrypt with the same format + # and salt and expect the result. + tom = "mypassword123!" + fmtsalt = users['tom'][0:users['tom'].rfind("$") + 1] + tom_enc = crypt.crypt(tom, fmtsalt) + self.assertEqual(tom_enc, users['tom']) + + harry_enc = ("$6$LF$9Z2p6rWK6TNC1DC6393ec0As.18KRAvKDbfsG" + "JEdWN3sRQRwpdfoh37EQ3yUh69tP4GSrGW5XKHxMLiKowJgm/") + dick_enc = "$1$ssisyfpf$YqvuJLfrrW6Cg/l53Pi1n1" + + # these should have been changed to random values. + self.assertNotEqual(harry_enc, users['harry']) + self.assertTrue(users['harry'].startswith("$")) + self.assertNotEqual(dick_enc, users['dick']) + self.assertTrue(users['dick'].startswith("$")) + + self.assertNotEqual(users['harry'], users['dick']) + + def test_shadow_expected_users(self): + """Test every tom, dick, and harry user in shadow""" + out = self.get_data_file('shadow') + self.assertIn('tom:', out) + self.assertIn('dick:', out) + self.assertIn('harry:', out) + self.assertIn('jane:', out) + + def test_sshd_config(self): + """Test sshd config allows passwords""" + out = self.get_data_file('sshd_config') + self.assertIn('PasswordAuthentication yes', out) + # vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/set_password_list.py b/tests/cloud_tests/testcases/modules/set_password_list.py index b764362f..6819d259 100644 --- a/tests/cloud_tests/testcases/modules/set_password_list.py +++ b/tests/cloud_tests/testcases/modules/set_password_list.py @@ -4,22 +4,8 @@ from tests.cloud_tests.testcases import base -class TestPasswordList(base.CloudTestCase): - """Test password module""" - - # TODO: Verify dick and harry passwords are random - # TODO: Verify tom's password was changed - - def test_shadow(self): - """Test every tom, dick, and harry user in shadow""" - out = self.get_data_file('shadow') - self.assertIn('tom:', out) - self.assertIn('dick:', out) - self.assertIn('harry:', out) - - def test_sshd_config(self): - """Test sshd config allows passwords""" - out = self.get_data_file('sshd_config') - self.assertIn('PasswordAuthentication yes', out) +class TestPasswordList(base.PasswordListTest, base.CloudTestCase): + """Test password setting via list in chpasswd/list""" + __test__ = True # vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/set_password_list_string.py b/tests/cloud_tests/testcases/modules/set_password_list_string.py new file mode 100644 index 00000000..2c34fada --- /dev/null +++ b/tests/cloud_tests/testcases/modules/set_password_list_string.py @@ -0,0 +1,11 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""cloud-init Integration Test Verify Script""" +from tests.cloud_tests.testcases import base + + +class TestPasswordListString(base.PasswordListTest, base.CloudTestCase): + """Test password setting via string in chpasswd/list""" + __test__ = True + +# vi: ts=4 expandtab -- cgit v1.2.3 From 5beecdf88b630a397b3722ddb299e9a37ff02737 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 15 Mar 2017 14:33:45 -0400 Subject: net: add renderers for automatically selecting the renderer. Previously, the distro had hard coded which network renderer it would use. This adds support for just picking the right renderer based on what is available. Now, that can be set via a priority in system_info, but should generally work. That config looks like: system_info: network: renderers: ["eni", "sysconfig"] When no renderers are found, a specific RendererNotFoundError is raised. stages.py is modified to catch that and log it at error level. This path should not really be exercised, but could occur if for example an Ubuntu system did not have ifupdown, or a rhel system did not have sysconfig. In such a system previously we would have quietly rendered ENI configuration but that would have been ignored. This is one step better in that we at least log the error. --- cloudinit/distros/__init__.py | 13 +++++++++++ cloudinit/distros/debian.py | 23 +++++++++---------- cloudinit/distros/rhel.py | 7 +----- cloudinit/net/__init__.py | 5 +++++ cloudinit/net/eni.py | 14 ++++++++++++ cloudinit/net/renderer.py | 5 +++++ cloudinit/net/renderers.py | 51 +++++++++++++++++++++++++++++++++++++++++++ cloudinit/net/sysconfig.py | 17 +++++++++++++++ cloudinit/settings.py | 1 + cloudinit/stages.py | 6 ++++- cloudinit/util.py | 43 ++++++++++++++++++++++++------------ tests/unittests/test_net.py | 45 ++++++++++++++++++++++++++++++++++++++ 12 files changed, 196 insertions(+), 34 deletions(-) create mode 100644 cloudinit/net/renderers.py (limited to 'tests') diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index f3d395b9..803ac74e 100755 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -22,6 +22,7 @@ from cloudinit import log as logging from cloudinit import net from cloudinit.net import eni from cloudinit.net import network_state +from cloudinit.net import renderers from cloudinit import ssh_util from cloudinit import type_utils from cloudinit import util @@ -50,6 +51,7 @@ class Distro(object): hostname_conf_fn = "/etc/hostname" tz_zone_dir = "/usr/share/zoneinfo" init_cmd = ['service'] # systemctl, service etc + renderer_configs = {} def __init__(self, name, cfg, paths): self._paths = paths @@ -69,6 +71,17 @@ class Distro(object): def _write_network_config(self, settings): raise NotImplementedError() + def _supported_write_network_config(self, network_config): + priority = util.get_cfg_by_path( + self._cfg, ('network', 'renderers'), None) + + name, render_cls = renderers.select(priority=priority) + LOG.debug("Selected renderer '%s' from priority list: %s", + name, priority) + renderer = render_cls(config=self.renderer_configs.get(name)) + renderer.render_network_config(network_config=network_config) + return [] + def _find_tz_file(self, tz): tz_file = os.path.join(self.tz_zone_dir, str(tz)) if not os.path.isfile(tz_file): diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 48ccec8c..1101f02d 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -13,8 +13,6 @@ import os from cloudinit import distros from cloudinit import helpers from cloudinit import log as logging -from cloudinit.net import eni -from cloudinit.net.network_state import parse_net_config_data from cloudinit import util from cloudinit.distros.parsers.hostname import HostnameConf @@ -38,11 +36,18 @@ ENI_HEADER = """# This file is generated from information provided by # network: {config: disabled} """ +NETWORK_CONF_FN = "/etc/network/interfaces.d/50-cloud-init.cfg" + class Distro(distros.Distro): hostname_conf_fn = "/etc/hostname" locale_conf_fn = "/etc/default/locale" - network_conf_fn = "/etc/network/interfaces.d/50-cloud-init.cfg" + renderer_configs = { + 'eni': { + 'eni_path': NETWORK_CONF_FN, + 'eni_header': ENI_HEADER, + } + } def __init__(self, name, cfg, paths): distros.Distro.__init__(self, name, cfg, paths) @@ -51,12 +56,6 @@ class Distro(distros.Distro): # should only happen say once per instance...) self._runner = helpers.Runners(paths) self.osfamily = 'debian' - self._net_renderer = eni.Renderer({ - 'eni_path': self.network_conf_fn, - 'eni_header': ENI_HEADER, - 'links_path_prefix': None, - 'netrules_path': None, - }) def apply_locale(self, locale, out_fn=None): if not out_fn: @@ -76,14 +75,12 @@ class Distro(distros.Distro): self.package_command('install', pkgs=pkglist) def _write_network(self, settings): - util.write_file(self.network_conf_fn, settings) + util.write_file(NETWORK_CONF_FN, settings) return ['all'] def _write_network_config(self, netconfig): - ns = parse_net_config_data(netconfig) - self._net_renderer.render_network_state("/", ns) _maybe_remove_legacy_eth0() - return [] + return self._supported_write_network_config(netconfig) def _bring_up_interfaces(self, device_names): use_all = False diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py index 7498c63a..372c7d0f 100644 --- a/cloudinit/distros/rhel.py +++ b/cloudinit/distros/rhel.py @@ -11,8 +11,6 @@ from cloudinit import distros from cloudinit import helpers from cloudinit import log as logging -from cloudinit.net.network_state import parse_net_config_data -from cloudinit.net import sysconfig from cloudinit import util from cloudinit.distros import net_util @@ -49,16 +47,13 @@ class Distro(distros.Distro): # should only happen say once per instance...) self._runner = helpers.Runners(paths) self.osfamily = 'redhat' - self._net_renderer = sysconfig.Renderer() cfg['ssh_svcname'] = 'sshd' def install_packages(self, pkglist): self.package_command('install', pkgs=pkglist) def _write_network_config(self, netconfig): - ns = parse_net_config_data(netconfig) - self._net_renderer.render_network_state("/", ns) - return [] + return self._supported_write_network_config(netconfig) def _write_network(self, settings): # TODO(harlowja) fix this... since this is the ubuntu format diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index ea649cc2..1cf98ef5 100755 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -364,4 +364,9 @@ def get_interfaces_by_mac(devs=None): ret[mac] = name return ret + +class RendererNotFoundError(RuntimeError): + pass + + # vi: ts=4 expandtab diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index efa034bf..9d39a2b0 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -500,4 +500,18 @@ def network_state_to_eni(network_state, header=None, render_hwaddress=False): network_state, render_hwaddress=render_hwaddress) return header + contents + +def available(target=None): + expected = ['ifquery', 'ifup', 'ifdown'] + search = ['/sbin', '/usr/sbin'] + for p in expected: + if not util.which(p, search=search, target=target): + return False + eni = util.target_path(target, 'etc/network/interfaces') + if not os.path.is_file(eni): + return False + + return True + + # vi: ts=4 expandtab diff --git a/cloudinit/net/renderer.py b/cloudinit/net/renderer.py index 3a192436..a5b2b573 100644 --- a/cloudinit/net/renderer.py +++ b/cloudinit/net/renderer.py @@ -7,6 +7,7 @@ import six +from .network_state import parse_net_config_data from .udev import generate_udev_rule @@ -36,4 +37,8 @@ class Renderer(object): iface['mac_address'])) return content.getvalue() + def render_network_config(self, network_config, target=None): + return self.render_network_state( + network_state=parse_net_config_data(network_config), target=target) + # vi: ts=4 expandtab diff --git a/cloudinit/net/renderers.py b/cloudinit/net/renderers.py new file mode 100644 index 00000000..5ad84553 --- /dev/null +++ b/cloudinit/net/renderers.py @@ -0,0 +1,51 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from . import eni +from . import RendererNotFoundError +from . import sysconfig + +NAME_TO_RENDERER = { + "eni": eni, + "sysconfig": sysconfig, +} + +DEFAULT_PRIORITY = ["eni", "sysconfig"] + + +def search(priority=None, target=None, first=False): + if priority is None: + priority = DEFAULT_PRIORITY + + available = NAME_TO_RENDERER + + unknown = [i for i in priority if i not in available] + if unknown: + raise ValueError( + "Unknown renderers provided in priority list: %s" % unknown) + + found = [] + for name in priority: + render_mod = available[name] + if render_mod.available(target): + cur = (name, render_mod.Renderer) + if first: + return cur + found.append(cur) + + return found + + +def select(priority=None, target=None): + found = search(priority, target=target, first=True) + if not found: + if priority is None: + priority = DEFAULT_PRIORITY + tmsg = "" + if target and target != "/": + tmsg = " in target=%s" % target + raise RendererNotFoundError( + "No available network renderers found%s. Searched " + "through list: %s" % (tmsg, priority)) + return found + +# vi: ts=4 expandtab diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index 4eeaaa8a..117b515c 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -404,4 +404,21 @@ class Renderer(renderer.Renderer): netrules_path = util.target_path(target, self.netrules_path) util.write_file(netrules_path, netrules_content) + +def available(target=None): + expected = ['ifup', 'ifdown'] + search = ['/sbin', '/usr/sbin'] + for p in expected: + if not util.which(p, search=search, target=target): + return False + + expected_paths = [ + 'etc/sysconfig/network-scripts/network-functions', + 'etc/sysconfig/network-scripts/ifdown-eth'] + for p in expected_paths: + if not os.path.isfile(util.target_path(target, p)): + return False + return True + + # vi: ts=4 expandtab diff --git a/cloudinit/settings.py b/cloudinit/settings.py index 692ff5e5..dbafead5 100644 --- a/cloudinit/settings.py +++ b/cloudinit/settings.py @@ -46,6 +46,7 @@ CFG_BUILTIN = { 'templates_dir': '/etc/cloud/templates/', }, 'distro': 'ubuntu', + 'network': {'renderers': None}, }, 'vendor_data': {'enabled': True, 'prefix': []}, } diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 5bed9032..12165433 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -646,9 +646,13 @@ class Init(object): src, bring_up, netcfg) try: return self.distro.apply_network_config(netcfg, bring_up=bring_up) + except net.RendererNotFoundError as e: + LOG.error("Unable to render networking. Network config is " + "likely broken: %s", e) + return except NotImplementedError: LOG.warn("distro '%s' does not implement apply_network_config. " - "networking may not be configured properly." % + "networking may not be configured properly.", self.distro) return diff --git a/cloudinit/util.py b/cloudinit/util.py index 7196a7ca..82f2f76b 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2099,21 +2099,36 @@ def get_mount_info(path, log=LOG): return parse_mount(path) -def which(program): - # Return path of program for execution if found in path - def is_exe(fpath): - return os.path.isfile(fpath) and os.access(fpath, os.X_OK) - - _fpath, _ = os.path.split(program) - if _fpath: - if is_exe(program): +def is_exe(fpath): + # return boolean indicating if fpath exists and is executable. + return os.path.isfile(fpath) and os.access(fpath, os.X_OK) + + +def which(program, search=None, target=None): + target = target_path(target) + + if os.path.sep in program: + # if program had a '/' in it, then do not search PATH + # 'which' does consider cwd here. (cd / && which bin/ls) = bin/ls + # so effectively we set cwd to / (or target) + if is_exe(target_path(target, program)): return program - else: - for path in os.environ.get("PATH", "").split(os.pathsep): - path = path.strip('"') - exe_file = os.path.join(path, program) - if is_exe(exe_file): - return exe_file + + if search is None: + paths = [p.strip('"') for p in + os.environ.get("PATH", "").split(os.pathsep)] + if target == "/": + search = paths + else: + search = [p for p in paths if p.startswith("/")] + + # normalize path input + search = [os.path.abspath(p) for p in search] + + for path in search: + ppath = os.path.sep.join((path, program)) + if is_exe(target_path(target, ppath)): + return ppath return None diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index dca44b37..902204a0 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -4,6 +4,7 @@ from cloudinit import net from cloudinit.net import cmdline from cloudinit.net import eni from cloudinit.net import network_state +from cloudinit.net import renderers from cloudinit.net import sysconfig from cloudinit.sources.helpers import openstack from cloudinit import util @@ -1050,6 +1051,50 @@ class TestEniRoundTrip(CiTestCase): expected, [line for line in found if line]) +class TestNetRenderers(CiTestCase): + @mock.patch("cloudinit.net.renderers.sysconfig.available") + @mock.patch("cloudinit.net.renderers.eni.available") + def test_eni_and_sysconfig_available(self, m_eni_avail, m_sysc_avail): + m_eni_avail.return_value = True + m_sysc_avail.return_value = True + found = renderers.search(priority=['sysconfig', 'eni'], first=False) + names = [f[0] for f in found] + self.assertEqual(['sysconfig', 'eni'], names) + + @mock.patch("cloudinit.net.renderers.eni.available") + def test_search_returns_empty_on_none(self, m_eni_avail): + m_eni_avail.return_value = False + found = renderers.search(priority=['eni'], first=False) + self.assertEqual([], found) + + @mock.patch("cloudinit.net.renderers.sysconfig.available") + @mock.patch("cloudinit.net.renderers.eni.available") + def test_first_in_priority(self, m_eni_avail, m_sysc_avail): + # available should only be called until one is found. + m_eni_avail.return_value = True + m_sysc_avail.side_effect = Exception("Should not call me") + found = renderers.search(priority=['eni', 'sysconfig'], first=True) + self.assertEqual(['eni'], [found[0]]) + + @mock.patch("cloudinit.net.renderers.sysconfig.available") + @mock.patch("cloudinit.net.renderers.eni.available") + def test_select_positive(self, m_eni_avail, m_sysc_avail): + m_eni_avail.return_value = True + m_sysc_avail.return_value = False + found = renderers.select(priority=['sysconfig', 'eni']) + self.assertEqual('eni', found[0]) + + @mock.patch("cloudinit.net.renderers.sysconfig.available") + @mock.patch("cloudinit.net.renderers.eni.available") + def test_select_none_found_raises(self, m_eni_avail, m_sysc_avail): + # if select finds nothing, should raise exception. + m_eni_avail.return_value = False + m_sysc_avail.return_value = False + + self.assertRaises(net.RendererNotFoundError, renderers.select, + priority=['sysconfig', 'eni']) + + def _gzip_data(data): with io.BytesIO() as iobuf: gzfp = gzip.GzipFile(mode="wb", fileobj=iobuf) -- cgit v1.2.3 From ef18b8ac4cf7e3dfd98830fbdb298380a192a0fc Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Sun, 19 Mar 2017 08:39:01 -0500 Subject: cloudinit.net: add network config v2 parsing and rendering Network configuration version 2 format is implemented in a package called netplan (nplan)[1] which allows consolidated network config for multiple network controllers. - Add a new netplan renderer - Update default policy, placing eni and sysconfig first This requires explicit policy to enable netplan over eni on systems which have both (Yakkety, Zesty, UC16) - Allow any network state (parsed from any format cloud-init supports) to render to v2 if system supports netplan. - Move eni's _subnet_is_ipv6 to common code for use by other renderers - Make sysconfig renderer always emit /etc/syconfig/network configuration - Update cloud-init.service systemd unit to also wait on systemd-networkd-wait-online.service 1. https://lists.ubuntu.com/archives/ubuntu-devel/2016-July/039464.html --- cloudinit/distros/debian.py | 16 +- cloudinit/net/eni.py | 15 +- cloudinit/net/netplan.py | 373 +++++++++++++++++++++++++ cloudinit/net/network_state.py | 312 ++++++++++++++++++++- cloudinit/net/renderers.py | 4 +- cloudinit/net/sysconfig.py | 20 +- cloudinit/util.py | 9 + systemd/cloud-init.service | 1 + tests/unittests/test_distros/test_netconfig.py | 351 ++++++++++++++++++++++- tests/unittests/test_net.py | 301 ++++++++++++++++++++ tools/net-convert.py | 84 ++++++ 11 files changed, 1450 insertions(+), 36 deletions(-) create mode 100644 cloudinit/net/netplan.py create mode 100755 tools/net-convert.py (limited to 'tests') diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 1101f02d..3f0f9d53 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -42,11 +42,16 @@ NETWORK_CONF_FN = "/etc/network/interfaces.d/50-cloud-init.cfg" class Distro(distros.Distro): hostname_conf_fn = "/etc/hostname" locale_conf_fn = "/etc/default/locale" + network_conf_fn = { + "eni": "/etc/network/interfaces.d/50-cloud-init.cfg", + "netplan": "/etc/netplan/50-cloud-init.yaml" + } renderer_configs = { - 'eni': { - 'eni_path': NETWORK_CONF_FN, - 'eni_header': ENI_HEADER, - } + "eni": {"eni_path": network_conf_fn["eni"], + "eni_header": ENI_HEADER}, + "netplan": {"netplan_path": network_conf_fn["netplan"], + "netplan_header": ENI_HEADER, + "postcmds": True} } def __init__(self, name, cfg, paths): @@ -75,7 +80,8 @@ class Distro(distros.Distro): self.package_command('install', pkgs=pkglist) def _write_network(self, settings): - util.write_file(NETWORK_CONF_FN, settings) + # this is a legacy method, it will always write eni + util.write_file(self.network_conf_fn["eni"], settings) return ['all'] def _write_network_config(self, netconfig): diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index f471e05f..9819d4f5 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -8,6 +8,7 @@ import re from . import ParserError from . import renderer +from .network_state import subnet_is_ipv6 from cloudinit import util @@ -111,16 +112,6 @@ def _iface_start_entry(iface, index, render_hwaddress=False): return lines -def _subnet_is_ipv6(subnet): - # 'static6' or 'dhcp6' - if subnet['type'].endswith('6'): - # This is a request for DHCPv6. - return True - elif subnet['type'] == 'static' and ":" in subnet['address']: - return True - return False - - def _parse_deb_config_data(ifaces, contents, src_dir, src_path): """Parses the file contents, placing result into ifaces. @@ -370,7 +361,7 @@ class Renderer(renderer.Renderer): iface['mode'] = subnet['type'] iface['control'] = subnet.get('control', 'auto') subnet_inet = 'inet' - if _subnet_is_ipv6(subnet): + if subnet_is_ipv6(subnet): subnet_inet += '6' iface['inet'] = subnet_inet if subnet['type'].startswith('dhcp'): @@ -486,7 +477,7 @@ class Renderer(renderer.Renderer): def network_state_to_eni(network_state, header=None, render_hwaddress=False): # render the provided network state, return a string of equivalent eni eni_path = 'etc/network/interfaces' - renderer = Renderer({ + renderer = Renderer(config={ 'eni_path': eni_path, 'eni_header': header, 'links_path_prefix': None, diff --git a/cloudinit/net/netplan.py b/cloudinit/net/netplan.py new file mode 100644 index 00000000..cd93b21c --- /dev/null +++ b/cloudinit/net/netplan.py @@ -0,0 +1,373 @@ +# This file is part of cloud-init. See LICENSE file ... + +import copy +import os + +from . import renderer +from .network_state import subnet_is_ipv6 + +from cloudinit import util +from cloudinit.net import SYS_CLASS_NET, get_devicelist + + +NET_CONFIG_TO_V2 = { + 'bond': {'bond-ad-select': 'ad-select', + 'bond-arp-interval': 'arp-interval', + 'bond-arp-ip-target': 'arp-ip-target', + 'bond-arp-validate': 'arp-validate', + 'bond-downdelay': 'down-delay', + 'bond-fail-over-mac': 'fail-over-mac-policy', + 'bond-lacp-rate': 'lacp-rate', + 'bond-miimon': 'mii-monitor-interval', + 'bond-min-links': 'min-links', + 'bond-mode': 'mode', + 'bond-num-grat-arp': 'gratuitious-arp', + 'bond-primary-reselect': 'primary-reselect-policy', + 'bond-updelay': 'up-delay', + 'bond-xmit_hash_policy': 'transmit_hash_policy'}, + 'bridge': {'bridge_ageing': 'ageing-time', + 'bridge_bridgeprio': 'priority', + 'bridge_fd': 'forward-delay', + 'bridge_gcint': None, + 'bridge_hello': 'hello-time', + 'bridge_maxage': 'max-age', + 'bridge_maxwait': None, + 'bridge_pathcost': 'path-cost', + 'bridge_portprio': None, + 'bridge_waitport': None}} + + +def _get_params_dict_by_match(config, match): + return dict((key, value) for (key, value) in config.items() + if key.startswith(match)) + + +def _extract_addresses(config, entry): + """This method parse a cloudinit.net.network_state dictionary (config) and + maps netstate keys/values into a dictionary (entry) to represent + netplan yaml. + + An example config dictionary might look like: + + {'mac_address': '52:54:00:12:34:00', + 'name': 'interface0', + 'subnets': [ + {'address': '192.168.1.2/24', + 'mtu': 1501, + 'type': 'static'}, + {'address': '2001:4800:78ff:1b:be76:4eff:fe06:1000", + 'mtu': 1480, + 'netmask': 64, + 'type': 'static'}], + 'type: physical' + } + + An entry dictionary looks like: + + {'set-name': 'interface0', + 'match': {'macaddress': '52:54:00:12:34:00'}, + 'mtu': 1501} + + After modification returns + + {'set-name': 'interface0', + 'match': {'macaddress': '52:54:00:12:34:00'}, + 'mtu': 1501, + 'address': ['192.168.1.2/24', '2001:4800:78ff:1b:be76:4eff:fe06:1000"], + 'mtu6': 1480} + + """ + + def _listify(obj, token=' '): + "Helper to convert strings to list of strings, handle single string" + if not obj or type(obj) not in [str]: + return obj + if token in obj: + return obj.split(token) + else: + return [obj, ] + + addresses = [] + routes = [] + nameservers = [] + searchdomains = [] + subnets = config.get('subnets', []) + if subnets is None: + subnets = [] + for subnet in subnets: + sn_type = subnet.get('type') + if sn_type.startswith('dhcp'): + if sn_type == 'dhcp': + sn_type += '4' + entry.update({sn_type: True}) + elif sn_type in ['static']: + addr = "%s" % subnet.get('address') + if 'netmask' in subnet: + addr += "/%s" % subnet.get('netmask') + if 'gateway' in subnet and subnet.get('gateway'): + gateway = subnet.get('gateway') + if ":" in gateway: + entry.update({'gateway6': gateway}) + else: + entry.update({'gateway4': gateway}) + if 'dns_nameservers' in subnet: + nameservers += _listify(subnet.get('dns_nameservers', [])) + if 'dns_search' in subnet: + searchdomains += _listify(subnet.get('dns_search', [])) + if 'mtu' in subnet: + mtukey = 'mtu' + if subnet_is_ipv6(subnet): + mtukey += '6' + entry.update({mtukey: subnet.get('mtu')}) + for route in subnet.get('routes', []): + to_net = "%s/%s" % (route.get('network'), + route.get('netmask')) + route = { + 'via': route.get('gateway'), + 'to': to_net, + } + if 'metric' in route: + route.update({'metric': route.get('metric', 100)}) + routes.append(route) + + addresses.append(addr) + + if len(addresses) > 0: + entry.update({'addresses': addresses}) + if len(routes) > 0: + entry.update({'routes': routes}) + if len(nameservers) > 0: + ns = {'addresses': nameservers} + entry.update({'nameservers': ns}) + if len(searchdomains) > 0: + ns = entry.get('nameservers', {}) + ns.update({'search': searchdomains}) + entry.update({'nameservers': ns}) + + +def _extract_bond_slaves_by_name(interfaces, entry, bond_master): + bond_slave_names = sorted([name for (name, cfg) in interfaces.items() + if cfg.get('bond-master', None) == bond_master]) + if len(bond_slave_names) > 0: + entry.update({'interfaces': bond_slave_names}) + + +class Renderer(renderer.Renderer): + """Renders network information in a /etc/netplan/network.yaml format.""" + + NETPLAN_GENERATE = ['netplan', 'generate'] + + def __init__(self, config=None): + if not config: + config = {} + self.netplan_path = config.get('netplan_path', + 'etc/netplan/50-cloud-init.yaml') + self.netplan_header = config.get('netplan_header', None) + self._postcmds = config.get('postcmds', False) + + def render_network_state(self, target, network_state): + # check network state for version + # if v2, then extract network_state.config + # else render_v2_from_state + fpnplan = os.path.join(target, self.netplan_path) + util.ensure_dir(os.path.dirname(fpnplan)) + header = self.netplan_header if self.netplan_header else "" + + # render from state + content = self._render_content(network_state) + + # ensure we poke udev to run net_setup_link + if not header.endswith("\n"): + header += "\n" + util.write_file(fpnplan, header + content) + + self._netplan_generate(run=self._postcmds) + self._net_setup_link(run=self._postcmds) + + def _netplan_generate(self, run=False): + if not run: + print("netplan postcmd disabled") + return + util.subp(self.NETPLAN_GENERATE, capture=True) + + def _net_setup_link(self, run=False): + """To ensure device link properties are applied, we poke + udev to re-evaluate networkd .link files and call + the setup_link udev builtin command + """ + if not run: + print("netsetup postcmd disabled") + return + setup_lnk = ['udevadm', 'test-builtin', 'net_setup_link'] + for cmd in [setup_lnk + [SYS_CLASS_NET + iface] + for iface in get_devicelist() if + os.path.islink(SYS_CLASS_NET + iface)]: + print(cmd) + util.subp(cmd, capture=True) + + def _render_content(self, network_state): + print('rendering v2 for victory!') + ethernets = {} + wifis = {} + bridges = {} + bonds = {} + vlans = {} + content = [] + + interfaces = network_state._network_state.get('interfaces', []) + + nameservers = network_state.dns_nameservers + searchdomains = network_state.dns_searchdomains + + for config in network_state.iter_interfaces(): + ifname = config.get('name') + # filter None entries up front so we can do simple if key in dict + ifcfg = dict((key, value) for (key, value) in config.items() + if value) + + if_type = ifcfg.get('type') + if if_type == 'physical': + # required_keys = ['name', 'mac_address'] + eth = { + 'set-name': ifname, + 'match': ifcfg.get('match', None), + } + if eth['match'] is None: + macaddr = ifcfg.get('mac_address', None) + if macaddr is not None: + eth['match'] = {'macaddress': macaddr.lower()} + else: + del eth['match'] + del eth['set-name'] + if 'mtu' in ifcfg: + eth['mtu'] = ifcfg.get('mtu') + + _extract_addresses(ifcfg, eth) + ethernets.update({ifname: eth}) + + elif if_type == 'bond': + # required_keys = ['name', 'bond_interfaces'] + bond = {} + bond_config = {} + # extract bond params and drop the bond_ prefix as it's + # redundent in v2 yaml format + v2_bond_map = NET_CONFIG_TO_V2.get('bond') + for match in ['bond_', 'bond-']: + bond_params = _get_params_dict_by_match(ifcfg, match) + for (param, value) in bond_params.items(): + newname = v2_bond_map.get(param) + if newname is None: + continue + bond_config.update({newname: value}) + + if len(bond_config) > 0: + bond.update({'parameters': bond_config}) + slave_interfaces = ifcfg.get('bond-slaves') + if slave_interfaces == 'none': + _extract_bond_slaves_by_name(interfaces, bond, ifname) + _extract_addresses(ifcfg, bond) + bonds.update({ifname: bond}) + + elif if_type == 'bridge': + # required_keys = ['name', 'bridge_ports'] + ports = sorted(copy.copy(ifcfg.get('bridge_ports'))) + bridge = { + 'interfaces': ports, + } + # extract bridge params and drop the bridge prefix as it's + # redundent in v2 yaml format + match_prefix = 'bridge_' + params = _get_params_dict_by_match(ifcfg, match_prefix) + br_config = {} + + # v2 yaml uses different names for the keys + # and at least one value format change + v2_bridge_map = NET_CONFIG_TO_V2.get('bridge') + for (param, value) in params.items(): + newname = v2_bridge_map.get(param) + if newname is None: + continue + br_config.update({newname: value}) + if newname == 'path-cost': + # -> : int() + newvalue = {} + for costval in value: + (port, cost) = costval.split() + newvalue[port] = int(cost) + br_config.update({newname: newvalue}) + if len(br_config) > 0: + bridge.update({'parameters': br_config}) + _extract_addresses(ifcfg, bridge) + bridges.update({ifname: bridge}) + + elif if_type == 'vlan': + # required_keys = ['name', 'vlan_id', 'vlan-raw-device'] + vlan = { + 'id': ifcfg.get('vlan_id'), + 'link': ifcfg.get('vlan-raw-device') + } + + _extract_addresses(ifcfg, vlan) + vlans.update({ifname: vlan}) + + # inject global nameserver values under each physical interface + if nameservers: + for _eth, cfg in ethernets.items(): + nscfg = cfg.get('nameservers', {}) + addresses = nscfg.get('addresses', []) + addresses += nameservers + nscfg.update({'addresses': addresses}) + cfg.update({'nameservers': nscfg}) + + if searchdomains: + for _eth, cfg in ethernets.items(): + nscfg = cfg.get('nameservers', {}) + search = nscfg.get('search', []) + search += searchdomains + nscfg.update({'search': search}) + cfg.update({'nameservers': nscfg}) + + # workaround yaml dictionary key sorting when dumping + def _render_section(name, section): + if section: + dump = util.yaml_dumps({name: section}, + explicit_start=False, + explicit_end=False) + txt = util.indent(dump, ' ' * 4) + return [txt] + return [] + + content.append("network:\n version: 2\n") + content += _render_section('ethernets', ethernets) + content += _render_section('wifis', wifis) + content += _render_section('bonds', bonds) + content += _render_section('bridges', bridges) + content += _render_section('vlans', vlans) + + return "".join(content) + + +def available(target=None): + expected = ['netplan'] + search = ['/usr/sbin', '/sbin'] + for p in expected: + if not util.which(p, search=search, target=target): + return False + return True + + +def network_state_to_netplan(network_state, header=None): + # render the provided network state, return a string of equivalent eni + netplan_path = 'etc/network/50-cloud-init.yaml' + renderer = Renderer({ + 'netplan_path': netplan_path, + 'netplan_header': header, + }) + if not header: + header = "" + if not header.endswith("\n"): + header += "\n" + contents = renderer._render_content(network_state) + return header + contents + +# vi: ts=4 expandtab diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 90b2835a..701aaa4e 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Canonical Ltd. +# Copyright (C) 2017 Canonical Ltd. # # Author: Ryan Harper # @@ -18,6 +18,10 @@ NETWORK_STATE_VERSION = 1 NETWORK_STATE_REQUIRED_KEYS = { 1: ['version', 'config', 'network_state'], } +NETWORK_V2_KEY_FILTER = [ + 'addresses', 'dhcp4', 'dhcp6', 'gateway4', 'gateway6', 'interfaces', + 'match', 'mtu', 'nameservers', 'renderer', 'set-name', 'wakeonlan' +] def parse_net_config_data(net_config, skip_broken=True): @@ -26,11 +30,18 @@ def parse_net_config_data(net_config, skip_broken=True): :param net_config: curtin network config dict """ state = None - if 'version' in net_config and 'config' in net_config: - nsi = NetworkStateInterpreter(version=net_config.get('version'), - config=net_config.get('config')) + version = net_config.get('version') + config = net_config.get('config') + if version == 2: + # v2 does not have explicit 'config' key so we + # pass the whole net-config as-is + config = net_config + + if version and config: + nsi = NetworkStateInterpreter(version=version, config=config) nsi.parse_config(skip_broken=skip_broken) - state = nsi.network_state + state = nsi.get_network_state() + return state @@ -106,6 +117,7 @@ class NetworkState(object): def __init__(self, network_state, version=NETWORK_STATE_VERSION): self._network_state = copy.deepcopy(network_state) self._version = version + self.use_ipv6 = network_state.get('use_ipv6', False) @property def version(self): @@ -152,7 +164,8 @@ class NetworkStateInterpreter(object): 'dns': { 'nameservers': [], 'search': [], - } + }, + 'use_ipv6': False, } def __init__(self, version=NETWORK_STATE_VERSION, config=None): @@ -165,6 +178,14 @@ class NetworkStateInterpreter(object): def network_state(self): return NetworkState(self._network_state, version=self._version) + @property + def use_ipv6(self): + return self._network_state.get('use_ipv6') + + @use_ipv6.setter + def use_ipv6(self, val): + self._network_state.update({'use_ipv6': val}) + def dump(self): state = { 'version': self._version, @@ -192,8 +213,22 @@ class NetworkStateInterpreter(object): def dump_network_state(self): return util.yaml_dumps(self._network_state) + def as_dict(self): + return {'version': self.version, 'config': self.config} + + def get_network_state(self): + ns = self.network_state + return ns + def parse_config(self, skip_broken=True): - # rebuild network state + if self._version == 1: + self.parse_config_v1(skip_broken=skip_broken) + self._parsed = True + elif self._version == 2: + self.parse_config_v2(skip_broken=skip_broken) + self._parsed = True + + def parse_config_v1(self, skip_broken=True): for command in self._config: command_type = command['type'] try: @@ -211,6 +246,26 @@ class NetworkStateInterpreter(object): exc_info=True) LOG.debug(self.dump_network_state()) + def parse_config_v2(self, skip_broken=True): + for command_type, command in self._config.items(): + if command_type == 'version': + continue + try: + handler = self.command_handlers[command_type] + except KeyError: + raise RuntimeError("No handler found for" + " command '%s'" % command_type) + try: + handler(self, command) + self._v2_common(command) + except InvalidCommand: + if not skip_broken: + raise + else: + LOG.warn("Skipping invalid command: %s", command, + exc_info=True) + LOG.debug(self.dump_network_state()) + @ensure_command_keys(['name']) def handle_loopback(self, command): return self.handle_physical(command) @@ -238,11 +293,16 @@ class NetworkStateInterpreter(object): if subnets: for subnet in subnets: if subnet['type'] == 'static': + if ':' in subnet['address']: + self.use_ipv6 = True if 'netmask' in subnet and ':' in subnet['address']: subnet['netmask'] = mask2cidr(subnet['netmask']) for route in subnet.get('routes', []): if 'netmask' in route: route['netmask'] = mask2cidr(route['netmask']) + elif subnet['type'].endswith('6'): + self.use_ipv6 = True + iface.update({ 'name': command.get('name'), 'type': command.get('type'), @@ -327,7 +387,7 @@ class NetworkStateInterpreter(object): bond_if.update({param: val}) self._network_state['interfaces'].update({ifname: bond_if}) - @ensure_command_keys(['name', 'bridge_interfaces', 'params']) + @ensure_command_keys(['name', 'bridge_interfaces']) def handle_bridge(self, command): ''' auto br0 @@ -373,7 +433,7 @@ class NetworkStateInterpreter(object): self.handle_physical(command) iface = interfaces.get(command.get('name'), {}) iface['bridge_ports'] = command['bridge_interfaces'] - for param, val in command.get('params').items(): + for param, val in command.get('params', {}).items(): iface.update({param: val}) interfaces.update({iface['name']: iface}) @@ -407,6 +467,240 @@ class NetworkStateInterpreter(object): } routes.append(route) + # V2 handlers + def handle_bonds(self, command): + ''' + v2_command = { + bond0: { + 'interfaces': ['interface0', 'interface1'], + 'miimon': 100, + 'mode': '802.3ad', + 'xmit_hash_policy': 'layer3+4'}, + bond1: { + 'bond-slaves': ['interface2', 'interface7'], + 'mode': 1 + } + } + + v1_command = { + 'type': 'bond' + 'name': 'bond0', + 'bond_interfaces': [interface0, interface1], + 'params': { + 'bond-mode': '802.3ad', + 'bond_miimon: 100, + 'bond_xmit_hash_policy': 'layer3+4', + } + } + + ''' + self._handle_bond_bridge(command, cmd_type='bond') + + def handle_bridges(self, command): + + ''' + v2_command = { + br0: { + 'interfaces': ['interface0', 'interface1'], + 'fd': 0, + 'stp': 'off', + 'maxwait': 0, + } + } + + v1_command = { + 'type': 'bridge' + 'name': 'br0', + 'bridge_interfaces': [interface0, interface1], + 'params': { + 'bridge_stp': 'off', + 'bridge_fd: 0, + 'bridge_maxwait': 0 + } + } + + ''' + self._handle_bond_bridge(command, cmd_type='bridge') + + def handle_ethernets(self, command): + ''' + ethernets: + eno1: + match: + macaddress: 00:11:22:33:44:55 + wakeonlan: true + dhcp4: true + dhcp6: false + addresses: + - 192.168.14.2/24 + - 2001:1::1/64 + gateway4: 192.168.14.1 + gateway6: 2001:1::2 + nameservers: + search: [foo.local, bar.local] + addresses: [8.8.8.8, 8.8.4.4] + lom: + match: + driver: ixgbe + set-name: lom1 + dhcp6: true + switchports: + match: + name: enp2* + mtu: 1280 + + command = { + 'type': 'physical', + 'mac_address': 'c0:d6:9f:2c:e8:80', + 'name': 'eth0', + 'subnets': [ + {'type': 'dhcp4'} + ] + } + ''' + for eth, cfg in command.items(): + phy_cmd = { + 'type': 'physical', + 'name': cfg.get('set-name', eth), + } + mac_address = cfg.get('match', {}).get('macaddress', None) + if not mac_address: + LOG.debug('NetworkState Version2: missing "macaddress" info ' + 'in config entry: %s: %s', eth, str(cfg)) + + for key in ['mtu', 'match', 'wakeonlan']: + if key in cfg: + phy_cmd.update({key: cfg.get(key)}) + + subnets = self._v2_to_v1_ipcfg(cfg) + if len(subnets) > 0: + phy_cmd.update({'subnets': subnets}) + + LOG.debug('v2(ethernets) -> v1(physical):\n%s', phy_cmd) + self.handle_physical(phy_cmd) + + def handle_vlans(self, command): + ''' + v2_vlans = { + 'eth0.123': { + 'id': 123, + 'link': 'eth0', + 'dhcp4': True, + } + } + + v1_command = { + 'type': 'vlan', + 'name': 'eth0.123', + 'vlan_link': 'eth0', + 'vlan_id': 123, + 'subnets': [{'type': 'dhcp4'}], + } + ''' + for vlan, cfg in command.items(): + vlan_cmd = { + 'type': 'vlan', + 'name': vlan, + 'vlan_id': cfg.get('id'), + 'vlan_link': cfg.get('link'), + } + subnets = self._v2_to_v1_ipcfg(cfg) + if len(subnets) > 0: + vlan_cmd.update({'subnets': subnets}) + LOG.debug('v2(vlans) -> v1(vlan):\n%s', vlan_cmd) + self.handle_vlan(vlan_cmd) + + def handle_wifis(self, command): + raise NotImplemented('NetworkState V2: Skipping wifi configuration') + + def _v2_common(self, cfg): + LOG.debug('v2_common: handling config:\n%s', cfg) + if 'nameservers' in cfg: + search = cfg.get('nameservers').get('search', []) + dns = cfg.get('nameservers').get('addresses', []) + name_cmd = {'type': 'nameserver'} + if len(search) > 0: + name_cmd.update({'search': search}) + if len(dns) > 0: + name_cmd.update({'addresses': dns}) + LOG.debug('v2(nameserver) -> v1(nameserver):\n%s', name_cmd) + self.handle_nameserver(name_cmd) + + def _handle_bond_bridge(self, command, cmd_type=None): + """Common handler for bond and bridge types""" + for item_name, item_cfg in command.items(): + item_params = dict((key, value) for (key, value) in + item_cfg.items() if key not in + NETWORK_V2_KEY_FILTER) + v1_cmd = { + 'type': cmd_type, + 'name': item_name, + cmd_type + '_interfaces': item_cfg.get('interfaces'), + 'params': item_params, + } + subnets = self._v2_to_v1_ipcfg(item_cfg) + if len(subnets) > 0: + v1_cmd.update({'subnets': subnets}) + + LOG.debug('v2(%ss) -> v1(%s):\n%s', cmd_type, cmd_type, v1_cmd) + self.handle_bridge(v1_cmd) + + def _v2_to_v1_ipcfg(self, cfg): + """Common ipconfig extraction from v2 to v1 subnets array.""" + + subnets = [] + if 'dhcp4' in cfg: + subnets.append({'type': 'dhcp4'}) + if 'dhcp6' in cfg: + self.use_ipv6 = True + subnets.append({'type': 'dhcp6'}) + + gateway4 = None + gateway6 = None + for address in cfg.get('addresses', []): + subnet = { + 'type': 'static', + 'address': address, + } + + routes = [] + for route in cfg.get('routes', []): + route_addr = route.get('to') + if "/" in route_addr: + route_addr, route_cidr = route_addr.split("/") + route_netmask = cidr2mask(route_cidr) + subnet_route = { + 'address': route_addr, + 'netmask': route_netmask, + 'gateway': route.get('via') + } + routes.append(subnet_route) + if len(routes) > 0: + subnet.update({'routes': routes}) + + if ":" in address: + if 'gateway6' in cfg and gateway6 is None: + gateway6 = cfg.get('gateway6') + subnet.update({'gateway': gateway6}) + else: + if 'gateway4' in cfg and gateway4 is None: + gateway4 = cfg.get('gateway4') + subnet.update({'gateway': gateway4}) + + subnets.append(subnet) + return subnets + + +def subnet_is_ipv6(subnet): + """Common helper for checking network_state subnets for ipv6.""" + # 'static6' or 'dhcp6' + if subnet['type'].endswith('6'): + # This is a request for DHCPv6. + return True + elif subnet['type'] == 'static' and ":" in subnet['address']: + return True + return False + def cidr2mask(cidr): mask = [0, 0, 0, 0] diff --git a/cloudinit/net/renderers.py b/cloudinit/net/renderers.py index 5ad84553..5117b4a5 100644 --- a/cloudinit/net/renderers.py +++ b/cloudinit/net/renderers.py @@ -1,15 +1,17 @@ # This file is part of cloud-init. See LICENSE file for license information. from . import eni +from . import netplan from . import RendererNotFoundError from . import sysconfig NAME_TO_RENDERER = { "eni": eni, + "netplan": netplan, "sysconfig": sysconfig, } -DEFAULT_PRIORITY = ["eni", "sysconfig"] +DEFAULT_PRIORITY = ["eni", "sysconfig", "netplan"] def search(priority=None, target=None, first=False): diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index 117b515c..504e4d02 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -9,6 +9,7 @@ from cloudinit.distros.parsers import resolv_conf from cloudinit import util from . import renderer +from .network_state import subnet_is_ipv6 def _make_header(sep='#'): @@ -194,7 +195,7 @@ class Renderer(renderer.Renderer): def __init__(self, config=None): if not config: config = {} - self.sysconf_dir = config.get('sysconf_dir', 'etc/sysconfig/') + self.sysconf_dir = config.get('sysconf_dir', 'etc/sysconfig') self.netrules_path = config.get( 'netrules_path', 'etc/udev/rules.d/70-persistent-net.rules') self.dns_path = config.get('dns_path', 'etc/resolv.conf') @@ -220,7 +221,7 @@ class Renderer(renderer.Renderer): iface_cfg['BOOTPROTO'] = 'dhcp' elif subnet_type == 'static': iface_cfg['BOOTPROTO'] = 'static' - if subnet.get('ipv6'): + if subnet_is_ipv6(subnet): iface_cfg['IPV6ADDR'] = subnet['address'] iface_cfg['IPV6INIT'] = True else: @@ -390,19 +391,28 @@ class Renderer(renderer.Renderer): return contents def render_network_state(self, network_state, target=None): + file_mode = 0o644 base_sysconf_dir = util.target_path(target, self.sysconf_dir) for path, data in self._render_sysconfig(base_sysconf_dir, network_state).items(): - util.write_file(path, data) + util.write_file(path, data, file_mode) if self.dns_path: dns_path = util.target_path(target, self.dns_path) resolv_content = self._render_dns(network_state, existing_dns_path=dns_path) - util.write_file(dns_path, resolv_content) + util.write_file(dns_path, resolv_content, file_mode) if self.netrules_path: netrules_content = self._render_persistent_net(network_state) netrules_path = util.target_path(target, self.netrules_path) - util.write_file(netrules_path, netrules_content) + util.write_file(netrules_path, netrules_content, file_mode) + + # always write /etc/sysconfig/network configuration + sysconfig_path = util.target_path(target, "etc/sysconfig/network") + netcfg = [_make_header(), 'NETWORKING=yes'] + if network_state.use_ipv6: + netcfg.append('NETWORKING_IPV6=yes') + netcfg.append('IPV6_AUTOCONF=no') + util.write_file(sysconfig_path, "\n".join(netcfg) + "\n", file_mode) def available(target=None): diff --git a/cloudinit/util.py b/cloudinit/util.py index 82f2f76b..33019579 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2373,4 +2373,13 @@ def system_is_snappy(): return True return False + +def indent(text, prefix): + """replacement for indent from textwrap that is not available in 2.7.""" + lines = [] + for line in text.splitlines(True): + lines.append(prefix + line) + return ''.join(lines) + + # vi: ts=4 expandtab diff --git a/systemd/cloud-init.service b/systemd/cloud-init.service index fb3b918c..39acc20a 100644 --- a/systemd/cloud-init.service +++ b/systemd/cloud-init.service @@ -5,6 +5,7 @@ Wants=cloud-init-local.service Wants=sshd-keygen.service Wants=sshd.service After=cloud-init-local.service +After=systemd-networkd-wait-online.service After=networking.service Before=network-online.target Before=sshd-keygen.service diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py index bde3bb50..b89b74ff 100644 --- a/tests/unittests/test_distros/test_netconfig.py +++ b/tests/unittests/test_distros/test_netconfig.py @@ -17,6 +17,7 @@ from ..helpers import TestCase from cloudinit import distros from cloudinit.distros.parsers.sys_conf import SysConf from cloudinit import helpers +from cloudinit.net import eni from cloudinit import settings from cloudinit import util @@ -28,10 +29,10 @@ iface lo inet loopback auto eth0 iface eth0 inet static address 192.168.1.5 - netmask 255.255.255.0 - network 192.168.0.0 broadcast 192.168.1.0 gateway 192.168.1.254 + netmask 255.255.255.0 + network 192.168.0.0 auto eth1 iface eth1 inet dhcp @@ -67,6 +68,100 @@ iface eth1 inet6 static gateway 2607:f0d0:1002:0011::1 ''' +V1_NET_CFG = {'config': [{'name': 'eth0', + + 'subnets': [{'address': '192.168.1.5', + 'broadcast': '192.168.1.0', + 'gateway': '192.168.1.254', + 'netmask': '255.255.255.0', + 'type': 'static'}], + 'type': 'physical'}, + {'name': 'eth1', + 'subnets': [{'control': 'auto', 'type': 'dhcp4'}], + 'type': 'physical'}], + 'version': 1} + +V1_NET_CFG_OUTPUT = """ +# This file is generated from information provided by +# the datasource. Changes to it will not persist across an instance. +# To disable cloud-init's network configuration capabilities, write a file +# /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg with the following: +# network: {config: disabled} +auto lo +iface lo inet loopback + +auto eth0 +iface eth0 inet static + address 192.168.1.5 + broadcast 192.168.1.0 + gateway 192.168.1.254 + netmask 255.255.255.0 + +auto eth1 +iface eth1 inet dhcp +""" + +V1_NET_CFG_IPV6 = {'config': [{'name': 'eth0', + 'subnets': [{'address': + '2607:f0d0:1002:0011::2', + 'gateway': + '2607:f0d0:1002:0011::1', + 'netmask': '64', + 'type': 'static'}], + 'type': 'physical'}, + {'name': 'eth1', + 'subnets': [{'control': 'auto', + 'type': 'dhcp4'}], + 'type': 'physical'}], + 'version': 1} + + +V1_TO_V2_NET_CFG_OUTPUT = """ +# This file is generated from information provided by +# the datasource. Changes to it will not persist across an instance. +# To disable cloud-init's network configuration capabilities, write a file +# /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg with the following: +# network: {config: disabled} +network: + version: 2 + ethernets: + eth0: + addresses: + - 192.168.1.5/255.255.255.0 + gateway4: 192.168.1.254 + eth1: + dhcp4: true +""" + +V2_NET_CFG = { + 'ethernets': { + 'eth7': { + 'addresses': ['192.168.1.5/255.255.255.0'], + 'gateway4': '192.168.1.254'}, + 'eth9': { + 'dhcp4': True} + }, + 'version': 2 +} + + +V2_TO_V2_NET_CFG_OUTPUT = """ +# This file is generated from information provided by +# the datasource. Changes to it will not persist across an instance. +# To disable cloud-init's network configuration capabilities, write a file +# /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg with the following: +# network: {config: disabled} +network: + version: 2 + ethernets: + eth7: + addresses: + - 192.168.1.5/255.255.255.0 + gateway4: 192.168.1.254 + eth9: + dhcp4: true +""" + class WriteBuffer(object): def __init__(self): @@ -83,12 +178,14 @@ class WriteBuffer(object): class TestNetCfgDistro(TestCase): - def _get_distro(self, dname): + def _get_distro(self, dname, renderers=None): cls = distros.fetch(dname) cfg = settings.CFG_BUILTIN cfg['system_info']['distro'] = dname + if renderers: + cfg['system_info']['network'] = {'renderers': renderers} paths = helpers.Paths({}) - return cls(dname, cfg, paths) + return cls(dname, cfg.get('system_info'), paths) def test_simple_write_ub(self): ub_distro = self._get_distro('ubuntu') @@ -116,6 +213,107 @@ class TestNetCfgDistro(TestCase): self.assertEqual(str(write_buf).strip(), BASE_NET_CFG.strip()) self.assertEqual(write_buf.mode, 0o644) + def test_apply_network_config_eni_ub(self): + ub_distro = self._get_distro('ubuntu') + with ExitStack() as mocks: + write_bufs = {} + + def replace_write(filename, content, mode=0o644, omode="wb"): + buf = WriteBuffer() + buf.mode = mode + buf.omode = omode + buf.write(content) + write_bufs[filename] = buf + + # eni availability checks + mocks.enter_context( + mock.patch.object(util, 'which', return_value=True)) + mocks.enter_context( + mock.patch.object(eni, 'available', return_value=True)) + mocks.enter_context( + mock.patch.object(util, 'ensure_dir')) + mocks.enter_context( + mock.patch.object(util, 'write_file', replace_write)) + mocks.enter_context( + mock.patch.object(os.path, 'isfile', return_value=False)) + + ub_distro.apply_network_config(V1_NET_CFG, False) + + self.assertEqual(len(write_bufs), 2) + eni_name = '/etc/network/interfaces.d/50-cloud-init.cfg' + self.assertIn(eni_name, write_bufs) + write_buf = write_bufs[eni_name] + self.assertEqual(str(write_buf).strip(), V1_NET_CFG_OUTPUT.strip()) + self.assertEqual(write_buf.mode, 0o644) + + def test_apply_network_config_v1_to_netplan_ub(self): + renderers = ['netplan'] + ub_distro = self._get_distro('ubuntu', renderers=renderers) + with ExitStack() as mocks: + write_bufs = {} + + def replace_write(filename, content, mode=0o644, omode="wb"): + buf = WriteBuffer() + buf.mode = mode + buf.omode = omode + buf.write(content) + write_bufs[filename] = buf + + mocks.enter_context( + mock.patch.object(util, 'which', return_value=True)) + mocks.enter_context( + mock.patch.object(util, 'write_file', replace_write)) + mocks.enter_context( + mock.patch.object(util, 'ensure_dir')) + mocks.enter_context( + mock.patch.object(util, 'subp', return_value=(0, 0))) + mocks.enter_context( + mock.patch.object(os.path, 'isfile', return_value=False)) + + ub_distro.apply_network_config(V1_NET_CFG, False) + + self.assertEqual(len(write_bufs), 1) + netplan_name = '/etc/netplan/50-cloud-init.yaml' + self.assertIn(netplan_name, write_bufs) + write_buf = write_bufs[netplan_name] + self.assertEqual(str(write_buf).strip(), + V1_TO_V2_NET_CFG_OUTPUT.strip()) + self.assertEqual(write_buf.mode, 0o644) + + def test_apply_network_config_v2_passthrough_ub(self): + renderers = ['netplan'] + ub_distro = self._get_distro('ubuntu', renderers=renderers) + with ExitStack() as mocks: + write_bufs = {} + + def replace_write(filename, content, mode=0o644, omode="wb"): + buf = WriteBuffer() + buf.mode = mode + buf.omode = omode + buf.write(content) + write_bufs[filename] = buf + + mocks.enter_context( + mock.patch.object(util, 'which', return_value=True)) + mocks.enter_context( + mock.patch.object(util, 'write_file', replace_write)) + mocks.enter_context( + mock.patch.object(util, 'ensure_dir')) + mocks.enter_context( + mock.patch.object(util, 'subp', return_value=(0, 0))) + mocks.enter_context( + mock.patch.object(os.path, 'isfile', return_value=False)) + + ub_distro.apply_network_config(V2_NET_CFG, False) + + self.assertEqual(len(write_bufs), 1) + netplan_name = '/etc/netplan/50-cloud-init.yaml' + self.assertIn(netplan_name, write_bufs) + write_buf = write_bufs[netplan_name] + self.assertEqual(str(write_buf).strip(), + V2_TO_V2_NET_CFG_OUTPUT.strip()) + self.assertEqual(write_buf.mode, 0o644) + def assertCfgEquals(self, blob1, blob2): b1 = dict(SysConf(blob1.strip().splitlines())) b2 = dict(SysConf(blob2.strip().splitlines())) @@ -191,6 +389,79 @@ ONBOOT=yes expected_buf = ''' # Created by cloud-init v. 0.7 NETWORKING=yes +''' + self.assertCfgEquals(expected_buf, str(write_buf)) + self.assertEqual(write_buf.mode, 0o644) + + def test_apply_network_config_rh(self): + renderers = ['sysconfig'] + rh_distro = self._get_distro('rhel', renderers=renderers) + + write_bufs = {} + + def replace_write(filename, content, mode=0o644, omode="wb"): + buf = WriteBuffer() + buf.mode = mode + buf.omode = omode + buf.write(content) + write_bufs[filename] = buf + + with ExitStack() as mocks: + # sysconfig availability checks + mocks.enter_context( + mock.patch.object(util, 'which', return_value=True)) + mocks.enter_context( + mock.patch.object(util, 'write_file', replace_write)) + mocks.enter_context( + mock.patch.object(util, 'load_file', return_value='')) + mocks.enter_context( + mock.patch.object(os.path, 'isfile', return_value=True)) + + rh_distro.apply_network_config(V1_NET_CFG, False) + + self.assertEqual(len(write_bufs), 5) + + # eth0 + self.assertIn('/etc/sysconfig/network-scripts/ifcfg-eth0', + write_bufs) + write_buf = write_bufs['/etc/sysconfig/network-scripts/ifcfg-eth0'] + expected_buf = ''' +# Created by cloud-init on instance boot automatically, do not edit. +# +BOOTPROTO=static +DEVICE=eth0 +IPADDR=192.168.1.5 +NETMASK=255.255.255.0 +NM_CONTROLLED=no +ONBOOT=yes +TYPE=Ethernet +USERCTL=no +''' + self.assertCfgEquals(expected_buf, str(write_buf)) + self.assertEqual(write_buf.mode, 0o644) + + # eth1 + self.assertIn('/etc/sysconfig/network-scripts/ifcfg-eth1', + write_bufs) + write_buf = write_bufs['/etc/sysconfig/network-scripts/ifcfg-eth1'] + expected_buf = ''' +# Created by cloud-init on instance boot automatically, do not edit. +# +BOOTPROTO=dhcp +DEVICE=eth1 +NM_CONTROLLED=no +ONBOOT=yes +TYPE=Ethernet +USERCTL=no +''' + self.assertCfgEquals(expected_buf, str(write_buf)) + self.assertEqual(write_buf.mode, 0o644) + + self.assertIn('/etc/sysconfig/network', write_bufs) + write_buf = write_bufs['/etc/sysconfig/network'] + expected_buf = ''' +# Created by cloud-init v. 0.7 +NETWORKING=yes ''' self.assertCfgEquals(expected_buf, str(write_buf)) self.assertEqual(write_buf.mode, 0o644) @@ -270,6 +541,78 @@ IPV6_DEFAULTGW="2607:f0d0:1002:0011::1" NETWORKING=yes NETWORKING_IPV6=yes IPV6_AUTOCONF=no +''' + self.assertCfgEquals(expected_buf, str(write_buf)) + self.assertEqual(write_buf.mode, 0o644) + + def test_apply_network_config_ipv6_rh(self): + renderers = ['sysconfig'] + rh_distro = self._get_distro('rhel', renderers=renderers) + + write_bufs = {} + + def replace_write(filename, content, mode=0o644, omode="wb"): + buf = WriteBuffer() + buf.mode = mode + buf.omode = omode + buf.write(content) + write_bufs[filename] = buf + + with ExitStack() as mocks: + mocks.enter_context( + mock.patch.object(util, 'which', return_value=True)) + mocks.enter_context( + mock.patch.object(util, 'write_file', replace_write)) + mocks.enter_context( + mock.patch.object(util, 'load_file', return_value='')) + mocks.enter_context( + mock.patch.object(os.path, 'isfile', return_value=True)) + + rh_distro.apply_network_config(V1_NET_CFG_IPV6, False) + + self.assertEqual(len(write_bufs), 5) + + self.assertIn('/etc/sysconfig/network-scripts/ifcfg-eth0', + write_bufs) + write_buf = write_bufs['/etc/sysconfig/network-scripts/ifcfg-eth0'] + expected_buf = ''' +# Created by cloud-init on instance boot automatically, do not edit. +# +BOOTPROTO=static +DEVICE=eth0 +IPV6ADDR=2607:f0d0:1002:0011::2 +IPV6INIT=yes +NETMASK=64 +NM_CONTROLLED=no +ONBOOT=yes +TYPE=Ethernet +USERCTL=no +''' + self.assertCfgEquals(expected_buf, str(write_buf)) + self.assertEqual(write_buf.mode, 0o644) + self.assertIn('/etc/sysconfig/network-scripts/ifcfg-eth1', + write_bufs) + write_buf = write_bufs['/etc/sysconfig/network-scripts/ifcfg-eth1'] + expected_buf = ''' +# Created by cloud-init on instance boot automatically, do not edit. +# +BOOTPROTO=dhcp +DEVICE=eth1 +NM_CONTROLLED=no +ONBOOT=yes +TYPE=Ethernet +USERCTL=no +''' + self.assertCfgEquals(expected_buf, str(write_buf)) + self.assertEqual(write_buf.mode, 0o644) + + self.assertIn('/etc/sysconfig/network', write_bufs) + write_buf = write_bufs['/etc/sysconfig/network'] + expected_buf = ''' +# Created by cloud-init v. 0.7 +NETWORKING=yes +NETWORKING_IPV6=yes +IPV6_AUTOCONF=no ''' self.assertCfgEquals(expected_buf, str(write_buf)) self.assertEqual(write_buf.mode, 0o644) diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 902204a0..4f07d804 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -3,6 +3,7 @@ from cloudinit import net from cloudinit.net import cmdline from cloudinit.net import eni +from cloudinit.net import netplan from cloudinit.net import network_state from cloudinit.net import renderers from cloudinit.net import sysconfig @@ -408,6 +409,41 @@ NETWORK_CONFIGS = { post-up route add default gw 65.61.151.37 || true pre-down route del default gw 65.61.151.37 || true """).rstrip(' '), + 'expected_netplan': textwrap.dedent(""" + network: + version: 2 + ethernets: + eth1: + match: + macaddress: cf:d6:af:48:e8:80 + nameservers: + addresses: + - 1.2.3.4 + - 5.6.7.8 + search: + - wark.maas + set-name: eth1 + eth99: + addresses: + - 192.168.21.3/24 + dhcp4: true + match: + macaddress: c0:d6:9f:2c:e8:80 + nameservers: + addresses: + - 8.8.8.8 + - 8.8.4.4 + - 1.2.3.4 + - 5.6.7.8 + search: + - barley.maas + - sach.maas + - wark.maas + routes: + - to: 0.0.0.0/0.0.0.0 + via: 65.61.151.37 + set-name: eth99 + """).rstrip(' '), 'yaml': textwrap.dedent(""" version: 1 config: @@ -450,6 +486,14 @@ NETWORK_CONFIGS = { # control-alias iface0 iface iface0 inet6 dhcp """).rstrip(' '), + 'expected_netplan': textwrap.dedent(""" + network: + version: 2 + ethernets: + iface0: + dhcp4: true + dhcp6: true + """).rstrip(' '), 'yaml': textwrap.dedent("""\ version: 1 config: @@ -524,6 +568,126 @@ iface eth0.101 inet static post-up route add -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true """), + 'expected_netplan': textwrap.dedent(""" + network: + version: 2 + ethernets: + eth0: + match: + macaddress: c0:d6:9f:2c:e8:80 + nameservers: + addresses: + - 8.8.8.8 + - 4.4.4.4 + - 8.8.4.4 + search: + - barley.maas + - wark.maas + - foobar.maas + set-name: eth0 + eth1: + match: + macaddress: aa:d6:9f:2c:e8:80 + nameservers: + addresses: + - 8.8.8.8 + - 4.4.4.4 + - 8.8.4.4 + search: + - barley.maas + - wark.maas + - foobar.maas + set-name: eth1 + eth2: + match: + macaddress: c0:bb:9f:2c:e8:80 + nameservers: + addresses: + - 8.8.8.8 + - 4.4.4.4 + - 8.8.4.4 + search: + - barley.maas + - wark.maas + - foobar.maas + set-name: eth2 + eth3: + match: + macaddress: 66:bb:9f:2c:e8:80 + nameservers: + addresses: + - 8.8.8.8 + - 4.4.4.4 + - 8.8.4.4 + search: + - barley.maas + - wark.maas + - foobar.maas + set-name: eth3 + eth4: + match: + macaddress: 98:bb:9f:2c:e8:80 + nameservers: + addresses: + - 8.8.8.8 + - 4.4.4.4 + - 8.8.4.4 + search: + - barley.maas + - wark.maas + - foobar.maas + set-name: eth4 + eth5: + dhcp4: true + match: + macaddress: 98:bb:9f:2c:e8:8a + nameservers: + addresses: + - 8.8.8.8 + - 4.4.4.4 + - 8.8.4.4 + search: + - barley.maas + - wark.maas + - foobar.maas + set-name: eth5 + bonds: + bond0: + dhcp6: true + interfaces: + - eth1 + - eth2 + parameters: + mode: active-backup + bridges: + br0: + addresses: + - 192.168.14.2/24 + - 2001:1::1/64 + interfaces: + - eth3 + - eth4 + vlans: + bond0.200: + dhcp4: true + id: 200 + link: bond0 + eth0.101: + addresses: + - 192.168.0.2/24 + - 192.168.2.10/24 + gateway4: 192.168.0.1 + id: 101 + link: eth0 + nameservers: + addresses: + - 192.168.0.10 + - 10.23.23.134 + search: + - barley.maas + - sacchromyces.maas + - brettanomyces.maas + """).rstrip(' '), 'yaml': textwrap.dedent(""" version: 1 config: @@ -808,6 +972,99 @@ iface eth0 inet dhcp expected, dir2dict(tmp_dir)['/etc/network/interfaces']) +class TestNetplanNetRendering(CiTestCase): + + @mock.patch("cloudinit.net.sys_dev_path") + @mock.patch("cloudinit.net.read_sys_net") + @mock.patch("cloudinit.net.get_devicelist") + def test_default_generation(self, mock_get_devicelist, + mock_read_sys_net, + mock_sys_dev_path): + tmp_dir = self.tmp_dir() + _setup_test(tmp_dir, mock_get_devicelist, + mock_read_sys_net, mock_sys_dev_path) + + network_cfg = net.generate_fallback_config() + ns = network_state.parse_net_config_data(network_cfg, + skip_broken=False) + + render_dir = os.path.join(tmp_dir, "render") + os.makedirs(render_dir) + + render_target = 'netplan.yaml' + renderer = netplan.Renderer( + {'netplan_path': render_target, 'postcmds': False}) + renderer.render_network_state(render_dir, ns) + + self.assertTrue(os.path.exists(os.path.join(render_dir, + render_target))) + with open(os.path.join(render_dir, render_target)) as fh: + contents = fh.read() + print(contents) + + expected = """ +network: + version: 2 + ethernets: + eth1000: + dhcp4: true + match: + macaddress: 07-1c-c6-75-a4-be + set-name: eth1000 +""" + self.assertEqual(expected.lstrip(), contents.lstrip()) + + +class TestNetplanPostcommands(CiTestCase): + mycfg = { + 'config': [{"type": "physical", "name": "eth0", + "mac_address": "c0:d6:9f:2c:e8:80", + "subnets": [{"type": "dhcp"}]}], + 'version': 1} + + @mock.patch.object(netplan.Renderer, '_netplan_generate') + @mock.patch.object(netplan.Renderer, '_net_setup_link') + def test_netplan_render_calls_postcmds(self, mock_netplan_generate, + mock_net_setup_link): + tmp_dir = self.tmp_dir() + ns = network_state.parse_net_config_data(self.mycfg, + skip_broken=False) + + render_dir = os.path.join(tmp_dir, "render") + os.makedirs(render_dir) + + render_target = 'netplan.yaml' + renderer = netplan.Renderer( + {'netplan_path': render_target, 'postcmds': True}) + renderer.render_network_state(render_dir, ns) + + mock_netplan_generate.assert_called_with(run=True) + mock_net_setup_link.assert_called_with(run=True) + + @mock.patch.object(netplan, "get_devicelist") + @mock.patch('cloudinit.util.subp') + def test_netplan_postcmds(self, mock_subp, mock_devlist): + mock_devlist.side_effect = [['lo']] + tmp_dir = self.tmp_dir() + ns = network_state.parse_net_config_data(self.mycfg, + skip_broken=False) + + render_dir = os.path.join(tmp_dir, "render") + os.makedirs(render_dir) + + render_target = 'netplan.yaml' + renderer = netplan.Renderer( + {'netplan_path': render_target, 'postcmds': True}) + renderer.render_network_state(render_dir, ns) + + expected = [ + mock.call(['netplan', 'generate'], capture=True), + mock.call(['udevadm', 'test-builtin', 'net_setup_link', + '/sys/class/net/lo'], capture=True), + ] + mock_subp.assert_has_calls(expected) + + class TestEniNetworkStateToEni(CiTestCase): mycfg = { 'config': [{"type": "physical", "name": "eth0", @@ -953,6 +1210,50 @@ class TestCmdlineReadKernelConfig(CiTestCase): self.assertEqual(found['config'], expected) +class TestNetplanRoundTrip(CiTestCase): + def _render_and_read(self, network_config=None, state=None, + netplan_path=None, dir=None): + if dir is None: + dir = self.tmp_dir() + + if network_config: + ns = network_state.parse_net_config_data(network_config) + elif state: + ns = state + else: + raise ValueError("Expected data or state, got neither") + + if netplan_path is None: + netplan_path = 'etc/netplan/50-cloud-init.yaml' + + renderer = netplan.Renderer( + config={'netplan_path': netplan_path}) + + renderer.render_network_state(dir, ns) + return dir2dict(dir) + + def testsimple_render_small_netplan(self): + entry = NETWORK_CONFIGS['small'] + files = self._render_and_read(network_config=yaml.load(entry['yaml'])) + self.assertEqual( + entry['expected_netplan'].splitlines(), + files['/etc/netplan/50-cloud-init.yaml'].splitlines()) + + def testsimple_render_v4_and_v6(self): + entry = NETWORK_CONFIGS['v4_and_v6'] + files = self._render_and_read(network_config=yaml.load(entry['yaml'])) + self.assertEqual( + entry['expected_netplan'].splitlines(), + files['/etc/netplan/50-cloud-init.yaml'].splitlines()) + + def testsimple_render_all(self): + entry = NETWORK_CONFIGS['all'] + files = self._render_and_read(network_config=yaml.load(entry['yaml'])) + self.assertEqual( + entry['expected_netplan'].splitlines(), + files['/etc/netplan/50-cloud-init.yaml'].splitlines()) + + class TestEniRoundTrip(CiTestCase): def _render_and_read(self, network_config=None, state=None, eni_path=None, links_prefix=None, netrules_path=None, dir=None): diff --git a/tools/net-convert.py b/tools/net-convert.py new file mode 100755 index 00000000..870da639 --- /dev/null +++ b/tools/net-convert.py @@ -0,0 +1,84 @@ +#!/usr/bin/python3 +# This file is part of cloud-init. See LICENSE file for license information. + +import argparse +import json +import os +import yaml + +from cloudinit.sources.helpers import openstack + +from cloudinit.net import eni +from cloudinit.net import network_state +from cloudinit.net import netplan +from cloudinit.net import sysconfig + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--network-data", "-p", type=open, + metavar="PATH", required=True) + parser.add_argument("--kind", "-k", + choices=['eni', 'network_data.json', 'yaml'], + required=True) + parser.add_argument("-d", "--directory", + metavar="PATH", + help="directory to place output in", + required=True) + parser.add_argument("-m", "--mac", + metavar="name,mac", + action='append', + help="interface name to mac mapping") + parser.add_argument("--output-kind", "-ok", + choices=['eni', 'netplan', 'sysconfig'], + required=True) + args = parser.parse_args() + + if not os.path.isdir(args.directory): + os.makedirs(args.directory) + + if args.mac: + known_macs = {} + for item in args.mac: + iface_name, iface_mac = item.split(",", 1) + known_macs[iface_mac] = iface_name + else: + known_macs = None + + net_data = args.network_data.read() + if args.kind == "eni": + pre_ns = eni.convert_eni_data(net_data) + ns = network_state.parse_net_config_data(pre_ns) + elif args.kind == "yaml": + pre_ns = yaml.load(net_data) + if 'network' in pre_ns: + pre_ns = pre_ns.get('network') + print("Input YAML") + print(yaml.dump(pre_ns, default_flow_style=False, indent=4)) + ns = network_state.parse_net_config_data(pre_ns) + else: + pre_ns = openstack.convert_net_json( + json.loads(net_data), known_macs=known_macs) + ns = network_state.parse_net_config_data(pre_ns) + + if not ns: + raise RuntimeError("No valid network_state object created from" + "input data") + + print("\nInternal State") + print(yaml.dump(ns, default_flow_style=False, indent=4)) + if args.output_kind == "eni": + r_cls = eni.Renderer + elif args.output_kind == "netplan": + r_cls = netplan.Renderer + else: + r_cls = sysconfig.Renderer + + r = r_cls() + r.render_network_state(ns, target=args.directory) + + +if __name__ == '__main__': + main() + +# vi: ts=4 expandtab -- cgit v1.2.3 From 192dbda37b50aa97f87976879dd177834a986123 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 21 Mar 2017 09:50:28 -0400 Subject: Fix unit test when running in a system deployed with cloud-init. test_netconfig.py:test_apply_network_config_eni_ub would attempt to remove any .link files that cloud-init had written. This was just a failure to mock out all of its interaction with the host. --- tests/unittests/test_distros/test_netconfig.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'tests') diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py index b89b74ff..88370669 100644 --- a/tests/unittests/test_distros/test_netconfig.py +++ b/tests/unittests/test_distros/test_netconfig.py @@ -236,6 +236,9 @@ class TestNetCfgDistro(TestCase): mock.patch.object(util, 'write_file', replace_write)) mocks.enter_context( mock.patch.object(os.path, 'isfile', return_value=False)) + mocks.enter_context( + mock.patch("cloudinit.net.eni.glob.glob", + return_value=[])) ub_distro.apply_network_config(V1_NET_CFG, False) -- cgit v1.2.3 From de63d664a664cb06cefcd7c68c5b3428279a9995 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 21 Mar 2017 16:33:14 -0400 Subject: advertise network config v2 support (NETWORK_CONFIG_V2) in features. ef18b8ac4c added support for handing network config v2 (aka netplan format). This just adds that feature to the list of supported features. --- cloudinit/version.py | 2 ++ tests/unittests/test_version.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 tests/unittests/test_version.py (limited to 'tests') diff --git a/cloudinit/version.py b/cloudinit/version.py index e61597d2..dff4af04 100644 --- a/cloudinit/version.py +++ b/cloudinit/version.py @@ -9,6 +9,8 @@ __VERSION__ = "0.7.9" FEATURES = [ # supports network config version 1 'NETWORK_CONFIG_V1', + # supports network config version 2 (netplan) + 'NETWORK_CONFIG_V2', ] diff --git a/tests/unittests/test_version.py b/tests/unittests/test_version.py new file mode 100644 index 00000000..1662ce09 --- /dev/null +++ b/tests/unittests/test_version.py @@ -0,0 +1,14 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from .helpers import CiTestCase +from cloudinit import version + + +class TestExportsFeatures(CiTestCase): + def test_has_network_config_v1(self): + self.assertIn('NETWORK_CONFIG_V1', version.FEATURES) + + def test_has_network_config_v2(self): + self.assertIn('NETWORK_CONFIG_V2', version.FEATURES) + +# vi: ts=4 expandtab -- cgit v1.2.3 From 4a2b2f87ec48c227eb8fb2091dba604457cf8de8 Mon Sep 17 00:00:00 2001 From: Jonathan Ballet Date: Thu, 23 Mar 2017 14:35:59 +0100 Subject: Fix filesystem creation when using "partition: auto" Accordingly to the documentation: The ``partition`` option may also be set to ``auto``, in which this module will search for the existance of a filesystem matching the ``label``, ``type`` and ``device`` of the ``fs_setup`` entry and will skip creating the filesystem if one is found. However, using this "auto" flag always recreates the partition no matter if it has been done before or not. This commit fixes a bug in which the "partition" attribute was always set to None although in some cases it should not. LP: #1634678 --- cloudinit/config/cc_disk_setup.py | 2 +- .../test_handler/test_handler_disk_setup.py | 44 ++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) (limited to 'tests') diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index 38df13ab..f39f0815 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -201,7 +201,7 @@ def update_fs_setup_devices(disk_setup, tformer): if part and 'partition' in definition: definition['_partition'] = definition['partition'] - definition['partition'] = part + definition['partition'] = part def value_splitter(values, start=None): diff --git a/tests/unittests/test_handler/test_handler_disk_setup.py b/tests/unittests/test_handler/test_handler_disk_setup.py index 227f0497..7ff39225 100644 --- a/tests/unittests/test_handler/test_handler_disk_setup.py +++ b/tests/unittests/test_handler/test_handler_disk_setup.py @@ -103,4 +103,48 @@ class TestGetPartitionMbrLayout(TestCase): ',{0},83\n,,82'.format(expected_partition_size), cc_disk_setup.get_partition_mbr_layout(disk_size, [33, [66, 82]])) + +class TestUpdateFsSetupDevices(TestCase): + def test_regression_1634678(self): + # Cf. https://bugs.launchpad.net/cloud-init/+bug/1634678 + fs_setup = { + 'partition': 'auto', + 'device': '/dev/xvdb1', + 'overwrite': False, + 'label': 'test', + 'filesystem': 'ext4' + } + + cc_disk_setup.update_fs_setup_devices([fs_setup], + lambda device: device) + + self.assertEqual({ + '_origname': '/dev/xvdb1', + 'partition': 'auto', + 'device': '/dev/xvdb1', + 'overwrite': False, + 'label': 'test', + 'filesystem': 'ext4' + }, fs_setup) + + def test_dotted_devname(self): + fs_setup = { + 'partition': 'auto', + 'device': 'ephemeral0.0', + 'label': 'test2', + 'filesystem': 'xfs' + } + + cc_disk_setup.update_fs_setup_devices([fs_setup], + lambda device: device) + + self.assertEqual({ + '_origname': 'ephemeral0.0', + '_partition': 'auto', + 'partition': '0', + 'device': 'ephemeral0', + 'label': 'test2', + 'filesystem': 'xfs' + }, fs_setup) + # vi: ts=4 expandtab -- cgit v1.2.3 From 21632972df034c200578e1fbc121a07f20bb8774 Mon Sep 17 00:00:00 2001 From: "Tore S. Lonoy" Date: Fri, 4 Nov 2016 11:38:31 +0100 Subject: Add support for setting hashed passwords This change will add support for hashed passwords in cc_set_passwords. It checks if a password is a hash with by checking that it matches in fairly safe way, and also that the password does not have a ":" in it. chpasswd needs to know if the password is hashed or not, so two lists is created so chpasswd is feed with the correct one. LP: #1570325 --- cloudinit/config/cc_set_passwords.py | 48 ++++++++++++++++------ doc/examples/cloud-config.txt | 9 +++- .../configs/modules/set_password_list.yaml | 3 ++ .../configs/modules/set_password_list_string.yaml | 3 ++ tests/cloud_tests/testcases/base.py | 4 ++ 5 files changed, 53 insertions(+), 14 deletions(-) (limited to 'tests') diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index 8440e593..eb0bdab0 100755 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -23,7 +23,8 @@ If the ``list`` key is provided, a list of ``username:password`` pairs can be specified. The usernames specified must already exist on the system, or have been created using the ``cc_users_groups`` module. A password can be randomly generated using -``username:RANDOM`` or ``username:R``. Password ssh authentication can be +``username:RANDOM`` or ``username:R``. A hashed password can be specified +using ``username:$6$salt$hash``. Password ssh authentication can be enabled, disabled, or left to system defaults using ``ssh_pwauth``. .. note:: @@ -60,8 +61,10 @@ enabled, disabled, or left to system defaults using ``ssh_pwauth``. - user2:RANDOM - user3:password3 - user4:R + - user4:$6$rL..$ej... """ +import re import sys from cloudinit.distros import ug_util @@ -112,24 +115,43 @@ def handle(_name, cfg, cloud, log, args): errors = [] if plist: plist_in = [] + hashed_plist_in = [] + hashed_users = [] randlist = [] users = [] + prog = re.compile(r'\$[1,2a,2y,5,6](\$.+){2}') for line in plist: u, p = line.split(':', 1) - if p == "R" or p == "RANDOM": - p = rand_user_password() - randlist.append("%s:%s" % (u, p)) - plist_in.append("%s:%s" % (u, p)) - users.append(u) + if prog.match(p) is not None and ":" not in p: + hashed_plist_in.append("%s:%s" % (u, p)) + hashed_users.append(u) + else: + if p == "R" or p == "RANDOM": + p = rand_user_password() + randlist.append("%s:%s" % (u, p)) + plist_in.append("%s:%s" % (u, p)) + users.append(u) ch_in = '\n'.join(plist_in) + '\n' - try: - log.debug("Changing password for %s:", users) - util.subp(['chpasswd'], ch_in) - except Exception as e: - errors.append(e) - util.logexc(log, "Failed to set passwords with chpasswd for %s", - users) + if users: + try: + log.debug("Changing password for %s:", users) + util.subp(['chpasswd'], ch_in) + except Exception as e: + errors.append(e) + util.logexc( + log, "Failed to set passwords with chpasswd for %s", users) + + hashed_ch_in = '\n'.join(hashed_plist_in) + '\n' + if hashed_users: + try: + log.debug("Setting hashed password for %s:", hashed_users) + util.subp(['chpasswd', '-e'], hashed_ch_in) + except Exception as e: + errors.append(e) + util.logexc( + log, "Failed to set hashed passwords with chpasswd for %s", + hashed_users) if len(randlist): blurb = ("Set the following 'random' passwords\n", diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index c03f1026..bd84c641 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -426,14 +426,21 @@ syslog_fix_perms: syslog:root # # there is also an option to set multiple users passwords, using 'chpasswd' # That looks like the following, with 'expire' set to 'True' by default. -# to not expire users passwords, set 'expire' to 'False': +# to not expire users passwords, set 'expire' to 'False'. Also possible +# to set hashed password, here account 'user3' has a password it set to +# 'cloud-init', hashed with SHA-256: # chpasswd: # list: | # user1:password1 # user2:RANDOM +# user3:$5$eriogqzq$Dg7PxHsKGzziuEGkZgkLvacjuEFeljJ.rLf.hZqKQLA # expire: True # ssh_pwauth: [ True, False, "" or "unchanged" ] # +# Hashed passwords can be generated in multiple ways, example with python3: +# python3 -c 'import crypt,getpass; print(crypt.crypt(getpass.getpass(), crypt.mksalt(crypt.METHOD_SHA512)))' +# Newer versions of 'mkpasswd' will also work: mkpasswd -m sha-512 password +# # So, a simple working example to allow login via ssh, and not expire # for the default user would look like: password: passw0rd diff --git a/tests/cloud_tests/configs/modules/set_password_list.yaml b/tests/cloud_tests/configs/modules/set_password_list.yaml index a1eadd75..a2a89c9d 100644 --- a/tests/cloud_tests/configs/modules/set_password_list.yaml +++ b/tests/cloud_tests/configs/modules/set_password_list.yaml @@ -21,11 +21,14 @@ cloud_config: | # sha256 gojanego passwd: "$5$iW$XsxmWCdpwIW8Yhv.Jn/R3uk6A4UaicfW5Xp7C9p9pg." lock_passwd: false + - name: "mikey" + lock_passwd: false chpasswd: list: - tom:mypassword123! - dick:RANDOM - harry:RANDOM + - mikey:$5$xZ$B2YGGEx2AOf4PeW48KC6.QyT1W2B4rZ9Qbltudtha89 collect_scripts: shadow: | #!/bin/bash diff --git a/tests/cloud_tests/configs/modules/set_password_list_string.yaml b/tests/cloud_tests/configs/modules/set_password_list_string.yaml index cbb71bee..c2a0f631 100644 --- a/tests/cloud_tests/configs/modules/set_password_list_string.yaml +++ b/tests/cloud_tests/configs/modules/set_password_list_string.yaml @@ -21,11 +21,14 @@ cloud_config: | # sha256 gojanego passwd: "$5$iW$XsxmWCdpwIW8Yhv.Jn/R3uk6A4UaicfW5Xp7C9p9pg." lock_passwd: false + - name: "mikey" + lock_passwd: false chpasswd: list: | tom:mypassword123! dick:RANDOM harry:RANDOM + mikey:$5$xZ$B2YGGEx2AOf4PeW48KC6.QyT1W2B4rZ9Qbltudtha89 collect_scripts: shadow: | #!/bin/bash diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py index 51ce2b41..64d5507a 100644 --- a/tests/cloud_tests/testcases/base.py +++ b/tests/cloud_tests/testcases/base.py @@ -98,6 +98,9 @@ class PasswordListTest(CloudTestCase): self.assertEqual([], dupes) self.assertEqual(jane_enc, users['jane']) + mikey_enc = "$5$xZ$B2YGGEx2AOf4PeW48KC6.QyT1W2B4rZ9Qbltudtha89" + self.assertEqual(mikey_enc, users['mikey']) + # shadow entry is $N$salt$, so we encrypt with the same format # and salt and expect the result. tom = "mypassword123!" @@ -124,6 +127,7 @@ class PasswordListTest(CloudTestCase): self.assertIn('dick:', out) self.assertIn('harry:', out) self.assertIn('jane:', out) + self.assertIn('mikey:', out) def test_sshd_config(self): """Test sshd config allows passwords""" -- cgit v1.2.3 From 328fe5ab399b1f5b48d1985f41fc2ef66e368922 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 27 Mar 2017 12:43:15 -0400 Subject: GCE: Search GCE in ds-identify, consider serial number in check. While documentation indicates that the smbios product name should contain 'Google Compute Engine', experimentation and bug reports indicate that is not always the case. The change here is to change the check for GCE to also consider a serial number that starts with 'GoogleCompute-'. Also, ds-identify was not currently searching for GCE if no config of datasource_list was found. Most images have a datasource_list defined. So update the list to include GCE. LP: #1674861 --- cloudinit/sources/DataSourceGCE.py | 18 ++++++++++++++++++ tests/unittests/test_datasource/test_gce.py | 14 +++++++++++++- tools/ds-identify | 14 +++++++++++++- 3 files changed, 44 insertions(+), 2 deletions(-) (limited to 'tests') diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index b1a1c8f2..637c9505 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -62,6 +62,9 @@ class DataSourceGCE(sources.DataSource): return public_key def get_data(self): + if not platform_reports_gce(): + return False + # url_map: (our-key, path, required, is_text) url_map = [ ('instance-id', ('instance/id',), True, True), @@ -144,6 +147,21 @@ class DataSourceGCE(sources.DataSource): return self.availability_zone.rsplit('-', 1)[0] +def platform_reports_gce(): + pname = util.read_dmi_data('system-product-name') or "N/A" + if pname == "Google Compute Engine": + return True + + # system-product-name is not always guaranteed (LP: #1674861) + serial = util.read_dmi_data('system-serial-number') or "N/A" + if serial.startswith("GoogleCloud-"): + return True + + LOG.debug("Not running on google cloud. product-name=%s serial=%s", + pname, serial) + return False + + # Used to match classes to dependencies datasources = [ (DataSourceGCE, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py index 4f83454e..3eaa58e3 100644 --- a/tests/unittests/test_datasource/test_gce.py +++ b/tests/unittests/test_datasource/test_gce.py @@ -5,6 +5,7 @@ # This file is part of cloud-init. See LICENSE file for license information. import httpretty +import mock import re from base64 import b64encode, b64decode @@ -71,6 +72,11 @@ class TestDataSourceGCE(test_helpers.HttprettyTestCase): self.ds = DataSourceGCE.DataSourceGCE( settings.CFG_BUILTIN, None, helpers.Paths({})) + self.m_platform_reports_gce = mock.patch( + 'cloudinit.sources.DataSourceGCE.platform_reports_gce', + return_value=True) + self.m_platform_reports_gce.start() + self.addCleanup(self.m_platform_reports_gce.stop) super(TestDataSourceGCE, self).setUp() def test_connection(self): @@ -153,7 +159,13 @@ class TestDataSourceGCE(test_helpers.HttprettyTestCase): def test_only_last_part_of_zone_used_for_availability_zone(self): _set_mock_metadata() - self.ds.get_data() + r = self.ds.get_data() + self.assertEqual(True, r) self.assertEqual('bar', self.ds.availability_zone) + def test_get_data_returns_false_if_not_on_gce(self): + self.m_platform_reports_gce.return_value = False + self.assertEqual(False, self.ds.get_data()) + + # vi: ts=4 expandtab diff --git a/tools/ds-identify b/tools/ds-identify index bf09a3ad..15d6600e 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -108,7 +108,7 @@ DI_DSNAME="" # this has to match the builtin list in cloud-init, it is what will # be searched if there is no setting found in config. DI_DSLIST_DEFAULT="MAAS ConfigDrive NoCloud AltCloud Azure Bigstep \ -CloudSigma CloudStack DigitalOcean Ec2 OpenNebula OpenStack OVF SmartOS" +CloudSigma CloudStack DigitalOcean Ec2 GCE OpenNebula OpenStack OVF SmartOS" DI_DSLIST="" DI_MODE="" DI_ON_FOUND="" @@ -383,6 +383,14 @@ dmi_product_name_matches() { return 1 } +dmi_product_serial_matches() { + is_container && return 1 + case "${DI_DMI_PRODUCT_SERIAL}" in + $1) return 0;; + esac + return 1 +} + dmi_product_name_is() { is_container && return 1 [ "${DI_DMI_PRODUCT_NAME}" = "$1" ] @@ -770,6 +778,10 @@ dscheck_GCE() { if dmi_product_name_is "Google Compute Engine"; then return ${DS_FOUND} fi + # product name is not guaranteed (LP: #1674861) + if dmi_product_serial_matches "GoogleCloud-*"; then + return ${DS_FOUND} + fi return ${DS_NOT_FOUND} } -- cgit v1.2.3 From d23543eb206326a53a59d86afba862edbd02c231 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 30 Mar 2017 12:21:43 -0400 Subject: net: in netplan renderer delete known image-builtin content. When rendering network configuration to netplan, remove known "builtin" configurations. The specific example here is Ubuntu Core that has netplan configuration in etc/netplan/00-snapd-config.yaml. We also delete the derived files since netplan will have created these derived files in its generator that runs well before cloud-init. LP: #1675576 --- cloudinit/net/netplan.py | 40 +++++++++++++++++++++++++++ tests/unittests/test_net.py | 67 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 106 insertions(+), 1 deletion(-) (limited to 'tests') diff --git a/cloudinit/net/netplan.py b/cloudinit/net/netplan.py index 7444ae33..825fe831 100644 --- a/cloudinit/net/netplan.py +++ b/cloudinit/net/netplan.py @@ -10,6 +10,21 @@ from cloudinit import log as logging from cloudinit import util from cloudinit.net import SYS_CLASS_NET, get_devicelist +KNOWN_SNAPD_CONFIG = b"""\ +# This is the initial network config. +# It can be overwritten by cloud-init or console-conf. +network: + version: 2 + ethernets: + all-en: + match: + name: "en*" + dhcp4: true + all-eth: + match: + name: "eth*" + dhcp4: true +""" LOG = logging.getLogger(__name__) NET_CONFIG_TO_V2 = { @@ -154,6 +169,28 @@ def _extract_bond_slaves_by_name(interfaces, entry, bond_master): entry.update({'interfaces': bond_slave_names}) +def _clean_default(target=None): + # clean out any known default files and derived files in target + # LP: #1675576 + tpath = util.target_path(target, "etc/netplan/00-snapd-config.yaml") + if not os.path.isfile(tpath): + return + content = util.load_file(tpath, decode=False) + if content != KNOWN_SNAPD_CONFIG: + return + + derived = [util.target_path(target, f) for f in ( + 'run/systemd/network/10-netplan-all-en.network', + 'run/systemd/network/10-netplan-all-eth.network', + 'run/systemd/generator/netplan.stamp')] + existing = [f for f in derived if os.path.isfile(f)] + LOG.debug("removing known config '%s' and derived existing files: %s", + tpath, existing) + + for f in [tpath] + existing: + os.unlink(f) + + class Renderer(renderer.Renderer): """Renders network information in a /etc/netplan/network.yaml format.""" @@ -166,6 +203,7 @@ class Renderer(renderer.Renderer): 'etc/netplan/50-cloud-init.yaml') self.netplan_header = config.get('netplan_header', None) self._postcmds = config.get('postcmds', False) + self.clean_default = config.get('clean_default', True) def render_network_state(self, target, network_state): # check network state for version @@ -182,6 +220,8 @@ class Renderer(renderer.Renderer): header += "\n" util.write_file(fpnplan, header + content) + if self.clean_default: + _clean_default(target=target) self._netplan_generate(run=self._postcmds) self._net_setup_link(run=self._postcmds) diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 4f07d804..bfd04ba0 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -974,12 +974,14 @@ iface eth0 inet dhcp class TestNetplanNetRendering(CiTestCase): + @mock.patch("cloudinit.net.netplan._clean_default") @mock.patch("cloudinit.net.sys_dev_path") @mock.patch("cloudinit.net.read_sys_net") @mock.patch("cloudinit.net.get_devicelist") def test_default_generation(self, mock_get_devicelist, mock_read_sys_net, - mock_sys_dev_path): + mock_sys_dev_path, + mock_clean_default): tmp_dir = self.tmp_dir() _setup_test(tmp_dir, mock_get_devicelist, mock_read_sys_net, mock_sys_dev_path) @@ -1013,6 +1015,69 @@ network: set-name: eth1000 """ self.assertEqual(expected.lstrip(), contents.lstrip()) + self.assertEqual(1, mock_clean_default.call_count) + + +class TestNetplanCleanDefault(CiTestCase): + snapd_known_path = 'etc/netplan/00-snapd-config.yaml' + snapd_known_content = textwrap.dedent("""\ + # This is the initial network config. + # It can be overwritten by cloud-init or console-conf. + network: + version: 2 + ethernets: + all-en: + match: + name: "en*" + dhcp4: true + all-eth: + match: + name: "eth*" + dhcp4: true + """) + stub_known = { + 'run/systemd/network/10-netplan-all-en.network': 'foo-en', + 'run/systemd/network/10-netplan-all-eth.network': 'foo-eth', + 'run/systemd/generator/netplan.stamp': 'stamp', + } + + def test_clean_known_config_cleaned(self): + content = {self.snapd_known_path: self.snapd_known_content, } + content.update(self.stub_known) + tmpd = self.tmp_dir() + files = sorted(populate_dir(tmpd, content)) + netplan._clean_default(target=tmpd) + found = [t for t in files if os.path.exists(t)] + self.assertEqual([], found) + + def test_clean_unknown_config_not_cleaned(self): + content = {self.snapd_known_path: self.snapd_known_content, } + content.update(self.stub_known) + content[self.snapd_known_path] += "# user put a comment\n" + tmpd = self.tmp_dir() + files = sorted(populate_dir(tmpd, content)) + netplan._clean_default(target=tmpd) + found = [t for t in files if os.path.exists(t)] + self.assertEqual(files, found) + + def test_clean_known_config_cleans_only_expected(self): + astamp = "run/systemd/generator/another.stamp" + anet = "run/systemd/network/10-netplan-all-lo.network" + ayaml = "etc/netplan/01-foo-config.yaml" + content = { + self.snapd_known_path: self.snapd_known_content, + astamp: "stamp", + anet: "network", + ayaml: "yaml", + } + content.update(self.stub_known) + + tmpd = self.tmp_dir() + files = sorted(populate_dir(tmpd, content)) + netplan._clean_default(target=tmpd) + found = [t for t in files if os.path.exists(t)] + expected = [util.target_path(tmpd, f) for f in (astamp, anet, ayaml)] + self.assertEqual(sorted(expected), found) class TestNetplanPostcommands(CiTestCase): -- cgit v1.2.3 From 5442b517ebe5508159a46d11d500fbc6ad854ba0 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 31 Mar 2017 13:55:27 -0400 Subject: tests: update OpenNebula and Digital Ocean to not rely on host interfaces. Mock the use use of get_interfaces_by_mac in Digital Ocean and OpenNebula. Its best to mock this for the tests as the results aren't expecting it to fail. Note, as it stands, OpenNebula relies on devices named 'eth0'. The metadata (context) does not provide mac addresses. --- tests/unittests/test_datasource/test_digitalocean.py | 14 ++++++++++++-- tests/unittests/test_datasource/test_opennebula.py | 9 +++++++-- 2 files changed, 19 insertions(+), 4 deletions(-) (limited to 'tests') diff --git a/tests/unittests/test_datasource/test_digitalocean.py b/tests/unittests/test_datasource/test_digitalocean.py index 9be6bc19..61d6e001 100644 --- a/tests/unittests/test_datasource/test_digitalocean.py +++ b/tests/unittests/test_datasource/test_digitalocean.py @@ -194,7 +194,12 @@ class TestDataSourceDigitalOcean(TestCase): class TestNetworkConvert(TestCase): - def _get_networking(self): + @mock.patch('cloudinit.net.get_interfaces_by_mac') + def _get_networking(self, m_get_by_mac): + m_get_by_mac.return_value = { + '04:01:57:d1:9e:01': 'ens1', '04:01:57:d1:9e:02': 'ens2', + 'b8:ae:ed:75:5f:9a': 'enp0s25', + 'ae:cc:08:7c:88:00': 'meta2p1'} netcfg = digitalocean.convert_network_configuration( DO_META['interfaces'], DO_META['dns']['nameservers']) self.assertIn('config', netcfg) @@ -302,10 +307,15 @@ class TestNetworkConvert(TestCase): self.assertEqual(ipv4_def.get('netmask'), subn_def.get('netmask')) self.assertNotIn('gateway', subn_def) - def test_convert_without_private(self): + @mock.patch('cloudinit.net.get_interfaces_by_mac') + def test_convert_without_private(self, m_get_by_mac): + m_get_by_mac.return_value = { + 'b8:ae:ed:75:5f:9a': 'enp0s25', + 'ae:cc:08:7c:88:00': 'meta2p1'} netcfg = digitalocean.convert_network_configuration( DO_META_2['interfaces'], DO_META_2['dns']['nameservers']) + # print(netcfg) byname = {} for i in netcfg['config']: if 'name' in i: diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index a266e952..bce66125 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -195,7 +195,9 @@ class TestOpenNebulaDataSource(TestCase): self.assertTrue('userdata' in results) self.assertEqual(USER_DATA, results['userdata']) - def test_hostname(self): + @mock.patch(DS_PATH + ".get_physical_nics_by_mac") + def test_hostname(self, m_get_phys_by_mac): + m_get_phys_by_mac.return_value = {'02:00:0a:12:01:01': 'eth0'} for k in ('HOSTNAME', 'PUBLIC_IP', 'IP_PUBLIC', 'ETH0_IP'): my_d = os.path.join(self.tmp, k) populate_context_dir(my_d, {k: PUBLIC_IP}) @@ -205,11 +207,14 @@ class TestOpenNebulaDataSource(TestCase): self.assertTrue('local-hostname' in results['metadata']) self.assertEqual(PUBLIC_IP, results['metadata']['local-hostname']) - def test_network_interfaces(self): + @mock.patch(DS_PATH + ".get_physical_nics_by_mac") + def test_network_interfaces(self, m_get_phys_by_mac): + m_get_phys_by_mac.return_value = {'02:00:0a:12:01:01': 'eth0'} populate_context_dir(self.seed_dir, {'ETH0_IP': '1.2.3.4'}) results = ds.read_context_disk_dir(self.seed_dir) self.assertTrue('network-interfaces' in results) + self.assertTrue('1.2.3.4' in results['network-interfaces']) def test_find_candidates(self): def my_devs_with(criteria): -- cgit v1.2.3 From bf7723e8092bb1f8a442aa2399dd870e130a27d9 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 31 Mar 2017 10:56:04 -0400 Subject: Fix bug that resulted in an attempt to rename bonds or vlans. When cloud-init ran in the init stage (after networking had come up). A bug could occur where cloud-init would attempt and fail to rename network devices that had "inherited" mac addresses. The intent of apply_network_config_names was always to rename only the devices that were "physical" per the network config. (This would include veth devices in a container). The bug was in creating the dictionary of interfaces by mac address. If there were multiple interfaces with the same mac address then renames could fail. This situation was guaranteed to occur with bonds or vlans or other devices that inherit their mac. The solution is to change get_interfaces_by_mac to skip interfaces that have an inherited mac. Also drop the 'devs' argument to get_interfaces_by_mac. It was non-obvious what the result should be if a device in the input list was filtered out. ie should the following have an entry for bond0 or not. get_interfaces_by_mac(devs=['bond0']) LP: #1669860 --- cloudinit/net/__init__.py | 78 ++++++++++++++++++++++++++++++++++----------- tests/unittests/test_net.py | 76 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+), 19 deletions(-) mode change 100755 => 100644 cloudinit/net/__init__.py (limited to 'tests') diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py old mode 100755 new mode 100644 index 1cf98ef5..346be5d3 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -82,6 +82,10 @@ def is_wireless(devname): return os.path.exists(sys_dev_path(devname, "wireless")) +def is_bridge(devname): + return os.path.exists(sys_dev_path(devname, "bridge")) + + def is_connected(devname): # is_connected isn't really as simple as that. 2 is # 'physically connected'. 3 is 'not connected'. but a wlan interface will @@ -132,7 +136,7 @@ def generate_fallback_config(): for interface in potential_interfaces: if interface.startswith("veth"): continue - if os.path.exists(sys_dev_path(interface, "bridge")): + if is_bridge(interface): # skip any bridges continue carrier = read_sys_net_int(interface, 'carrier') @@ -187,7 +191,11 @@ def apply_network_config_names(netcfg, strict_present=True, strict_busy=True): """read the network config and rename devices accordingly. if strict_present is false, then do not raise exception if no devices match. if strict_busy is false, then do not raise exception if the - device cannot be renamed because it is currently configured.""" + device cannot be renamed because it is currently configured. + + renames are only attempted for interfaces of type 'physical'. It is + expected that the network system will create other devices with the + correct name in place.""" renames = [] for ent in netcfg.get('config', {}): if ent.get('type') != 'physical': @@ -201,13 +209,35 @@ def apply_network_config_names(netcfg, strict_present=True, strict_busy=True): return _rename_interfaces(renames) +def interface_has_own_mac(ifname, strict=False): + """return True if the provided interface has its own address. + + Based on addr_assign_type in /sys. Return true for any interface + that does not have a 'stolen' address. Examples of such devices + are bonds or vlans that inherit their mac from another device. + Possible values are: + 0: permanent address 2: stolen from another device + 1: randomly generated 3: set using dev_set_mac_address""" + + assign_type = read_sys_net_int(ifname, "addr_assign_type") + if strict and assign_type is None: + raise ValueError("%s had no addr_assign_type.") + return assign_type in (0, 1, 3) + + def _get_current_rename_info(check_downable=True): - """Collect information necessary for rename_interfaces.""" - names = get_devicelist() + """Collect information necessary for rename_interfaces. + + returns a dictionary by mac address like: + {mac: + {'name': name + 'up': boolean: is_up(name), + 'downable': None or boolean indicating that the + device has only automatically assigned ip addrs.}} + """ bymac = {} - for n in names: - bymac[get_interface_mac(n)] = { - 'name': n, 'up': is_up(n), 'downable': None} + for mac, name in get_interfaces_by_mac().items(): + bymac[mac] = {'name': name, 'up': is_up(name), 'downable': None} if check_downable: nmatch = re.compile(r"[0-9]+:\s+(\w+)[@:]") @@ -346,22 +376,32 @@ def get_interface_mac(ifname): return read_sys_net_safe(ifname, path) -def get_interfaces_by_mac(devs=None): - """Build a dictionary of tuples {mac: name}""" - if devs is None: - try: - devs = get_devicelist() - except OSError as e: - if e.errno == errno.ENOENT: - devs = [] - else: - raise +def get_interfaces_by_mac(): + """Build a dictionary of tuples {mac: name}. + + Bridges and any devices that have a 'stolen' mac are excluded.""" + try: + devs = get_devicelist() + except OSError as e: + if e.errno == errno.ENOENT: + devs = [] + else: + raise ret = {} for name in devs: + if not interface_has_own_mac(name): + continue + if is_bridge(name): + continue mac = get_interface_mac(name) # some devices may not have a mac (tun0) - if mac: - ret[mac] = name + if not mac: + continue + if mac in ret: + raise RuntimeError( + "duplicate mac found! both '%s' and '%s' have mac '%s'" % + (name, ret[mac], mac)) + ret[mac] = name return ret diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index bfd04ba0..9cc5e4ab 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -1461,6 +1461,82 @@ class TestNetRenderers(CiTestCase): priority=['sysconfig', 'eni']) +class TestGetInterfacesByMac(CiTestCase): + _data = {'devices': ['enp0s1', 'enp0s2', 'bond1', 'bridge1', + 'bridge1-nic', 'tun0'], + 'bonds': ['bond1'], + 'bridges': ['bridge1'], + 'own_macs': ['enp0s1', 'enp0s2', 'bridge1-nic', 'bridge1'], + 'macs': {'enp0s1': 'aa:aa:aa:aa:aa:01', + 'enp0s2': 'aa:aa:aa:aa:aa:02', + 'bond1': 'aa:aa:aa:aa:aa:01', + 'bridge1': 'aa:aa:aa:aa:aa:03', + 'bridge1-nic': 'aa:aa:aa:aa:aa:03', + 'tun0': None}} + data = {} + + def _se_get_devicelist(self): + return self.data['devices'] + + def _se_get_interface_mac(self, name): + return self.data['macs'][name] + + def _se_is_bridge(self, name): + return name in self.data['bridges'] + + def _se_interface_has_own_mac(self, name): + return name in self.data['own_macs'] + + def _mock_setup(self): + self.data = copy.deepcopy(self._data) + mocks = ('get_devicelist', 'get_interface_mac', 'is_bridge', + 'interface_has_own_mac') + self.mocks = {} + for n in mocks: + m = mock.patch('cloudinit.net.' + n, + side_effect=getattr(self, '_se_' + n)) + self.addCleanup(m.stop) + self.mocks[n] = m.start() + + def test_raise_exception_on_duplicate_macs(self): + self._mock_setup() + self.data['macs']['bridge1-nic'] = self.data['macs']['enp0s1'] + self.assertRaises(RuntimeError, net.get_interfaces_by_mac) + + def test_excludes_any_without_mac_address(self): + self._mock_setup() + ret = net.get_interfaces_by_mac() + self.assertIn('tun0', self._se_get_devicelist()) + self.assertNotIn('tun0', ret.values()) + + def test_excludes_stolen_macs(self): + self._mock_setup() + ret = net.get_interfaces_by_mac() + self.mocks['interface_has_own_mac'].assert_has_calls( + [mock.call('enp0s1'), mock.call('bond1')], any_order=True) + self.assertEqual( + {'aa:aa:aa:aa:aa:01': 'enp0s1', 'aa:aa:aa:aa:aa:02': 'enp0s2', + 'aa:aa:aa:aa:aa:03': 'bridge1-nic'}, + ret) + + def test_excludes_bridges(self): + self._mock_setup() + # add a device 'b1', make all return they have their "own mac", + # set everything other than 'b1' to be a bridge. + # then expect b1 is the only thing left. + self.data['macs']['b1'] = 'aa:aa:aa:aa:aa:b1' + self.data['devices'].append('b1') + self.data['bonds'] = [] + self.data['own_macs'] = self.data['devices'] + self.data['bridges'] = [f for f in self.data['devices'] if f != "b1"] + ret = net.get_interfaces_by_mac() + self.assertEqual({'aa:aa:aa:aa:aa:b1': 'b1'}, ret) + self.mocks['is_bridge'].assert_has_calls( + [mock.call('bridge1'), mock.call('enp0s1'), mock.call('bond1'), + mock.call('b1')], + any_order=True) + + def _gzip_data(data): with io.BytesIO() as iobuf: gzfp = gzip.GzipFile(mode="wb", fileobj=iobuf) -- cgit v1.2.3