From 1d751a6f46f044e3c3827f3cef0e4a2e71d50fe7 Mon Sep 17 00:00:00 2001 From: Lars Kellogg-Stedman Date: Thu, 2 Mar 2017 11:08:26 -0500 Subject: net: support both ipv4 and ipv6 gateways in sysconfig. Previously, cloud-init would throw an exception if an interface had both ipv4 and ipv6 addresses and a default gateway for each address family. This change allows cloud-init to correctly configure interfaces in this situation. LP: #1669504 --- cloudinit/net/sysconfig.py | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) (limited to 'cloudinit/net') diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index 6e7739fb..06de660f 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -87,7 +87,8 @@ class Route(ConfigMap): def __init__(self, route_name, base_sysconf_dir): super(Route, self).__init__() self.last_idx = 1 - self.has_set_default = False + self.has_set_default_ipv4 = False + self.has_set_default_ipv6 = False self._route_name = route_name self._base_sysconf_dir = base_sysconf_dir @@ -95,7 +96,8 @@ class Route(ConfigMap): r = Route(self._route_name, self._base_sysconf_dir) r._conf = self._conf.copy() r.last_idx = self.last_idx - r.has_set_default = self.has_set_default + r.has_set_default_ipv4 = self.has_set_default_ipv4 + r.has_set_default_ipv6 = self.has_set_default_ipv6 return r @property @@ -119,10 +121,10 @@ class NetInterface(ConfigMap): super(NetInterface, self).__init__() self.children = [] self.routes = Route(iface_name, base_sysconf_dir) - self._kind = kind + self.kind = kind + self._iface_name = iface_name self._conf['DEVICE'] = iface_name - self._conf['TYPE'] = self.iface_types[kind] self._base_sysconf_dir = base_sysconf_dir @property @@ -140,6 +142,8 @@ class NetInterface(ConfigMap): @kind.setter def kind(self, kind): + if kind not in self.iface_types: + raise ValueError(kind) self._kind = kind self._conf['TYPE'] = self.iface_types[kind] @@ -173,7 +177,7 @@ class Renderer(renderer.Renderer): ('BOOTPROTO', 'none'), ]) - # If these keys exist, then there values will be used to form + # If these keys exist, then their values will be used to form # a BONDING_OPTS grouping; otherwise no grouping will be set. bond_tpl_opts = tuple([ ('bond_mode', "mode=%s"), @@ -199,6 +203,7 @@ class Renderer(renderer.Renderer): def _render_iface_shared(cls, iface, iface_cfg): for k, v in cls.iface_defaults: iface_cfg[k] = v + for (old_key, new_key) in [('mac_address', 'HWADDR'), ('mtu', 'MTU')]: old_value = iface.get(old_key) if old_value is not None: @@ -227,10 +232,20 @@ class Renderer(renderer.Renderer): if 'netmask' in subnet: iface_cfg['NETMASK'] = subnet['netmask'] for route in subnet.get('routes', []): + if subnet.get('ipv6'): + gw_cfg = 'IPV6_DEFAULTGW' + else: + gw_cfg = 'GATEWAY' + if _is_default_route(route): - if route_cfg.has_set_default: - raise ValueError("Duplicate declaration of default" - " route found for interface '%s'" + if ( + (subnet.get('ipv4') and + route_cfg.has_set_default_ipv4) or + (subnet.get('ipv6') and + route_cfg.has_set_default_ipv6) + ): + raise ValueError("Duplicate declaration of default " + "route found for interface '%s'" % (iface_cfg.name)) # NOTE(harlowja): ipv6 and ipv4 default gateways gw_key = 'GATEWAY0' @@ -242,7 +257,7 @@ class Renderer(renderer.Renderer): # also provided the default route? iface_cfg['DEFROUTE'] = True if 'gateway' in route: - iface_cfg['GATEWAY'] = route['gateway'] + iface_cfg[gw_cfg] = route['gateway'] route_cfg.has_set_default = True else: gw_key = 'GATEWAY%s' % route_cfg.last_idx -- cgit v1.2.3 From 1a2ca7530518d819cbab7287b12f942743427e38 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 15 Mar 2017 12:06:40 -0400 Subject: support 'loopback' as a device type. As reported in bug 1671927, sysconfig had an issue with rendering a loopback device. The problem was that some as yet unknown issue was causing the openstack config drive to parse the provided ENI file rather than reading the network_data.json. Parsing an ENI file would add a a 'lo' device of type 'physical', and sysconfig was failing to render that. The change here is: a.) add a 'loopback' type rather than 'physical' for network config. {'name': 'lo', 'type': 'loopback', 'subnets': ['type': 'loopback']} b.) support skipping that type in the eni and sysconfig renderers. c.) make network_state just piggy back on 'physical' renderer for loopback (this was what was happening before). Tests are added for eni and sysconfig renderer. --- cloudinit/net/eni.py | 16 +++++++++------ cloudinit/net/network_state.py | 4 ++++ cloudinit/net/sysconfig.py | 2 ++ tests/unittests/test_net.py | 44 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 6 deletions(-) (limited to 'cloudinit/net') diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index 5b249f1f..69ecbb5d 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -273,8 +273,11 @@ def _ifaces_to_net_config_data(ifaces): # devname is 'eth0' for name='eth0:1' devname = name.partition(":")[0] if devname not in devs: - devs[devname] = {'type': 'physical', 'name': devname, - 'subnets': []} + if devname == "lo": + dtype = "loopback" + else: + dtype = "physical" + devs[devname] = {'type': dtype, 'name': devname, 'subnets': []} # this isnt strictly correct, but some might specify # hwaddress on a nic for matching / declaring name. if 'hwaddress' in data: @@ -423,10 +426,11 @@ class Renderer(renderer.Renderer): bonding ''' order = { - 'physical': 0, - 'bond': 1, - 'bridge': 2, - 'vlan': 3, + 'loopback': 0, + 'physical': 1, + 'bond': 2, + 'bridge': 3, + 'vlan': 4, } sections = [] diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 11ef585b..90b2835a 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -211,6 +211,10 @@ class NetworkStateInterpreter(object): exc_info=True) LOG.debug(self.dump_network_state()) + @ensure_command_keys(['name']) + def handle_loopback(self, command): + return self.handle_physical(command) + @ensure_command_keys(['name']) def handle_physical(self, command): ''' diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index 06de660f..7f52db4a 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -368,6 +368,8 @@ class Renderer(renderer.Renderer): '''Given state, return /etc/sysconfig files + contents''' iface_contents = {} for iface in network_state.iter_interfaces(): + if iface['type'] == "loopback": + continue iface_name = iface['name'] iface_cfg = NetInterface(iface_name, base_sysconf_dir) cls._render_iface_shared(iface, iface_cfg) diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 8d253109..c6535e1c 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -637,6 +637,14 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true } } +CONFIG_V1_EXPLICIT_LOOPBACK = { + 'version': 1, + 'config': [{'name': 'eth0', 'type': 'physical', + 'subnets': [{'control': 'auto', 'type': 'dhcp'}]}, + {'name': 'lo', 'type': 'loopback', + 'subnets': [{'control': 'auto', 'type': 'loopback'}]}, + ]} + def _setup_test(tmp_dir, mock_get_devicelist, mock_read_sys_net, mock_sys_dev_path): @@ -722,6 +730,27 @@ USERCTL=no with open(os.path.join(render_dir, fn)) as fh: self.assertEqual(expected_content, fh.read()) + def test_config_with_explicit_loopback(self): + ns = network_state.parse_net_config_data(CONFIG_V1_EXPLICIT_LOOPBACK) + render_dir = self.tmp_path("render") + os.makedirs(render_dir) + renderer = sysconfig.Renderer() + renderer.render_network_state(render_dir, ns) + found = dir2dict(render_dir) + nspath = '/etc/sysconfig/network-scripts/' + self.assertNotIn(nspath + 'ifcfg-lo', found.keys()) + expected = """\ +# Created by cloud-init on instance boot automatically, do not edit. +# +BOOTPROTO=dhcp +DEVICE=eth0 +NM_CONTROLLED=no +ONBOOT=yes +TYPE=Ethernet +USERCTL=no +""" + self.assertEqual(expected, found[nspath + 'ifcfg-eth0']) + class TestEniNetRendering(CiTestCase): @@ -762,6 +791,21 @@ iface eth1000 inet dhcp """ self.assertEqual(expected.lstrip(), contents.lstrip()) + def test_config_with_explicit_loopback(self): + tmp_dir = self.tmp_dir() + ns = network_state.parse_net_config_data(CONFIG_V1_EXPLICIT_LOOPBACK) + renderer = eni.Renderer() + renderer.render_network_state(tmp_dir, ns) + expected = """\ +auto lo +iface lo inet loopback + +auto eth0 +iface eth0 inet dhcp +""" + self.assertEqual( + expected, dir2dict(tmp_dir)['/etc/network/interfaces']) + class TestEniNetworkStateToEni(CiTestCase): mycfg = { -- cgit v1.2.3 From a33447344eed897010603b3e8ea1fd122052de76 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 16 Mar 2017 14:30:15 -0400 Subject: render_network_state: switch arguments around, do not require target render_network_state should default to rendering on /. The changes here just make it so render_network_state does not require a target, but defaults to None, and uses target_path to handle that. --- cloudinit/net/eni.py | 8 ++++---- cloudinit/net/sysconfig.py | 8 ++++---- tests/unittests/test_datasource/test_configdrive.py | 7 ++++--- tests/unittests/test_net.py | 12 ++++++------ 4 files changed, 18 insertions(+), 17 deletions(-) (limited to 'cloudinit/net') diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index 69ecbb5d..efa034bf 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -448,14 +448,14 @@ class Renderer(renderer.Renderer): return '\n\n'.join(['\n'.join(s) for s in sections]) + "\n" - def render_network_state(self, target, network_state): - fpeni = os.path.join(target, self.eni_path) + def render_network_state(self, network_state, target=None): + fpeni = util.target_path(target, self.eni_path) util.ensure_dir(os.path.dirname(fpeni)) header = self.eni_header if self.eni_header else "" util.write_file(fpeni, header + self._render_interfaces(network_state)) if self.netrules_path: - netrules = os.path.join(target, self.netrules_path) + netrules = util.target_path(target, self.netrules_path) util.ensure_dir(os.path.dirname(netrules)) util.write_file(netrules, self._render_persistent_net(network_state)) @@ -465,7 +465,7 @@ class Renderer(renderer.Renderer): links_prefix=self.links_path_prefix) def _render_systemd_links(self, target, network_state, links_prefix): - fp_prefix = os.path.join(target, links_prefix) + fp_prefix = util.target_path(target, links_prefix) for f in glob.glob(fp_prefix + "*"): os.unlink(f) for iface in network_state.iter_interfaces(): diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index 7f52db4a..4eeaaa8a 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -389,19 +389,19 @@ class Renderer(renderer.Renderer): contents[iface_cfg.routes.path] = iface_cfg.routes.to_string() return contents - def render_network_state(self, target, network_state): - base_sysconf_dir = os.path.join(target, self.sysconf_dir) + def render_network_state(self, network_state, target=None): + base_sysconf_dir = util.target_path(target, self.sysconf_dir) for path, data in self._render_sysconfig(base_sysconf_dir, network_state).items(): util.write_file(path, data) if self.dns_path: - dns_path = os.path.join(target, self.dns_path) + dns_path = util.target_path(target, self.dns_path) resolv_content = self._render_dns(network_state, existing_dns_path=dns_path) util.write_file(dns_path, resolv_content) if self.netrules_path: netrules_content = self._render_persistent_net(network_state) - netrules_path = os.path.join(target, self.netrules_path) + netrules_path = util.target_path(target, self.netrules_path) util.write_file(netrules_path, netrules_content) # vi: ts=4 expandtab diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index 55153357..337be667 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -645,7 +645,7 @@ class TestConvertNetworkData(TestCase): routes) eni_renderer = eni.Renderer() eni_renderer.render_network_state( - self.tmp, network_state.parse_net_config_data(ncfg)) + network_state.parse_net_config_data(ncfg), self.tmp) with open(os.path.join(self.tmp, "etc", "network", "interfaces"), 'r') as f: eni_rendering = f.read() @@ -665,8 +665,9 @@ class TestConvertNetworkData(TestCase): ncfg = openstack.convert_net_json(NETWORK_DATA_BOND, known_macs=KNOWN_MACS) eni_renderer = eni.Renderer() + eni_renderer.render_network_state( - self.tmp, network_state.parse_net_config_data(ncfg)) + network_state.parse_net_config_data(ncfg), self.tmp) with open(os.path.join(self.tmp, "etc", "network", "interfaces"), 'r') as f: eni_rendering = f.read() @@ -697,7 +698,7 @@ class TestConvertNetworkData(TestCase): known_macs=KNOWN_MACS) eni_renderer = eni.Renderer() eni_renderer.render_network_state( - self.tmp, network_state.parse_net_config_data(ncfg)) + network_state.parse_net_config_data(ncfg), self.tmp) with open(os.path.join(self.tmp, "etc", "network", "interfaces"), 'r') as f: eni_rendering = f.read() diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index c6535e1c..dca44b37 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -697,7 +697,7 @@ class TestSysConfigRendering(CiTestCase): os.makedirs(render_dir) renderer = sysconfig.Renderer() - renderer.render_network_state(render_dir, ns) + renderer.render_network_state(ns, render_dir) render_file = 'etc/sysconfig/network-scripts/ifcfg-eth1000' with open(os.path.join(render_dir, render_file)) as fh: @@ -725,7 +725,7 @@ USERCTL=no ns = network_state.parse_net_config_data(network_cfg, skip_broken=False) renderer = sysconfig.Renderer() - renderer.render_network_state(render_dir, ns) + renderer.render_network_state(ns, render_dir) for fn, expected_content in os_sample.get('out_sysconfig', []): with open(os.path.join(render_dir, fn)) as fh: self.assertEqual(expected_content, fh.read()) @@ -735,7 +735,7 @@ USERCTL=no render_dir = self.tmp_path("render") os.makedirs(render_dir) renderer = sysconfig.Renderer() - renderer.render_network_state(render_dir, ns) + renderer.render_network_state(ns, render_dir) found = dir2dict(render_dir) nspath = '/etc/sysconfig/network-scripts/' self.assertNotIn(nspath + 'ifcfg-lo', found.keys()) @@ -775,7 +775,7 @@ class TestEniNetRendering(CiTestCase): {'links_path_prefix': None, 'eni_path': 'interfaces', 'netrules_path': None, }) - renderer.render_network_state(render_dir, ns) + renderer.render_network_state(ns, render_dir) self.assertTrue(os.path.exists(os.path.join(render_dir, 'interfaces'))) @@ -795,7 +795,7 @@ iface eth1000 inet dhcp tmp_dir = self.tmp_dir() ns = network_state.parse_net_config_data(CONFIG_V1_EXPLICIT_LOOPBACK) renderer = eni.Renderer() - renderer.render_network_state(tmp_dir, ns) + renderer.render_network_state(ns, tmp_dir) expected = """\ auto lo iface lo inet loopback @@ -972,7 +972,7 @@ class TestEniRoundTrip(CiTestCase): config={'eni_path': eni_path, 'links_path_prefix': links_prefix, 'netrules_path': netrules_path}) - renderer.render_network_state(dir, ns) + renderer.render_network_state(ns, dir) return dir2dict(dir) def testsimple_convert_and_render(self): -- cgit v1.2.3 From 5beecdf88b630a397b3722ddb299e9a37ff02737 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 15 Mar 2017 14:33:45 -0400 Subject: net: add renderers for automatically selecting the renderer. Previously, the distro had hard coded which network renderer it would use. This adds support for just picking the right renderer based on what is available. Now, that can be set via a priority in system_info, but should generally work. That config looks like: system_info: network: renderers: ["eni", "sysconfig"] When no renderers are found, a specific RendererNotFoundError is raised. stages.py is modified to catch that and log it at error level. This path should not really be exercised, but could occur if for example an Ubuntu system did not have ifupdown, or a rhel system did not have sysconfig. In such a system previously we would have quietly rendered ENI configuration but that would have been ignored. This is one step better in that we at least log the error. --- cloudinit/distros/__init__.py | 13 +++++++++++ cloudinit/distros/debian.py | 23 +++++++++---------- cloudinit/distros/rhel.py | 7 +----- cloudinit/net/__init__.py | 5 +++++ cloudinit/net/eni.py | 14 ++++++++++++ cloudinit/net/renderer.py | 5 +++++ cloudinit/net/renderers.py | 51 +++++++++++++++++++++++++++++++++++++++++++ cloudinit/net/sysconfig.py | 17 +++++++++++++++ cloudinit/settings.py | 1 + cloudinit/stages.py | 6 ++++- cloudinit/util.py | 43 ++++++++++++++++++++++++------------ tests/unittests/test_net.py | 45 ++++++++++++++++++++++++++++++++++++++ 12 files changed, 196 insertions(+), 34 deletions(-) create mode 100644 cloudinit/net/renderers.py (limited to 'cloudinit/net') diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index f3d395b9..803ac74e 100755 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -22,6 +22,7 @@ from cloudinit import log as logging from cloudinit import net from cloudinit.net import eni from cloudinit.net import network_state +from cloudinit.net import renderers from cloudinit import ssh_util from cloudinit import type_utils from cloudinit import util @@ -50,6 +51,7 @@ class Distro(object): hostname_conf_fn = "/etc/hostname" tz_zone_dir = "/usr/share/zoneinfo" init_cmd = ['service'] # systemctl, service etc + renderer_configs = {} def __init__(self, name, cfg, paths): self._paths = paths @@ -69,6 +71,17 @@ class Distro(object): def _write_network_config(self, settings): raise NotImplementedError() + def _supported_write_network_config(self, network_config): + priority = util.get_cfg_by_path( + self._cfg, ('network', 'renderers'), None) + + name, render_cls = renderers.select(priority=priority) + LOG.debug("Selected renderer '%s' from priority list: %s", + name, priority) + renderer = render_cls(config=self.renderer_configs.get(name)) + renderer.render_network_config(network_config=network_config) + return [] + def _find_tz_file(self, tz): tz_file = os.path.join(self.tz_zone_dir, str(tz)) if not os.path.isfile(tz_file): diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 48ccec8c..1101f02d 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -13,8 +13,6 @@ import os from cloudinit import distros from cloudinit import helpers from cloudinit import log as logging -from cloudinit.net import eni -from cloudinit.net.network_state import parse_net_config_data from cloudinit import util from cloudinit.distros.parsers.hostname import HostnameConf @@ -38,11 +36,18 @@ ENI_HEADER = """# This file is generated from information provided by # network: {config: disabled} """ +NETWORK_CONF_FN = "/etc/network/interfaces.d/50-cloud-init.cfg" + class Distro(distros.Distro): hostname_conf_fn = "/etc/hostname" locale_conf_fn = "/etc/default/locale" - network_conf_fn = "/etc/network/interfaces.d/50-cloud-init.cfg" + renderer_configs = { + 'eni': { + 'eni_path': NETWORK_CONF_FN, + 'eni_header': ENI_HEADER, + } + } def __init__(self, name, cfg, paths): distros.Distro.__init__(self, name, cfg, paths) @@ -51,12 +56,6 @@ class Distro(distros.Distro): # should only happen say once per instance...) self._runner = helpers.Runners(paths) self.osfamily = 'debian' - self._net_renderer = eni.Renderer({ - 'eni_path': self.network_conf_fn, - 'eni_header': ENI_HEADER, - 'links_path_prefix': None, - 'netrules_path': None, - }) def apply_locale(self, locale, out_fn=None): if not out_fn: @@ -76,14 +75,12 @@ class Distro(distros.Distro): self.package_command('install', pkgs=pkglist) def _write_network(self, settings): - util.write_file(self.network_conf_fn, settings) + util.write_file(NETWORK_CONF_FN, settings) return ['all'] def _write_network_config(self, netconfig): - ns = parse_net_config_data(netconfig) - self._net_renderer.render_network_state("/", ns) _maybe_remove_legacy_eth0() - return [] + return self._supported_write_network_config(netconfig) def _bring_up_interfaces(self, device_names): use_all = False diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py index 7498c63a..372c7d0f 100644 --- a/cloudinit/distros/rhel.py +++ b/cloudinit/distros/rhel.py @@ -11,8 +11,6 @@ from cloudinit import distros from cloudinit import helpers from cloudinit import log as logging -from cloudinit.net.network_state import parse_net_config_data -from cloudinit.net import sysconfig from cloudinit import util from cloudinit.distros import net_util @@ -49,16 +47,13 @@ class Distro(distros.Distro): # should only happen say once per instance...) self._runner = helpers.Runners(paths) self.osfamily = 'redhat' - self._net_renderer = sysconfig.Renderer() cfg['ssh_svcname'] = 'sshd' def install_packages(self, pkglist): self.package_command('install', pkgs=pkglist) def _write_network_config(self, netconfig): - ns = parse_net_config_data(netconfig) - self._net_renderer.render_network_state("/", ns) - return [] + return self._supported_write_network_config(netconfig) def _write_network(self, settings): # TODO(harlowja) fix this... since this is the ubuntu format diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index ea649cc2..1cf98ef5 100755 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -364,4 +364,9 @@ def get_interfaces_by_mac(devs=None): ret[mac] = name return ret + +class RendererNotFoundError(RuntimeError): + pass + + # vi: ts=4 expandtab diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index efa034bf..9d39a2b0 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -500,4 +500,18 @@ def network_state_to_eni(network_state, header=None, render_hwaddress=False): network_state, render_hwaddress=render_hwaddress) return header + contents + +def available(target=None): + expected = ['ifquery', 'ifup', 'ifdown'] + search = ['/sbin', '/usr/sbin'] + for p in expected: + if not util.which(p, search=search, target=target): + return False + eni = util.target_path(target, 'etc/network/interfaces') + if not os.path.is_file(eni): + return False + + return True + + # vi: ts=4 expandtab diff --git a/cloudinit/net/renderer.py b/cloudinit/net/renderer.py index 3a192436..a5b2b573 100644 --- a/cloudinit/net/renderer.py +++ b/cloudinit/net/renderer.py @@ -7,6 +7,7 @@ import six +from .network_state import parse_net_config_data from .udev import generate_udev_rule @@ -36,4 +37,8 @@ class Renderer(object): iface['mac_address'])) return content.getvalue() + def render_network_config(self, network_config, target=None): + return self.render_network_state( + network_state=parse_net_config_data(network_config), target=target) + # vi: ts=4 expandtab diff --git a/cloudinit/net/renderers.py b/cloudinit/net/renderers.py new file mode 100644 index 00000000..5ad84553 --- /dev/null +++ b/cloudinit/net/renderers.py @@ -0,0 +1,51 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from . import eni +from . import RendererNotFoundError +from . import sysconfig + +NAME_TO_RENDERER = { + "eni": eni, + "sysconfig": sysconfig, +} + +DEFAULT_PRIORITY = ["eni", "sysconfig"] + + +def search(priority=None, target=None, first=False): + if priority is None: + priority = DEFAULT_PRIORITY + + available = NAME_TO_RENDERER + + unknown = [i for i in priority if i not in available] + if unknown: + raise ValueError( + "Unknown renderers provided in priority list: %s" % unknown) + + found = [] + for name in priority: + render_mod = available[name] + if render_mod.available(target): + cur = (name, render_mod.Renderer) + if first: + return cur + found.append(cur) + + return found + + +def select(priority=None, target=None): + found = search(priority, target=target, first=True) + if not found: + if priority is None: + priority = DEFAULT_PRIORITY + tmsg = "" + if target and target != "/": + tmsg = " in target=%s" % target + raise RendererNotFoundError( + "No available network renderers found%s. Searched " + "through list: %s" % (tmsg, priority)) + return found + +# vi: ts=4 expandtab diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index 4eeaaa8a..117b515c 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -404,4 +404,21 @@ class Renderer(renderer.Renderer): netrules_path = util.target_path(target, self.netrules_path) util.write_file(netrules_path, netrules_content) + +def available(target=None): + expected = ['ifup', 'ifdown'] + search = ['/sbin', '/usr/sbin'] + for p in expected: + if not util.which(p, search=search, target=target): + return False + + expected_paths = [ + 'etc/sysconfig/network-scripts/network-functions', + 'etc/sysconfig/network-scripts/ifdown-eth'] + for p in expected_paths: + if not os.path.isfile(util.target_path(target, p)): + return False + return True + + # vi: ts=4 expandtab diff --git a/cloudinit/settings.py b/cloudinit/settings.py index 692ff5e5..dbafead5 100644 --- a/cloudinit/settings.py +++ b/cloudinit/settings.py @@ -46,6 +46,7 @@ CFG_BUILTIN = { 'templates_dir': '/etc/cloud/templates/', }, 'distro': 'ubuntu', + 'network': {'renderers': None}, }, 'vendor_data': {'enabled': True, 'prefix': []}, } diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 5bed9032..12165433 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -646,9 +646,13 @@ class Init(object): src, bring_up, netcfg) try: return self.distro.apply_network_config(netcfg, bring_up=bring_up) + except net.RendererNotFoundError as e: + LOG.error("Unable to render networking. Network config is " + "likely broken: %s", e) + return except NotImplementedError: LOG.warn("distro '%s' does not implement apply_network_config. " - "networking may not be configured properly." % + "networking may not be configured properly.", self.distro) return diff --git a/cloudinit/util.py b/cloudinit/util.py index 7196a7ca..82f2f76b 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2099,21 +2099,36 @@ def get_mount_info(path, log=LOG): return parse_mount(path) -def which(program): - # Return path of program for execution if found in path - def is_exe(fpath): - return os.path.isfile(fpath) and os.access(fpath, os.X_OK) - - _fpath, _ = os.path.split(program) - if _fpath: - if is_exe(program): +def is_exe(fpath): + # return boolean indicating if fpath exists and is executable. + return os.path.isfile(fpath) and os.access(fpath, os.X_OK) + + +def which(program, search=None, target=None): + target = target_path(target) + + if os.path.sep in program: + # if program had a '/' in it, then do not search PATH + # 'which' does consider cwd here. (cd / && which bin/ls) = bin/ls + # so effectively we set cwd to / (or target) + if is_exe(target_path(target, program)): return program - else: - for path in os.environ.get("PATH", "").split(os.pathsep): - path = path.strip('"') - exe_file = os.path.join(path, program) - if is_exe(exe_file): - return exe_file + + if search is None: + paths = [p.strip('"') for p in + os.environ.get("PATH", "").split(os.pathsep)] + if target == "/": + search = paths + else: + search = [p for p in paths if p.startswith("/")] + + # normalize path input + search = [os.path.abspath(p) for p in search] + + for path in search: + ppath = os.path.sep.join((path, program)) + if is_exe(target_path(target, ppath)): + return ppath return None diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index dca44b37..902204a0 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -4,6 +4,7 @@ from cloudinit import net from cloudinit.net import cmdline from cloudinit.net import eni from cloudinit.net import network_state +from cloudinit.net import renderers from cloudinit.net import sysconfig from cloudinit.sources.helpers import openstack from cloudinit import util @@ -1050,6 +1051,50 @@ class TestEniRoundTrip(CiTestCase): expected, [line for line in found if line]) +class TestNetRenderers(CiTestCase): + @mock.patch("cloudinit.net.renderers.sysconfig.available") + @mock.patch("cloudinit.net.renderers.eni.available") + def test_eni_and_sysconfig_available(self, m_eni_avail, m_sysc_avail): + m_eni_avail.return_value = True + m_sysc_avail.return_value = True + found = renderers.search(priority=['sysconfig', 'eni'], first=False) + names = [f[0] for f in found] + self.assertEqual(['sysconfig', 'eni'], names) + + @mock.patch("cloudinit.net.renderers.eni.available") + def test_search_returns_empty_on_none(self, m_eni_avail): + m_eni_avail.return_value = False + found = renderers.search(priority=['eni'], first=False) + self.assertEqual([], found) + + @mock.patch("cloudinit.net.renderers.sysconfig.available") + @mock.patch("cloudinit.net.renderers.eni.available") + def test_first_in_priority(self, m_eni_avail, m_sysc_avail): + # available should only be called until one is found. + m_eni_avail.return_value = True + m_sysc_avail.side_effect = Exception("Should not call me") + found = renderers.search(priority=['eni', 'sysconfig'], first=True) + self.assertEqual(['eni'], [found[0]]) + + @mock.patch("cloudinit.net.renderers.sysconfig.available") + @mock.patch("cloudinit.net.renderers.eni.available") + def test_select_positive(self, m_eni_avail, m_sysc_avail): + m_eni_avail.return_value = True + m_sysc_avail.return_value = False + found = renderers.select(priority=['sysconfig', 'eni']) + self.assertEqual('eni', found[0]) + + @mock.patch("cloudinit.net.renderers.sysconfig.available") + @mock.patch("cloudinit.net.renderers.eni.available") + def test_select_none_found_raises(self, m_eni_avail, m_sysc_avail): + # if select finds nothing, should raise exception. + m_eni_avail.return_value = False + m_sysc_avail.return_value = False + + self.assertRaises(net.RendererNotFoundError, renderers.select, + priority=['sysconfig', 'eni']) + + def _gzip_data(data): with io.BytesIO() as iobuf: gzfp = gzip.GzipFile(mode="wb", fileobj=iobuf) -- cgit v1.2.3 From 9040e78feb7c1bcf3a1dab0ee163efaa0d21612c Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Mon, 20 Mar 2017 12:18:48 -0600 Subject: net: Fix incorrect call to isfile Previous commit introduced a regression by calling os.path.is_file, a non-existent function. This changes that call to use os.path.isfile. LP: #1674317 --- cloudinit/net/eni.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/net') diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index 9d39a2b0..f471e05f 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -508,7 +508,7 @@ def available(target=None): if not util.which(p, search=search, target=target): return False eni = util.target_path(target, 'etc/network/interfaces') - if not os.path.is_file(eni): + if not os.path.isfile(eni): return False return True -- cgit v1.2.3 From ef18b8ac4cf7e3dfd98830fbdb298380a192a0fc Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Sun, 19 Mar 2017 08:39:01 -0500 Subject: cloudinit.net: add network config v2 parsing and rendering Network configuration version 2 format is implemented in a package called netplan (nplan)[1] which allows consolidated network config for multiple network controllers. - Add a new netplan renderer - Update default policy, placing eni and sysconfig first This requires explicit policy to enable netplan over eni on systems which have both (Yakkety, Zesty, UC16) - Allow any network state (parsed from any format cloud-init supports) to render to v2 if system supports netplan. - Move eni's _subnet_is_ipv6 to common code for use by other renderers - Make sysconfig renderer always emit /etc/syconfig/network configuration - Update cloud-init.service systemd unit to also wait on systemd-networkd-wait-online.service 1. https://lists.ubuntu.com/archives/ubuntu-devel/2016-July/039464.html --- cloudinit/distros/debian.py | 16 +- cloudinit/net/eni.py | 15 +- cloudinit/net/netplan.py | 373 +++++++++++++++++++++++++ cloudinit/net/network_state.py | 312 ++++++++++++++++++++- cloudinit/net/renderers.py | 4 +- cloudinit/net/sysconfig.py | 20 +- cloudinit/util.py | 9 + systemd/cloud-init.service | 1 + tests/unittests/test_distros/test_netconfig.py | 351 ++++++++++++++++++++++- tests/unittests/test_net.py | 301 ++++++++++++++++++++ tools/net-convert.py | 84 ++++++ 11 files changed, 1450 insertions(+), 36 deletions(-) create mode 100644 cloudinit/net/netplan.py create mode 100755 tools/net-convert.py (limited to 'cloudinit/net') diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 1101f02d..3f0f9d53 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -42,11 +42,16 @@ NETWORK_CONF_FN = "/etc/network/interfaces.d/50-cloud-init.cfg" class Distro(distros.Distro): hostname_conf_fn = "/etc/hostname" locale_conf_fn = "/etc/default/locale" + network_conf_fn = { + "eni": "/etc/network/interfaces.d/50-cloud-init.cfg", + "netplan": "/etc/netplan/50-cloud-init.yaml" + } renderer_configs = { - 'eni': { - 'eni_path': NETWORK_CONF_FN, - 'eni_header': ENI_HEADER, - } + "eni": {"eni_path": network_conf_fn["eni"], + "eni_header": ENI_HEADER}, + "netplan": {"netplan_path": network_conf_fn["netplan"], + "netplan_header": ENI_HEADER, + "postcmds": True} } def __init__(self, name, cfg, paths): @@ -75,7 +80,8 @@ class Distro(distros.Distro): self.package_command('install', pkgs=pkglist) def _write_network(self, settings): - util.write_file(NETWORK_CONF_FN, settings) + # this is a legacy method, it will always write eni + util.write_file(self.network_conf_fn["eni"], settings) return ['all'] def _write_network_config(self, netconfig): diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index f471e05f..9819d4f5 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -8,6 +8,7 @@ import re from . import ParserError from . import renderer +from .network_state import subnet_is_ipv6 from cloudinit import util @@ -111,16 +112,6 @@ def _iface_start_entry(iface, index, render_hwaddress=False): return lines -def _subnet_is_ipv6(subnet): - # 'static6' or 'dhcp6' - if subnet['type'].endswith('6'): - # This is a request for DHCPv6. - return True - elif subnet['type'] == 'static' and ":" in subnet['address']: - return True - return False - - def _parse_deb_config_data(ifaces, contents, src_dir, src_path): """Parses the file contents, placing result into ifaces. @@ -370,7 +361,7 @@ class Renderer(renderer.Renderer): iface['mode'] = subnet['type'] iface['control'] = subnet.get('control', 'auto') subnet_inet = 'inet' - if _subnet_is_ipv6(subnet): + if subnet_is_ipv6(subnet): subnet_inet += '6' iface['inet'] = subnet_inet if subnet['type'].startswith('dhcp'): @@ -486,7 +477,7 @@ class Renderer(renderer.Renderer): def network_state_to_eni(network_state, header=None, render_hwaddress=False): # render the provided network state, return a string of equivalent eni eni_path = 'etc/network/interfaces' - renderer = Renderer({ + renderer = Renderer(config={ 'eni_path': eni_path, 'eni_header': header, 'links_path_prefix': None, diff --git a/cloudinit/net/netplan.py b/cloudinit/net/netplan.py new file mode 100644 index 00000000..cd93b21c --- /dev/null +++ b/cloudinit/net/netplan.py @@ -0,0 +1,373 @@ +# This file is part of cloud-init. See LICENSE file ... + +import copy +import os + +from . import renderer +from .network_state import subnet_is_ipv6 + +from cloudinit import util +from cloudinit.net import SYS_CLASS_NET, get_devicelist + + +NET_CONFIG_TO_V2 = { + 'bond': {'bond-ad-select': 'ad-select', + 'bond-arp-interval': 'arp-interval', + 'bond-arp-ip-target': 'arp-ip-target', + 'bond-arp-validate': 'arp-validate', + 'bond-downdelay': 'down-delay', + 'bond-fail-over-mac': 'fail-over-mac-policy', + 'bond-lacp-rate': 'lacp-rate', + 'bond-miimon': 'mii-monitor-interval', + 'bond-min-links': 'min-links', + 'bond-mode': 'mode', + 'bond-num-grat-arp': 'gratuitious-arp', + 'bond-primary-reselect': 'primary-reselect-policy', + 'bond-updelay': 'up-delay', + 'bond-xmit_hash_policy': 'transmit_hash_policy'}, + 'bridge': {'bridge_ageing': 'ageing-time', + 'bridge_bridgeprio': 'priority', + 'bridge_fd': 'forward-delay', + 'bridge_gcint': None, + 'bridge_hello': 'hello-time', + 'bridge_maxage': 'max-age', + 'bridge_maxwait': None, + 'bridge_pathcost': 'path-cost', + 'bridge_portprio': None, + 'bridge_waitport': None}} + + +def _get_params_dict_by_match(config, match): + return dict((key, value) for (key, value) in config.items() + if key.startswith(match)) + + +def _extract_addresses(config, entry): + """This method parse a cloudinit.net.network_state dictionary (config) and + maps netstate keys/values into a dictionary (entry) to represent + netplan yaml. + + An example config dictionary might look like: + + {'mac_address': '52:54:00:12:34:00', + 'name': 'interface0', + 'subnets': [ + {'address': '192.168.1.2/24', + 'mtu': 1501, + 'type': 'static'}, + {'address': '2001:4800:78ff:1b:be76:4eff:fe06:1000", + 'mtu': 1480, + 'netmask': 64, + 'type': 'static'}], + 'type: physical' + } + + An entry dictionary looks like: + + {'set-name': 'interface0', + 'match': {'macaddress': '52:54:00:12:34:00'}, + 'mtu': 1501} + + After modification returns + + {'set-name': 'interface0', + 'match': {'macaddress': '52:54:00:12:34:00'}, + 'mtu': 1501, + 'address': ['192.168.1.2/24', '2001:4800:78ff:1b:be76:4eff:fe06:1000"], + 'mtu6': 1480} + + """ + + def _listify(obj, token=' '): + "Helper to convert strings to list of strings, handle single string" + if not obj or type(obj) not in [str]: + return obj + if token in obj: + return obj.split(token) + else: + return [obj, ] + + addresses = [] + routes = [] + nameservers = [] + searchdomains = [] + subnets = config.get('subnets', []) + if subnets is None: + subnets = [] + for subnet in subnets: + sn_type = subnet.get('type') + if sn_type.startswith('dhcp'): + if sn_type == 'dhcp': + sn_type += '4' + entry.update({sn_type: True}) + elif sn_type in ['static']: + addr = "%s" % subnet.get('address') + if 'netmask' in subnet: + addr += "/%s" % subnet.get('netmask') + if 'gateway' in subnet and subnet.get('gateway'): + gateway = subnet.get('gateway') + if ":" in gateway: + entry.update({'gateway6': gateway}) + else: + entry.update({'gateway4': gateway}) + if 'dns_nameservers' in subnet: + nameservers += _listify(subnet.get('dns_nameservers', [])) + if 'dns_search' in subnet: + searchdomains += _listify(subnet.get('dns_search', [])) + if 'mtu' in subnet: + mtukey = 'mtu' + if subnet_is_ipv6(subnet): + mtukey += '6' + entry.update({mtukey: subnet.get('mtu')}) + for route in subnet.get('routes', []): + to_net = "%s/%s" % (route.get('network'), + route.get('netmask')) + route = { + 'via': route.get('gateway'), + 'to': to_net, + } + if 'metric' in route: + route.update({'metric': route.get('metric', 100)}) + routes.append(route) + + addresses.append(addr) + + if len(addresses) > 0: + entry.update({'addresses': addresses}) + if len(routes) > 0: + entry.update({'routes': routes}) + if len(nameservers) > 0: + ns = {'addresses': nameservers} + entry.update({'nameservers': ns}) + if len(searchdomains) > 0: + ns = entry.get('nameservers', {}) + ns.update({'search': searchdomains}) + entry.update({'nameservers': ns}) + + +def _extract_bond_slaves_by_name(interfaces, entry, bond_master): + bond_slave_names = sorted([name for (name, cfg) in interfaces.items() + if cfg.get('bond-master', None) == bond_master]) + if len(bond_slave_names) > 0: + entry.update({'interfaces': bond_slave_names}) + + +class Renderer(renderer.Renderer): + """Renders network information in a /etc/netplan/network.yaml format.""" + + NETPLAN_GENERATE = ['netplan', 'generate'] + + def __init__(self, config=None): + if not config: + config = {} + self.netplan_path = config.get('netplan_path', + 'etc/netplan/50-cloud-init.yaml') + self.netplan_header = config.get('netplan_header', None) + self._postcmds = config.get('postcmds', False) + + def render_network_state(self, target, network_state): + # check network state for version + # if v2, then extract network_state.config + # else render_v2_from_state + fpnplan = os.path.join(target, self.netplan_path) + util.ensure_dir(os.path.dirname(fpnplan)) + header = self.netplan_header if self.netplan_header else "" + + # render from state + content = self._render_content(network_state) + + # ensure we poke udev to run net_setup_link + if not header.endswith("\n"): + header += "\n" + util.write_file(fpnplan, header + content) + + self._netplan_generate(run=self._postcmds) + self._net_setup_link(run=self._postcmds) + + def _netplan_generate(self, run=False): + if not run: + print("netplan postcmd disabled") + return + util.subp(self.NETPLAN_GENERATE, capture=True) + + def _net_setup_link(self, run=False): + """To ensure device link properties are applied, we poke + udev to re-evaluate networkd .link files and call + the setup_link udev builtin command + """ + if not run: + print("netsetup postcmd disabled") + return + setup_lnk = ['udevadm', 'test-builtin', 'net_setup_link'] + for cmd in [setup_lnk + [SYS_CLASS_NET + iface] + for iface in get_devicelist() if + os.path.islink(SYS_CLASS_NET + iface)]: + print(cmd) + util.subp(cmd, capture=True) + + def _render_content(self, network_state): + print('rendering v2 for victory!') + ethernets = {} + wifis = {} + bridges = {} + bonds = {} + vlans = {} + content = [] + + interfaces = network_state._network_state.get('interfaces', []) + + nameservers = network_state.dns_nameservers + searchdomains = network_state.dns_searchdomains + + for config in network_state.iter_interfaces(): + ifname = config.get('name') + # filter None entries up front so we can do simple if key in dict + ifcfg = dict((key, value) for (key, value) in config.items() + if value) + + if_type = ifcfg.get('type') + if if_type == 'physical': + # required_keys = ['name', 'mac_address'] + eth = { + 'set-name': ifname, + 'match': ifcfg.get('match', None), + } + if eth['match'] is None: + macaddr = ifcfg.get('mac_address', None) + if macaddr is not None: + eth['match'] = {'macaddress': macaddr.lower()} + else: + del eth['match'] + del eth['set-name'] + if 'mtu' in ifcfg: + eth['mtu'] = ifcfg.get('mtu') + + _extract_addresses(ifcfg, eth) + ethernets.update({ifname: eth}) + + elif if_type == 'bond': + # required_keys = ['name', 'bond_interfaces'] + bond = {} + bond_config = {} + # extract bond params and drop the bond_ prefix as it's + # redundent in v2 yaml format + v2_bond_map = NET_CONFIG_TO_V2.get('bond') + for match in ['bond_', 'bond-']: + bond_params = _get_params_dict_by_match(ifcfg, match) + for (param, value) in bond_params.items(): + newname = v2_bond_map.get(param) + if newname is None: + continue + bond_config.update({newname: value}) + + if len(bond_config) > 0: + bond.update({'parameters': bond_config}) + slave_interfaces = ifcfg.get('bond-slaves') + if slave_interfaces == 'none': + _extract_bond_slaves_by_name(interfaces, bond, ifname) + _extract_addresses(ifcfg, bond) + bonds.update({ifname: bond}) + + elif if_type == 'bridge': + # required_keys = ['name', 'bridge_ports'] + ports = sorted(copy.copy(ifcfg.get('bridge_ports'))) + bridge = { + 'interfaces': ports, + } + # extract bridge params and drop the bridge prefix as it's + # redundent in v2 yaml format + match_prefix = 'bridge_' + params = _get_params_dict_by_match(ifcfg, match_prefix) + br_config = {} + + # v2 yaml uses different names for the keys + # and at least one value format change + v2_bridge_map = NET_CONFIG_TO_V2.get('bridge') + for (param, value) in params.items(): + newname = v2_bridge_map.get(param) + if newname is None: + continue + br_config.update({newname: value}) + if newname == 'path-cost': + # -> : int() + newvalue = {} + for costval in value: + (port, cost) = costval.split() + newvalue[port] = int(cost) + br_config.update({newname: newvalue}) + if len(br_config) > 0: + bridge.update({'parameters': br_config}) + _extract_addresses(ifcfg, bridge) + bridges.update({ifname: bridge}) + + elif if_type == 'vlan': + # required_keys = ['name', 'vlan_id', 'vlan-raw-device'] + vlan = { + 'id': ifcfg.get('vlan_id'), + 'link': ifcfg.get('vlan-raw-device') + } + + _extract_addresses(ifcfg, vlan) + vlans.update({ifname: vlan}) + + # inject global nameserver values under each physical interface + if nameservers: + for _eth, cfg in ethernets.items(): + nscfg = cfg.get('nameservers', {}) + addresses = nscfg.get('addresses', []) + addresses += nameservers + nscfg.update({'addresses': addresses}) + cfg.update({'nameservers': nscfg}) + + if searchdomains: + for _eth, cfg in ethernets.items(): + nscfg = cfg.get('nameservers', {}) + search = nscfg.get('search', []) + search += searchdomains + nscfg.update({'search': search}) + cfg.update({'nameservers': nscfg}) + + # workaround yaml dictionary key sorting when dumping + def _render_section(name, section): + if section: + dump = util.yaml_dumps({name: section}, + explicit_start=False, + explicit_end=False) + txt = util.indent(dump, ' ' * 4) + return [txt] + return [] + + content.append("network:\n version: 2\n") + content += _render_section('ethernets', ethernets) + content += _render_section('wifis', wifis) + content += _render_section('bonds', bonds) + content += _render_section('bridges', bridges) + content += _render_section('vlans', vlans) + + return "".join(content) + + +def available(target=None): + expected = ['netplan'] + search = ['/usr/sbin', '/sbin'] + for p in expected: + if not util.which(p, search=search, target=target): + return False + return True + + +def network_state_to_netplan(network_state, header=None): + # render the provided network state, return a string of equivalent eni + netplan_path = 'etc/network/50-cloud-init.yaml' + renderer = Renderer({ + 'netplan_path': netplan_path, + 'netplan_header': header, + }) + if not header: + header = "" + if not header.endswith("\n"): + header += "\n" + contents = renderer._render_content(network_state) + return header + contents + +# vi: ts=4 expandtab diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 90b2835a..701aaa4e 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Canonical Ltd. +# Copyright (C) 2017 Canonical Ltd. # # Author: Ryan Harper # @@ -18,6 +18,10 @@ NETWORK_STATE_VERSION = 1 NETWORK_STATE_REQUIRED_KEYS = { 1: ['version', 'config', 'network_state'], } +NETWORK_V2_KEY_FILTER = [ + 'addresses', 'dhcp4', 'dhcp6', 'gateway4', 'gateway6', 'interfaces', + 'match', 'mtu', 'nameservers', 'renderer', 'set-name', 'wakeonlan' +] def parse_net_config_data(net_config, skip_broken=True): @@ -26,11 +30,18 @@ def parse_net_config_data(net_config, skip_broken=True): :param net_config: curtin network config dict """ state = None - if 'version' in net_config and 'config' in net_config: - nsi = NetworkStateInterpreter(version=net_config.get('version'), - config=net_config.get('config')) + version = net_config.get('version') + config = net_config.get('config') + if version == 2: + # v2 does not have explicit 'config' key so we + # pass the whole net-config as-is + config = net_config + + if version and config: + nsi = NetworkStateInterpreter(version=version, config=config) nsi.parse_config(skip_broken=skip_broken) - state = nsi.network_state + state = nsi.get_network_state() + return state @@ -106,6 +117,7 @@ class NetworkState(object): def __init__(self, network_state, version=NETWORK_STATE_VERSION): self._network_state = copy.deepcopy(network_state) self._version = version + self.use_ipv6 = network_state.get('use_ipv6', False) @property def version(self): @@ -152,7 +164,8 @@ class NetworkStateInterpreter(object): 'dns': { 'nameservers': [], 'search': [], - } + }, + 'use_ipv6': False, } def __init__(self, version=NETWORK_STATE_VERSION, config=None): @@ -165,6 +178,14 @@ class NetworkStateInterpreter(object): def network_state(self): return NetworkState(self._network_state, version=self._version) + @property + def use_ipv6(self): + return self._network_state.get('use_ipv6') + + @use_ipv6.setter + def use_ipv6(self, val): + self._network_state.update({'use_ipv6': val}) + def dump(self): state = { 'version': self._version, @@ -192,8 +213,22 @@ class NetworkStateInterpreter(object): def dump_network_state(self): return util.yaml_dumps(self._network_state) + def as_dict(self): + return {'version': self.version, 'config': self.config} + + def get_network_state(self): + ns = self.network_state + return ns + def parse_config(self, skip_broken=True): - # rebuild network state + if self._version == 1: + self.parse_config_v1(skip_broken=skip_broken) + self._parsed = True + elif self._version == 2: + self.parse_config_v2(skip_broken=skip_broken) + self._parsed = True + + def parse_config_v1(self, skip_broken=True): for command in self._config: command_type = command['type'] try: @@ -211,6 +246,26 @@ class NetworkStateInterpreter(object): exc_info=True) LOG.debug(self.dump_network_state()) + def parse_config_v2(self, skip_broken=True): + for command_type, command in self._config.items(): + if command_type == 'version': + continue + try: + handler = self.command_handlers[command_type] + except KeyError: + raise RuntimeError("No handler found for" + " command '%s'" % command_type) + try: + handler(self, command) + self._v2_common(command) + except InvalidCommand: + if not skip_broken: + raise + else: + LOG.warn("Skipping invalid command: %s", command, + exc_info=True) + LOG.debug(self.dump_network_state()) + @ensure_command_keys(['name']) def handle_loopback(self, command): return self.handle_physical(command) @@ -238,11 +293,16 @@ class NetworkStateInterpreter(object): if subnets: for subnet in subnets: if subnet['type'] == 'static': + if ':' in subnet['address']: + self.use_ipv6 = True if 'netmask' in subnet and ':' in subnet['address']: subnet['netmask'] = mask2cidr(subnet['netmask']) for route in subnet.get('routes', []): if 'netmask' in route: route['netmask'] = mask2cidr(route['netmask']) + elif subnet['type'].endswith('6'): + self.use_ipv6 = True + iface.update({ 'name': command.get('name'), 'type': command.get('type'), @@ -327,7 +387,7 @@ class NetworkStateInterpreter(object): bond_if.update({param: val}) self._network_state['interfaces'].update({ifname: bond_if}) - @ensure_command_keys(['name', 'bridge_interfaces', 'params']) + @ensure_command_keys(['name', 'bridge_interfaces']) def handle_bridge(self, command): ''' auto br0 @@ -373,7 +433,7 @@ class NetworkStateInterpreter(object): self.handle_physical(command) iface = interfaces.get(command.get('name'), {}) iface['bridge_ports'] = command['bridge_interfaces'] - for param, val in command.get('params').items(): + for param, val in command.get('params', {}).items(): iface.update({param: val}) interfaces.update({iface['name']: iface}) @@ -407,6 +467,240 @@ class NetworkStateInterpreter(object): } routes.append(route) + # V2 handlers + def handle_bonds(self, command): + ''' + v2_command = { + bond0: { + 'interfaces': ['interface0', 'interface1'], + 'miimon': 100, + 'mode': '802.3ad', + 'xmit_hash_policy': 'layer3+4'}, + bond1: { + 'bond-slaves': ['interface2', 'interface7'], + 'mode': 1 + } + } + + v1_command = { + 'type': 'bond' + 'name': 'bond0', + 'bond_interfaces': [interface0, interface1], + 'params': { + 'bond-mode': '802.3ad', + 'bond_miimon: 100, + 'bond_xmit_hash_policy': 'layer3+4', + } + } + + ''' + self._handle_bond_bridge(command, cmd_type='bond') + + def handle_bridges(self, command): + + ''' + v2_command = { + br0: { + 'interfaces': ['interface0', 'interface1'], + 'fd': 0, + 'stp': 'off', + 'maxwait': 0, + } + } + + v1_command = { + 'type': 'bridge' + 'name': 'br0', + 'bridge_interfaces': [interface0, interface1], + 'params': { + 'bridge_stp': 'off', + 'bridge_fd: 0, + 'bridge_maxwait': 0 + } + } + + ''' + self._handle_bond_bridge(command, cmd_type='bridge') + + def handle_ethernets(self, command): + ''' + ethernets: + eno1: + match: + macaddress: 00:11:22:33:44:55 + wakeonlan: true + dhcp4: true + dhcp6: false + addresses: + - 192.168.14.2/24 + - 2001:1::1/64 + gateway4: 192.168.14.1 + gateway6: 2001:1::2 + nameservers: + search: [foo.local, bar.local] + addresses: [8.8.8.8, 8.8.4.4] + lom: + match: + driver: ixgbe + set-name: lom1 + dhcp6: true + switchports: + match: + name: enp2* + mtu: 1280 + + command = { + 'type': 'physical', + 'mac_address': 'c0:d6:9f:2c:e8:80', + 'name': 'eth0', + 'subnets': [ + {'type': 'dhcp4'} + ] + } + ''' + for eth, cfg in command.items(): + phy_cmd = { + 'type': 'physical', + 'name': cfg.get('set-name', eth), + } + mac_address = cfg.get('match', {}).get('macaddress', None) + if not mac_address: + LOG.debug('NetworkState Version2: missing "macaddress" info ' + 'in config entry: %s: %s', eth, str(cfg)) + + for key in ['mtu', 'match', 'wakeonlan']: + if key in cfg: + phy_cmd.update({key: cfg.get(key)}) + + subnets = self._v2_to_v1_ipcfg(cfg) + if len(subnets) > 0: + phy_cmd.update({'subnets': subnets}) + + LOG.debug('v2(ethernets) -> v1(physical):\n%s', phy_cmd) + self.handle_physical(phy_cmd) + + def handle_vlans(self, command): + ''' + v2_vlans = { + 'eth0.123': { + 'id': 123, + 'link': 'eth0', + 'dhcp4': True, + } + } + + v1_command = { + 'type': 'vlan', + 'name': 'eth0.123', + 'vlan_link': 'eth0', + 'vlan_id': 123, + 'subnets': [{'type': 'dhcp4'}], + } + ''' + for vlan, cfg in command.items(): + vlan_cmd = { + 'type': 'vlan', + 'name': vlan, + 'vlan_id': cfg.get('id'), + 'vlan_link': cfg.get('link'), + } + subnets = self._v2_to_v1_ipcfg(cfg) + if len(subnets) > 0: + vlan_cmd.update({'subnets': subnets}) + LOG.debug('v2(vlans) -> v1(vlan):\n%s', vlan_cmd) + self.handle_vlan(vlan_cmd) + + def handle_wifis(self, command): + raise NotImplemented('NetworkState V2: Skipping wifi configuration') + + def _v2_common(self, cfg): + LOG.debug('v2_common: handling config:\n%s', cfg) + if 'nameservers' in cfg: + search = cfg.get('nameservers').get('search', []) + dns = cfg.get('nameservers').get('addresses', []) + name_cmd = {'type': 'nameserver'} + if len(search) > 0: + name_cmd.update({'search': search}) + if len(dns) > 0: + name_cmd.update({'addresses': dns}) + LOG.debug('v2(nameserver) -> v1(nameserver):\n%s', name_cmd) + self.handle_nameserver(name_cmd) + + def _handle_bond_bridge(self, command, cmd_type=None): + """Common handler for bond and bridge types""" + for item_name, item_cfg in command.items(): + item_params = dict((key, value) for (key, value) in + item_cfg.items() if key not in + NETWORK_V2_KEY_FILTER) + v1_cmd = { + 'type': cmd_type, + 'name': item_name, + cmd_type + '_interfaces': item_cfg.get('interfaces'), + 'params': item_params, + } + subnets = self._v2_to_v1_ipcfg(item_cfg) + if len(subnets) > 0: + v1_cmd.update({'subnets': subnets}) + + LOG.debug('v2(%ss) -> v1(%s):\n%s', cmd_type, cmd_type, v1_cmd) + self.handle_bridge(v1_cmd) + + def _v2_to_v1_ipcfg(self, cfg): + """Common ipconfig extraction from v2 to v1 subnets array.""" + + subnets = [] + if 'dhcp4' in cfg: + subnets.append({'type': 'dhcp4'}) + if 'dhcp6' in cfg: + self.use_ipv6 = True + subnets.append({'type': 'dhcp6'}) + + gateway4 = None + gateway6 = None + for address in cfg.get('addresses', []): + subnet = { + 'type': 'static', + 'address': address, + } + + routes = [] + for route in cfg.get('routes', []): + route_addr = route.get('to') + if "/" in route_addr: + route_addr, route_cidr = route_addr.split("/") + route_netmask = cidr2mask(route_cidr) + subnet_route = { + 'address': route_addr, + 'netmask': route_netmask, + 'gateway': route.get('via') + } + routes.append(subnet_route) + if len(routes) > 0: + subnet.update({'routes': routes}) + + if ":" in address: + if 'gateway6' in cfg and gateway6 is None: + gateway6 = cfg.get('gateway6') + subnet.update({'gateway': gateway6}) + else: + if 'gateway4' in cfg and gateway4 is None: + gateway4 = cfg.get('gateway4') + subnet.update({'gateway': gateway4}) + + subnets.append(subnet) + return subnets + + +def subnet_is_ipv6(subnet): + """Common helper for checking network_state subnets for ipv6.""" + # 'static6' or 'dhcp6' + if subnet['type'].endswith('6'): + # This is a request for DHCPv6. + return True + elif subnet['type'] == 'static' and ":" in subnet['address']: + return True + return False + def cidr2mask(cidr): mask = [0, 0, 0, 0] diff --git a/cloudinit/net/renderers.py b/cloudinit/net/renderers.py index 5ad84553..5117b4a5 100644 --- a/cloudinit/net/renderers.py +++ b/cloudinit/net/renderers.py @@ -1,15 +1,17 @@ # This file is part of cloud-init. See LICENSE file for license information. from . import eni +from . import netplan from . import RendererNotFoundError from . import sysconfig NAME_TO_RENDERER = { "eni": eni, + "netplan": netplan, "sysconfig": sysconfig, } -DEFAULT_PRIORITY = ["eni", "sysconfig"] +DEFAULT_PRIORITY = ["eni", "sysconfig", "netplan"] def search(priority=None, target=None, first=False): diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index 117b515c..504e4d02 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -9,6 +9,7 @@ from cloudinit.distros.parsers import resolv_conf from cloudinit import util from . import renderer +from .network_state import subnet_is_ipv6 def _make_header(sep='#'): @@ -194,7 +195,7 @@ class Renderer(renderer.Renderer): def __init__(self, config=None): if not config: config = {} - self.sysconf_dir = config.get('sysconf_dir', 'etc/sysconfig/') + self.sysconf_dir = config.get('sysconf_dir', 'etc/sysconfig') self.netrules_path = config.get( 'netrules_path', 'etc/udev/rules.d/70-persistent-net.rules') self.dns_path = config.get('dns_path', 'etc/resolv.conf') @@ -220,7 +221,7 @@ class Renderer(renderer.Renderer): iface_cfg['BOOTPROTO'] = 'dhcp' elif subnet_type == 'static': iface_cfg['BOOTPROTO'] = 'static' - if subnet.get('ipv6'): + if subnet_is_ipv6(subnet): iface_cfg['IPV6ADDR'] = subnet['address'] iface_cfg['IPV6INIT'] = True else: @@ -390,19 +391,28 @@ class Renderer(renderer.Renderer): return contents def render_network_state(self, network_state, target=None): + file_mode = 0o644 base_sysconf_dir = util.target_path(target, self.sysconf_dir) for path, data in self._render_sysconfig(base_sysconf_dir, network_state).items(): - util.write_file(path, data) + util.write_file(path, data, file_mode) if self.dns_path: dns_path = util.target_path(target, self.dns_path) resolv_content = self._render_dns(network_state, existing_dns_path=dns_path) - util.write_file(dns_path, resolv_content) + util.write_file(dns_path, resolv_content, file_mode) if self.netrules_path: netrules_content = self._render_persistent_net(network_state) netrules_path = util.target_path(target, self.netrules_path) - util.write_file(netrules_path, netrules_content) + util.write_file(netrules_path, netrules_content, file_mode) + + # always write /etc/sysconfig/network configuration + sysconfig_path = util.target_path(target, "etc/sysconfig/network") + netcfg = [_make_header(), 'NETWORKING=yes'] + if network_state.use_ipv6: + netcfg.append('NETWORKING_IPV6=yes') + netcfg.append('IPV6_AUTOCONF=no') + util.write_file(sysconfig_path, "\n".join(netcfg) + "\n", file_mode) def available(target=None): diff --git a/cloudinit/util.py b/cloudinit/util.py index 82f2f76b..33019579 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2373,4 +2373,13 @@ def system_is_snappy(): return True return False + +def indent(text, prefix): + """replacement for indent from textwrap that is not available in 2.7.""" + lines = [] + for line in text.splitlines(True): + lines.append(prefix + line) + return ''.join(lines) + + # vi: ts=4 expandtab diff --git a/systemd/cloud-init.service b/systemd/cloud-init.service index fb3b918c..39acc20a 100644 --- a/systemd/cloud-init.service +++ b/systemd/cloud-init.service @@ -5,6 +5,7 @@ Wants=cloud-init-local.service Wants=sshd-keygen.service Wants=sshd.service After=cloud-init-local.service +After=systemd-networkd-wait-online.service After=networking.service Before=network-online.target Before=sshd-keygen.service diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py index bde3bb50..b89b74ff 100644 --- a/tests/unittests/test_distros/test_netconfig.py +++ b/tests/unittests/test_distros/test_netconfig.py @@ -17,6 +17,7 @@ from ..helpers import TestCase from cloudinit import distros from cloudinit.distros.parsers.sys_conf import SysConf from cloudinit import helpers +from cloudinit.net import eni from cloudinit import settings from cloudinit import util @@ -28,10 +29,10 @@ iface lo inet loopback auto eth0 iface eth0 inet static address 192.168.1.5 - netmask 255.255.255.0 - network 192.168.0.0 broadcast 192.168.1.0 gateway 192.168.1.254 + netmask 255.255.255.0 + network 192.168.0.0 auto eth1 iface eth1 inet dhcp @@ -67,6 +68,100 @@ iface eth1 inet6 static gateway 2607:f0d0:1002:0011::1 ''' +V1_NET_CFG = {'config': [{'name': 'eth0', + + 'subnets': [{'address': '192.168.1.5', + 'broadcast': '192.168.1.0', + 'gateway': '192.168.1.254', + 'netmask': '255.255.255.0', + 'type': 'static'}], + 'type': 'physical'}, + {'name': 'eth1', + 'subnets': [{'control': 'auto', 'type': 'dhcp4'}], + 'type': 'physical'}], + 'version': 1} + +V1_NET_CFG_OUTPUT = """ +# This file is generated from information provided by +# the datasource. Changes to it will not persist across an instance. +# To disable cloud-init's network configuration capabilities, write a file +# /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg with the following: +# network: {config: disabled} +auto lo +iface lo inet loopback + +auto eth0 +iface eth0 inet static + address 192.168.1.5 + broadcast 192.168.1.0 + gateway 192.168.1.254 + netmask 255.255.255.0 + +auto eth1 +iface eth1 inet dhcp +""" + +V1_NET_CFG_IPV6 = {'config': [{'name': 'eth0', + 'subnets': [{'address': + '2607:f0d0:1002:0011::2', + 'gateway': + '2607:f0d0:1002:0011::1', + 'netmask': '64', + 'type': 'static'}], + 'type': 'physical'}, + {'name': 'eth1', + 'subnets': [{'control': 'auto', + 'type': 'dhcp4'}], + 'type': 'physical'}], + 'version': 1} + + +V1_TO_V2_NET_CFG_OUTPUT = """ +# This file is generated from information provided by +# the datasource. Changes to it will not persist across an instance. +# To disable cloud-init's network configuration capabilities, write a file +# /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg with the following: +# network: {config: disabled} +network: + version: 2 + ethernets: + eth0: + addresses: + - 192.168.1.5/255.255.255.0 + gateway4: 192.168.1.254 + eth1: + dhcp4: true +""" + +V2_NET_CFG = { + 'ethernets': { + 'eth7': { + 'addresses': ['192.168.1.5/255.255.255.0'], + 'gateway4': '192.168.1.254'}, + 'eth9': { + 'dhcp4': True} + }, + 'version': 2 +} + + +V2_TO_V2_NET_CFG_OUTPUT = """ +# This file is generated from information provided by +# the datasource. Changes to it will not persist across an instance. +# To disable cloud-init's network configuration capabilities, write a file +# /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg with the following: +# network: {config: disabled} +network: + version: 2 + ethernets: + eth7: + addresses: + - 192.168.1.5/255.255.255.0 + gateway4: 192.168.1.254 + eth9: + dhcp4: true +""" + class WriteBuffer(object): def __init__(self): @@ -83,12 +178,14 @@ class WriteBuffer(object): class TestNetCfgDistro(TestCase): - def _get_distro(self, dname): + def _get_distro(self, dname, renderers=None): cls = distros.fetch(dname) cfg = settings.CFG_BUILTIN cfg['system_info']['distro'] = dname + if renderers: + cfg['system_info']['network'] = {'renderers': renderers} paths = helpers.Paths({}) - return cls(dname, cfg, paths) + return cls(dname, cfg.get('system_info'), paths) def test_simple_write_ub(self): ub_distro = self._get_distro('ubuntu') @@ -116,6 +213,107 @@ class TestNetCfgDistro(TestCase): self.assertEqual(str(write_buf).strip(), BASE_NET_CFG.strip()) self.assertEqual(write_buf.mode, 0o644) + def test_apply_network_config_eni_ub(self): + ub_distro = self._get_distro('ubuntu') + with ExitStack() as mocks: + write_bufs = {} + + def replace_write(filename, content, mode=0o644, omode="wb"): + buf = WriteBuffer() + buf.mode = mode + buf.omode = omode + buf.write(content) + write_bufs[filename] = buf + + # eni availability checks + mocks.enter_context( + mock.patch.object(util, 'which', return_value=True)) + mocks.enter_context( + mock.patch.object(eni, 'available', return_value=True)) + mocks.enter_context( + mock.patch.object(util, 'ensure_dir')) + mocks.enter_context( + mock.patch.object(util, 'write_file', replace_write)) + mocks.enter_context( + mock.patch.object(os.path, 'isfile', return_value=False)) + + ub_distro.apply_network_config(V1_NET_CFG, False) + + self.assertEqual(len(write_bufs), 2) + eni_name = '/etc/network/interfaces.d/50-cloud-init.cfg' + self.assertIn(eni_name, write_bufs) + write_buf = write_bufs[eni_name] + self.assertEqual(str(write_buf).strip(), V1_NET_CFG_OUTPUT.strip()) + self.assertEqual(write_buf.mode, 0o644) + + def test_apply_network_config_v1_to_netplan_ub(self): + renderers = ['netplan'] + ub_distro = self._get_distro('ubuntu', renderers=renderers) + with ExitStack() as mocks: + write_bufs = {} + + def replace_write(filename, content, mode=0o644, omode="wb"): + buf = WriteBuffer() + buf.mode = mode + buf.omode = omode + buf.write(content) + write_bufs[filename] = buf + + mocks.enter_context( + mock.patch.object(util, 'which', return_value=True)) + mocks.enter_context( + mock.patch.object(util, 'write_file', replace_write)) + mocks.enter_context( + mock.patch.object(util, 'ensure_dir')) + mocks.enter_context( + mock.patch.object(util, 'subp', return_value=(0, 0))) + mocks.enter_context( + mock.patch.object(os.path, 'isfile', return_value=False)) + + ub_distro.apply_network_config(V1_NET_CFG, False) + + self.assertEqual(len(write_bufs), 1) + netplan_name = '/etc/netplan/50-cloud-init.yaml' + self.assertIn(netplan_name, write_bufs) + write_buf = write_bufs[netplan_name] + self.assertEqual(str(write_buf).strip(), + V1_TO_V2_NET_CFG_OUTPUT.strip()) + self.assertEqual(write_buf.mode, 0o644) + + def test_apply_network_config_v2_passthrough_ub(self): + renderers = ['netplan'] + ub_distro = self._get_distro('ubuntu', renderers=renderers) + with ExitStack() as mocks: + write_bufs = {} + + def replace_write(filename, content, mode=0o644, omode="wb"): + buf = WriteBuffer() + buf.mode = mode + buf.omode = omode + buf.write(content) + write_bufs[filename] = buf + + mocks.enter_context( + mock.patch.object(util, 'which', return_value=True)) + mocks.enter_context( + mock.patch.object(util, 'write_file', replace_write)) + mocks.enter_context( + mock.patch.object(util, 'ensure_dir')) + mocks.enter_context( + mock.patch.object(util, 'subp', return_value=(0, 0))) + mocks.enter_context( + mock.patch.object(os.path, 'isfile', return_value=False)) + + ub_distro.apply_network_config(V2_NET_CFG, False) + + self.assertEqual(len(write_bufs), 1) + netplan_name = '/etc/netplan/50-cloud-init.yaml' + self.assertIn(netplan_name, write_bufs) + write_buf = write_bufs[netplan_name] + self.assertEqual(str(write_buf).strip(), + V2_TO_V2_NET_CFG_OUTPUT.strip()) + self.assertEqual(write_buf.mode, 0o644) + def assertCfgEquals(self, blob1, blob2): b1 = dict(SysConf(blob1.strip().splitlines())) b2 = dict(SysConf(blob2.strip().splitlines())) @@ -191,6 +389,79 @@ ONBOOT=yes expected_buf = ''' # Created by cloud-init v. 0.7 NETWORKING=yes +''' + self.assertCfgEquals(expected_buf, str(write_buf)) + self.assertEqual(write_buf.mode, 0o644) + + def test_apply_network_config_rh(self): + renderers = ['sysconfig'] + rh_distro = self._get_distro('rhel', renderers=renderers) + + write_bufs = {} + + def replace_write(filename, content, mode=0o644, omode="wb"): + buf = WriteBuffer() + buf.mode = mode + buf.omode = omode + buf.write(content) + write_bufs[filename] = buf + + with ExitStack() as mocks: + # sysconfig availability checks + mocks.enter_context( + mock.patch.object(util, 'which', return_value=True)) + mocks.enter_context( + mock.patch.object(util, 'write_file', replace_write)) + mocks.enter_context( + mock.patch.object(util, 'load_file', return_value='')) + mocks.enter_context( + mock.patch.object(os.path, 'isfile', return_value=True)) + + rh_distro.apply_network_config(V1_NET_CFG, False) + + self.assertEqual(len(write_bufs), 5) + + # eth0 + self.assertIn('/etc/sysconfig/network-scripts/ifcfg-eth0', + write_bufs) + write_buf = write_bufs['/etc/sysconfig/network-scripts/ifcfg-eth0'] + expected_buf = ''' +# Created by cloud-init on instance boot automatically, do not edit. +# +BOOTPROTO=static +DEVICE=eth0 +IPADDR=192.168.1.5 +NETMASK=255.255.255.0 +NM_CONTROLLED=no +ONBOOT=yes +TYPE=Ethernet +USERCTL=no +''' + self.assertCfgEquals(expected_buf, str(write_buf)) + self.assertEqual(write_buf.mode, 0o644) + + # eth1 + self.assertIn('/etc/sysconfig/network-scripts/ifcfg-eth1', + write_bufs) + write_buf = write_bufs['/etc/sysconfig/network-scripts/ifcfg-eth1'] + expected_buf = ''' +# Created by cloud-init on instance boot automatically, do not edit. +# +BOOTPROTO=dhcp +DEVICE=eth1 +NM_CONTROLLED=no +ONBOOT=yes +TYPE=Ethernet +USERCTL=no +''' + self.assertCfgEquals(expected_buf, str(write_buf)) + self.assertEqual(write_buf.mode, 0o644) + + self.assertIn('/etc/sysconfig/network', write_bufs) + write_buf = write_bufs['/etc/sysconfig/network'] + expected_buf = ''' +# Created by cloud-init v. 0.7 +NETWORKING=yes ''' self.assertCfgEquals(expected_buf, str(write_buf)) self.assertEqual(write_buf.mode, 0o644) @@ -270,6 +541,78 @@ IPV6_DEFAULTGW="2607:f0d0:1002:0011::1" NETWORKING=yes NETWORKING_IPV6=yes IPV6_AUTOCONF=no +''' + self.assertCfgEquals(expected_buf, str(write_buf)) + self.assertEqual(write_buf.mode, 0o644) + + def test_apply_network_config_ipv6_rh(self): + renderers = ['sysconfig'] + rh_distro = self._get_distro('rhel', renderers=renderers) + + write_bufs = {} + + def replace_write(filename, content, mode=0o644, omode="wb"): + buf = WriteBuffer() + buf.mode = mode + buf.omode = omode + buf.write(content) + write_bufs[filename] = buf + + with ExitStack() as mocks: + mocks.enter_context( + mock.patch.object(util, 'which', return_value=True)) + mocks.enter_context( + mock.patch.object(util, 'write_file', replace_write)) + mocks.enter_context( + mock.patch.object(util, 'load_file', return_value='')) + mocks.enter_context( + mock.patch.object(os.path, 'isfile', return_value=True)) + + rh_distro.apply_network_config(V1_NET_CFG_IPV6, False) + + self.assertEqual(len(write_bufs), 5) + + self.assertIn('/etc/sysconfig/network-scripts/ifcfg-eth0', + write_bufs) + write_buf = write_bufs['/etc/sysconfig/network-scripts/ifcfg-eth0'] + expected_buf = ''' +# Created by cloud-init on instance boot automatically, do not edit. +# +BOOTPROTO=static +DEVICE=eth0 +IPV6ADDR=2607:f0d0:1002:0011::2 +IPV6INIT=yes +NETMASK=64 +NM_CONTROLLED=no +ONBOOT=yes +TYPE=Ethernet +USERCTL=no +''' + self.assertCfgEquals(expected_buf, str(write_buf)) + self.assertEqual(write_buf.mode, 0o644) + self.assertIn('/etc/sysconfig/network-scripts/ifcfg-eth1', + write_bufs) + write_buf = write_bufs['/etc/sysconfig/network-scripts/ifcfg-eth1'] + expected_buf = ''' +# Created by cloud-init on instance boot automatically, do not edit. +# +BOOTPROTO=dhcp +DEVICE=eth1 +NM_CONTROLLED=no +ONBOOT=yes +TYPE=Ethernet +USERCTL=no +''' + self.assertCfgEquals(expected_buf, str(write_buf)) + self.assertEqual(write_buf.mode, 0o644) + + self.assertIn('/etc/sysconfig/network', write_bufs) + write_buf = write_bufs['/etc/sysconfig/network'] + expected_buf = ''' +# Created by cloud-init v. 0.7 +NETWORKING=yes +NETWORKING_IPV6=yes +IPV6_AUTOCONF=no ''' self.assertCfgEquals(expected_buf, str(write_buf)) self.assertEqual(write_buf.mode, 0o644) diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 902204a0..4f07d804 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -3,6 +3,7 @@ from cloudinit import net from cloudinit.net import cmdline from cloudinit.net import eni +from cloudinit.net import netplan from cloudinit.net import network_state from cloudinit.net import renderers from cloudinit.net import sysconfig @@ -408,6 +409,41 @@ NETWORK_CONFIGS = { post-up route add default gw 65.61.151.37 || true pre-down route del default gw 65.61.151.37 || true """).rstrip(' '), + 'expected_netplan': textwrap.dedent(""" + network: + version: 2 + ethernets: + eth1: + match: + macaddress: cf:d6:af:48:e8:80 + nameservers: + addresses: + - 1.2.3.4 + - 5.6.7.8 + search: + - wark.maas + set-name: eth1 + eth99: + addresses: + - 192.168.21.3/24 + dhcp4: true + match: + macaddress: c0:d6:9f:2c:e8:80 + nameservers: + addresses: + - 8.8.8.8 + - 8.8.4.4 + - 1.2.3.4 + - 5.6.7.8 + search: + - barley.maas + - sach.maas + - wark.maas + routes: + - to: 0.0.0.0/0.0.0.0 + via: 65.61.151.37 + set-name: eth99 + """).rstrip(' '), 'yaml': textwrap.dedent(""" version: 1 config: @@ -450,6 +486,14 @@ NETWORK_CONFIGS = { # control-alias iface0 iface iface0 inet6 dhcp """).rstrip(' '), + 'expected_netplan': textwrap.dedent(""" + network: + version: 2 + ethernets: + iface0: + dhcp4: true + dhcp6: true + """).rstrip(' '), 'yaml': textwrap.dedent("""\ version: 1 config: @@ -524,6 +568,126 @@ iface eth0.101 inet static post-up route add -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true """), + 'expected_netplan': textwrap.dedent(""" + network: + version: 2 + ethernets: + eth0: + match: + macaddress: c0:d6:9f:2c:e8:80 + nameservers: + addresses: + - 8.8.8.8 + - 4.4.4.4 + - 8.8.4.4 + search: + - barley.maas + - wark.maas + - foobar.maas + set-name: eth0 + eth1: + match: + macaddress: aa:d6:9f:2c:e8:80 + nameservers: + addresses: + - 8.8.8.8 + - 4.4.4.4 + - 8.8.4.4 + search: + - barley.maas + - wark.maas + - foobar.maas + set-name: eth1 + eth2: + match: + macaddress: c0:bb:9f:2c:e8:80 + nameservers: + addresses: + - 8.8.8.8 + - 4.4.4.4 + - 8.8.4.4 + search: + - barley.maas + - wark.maas + - foobar.maas + set-name: eth2 + eth3: + match: + macaddress: 66:bb:9f:2c:e8:80 + nameservers: + addresses: + - 8.8.8.8 + - 4.4.4.4 + - 8.8.4.4 + search: + - barley.maas + - wark.maas + - foobar.maas + set-name: eth3 + eth4: + match: + macaddress: 98:bb:9f:2c:e8:80 + nameservers: + addresses: + - 8.8.8.8 + - 4.4.4.4 + - 8.8.4.4 + search: + - barley.maas + - wark.maas + - foobar.maas + set-name: eth4 + eth5: + dhcp4: true + match: + macaddress: 98:bb:9f:2c:e8:8a + nameservers: + addresses: + - 8.8.8.8 + - 4.4.4.4 + - 8.8.4.4 + search: + - barley.maas + - wark.maas + - foobar.maas + set-name: eth5 + bonds: + bond0: + dhcp6: true + interfaces: + - eth1 + - eth2 + parameters: + mode: active-backup + bridges: + br0: + addresses: + - 192.168.14.2/24 + - 2001:1::1/64 + interfaces: + - eth3 + - eth4 + vlans: + bond0.200: + dhcp4: true + id: 200 + link: bond0 + eth0.101: + addresses: + - 192.168.0.2/24 + - 192.168.2.10/24 + gateway4: 192.168.0.1 + id: 101 + link: eth0 + nameservers: + addresses: + - 192.168.0.10 + - 10.23.23.134 + search: + - barley.maas + - sacchromyces.maas + - brettanomyces.maas + """).rstrip(' '), 'yaml': textwrap.dedent(""" version: 1 config: @@ -808,6 +972,99 @@ iface eth0 inet dhcp expected, dir2dict(tmp_dir)['/etc/network/interfaces']) +class TestNetplanNetRendering(CiTestCase): + + @mock.patch("cloudinit.net.sys_dev_path") + @mock.patch("cloudinit.net.read_sys_net") + @mock.patch("cloudinit.net.get_devicelist") + def test_default_generation(self, mock_get_devicelist, + mock_read_sys_net, + mock_sys_dev_path): + tmp_dir = self.tmp_dir() + _setup_test(tmp_dir, mock_get_devicelist, + mock_read_sys_net, mock_sys_dev_path) + + network_cfg = net.generate_fallback_config() + ns = network_state.parse_net_config_data(network_cfg, + skip_broken=False) + + render_dir = os.path.join(tmp_dir, "render") + os.makedirs(render_dir) + + render_target = 'netplan.yaml' + renderer = netplan.Renderer( + {'netplan_path': render_target, 'postcmds': False}) + renderer.render_network_state(render_dir, ns) + + self.assertTrue(os.path.exists(os.path.join(render_dir, + render_target))) + with open(os.path.join(render_dir, render_target)) as fh: + contents = fh.read() + print(contents) + + expected = """ +network: + version: 2 + ethernets: + eth1000: + dhcp4: true + match: + macaddress: 07-1c-c6-75-a4-be + set-name: eth1000 +""" + self.assertEqual(expected.lstrip(), contents.lstrip()) + + +class TestNetplanPostcommands(CiTestCase): + mycfg = { + 'config': [{"type": "physical", "name": "eth0", + "mac_address": "c0:d6:9f:2c:e8:80", + "subnets": [{"type": "dhcp"}]}], + 'version': 1} + + @mock.patch.object(netplan.Renderer, '_netplan_generate') + @mock.patch.object(netplan.Renderer, '_net_setup_link') + def test_netplan_render_calls_postcmds(self, mock_netplan_generate, + mock_net_setup_link): + tmp_dir = self.tmp_dir() + ns = network_state.parse_net_config_data(self.mycfg, + skip_broken=False) + + render_dir = os.path.join(tmp_dir, "render") + os.makedirs(render_dir) + + render_target = 'netplan.yaml' + renderer = netplan.Renderer( + {'netplan_path': render_target, 'postcmds': True}) + renderer.render_network_state(render_dir, ns) + + mock_netplan_generate.assert_called_with(run=True) + mock_net_setup_link.assert_called_with(run=True) + + @mock.patch.object(netplan, "get_devicelist") + @mock.patch('cloudinit.util.subp') + def test_netplan_postcmds(self, mock_subp, mock_devlist): + mock_devlist.side_effect = [['lo']] + tmp_dir = self.tmp_dir() + ns = network_state.parse_net_config_data(self.mycfg, + skip_broken=False) + + render_dir = os.path.join(tmp_dir, "render") + os.makedirs(render_dir) + + render_target = 'netplan.yaml' + renderer = netplan.Renderer( + {'netplan_path': render_target, 'postcmds': True}) + renderer.render_network_state(render_dir, ns) + + expected = [ + mock.call(['netplan', 'generate'], capture=True), + mock.call(['udevadm', 'test-builtin', 'net_setup_link', + '/sys/class/net/lo'], capture=True), + ] + mock_subp.assert_has_calls(expected) + + class TestEniNetworkStateToEni(CiTestCase): mycfg = { 'config': [{"type": "physical", "name": "eth0", @@ -953,6 +1210,50 @@ class TestCmdlineReadKernelConfig(CiTestCase): self.assertEqual(found['config'], expected) +class TestNetplanRoundTrip(CiTestCase): + def _render_and_read(self, network_config=None, state=None, + netplan_path=None, dir=None): + if dir is None: + dir = self.tmp_dir() + + if network_config: + ns = network_state.parse_net_config_data(network_config) + elif state: + ns = state + else: + raise ValueError("Expected data or state, got neither") + + if netplan_path is None: + netplan_path = 'etc/netplan/50-cloud-init.yaml' + + renderer = netplan.Renderer( + config={'netplan_path': netplan_path}) + + renderer.render_network_state(dir, ns) + return dir2dict(dir) + + def testsimple_render_small_netplan(self): + entry = NETWORK_CONFIGS['small'] + files = self._render_and_read(network_config=yaml.load(entry['yaml'])) + self.assertEqual( + entry['expected_netplan'].splitlines(), + files['/etc/netplan/50-cloud-init.yaml'].splitlines()) + + def testsimple_render_v4_and_v6(self): + entry = NETWORK_CONFIGS['v4_and_v6'] + files = self._render_and_read(network_config=yaml.load(entry['yaml'])) + self.assertEqual( + entry['expected_netplan'].splitlines(), + files['/etc/netplan/50-cloud-init.yaml'].splitlines()) + + def testsimple_render_all(self): + entry = NETWORK_CONFIGS['all'] + files = self._render_and_read(network_config=yaml.load(entry['yaml'])) + self.assertEqual( + entry['expected_netplan'].splitlines(), + files['/etc/netplan/50-cloud-init.yaml'].splitlines()) + + class TestEniRoundTrip(CiTestCase): def _render_and_read(self, network_config=None, state=None, eni_path=None, links_prefix=None, netrules_path=None, dir=None): diff --git a/tools/net-convert.py b/tools/net-convert.py new file mode 100755 index 00000000..870da639 --- /dev/null +++ b/tools/net-convert.py @@ -0,0 +1,84 @@ +#!/usr/bin/python3 +# This file is part of cloud-init. See LICENSE file for license information. + +import argparse +import json +import os +import yaml + +from cloudinit.sources.helpers import openstack + +from cloudinit.net import eni +from cloudinit.net import network_state +from cloudinit.net import netplan +from cloudinit.net import sysconfig + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--network-data", "-p", type=open, + metavar="PATH", required=True) + parser.add_argument("--kind", "-k", + choices=['eni', 'network_data.json', 'yaml'], + required=True) + parser.add_argument("-d", "--directory", + metavar="PATH", + help="directory to place output in", + required=True) + parser.add_argument("-m", "--mac", + metavar="name,mac", + action='append', + help="interface name to mac mapping") + parser.add_argument("--output-kind", "-ok", + choices=['eni', 'netplan', 'sysconfig'], + required=True) + args = parser.parse_args() + + if not os.path.isdir(args.directory): + os.makedirs(args.directory) + + if args.mac: + known_macs = {} + for item in args.mac: + iface_name, iface_mac = item.split(",", 1) + known_macs[iface_mac] = iface_name + else: + known_macs = None + + net_data = args.network_data.read() + if args.kind == "eni": + pre_ns = eni.convert_eni_data(net_data) + ns = network_state.parse_net_config_data(pre_ns) + elif args.kind == "yaml": + pre_ns = yaml.load(net_data) + if 'network' in pre_ns: + pre_ns = pre_ns.get('network') + print("Input YAML") + print(yaml.dump(pre_ns, default_flow_style=False, indent=4)) + ns = network_state.parse_net_config_data(pre_ns) + else: + pre_ns = openstack.convert_net_json( + json.loads(net_data), known_macs=known_macs) + ns = network_state.parse_net_config_data(pre_ns) + + if not ns: + raise RuntimeError("No valid network_state object created from" + "input data") + + print("\nInternal State") + print(yaml.dump(ns, default_flow_style=False, indent=4)) + if args.output_kind == "eni": + r_cls = eni.Renderer + elif args.output_kind == "netplan": + r_cls = netplan.Renderer + else: + r_cls = sysconfig.Renderer + + r = r_cls() + r.render_network_state(ns, target=args.directory) + + +if __name__ == '__main__': + main() + +# vi: ts=4 expandtab -- cgit v1.2.3 From 35cf3415f9748c880db4d3c004f3410c3aa2cab2 Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Tue, 21 Mar 2017 14:18:46 -0600 Subject: test: add running of pylint Now tox will run pylint. The .pylintrc file sets pylint to only produce errors, and will ignore certain classes that are known problematic (six). --- .pylintrc | 39 +++++++++++++++++++++++++++++++ cloudinit/net/network_state.py | 5 ++-- cloudinit/net/renderer.py | 5 ++++ cloudinit/sources/DataSourceAltCloud.py | 3 +-- cloudinit/sources/DataSourceOpenNebula.py | 4 ++-- cloudinit/sources/__init__.py | 2 +- cloudinit/url_helper.py | 2 +- tox.ini | 10 +++++++- 8 files changed, 61 insertions(+), 9 deletions(-) create mode 100644 .pylintrc (limited to 'cloudinit/net') diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 00000000..b8cda03c --- /dev/null +++ b/.pylintrc @@ -0,0 +1,39 @@ +[MASTER] + +# --go-faster, use multiple processes to speed up Pylint +jobs=4 + + +[MESSAGES CONTROL] + +# Errors only +disable=C, F, I, R, W + + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +output-format=colorized + +# Just the errors please, no full report +reports=no + + +[TYPECHECK] + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules=six.moves,pkg_resources + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members=types,http.client,command_handlers + diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 701aaa4e..692b6007 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -214,7 +214,7 @@ class NetworkStateInterpreter(object): return util.yaml_dumps(self._network_state) def as_dict(self): - return {'version': self.version, 'config': self.config} + return {'version': self._version, 'config': self._config} def get_network_state(self): ns = self.network_state @@ -611,7 +611,8 @@ class NetworkStateInterpreter(object): self.handle_vlan(vlan_cmd) def handle_wifis(self, command): - raise NotImplemented('NetworkState V2: Skipping wifi configuration') + raise NotImplementedError("NetworkState V2: " + "Skipping wifi configuration") def _v2_common(self, cfg): LOG.debug('v2_common: handling config:\n%s', cfg) diff --git a/cloudinit/net/renderer.py b/cloudinit/net/renderer.py index a5b2b573..c68658dc 100644 --- a/cloudinit/net/renderer.py +++ b/cloudinit/net/renderer.py @@ -5,6 +5,7 @@ # # This file is part of cloud-init. See LICENSE file for license information. +import abc import six from .network_state import parse_net_config_data @@ -37,6 +38,10 @@ class Renderer(object): iface['mac_address'])) return content.getvalue() + @abc.abstractmethod + def render_network_state(self, network_state, target=None): + """Render network state.""" + def render_network_config(self, network_config, target=None): return self.render_network_state( network_state=parse_net_config_data(network_config), target=target) diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py index c2b0eac2..8528fa10 100644 --- a/cloudinit/sources/DataSourceAltCloud.py +++ b/cloudinit/sources/DataSourceAltCloud.py @@ -201,8 +201,7 @@ class DataSourceAltCloud(sources.DataSource): util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err) return False except OSError as _err: - util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), - _err.message) + util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err) return False try: diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 1f1baf46..cd75e6ea 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -286,12 +286,12 @@ def parse_shell_config(content, keylist=None, bash=None, asuser=None, output = output[0:-1] # remove trailing null # go through output. First _start_ is for 'preset', second for 'target'. - # Add to target only things were changed and not in volitile + # Add to ret only things were changed and not in excluded. for line in output.split("\x00"): try: (key, val) = line.split("=", 1) if target is preset: - target[key] = val + preset[key] = val elif (key not in excluded and (key in keylist_in or preset.get(key) != val)): ret[key] = val diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 18294505..5c99437e 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -50,7 +50,7 @@ class DataSource(object): self.distro = distro self.paths = paths self.userdata = None - self.metadata = None + self.metadata = {} self.userdata_raw = None self.vendordata = None self.vendordata_raw = None diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 312b0460..2f6a158e 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -45,7 +45,7 @@ try: from distutils.version import LooseVersion import pkg_resources _REQ = pkg_resources.get_distribution('requests') - _REQ_VER = LooseVersion(_REQ.version) + _REQ_VER = LooseVersion(_REQ.version) # pylint: disable=no-member if _REQ_VER >= LooseVersion('0.8.8'): SSL_ENABLED = True if _REQ_VER >= LooseVersion('0.7.0') and _REQ_VER < LooseVersion('1.0.0'): diff --git a/tox.ini b/tox.ini index f016f206..bf9046af 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27, py3, flake8, xenial +envlist = py27, py3, flake8, xenial, pylint recreate = True [testenv] @@ -17,6 +17,10 @@ commands = {envpython} -m flake8 {posargs:cloudinit/ tests/ tools/} setenv = LC_ALL = en_US.utf-8 +[testenv:pylint] +deps = pylint==1.6.5 +commands = {envpython} -m pylint {posargs:cloudinit} + [testenv:py3] basepython = python3 commands = {envpython} -m nose {posargs:--with-coverage \ @@ -88,6 +92,10 @@ deps = pycodestyle commands = {envpython} -m pyflakes {posargs:cloudinit/ tests/ tools/} deps = pyflakes +[testenv:tip-pylint] +commands = {envpython} -m pylint {posargs:cloudinit} +deps = pylint + [testenv:citest] basepython = python3 commands = {envpython} -m tests.cloud_tests {posargs} -- cgit v1.2.3 From 18762d706a2527b8a9ae94e4497b5c3f4a7c845e Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Tue, 28 Mar 2017 13:44:37 -0500 Subject: netplan: remove debugging prints, add debug logging Remove debugging print statements. Change a few to use logging.debug() where useful. --- cloudinit/net/netplan.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'cloudinit/net') diff --git a/cloudinit/net/netplan.py b/cloudinit/net/netplan.py index cd93b21c..7444ae33 100644 --- a/cloudinit/net/netplan.py +++ b/cloudinit/net/netplan.py @@ -6,10 +6,12 @@ import os from . import renderer from .network_state import subnet_is_ipv6 +from cloudinit import log as logging from cloudinit import util from cloudinit.net import SYS_CLASS_NET, get_devicelist +LOG = logging.getLogger(__name__) NET_CONFIG_TO_V2 = { 'bond': {'bond-ad-select': 'ad-select', 'bond-arp-interval': 'arp-interval', @@ -176,7 +178,6 @@ class Renderer(renderer.Renderer): # render from state content = self._render_content(network_state) - # ensure we poke udev to run net_setup_link if not header.endswith("\n"): header += "\n" util.write_file(fpnplan, header + content) @@ -186,7 +187,7 @@ class Renderer(renderer.Renderer): def _netplan_generate(self, run=False): if not run: - print("netplan postcmd disabled") + LOG.debug("netplan generate postcmd disabled") return util.subp(self.NETPLAN_GENERATE, capture=True) @@ -196,17 +197,15 @@ class Renderer(renderer.Renderer): the setup_link udev builtin command """ if not run: - print("netsetup postcmd disabled") + LOG.debug("netplan net_setup_link postcmd disabled") return setup_lnk = ['udevadm', 'test-builtin', 'net_setup_link'] for cmd in [setup_lnk + [SYS_CLASS_NET + iface] for iface in get_devicelist() if os.path.islink(SYS_CLASS_NET + iface)]: - print(cmd) util.subp(cmd, capture=True) def _render_content(self, network_state): - print('rendering v2 for victory!') ethernets = {} wifis = {} bridges = {} -- cgit v1.2.3 From d23543eb206326a53a59d86afba862edbd02c231 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 30 Mar 2017 12:21:43 -0400 Subject: net: in netplan renderer delete known image-builtin content. When rendering network configuration to netplan, remove known "builtin" configurations. The specific example here is Ubuntu Core that has netplan configuration in etc/netplan/00-snapd-config.yaml. We also delete the derived files since netplan will have created these derived files in its generator that runs well before cloud-init. LP: #1675576 --- cloudinit/net/netplan.py | 40 +++++++++++++++++++++++++++ tests/unittests/test_net.py | 67 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 106 insertions(+), 1 deletion(-) (limited to 'cloudinit/net') diff --git a/cloudinit/net/netplan.py b/cloudinit/net/netplan.py index 7444ae33..825fe831 100644 --- a/cloudinit/net/netplan.py +++ b/cloudinit/net/netplan.py @@ -10,6 +10,21 @@ from cloudinit import log as logging from cloudinit import util from cloudinit.net import SYS_CLASS_NET, get_devicelist +KNOWN_SNAPD_CONFIG = b"""\ +# This is the initial network config. +# It can be overwritten by cloud-init or console-conf. +network: + version: 2 + ethernets: + all-en: + match: + name: "en*" + dhcp4: true + all-eth: + match: + name: "eth*" + dhcp4: true +""" LOG = logging.getLogger(__name__) NET_CONFIG_TO_V2 = { @@ -154,6 +169,28 @@ def _extract_bond_slaves_by_name(interfaces, entry, bond_master): entry.update({'interfaces': bond_slave_names}) +def _clean_default(target=None): + # clean out any known default files and derived files in target + # LP: #1675576 + tpath = util.target_path(target, "etc/netplan/00-snapd-config.yaml") + if not os.path.isfile(tpath): + return + content = util.load_file(tpath, decode=False) + if content != KNOWN_SNAPD_CONFIG: + return + + derived = [util.target_path(target, f) for f in ( + 'run/systemd/network/10-netplan-all-en.network', + 'run/systemd/network/10-netplan-all-eth.network', + 'run/systemd/generator/netplan.stamp')] + existing = [f for f in derived if os.path.isfile(f)] + LOG.debug("removing known config '%s' and derived existing files: %s", + tpath, existing) + + for f in [tpath] + existing: + os.unlink(f) + + class Renderer(renderer.Renderer): """Renders network information in a /etc/netplan/network.yaml format.""" @@ -166,6 +203,7 @@ class Renderer(renderer.Renderer): 'etc/netplan/50-cloud-init.yaml') self.netplan_header = config.get('netplan_header', None) self._postcmds = config.get('postcmds', False) + self.clean_default = config.get('clean_default', True) def render_network_state(self, target, network_state): # check network state for version @@ -182,6 +220,8 @@ class Renderer(renderer.Renderer): header += "\n" util.write_file(fpnplan, header + content) + if self.clean_default: + _clean_default(target=target) self._netplan_generate(run=self._postcmds) self._net_setup_link(run=self._postcmds) diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 4f07d804..bfd04ba0 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -974,12 +974,14 @@ iface eth0 inet dhcp class TestNetplanNetRendering(CiTestCase): + @mock.patch("cloudinit.net.netplan._clean_default") @mock.patch("cloudinit.net.sys_dev_path") @mock.patch("cloudinit.net.read_sys_net") @mock.patch("cloudinit.net.get_devicelist") def test_default_generation(self, mock_get_devicelist, mock_read_sys_net, - mock_sys_dev_path): + mock_sys_dev_path, + mock_clean_default): tmp_dir = self.tmp_dir() _setup_test(tmp_dir, mock_get_devicelist, mock_read_sys_net, mock_sys_dev_path) @@ -1013,6 +1015,69 @@ network: set-name: eth1000 """ self.assertEqual(expected.lstrip(), contents.lstrip()) + self.assertEqual(1, mock_clean_default.call_count) + + +class TestNetplanCleanDefault(CiTestCase): + snapd_known_path = 'etc/netplan/00-snapd-config.yaml' + snapd_known_content = textwrap.dedent("""\ + # This is the initial network config. + # It can be overwritten by cloud-init or console-conf. + network: + version: 2 + ethernets: + all-en: + match: + name: "en*" + dhcp4: true + all-eth: + match: + name: "eth*" + dhcp4: true + """) + stub_known = { + 'run/systemd/network/10-netplan-all-en.network': 'foo-en', + 'run/systemd/network/10-netplan-all-eth.network': 'foo-eth', + 'run/systemd/generator/netplan.stamp': 'stamp', + } + + def test_clean_known_config_cleaned(self): + content = {self.snapd_known_path: self.snapd_known_content, } + content.update(self.stub_known) + tmpd = self.tmp_dir() + files = sorted(populate_dir(tmpd, content)) + netplan._clean_default(target=tmpd) + found = [t for t in files if os.path.exists(t)] + self.assertEqual([], found) + + def test_clean_unknown_config_not_cleaned(self): + content = {self.snapd_known_path: self.snapd_known_content, } + content.update(self.stub_known) + content[self.snapd_known_path] += "# user put a comment\n" + tmpd = self.tmp_dir() + files = sorted(populate_dir(tmpd, content)) + netplan._clean_default(target=tmpd) + found = [t for t in files if os.path.exists(t)] + self.assertEqual(files, found) + + def test_clean_known_config_cleans_only_expected(self): + astamp = "run/systemd/generator/another.stamp" + anet = "run/systemd/network/10-netplan-all-lo.network" + ayaml = "etc/netplan/01-foo-config.yaml" + content = { + self.snapd_known_path: self.snapd_known_content, + astamp: "stamp", + anet: "network", + ayaml: "yaml", + } + content.update(self.stub_known) + + tmpd = self.tmp_dir() + files = sorted(populate_dir(tmpd, content)) + netplan._clean_default(target=tmpd) + found = [t for t in files if os.path.exists(t)] + expected = [util.target_path(tmpd, f) for f in (astamp, anet, ayaml)] + self.assertEqual(sorted(expected), found) class TestNetplanPostcommands(CiTestCase): -- cgit v1.2.3 From bf7723e8092bb1f8a442aa2399dd870e130a27d9 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 31 Mar 2017 10:56:04 -0400 Subject: Fix bug that resulted in an attempt to rename bonds or vlans. When cloud-init ran in the init stage (after networking had come up). A bug could occur where cloud-init would attempt and fail to rename network devices that had "inherited" mac addresses. The intent of apply_network_config_names was always to rename only the devices that were "physical" per the network config. (This would include veth devices in a container). The bug was in creating the dictionary of interfaces by mac address. If there were multiple interfaces with the same mac address then renames could fail. This situation was guaranteed to occur with bonds or vlans or other devices that inherit their mac. The solution is to change get_interfaces_by_mac to skip interfaces that have an inherited mac. Also drop the 'devs' argument to get_interfaces_by_mac. It was non-obvious what the result should be if a device in the input list was filtered out. ie should the following have an entry for bond0 or not. get_interfaces_by_mac(devs=['bond0']) LP: #1669860 --- cloudinit/net/__init__.py | 78 ++++++++++++++++++++++++++++++++++----------- tests/unittests/test_net.py | 76 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+), 19 deletions(-) mode change 100755 => 100644 cloudinit/net/__init__.py (limited to 'cloudinit/net') diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py old mode 100755 new mode 100644 index 1cf98ef5..346be5d3 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -82,6 +82,10 @@ def is_wireless(devname): return os.path.exists(sys_dev_path(devname, "wireless")) +def is_bridge(devname): + return os.path.exists(sys_dev_path(devname, "bridge")) + + def is_connected(devname): # is_connected isn't really as simple as that. 2 is # 'physically connected'. 3 is 'not connected'. but a wlan interface will @@ -132,7 +136,7 @@ def generate_fallback_config(): for interface in potential_interfaces: if interface.startswith("veth"): continue - if os.path.exists(sys_dev_path(interface, "bridge")): + if is_bridge(interface): # skip any bridges continue carrier = read_sys_net_int(interface, 'carrier') @@ -187,7 +191,11 @@ def apply_network_config_names(netcfg, strict_present=True, strict_busy=True): """read the network config and rename devices accordingly. if strict_present is false, then do not raise exception if no devices match. if strict_busy is false, then do not raise exception if the - device cannot be renamed because it is currently configured.""" + device cannot be renamed because it is currently configured. + + renames are only attempted for interfaces of type 'physical'. It is + expected that the network system will create other devices with the + correct name in place.""" renames = [] for ent in netcfg.get('config', {}): if ent.get('type') != 'physical': @@ -201,13 +209,35 @@ def apply_network_config_names(netcfg, strict_present=True, strict_busy=True): return _rename_interfaces(renames) +def interface_has_own_mac(ifname, strict=False): + """return True if the provided interface has its own address. + + Based on addr_assign_type in /sys. Return true for any interface + that does not have a 'stolen' address. Examples of such devices + are bonds or vlans that inherit their mac from another device. + Possible values are: + 0: permanent address 2: stolen from another device + 1: randomly generated 3: set using dev_set_mac_address""" + + assign_type = read_sys_net_int(ifname, "addr_assign_type") + if strict and assign_type is None: + raise ValueError("%s had no addr_assign_type.") + return assign_type in (0, 1, 3) + + def _get_current_rename_info(check_downable=True): - """Collect information necessary for rename_interfaces.""" - names = get_devicelist() + """Collect information necessary for rename_interfaces. + + returns a dictionary by mac address like: + {mac: + {'name': name + 'up': boolean: is_up(name), + 'downable': None or boolean indicating that the + device has only automatically assigned ip addrs.}} + """ bymac = {} - for n in names: - bymac[get_interface_mac(n)] = { - 'name': n, 'up': is_up(n), 'downable': None} + for mac, name in get_interfaces_by_mac().items(): + bymac[mac] = {'name': name, 'up': is_up(name), 'downable': None} if check_downable: nmatch = re.compile(r"[0-9]+:\s+(\w+)[@:]") @@ -346,22 +376,32 @@ def get_interface_mac(ifname): return read_sys_net_safe(ifname, path) -def get_interfaces_by_mac(devs=None): - """Build a dictionary of tuples {mac: name}""" - if devs is None: - try: - devs = get_devicelist() - except OSError as e: - if e.errno == errno.ENOENT: - devs = [] - else: - raise +def get_interfaces_by_mac(): + """Build a dictionary of tuples {mac: name}. + + Bridges and any devices that have a 'stolen' mac are excluded.""" + try: + devs = get_devicelist() + except OSError as e: + if e.errno == errno.ENOENT: + devs = [] + else: + raise ret = {} for name in devs: + if not interface_has_own_mac(name): + continue + if is_bridge(name): + continue mac = get_interface_mac(name) # some devices may not have a mac (tun0) - if mac: - ret[mac] = name + if not mac: + continue + if mac in ret: + raise RuntimeError( + "duplicate mac found! both '%s' and '%s' have mac '%s'" % + (name, ret[mac], mac)) + ret[mac] = name return ret diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index bfd04ba0..9cc5e4ab 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -1461,6 +1461,82 @@ class TestNetRenderers(CiTestCase): priority=['sysconfig', 'eni']) +class TestGetInterfacesByMac(CiTestCase): + _data = {'devices': ['enp0s1', 'enp0s2', 'bond1', 'bridge1', + 'bridge1-nic', 'tun0'], + 'bonds': ['bond1'], + 'bridges': ['bridge1'], + 'own_macs': ['enp0s1', 'enp0s2', 'bridge1-nic', 'bridge1'], + 'macs': {'enp0s1': 'aa:aa:aa:aa:aa:01', + 'enp0s2': 'aa:aa:aa:aa:aa:02', + 'bond1': 'aa:aa:aa:aa:aa:01', + 'bridge1': 'aa:aa:aa:aa:aa:03', + 'bridge1-nic': 'aa:aa:aa:aa:aa:03', + 'tun0': None}} + data = {} + + def _se_get_devicelist(self): + return self.data['devices'] + + def _se_get_interface_mac(self, name): + return self.data['macs'][name] + + def _se_is_bridge(self, name): + return name in self.data['bridges'] + + def _se_interface_has_own_mac(self, name): + return name in self.data['own_macs'] + + def _mock_setup(self): + self.data = copy.deepcopy(self._data) + mocks = ('get_devicelist', 'get_interface_mac', 'is_bridge', + 'interface_has_own_mac') + self.mocks = {} + for n in mocks: + m = mock.patch('cloudinit.net.' + n, + side_effect=getattr(self, '_se_' + n)) + self.addCleanup(m.stop) + self.mocks[n] = m.start() + + def test_raise_exception_on_duplicate_macs(self): + self._mock_setup() + self.data['macs']['bridge1-nic'] = self.data['macs']['enp0s1'] + self.assertRaises(RuntimeError, net.get_interfaces_by_mac) + + def test_excludes_any_without_mac_address(self): + self._mock_setup() + ret = net.get_interfaces_by_mac() + self.assertIn('tun0', self._se_get_devicelist()) + self.assertNotIn('tun0', ret.values()) + + def test_excludes_stolen_macs(self): + self._mock_setup() + ret = net.get_interfaces_by_mac() + self.mocks['interface_has_own_mac'].assert_has_calls( + [mock.call('enp0s1'), mock.call('bond1')], any_order=True) + self.assertEqual( + {'aa:aa:aa:aa:aa:01': 'enp0s1', 'aa:aa:aa:aa:aa:02': 'enp0s2', + 'aa:aa:aa:aa:aa:03': 'bridge1-nic'}, + ret) + + def test_excludes_bridges(self): + self._mock_setup() + # add a device 'b1', make all return they have their "own mac", + # set everything other than 'b1' to be a bridge. + # then expect b1 is the only thing left. + self.data['macs']['b1'] = 'aa:aa:aa:aa:aa:b1' + self.data['devices'].append('b1') + self.data['bonds'] = [] + self.data['own_macs'] = self.data['devices'] + self.data['bridges'] = [f for f in self.data['devices'] if f != "b1"] + ret = net.get_interfaces_by_mac() + self.assertEqual({'aa:aa:aa:aa:aa:b1': 'b1'}, ret) + self.mocks['is_bridge'].assert_has_calls( + [mock.call('bridge1'), mock.call('enp0s1'), mock.call('bond1'), + mock.call('b1')], + any_order=True) + + def _gzip_data(data): with io.BytesIO() as iobuf: gzfp = gzip.GzipFile(mode="wb", fileobj=iobuf) -- cgit v1.2.3