From 5bec6b0e2a2ce5fd03bb04f441536fc130e67997 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 24 Oct 2019 20:02:15 +0000 Subject: Fix usages of yaml, and move yaml_dump to safeyaml.dumps. Here we replace uses of the pyyaml module directly with functions provided by cloudinit.safeyaml. Also, change/move cloudinit.util.yaml_dumps to cloudinit.safeyaml.dumps LP: #1849640 --- cloudinit/cmd/devel/net_convert.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) (limited to 'cloudinit/cmd/devel/net_convert.py') diff --git a/cloudinit/cmd/devel/net_convert.py b/cloudinit/cmd/devel/net_convert.py index 1ad7e0bd..9b768304 100755 --- a/cloudinit/cmd/devel/net_convert.py +++ b/cloudinit/cmd/devel/net_convert.py @@ -5,13 +5,12 @@ import argparse import json import os import sys -import yaml from cloudinit.sources.helpers import openstack from cloudinit.sources import DataSourceAzure as azure from cloudinit.sources import DataSourceOVF as ovf -from cloudinit import distros +from cloudinit import distros, safeyaml from cloudinit.net import eni, netplan, network_state, sysconfig from cloudinit import log @@ -78,13 +77,12 @@ def handle_args(name, args): if args.kind == "eni": pre_ns = eni.convert_eni_data(net_data) elif args.kind == "yaml": - pre_ns = yaml.load(net_data) + pre_ns = safeyaml.load(net_data) if 'network' in pre_ns: pre_ns = pre_ns.get('network') if args.debug: sys.stderr.write('\n'.join( - ["Input YAML", - yaml.dump(pre_ns, default_flow_style=False, indent=4), ""])) + ["Input YAML", safeyaml.dumps(pre_ns), ""])) elif args.kind == 'network_data.json': pre_ns = openstack.convert_net_json( json.loads(net_data), known_macs=known_macs) @@ -100,9 +98,8 @@ def handle_args(name, args): "input data") if args.debug: - sys.stderr.write('\n'.join([ - "", "Internal State", - yaml.dump(ns, default_flow_style=False, indent=4), ""])) + sys.stderr.write('\n'.join( + ["", "Internal State", safeyaml.dumps(ns), ""])) distro_cls = distros.fetch(args.distro) distro = distro_cls(args.distro, {}, None) config = {} -- cgit v1.2.3 From f1c788e2bb7c86069d43a015267facfb8aefcdf0 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Thu, 24 Oct 2019 20:16:47 +0000 Subject: net/netplan: use ipv6-mtu key for specifying ipv6 mtu values netplan introduced an 'info' subcommand which emits yaml describing implemented features that indicate new or changed fields and values in the yaml that it accepts. Previously, cloud-init emitted the key 'mtu6' for ipv6 MTU values. This is not correct and netplan will fail to parse these values. Netplan as of 0.98 supports both the info subcommand and the ipv6-mtu key. This branch modifies the netplan renderer to collect the netplan info output into a 'features' property which is a list of available feature flags which the renderer can use to modify its output. If the command is not available, no feature flags are set and cloud-init will render IPv6 MTU values just as MTU for the subnet. --- cloudinit/cmd/devel/net_convert.py | 2 ++ cloudinit/net/netplan.py | 35 +++++++++++++++++++++++++++-------- tests/unittests/test_net.py | 27 +++++++++++++++++++++++++-- 3 files changed, 54 insertions(+), 10 deletions(-) (limited to 'cloudinit/cmd/devel/net_convert.py') diff --git a/cloudinit/cmd/devel/net_convert.py b/cloudinit/cmd/devel/net_convert.py index 9b768304..2d27a76a 100755 --- a/cloudinit/cmd/devel/net_convert.py +++ b/cloudinit/cmd/devel/net_convert.py @@ -113,6 +113,8 @@ def handle_args(name, args): config['postcmds'] = False # trim leading slash config['netplan_path'] = config['netplan_path'][1:] + # enable some netplan features + config['features'] = ['dhcp-use-domains', 'ipv6-mtu'] else: r_cls = sysconfig.Renderer config = distro.renderer_configs.get('sysconfig') diff --git a/cloudinit/net/netplan.py b/cloudinit/net/netplan.py index 54be1221..749d46f8 100644 --- a/cloudinit/net/netplan.py +++ b/cloudinit/net/netplan.py @@ -35,7 +35,7 @@ def _get_params_dict_by_match(config, match): if key.startswith(match)) -def _extract_addresses(config, entry, ifname): +def _extract_addresses(config, entry, ifname, features=None): """This method parse a cloudinit.net.network_state dictionary (config) and maps netstate keys/values into a dictionary (entry) to represent netplan yaml. @@ -67,7 +67,7 @@ def _extract_addresses(config, entry, ifname): 'match': {'macaddress': '52:54:00:12:34:00'}, 'mtu': 1501, 'address': ['192.168.1.2/24', '2001:4800:78ff:1b:be76:4eff:fe06:1000"], - 'mtu6': 1480} + 'ipv6-mtu': 1480} """ @@ -80,6 +80,8 @@ def _extract_addresses(config, entry, ifname): else: return [obj, ] + if features is None: + features = [] addresses = [] routes = [] nameservers = [] @@ -109,8 +111,8 @@ def _extract_addresses(config, entry, ifname): searchdomains += _listify(subnet.get('dns_search', [])) if 'mtu' in subnet: mtukey = 'mtu' - if subnet_is_ipv6(subnet): - mtukey += '6' + if subnet_is_ipv6(subnet) and 'ipv6-mtu' in features: + mtukey = 'ipv6-mtu' entry.update({mtukey: subnet.get('mtu')}) for route in subnet.get('routes', []): to_net = "%s/%s" % (route.get('network'), @@ -180,6 +182,7 @@ class Renderer(renderer.Renderer): """Renders network information in a /etc/netplan/network.yaml format.""" NETPLAN_GENERATE = ['netplan', 'generate'] + NETPLAN_INFO = ['netplan', 'info'] def __init__(self, config=None): if not config: @@ -189,6 +192,22 @@ class Renderer(renderer.Renderer): self.netplan_header = config.get('netplan_header', None) self._postcmds = config.get('postcmds', False) self.clean_default = config.get('clean_default', True) + self._features = config.get('features', None) + + @property + def features(self): + if self._features is None: + try: + info_blob, _err = util.subp(self.NETPLAN_INFO, capture=True) + info = util.load_yaml(info_blob) + self._features = info['netplan.io']['features'] + except util.ProcessExecutionError: + # if the info subcommand is not present then we don't have any + # new features + pass + except (TypeError, KeyError) as e: + LOG.debug('Failed to list features from netplan info: %s', e) + return self._features def render_network_state(self, network_state, templates=None, target=None): # check network state for version @@ -272,7 +291,7 @@ class Renderer(renderer.Renderer): else: del eth['match'] del eth['set-name'] - _extract_addresses(ifcfg, eth, ifname) + _extract_addresses(ifcfg, eth, ifname, self.features) ethernets.update({ifname: eth}) elif if_type == 'bond': @@ -297,7 +316,7 @@ class Renderer(renderer.Renderer): slave_interfaces = ifcfg.get('bond-slaves') if slave_interfaces == 'none': _extract_bond_slaves_by_name(interfaces, bond, ifname) - _extract_addresses(ifcfg, bond, ifname) + _extract_addresses(ifcfg, bond, ifname, self.features) bonds.update({ifname: bond}) elif if_type == 'bridge': @@ -332,7 +351,7 @@ class Renderer(renderer.Renderer): bridge.update({'parameters': br_config}) if ifcfg.get('mac_address'): bridge['macaddress'] = ifcfg.get('mac_address').lower() - _extract_addresses(ifcfg, bridge, ifname) + _extract_addresses(ifcfg, bridge, ifname, self.features) bridges.update({ifname: bridge}) elif if_type == 'vlan': @@ -344,7 +363,7 @@ class Renderer(renderer.Renderer): macaddr = ifcfg.get('mac_address', None) if macaddr is not None: vlan['macaddress'] = macaddr.lower() - _extract_addresses(ifcfg, vlan, ifname) + _extract_addresses(ifcfg, vlan, ifname, self.features) vlans.update({ifname: vlan}) # inject global nameserver values under each all interface which diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index d2201998..21604b12 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -996,8 +996,8 @@ NETWORK_CONFIGS = { addresses: - 192.168.14.2/24 - 2001:1::1/64 + ipv6-mtu: 1500 mtu: 9000 - mtu6: 1500 """).rstrip(' '), 'yaml': textwrap.dedent("""\ version: 1 @@ -3585,7 +3585,9 @@ class TestNetplanPostcommands(CiTestCase): @mock.patch.object(netplan.Renderer, '_netplan_generate') @mock.patch.object(netplan.Renderer, '_net_setup_link') - def test_netplan_render_calls_postcmds(self, mock_netplan_generate, + @mock.patch('cloudinit.util.subp') + def test_netplan_render_calls_postcmds(self, mock_subp, + mock_netplan_generate, mock_net_setup_link): tmp_dir = self.tmp_dir() ns = network_state.parse_net_config_data(self.mycfg, @@ -3597,6 +3599,7 @@ class TestNetplanPostcommands(CiTestCase): render_target = 'netplan.yaml' renderer = netplan.Renderer( {'netplan_path': render_target, 'postcmds': True}) + mock_subp.side_effect = iter([util.ProcessExecutionError]) renderer.render_network_state(ns, target=render_dir) mock_netplan_generate.assert_called_with(run=True) @@ -3619,7 +3622,13 @@ class TestNetplanPostcommands(CiTestCase): render_target = 'netplan.yaml' renderer = netplan.Renderer( {'netplan_path': render_target, 'postcmds': True}) + mock_subp.side_effect = iter([ + util.ProcessExecutionError, + ('', ''), + ('', ''), + ]) expected = [ + mock.call(['netplan', 'info'], capture=True), mock.call(['netplan', 'generate'], capture=True), mock.call(['udevadm', 'test-builtin', 'net_setup_link', '/sys/class/net/lo'], capture=True), @@ -3875,6 +3884,20 @@ class TestReadInitramfsConfig(CiTestCase): class TestNetplanRoundTrip(CiTestCase): + + NETPLAN_INFO_OUT = textwrap.dedent(""" + netplan.io: + features: + - dhcp-use-domains + - ipv6-mtu + website: https://netplan.io/ + """) + + def setUp(self): + super(TestNetplanRoundTrip, self).setUp() + self.add_patch('cloudinit.net.netplan.util.subp', 'm_subp') + self.m_subp.return_value = (self.NETPLAN_INFO_OUT, '') + def _render_and_read(self, network_config=None, state=None, netplan_path=None, target=None): if target is None: -- cgit v1.2.3 From 88b09643787c55b700cfadc25bbe0ddbfe0368e6 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 28 Jan 2020 10:45:27 -0500 Subject: cloudinit/cmd/devel/net_convert.py: add missing space (#191) --- cloudinit/cmd/devel/net_convert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/cmd/devel/net_convert.py') diff --git a/cloudinit/cmd/devel/net_convert.py b/cloudinit/cmd/devel/net_convert.py index 2d27a76a..80d217ca 100755 --- a/cloudinit/cmd/devel/net_convert.py +++ b/cloudinit/cmd/devel/net_convert.py @@ -95,7 +95,7 @@ def handle_args(name, args): ns = network_state.parse_net_config_data(pre_ns) if not ns: raise RuntimeError("No valid network_state object created from" - "input data") + " input data") if args.debug: sys.stderr.write('\n'.join( -- cgit v1.2.3