From a2249942522d140a063acdc007aced991d4f0588 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 5 May 2016 14:57:48 -0700 Subject: Work on refactoring (and adding) network conversion tests --- cloudinit/net/network_state.py | 68 ------------------------------------------ 1 file changed, 68 deletions(-) (limited to 'cloudinit/net/network_state.py') diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index e32d2cdf..d08e94fe 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -376,71 +376,3 @@ def mask2cidr(mask): return ipv4mask2cidr(mask) else: return mask - - -if __name__ == '__main__': - import sys - import random - from cloudinit import net - - def load_config(nc): - version = nc.get('version') - config = nc.get('config') - return (version, config) - - def test_parse(network_config): - (version, config) = load_config(network_config) - ns1 = NetworkState(version=version, config=config) - ns1.parse_config() - random.shuffle(config) - ns2 = NetworkState(version=version, config=config) - ns2.parse_config() - print("----NS1-----") - print(ns1.dump_network_state()) - print() - print("----NS2-----") - print(ns2.dump_network_state()) - print("NS1 == NS2 ?=> {}".format( - ns1.network_state == ns2.network_state)) - eni = net.render_interfaces(ns2.network_state) - print(eni) - udev_rules = net.render_persistent_net(ns2.network_state) - print(udev_rules) - - def test_dump_and_load(network_config): - print("Loading network_config into NetworkState") - (version, config) = load_config(network_config) - ns1 = NetworkState(version=version, config=config) - ns1.parse_config() - print("Dumping state to file") - ns1_dump = ns1.dump() - ns1_state = "/tmp/ns1.state" - with open(ns1_state, "w+") as f: - f.write(ns1_dump) - - print("Loading state from file") - ns2 = from_state_file(ns1_state) - print("NS1 == NS2 ?=> {}".format( - ns1.network_state == ns2.network_state)) - - def test_output(network_config): - (version, config) = load_config(network_config) - ns1 = NetworkState(version=version, config=config) - ns1.parse_config() - random.shuffle(config) - ns2 = NetworkState(version=version, config=config) - ns2.parse_config() - print("NS1 == NS2 ?=> {}".format( - ns1.network_state == ns2.network_state)) - eni_1 = net.render_interfaces(ns1.network_state) - eni_2 = net.render_interfaces(ns2.network_state) - print(eni_1) - print(eni_2) - print("eni_1 == eni_2 ?=> {}".format( - eni_1 == eni_2)) - - y = util.read_conf(sys.argv[1]) - network_config = y.get('network') - test_parse(network_config) - test_dump_and_load(network_config) - test_output(network_config) -- cgit v1.2.3 From db9d958a0e76c2c59298041a1355aba97fda000f Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 5 May 2016 16:38:53 -0700 Subject: Add the bridge net type --- cloudinit/net/__init__.py | 2 +- cloudinit/net/network_state.py | 39 +++++++++++++++++++++------------- cloudinit/sources/helpers/openstack.py | 5 +++++ 3 files changed, 30 insertions(+), 16 deletions(-) (limited to 'cloudinit/net/network_state.py') diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 31544fd8..cc154c57 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -262,7 +262,7 @@ def parse_deb_config(path): def parse_net_config_data(net_config): - """Parses the config, returns NetworkState dictionary + """Parses the config, returns NetworkState object :param net_config: curtin network config dict """ diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index d08e94fe..27d35256 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -15,6 +15,8 @@ # You should have received a copy of the GNU Affero General Public License # along with Curtin. If not, see . +import six + from cloudinit import log as logging from cloudinit import util from cloudinit.util import yaml_dumps as dump_config @@ -32,11 +34,31 @@ def from_state_file(state_file): state = util.read_conf(state_file) network_state = NetworkState() network_state.load(state) - return network_state -class NetworkState: +class CommandHandlerMeta(type): + """Metaclass that dynamically creates a 'command_handlers' attribute. + + This will scan the to-be-created class for methods that start with + 'handle_' and on finding those will populate a class attribute mapping + so that those methods can be quickly located and called. + """ + def __new__(cls, name, parents, dct): + command_handlers = {} + for attr_name, attr in six.iteritems(dct): + if six.callable(attr) and attr_name.startswith('handle_'): + handles_what = attr_name[len('handle_'):] + if handles_what: + command_handlers[handles_what] = attr + dct['command_handlers'] = command_handlers + return super(CommandHandlerMeta, cls).__new__(cls, name, + parents, dct) + + +@six.add_metaclass(CommandHandlerMeta) +class NetworkState(object): + def __init__(self, version=NETWORK_STATE_VERSION, config=None): self.version = version self.config = config @@ -48,18 +70,6 @@ class NetworkState: 'search': [], } } - self.command_handlers = self.get_command_handlers() - - def get_command_handlers(self): - METHOD_PREFIX = 'handle_' - methods = filter(lambda x: callable(getattr(self, x)) and - x.startswith(METHOD_PREFIX), dir(self)) - handlers = {} - for m in methods: - key = m.replace(METHOD_PREFIX, '') - handlers[key] = getattr(self, m) - - return handlers def dump(self): state = { @@ -83,7 +93,6 @@ class NetworkState: # v1 - direct attr mapping, except version for key in [k for k in required_keys if k not in ['version']]: setattr(self, key, state[key]) - self.command_handlers = self.get_command_handlers() def dump_network_state(self): return dump_config(self.network_state) diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 475ccab3..845ea971 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -576,6 +576,11 @@ def convert_net_json(network_json): 'vlan_id': link['vlan_id'], 'mac_address': link['vlan_mac_address'], }) + elif link['type'] in ['bridge']: + cfg.update({ + 'type': 'bridge', + 'mac_address': link['ethernet_mac_address'], + 'mtu': link['mtu']}) else: raise ValueError( 'Unknown network_data link type: %s' % link['type']) -- cgit v1.2.3 From 2c32c855278c22b77b6ac3bfca23fa2fa6bc6330 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 5 May 2016 17:03:08 -0700 Subject: Use a decorator vs repeated key checks. --- cloudinit/net/network_state.py | 136 ++++++++++++++++++++--------------------- 1 file changed, 66 insertions(+), 70 deletions(-) (limited to 'cloudinit/net/network_state.py') diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 27d35256..73be84e1 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -15,6 +15,8 @@ # You should have received a copy of the GNU Affero General Public License # along with Curtin. If not, see . +import copy + import six from cloudinit import log as logging @@ -37,6 +39,37 @@ def from_state_file(state_file): return network_state +class InvalidCommand(Exception): + pass + + +def ensure_command_keys(required_keys): + required_keys = frozenset(required_keys) + + def extract_missing(command): + missing_keys = set() + for key in required_keys: + if key not in command: + missing_keys.add(key) + return missing_keys + + def wrapper(func): + + @six.wraps(func) + def decorator(self, command, *args, **kwargs): + if required_keys: + missing_keys = extract_missing(command) + if missing_keys: + raise InvalidCommand("Command missing %s of required" + " keys %s" % (missing_keys, + required_keys)) + return func(self, command, *args, **kwargs) + + return decorator + + return wrapper + + class CommandHandlerMeta(type): """Metaclass that dynamically creates a 'command_handlers' attribute. @@ -59,17 +92,19 @@ class CommandHandlerMeta(type): @six.add_metaclass(CommandHandlerMeta) class NetworkState(object): + initial_network_state = { + 'interfaces': {}, + 'routes': [], + 'dns': { + 'nameservers': [], + 'search': [], + } + } + def __init__(self, version=NETWORK_STATE_VERSION, config=None): self.version = version self.config = config - self.network_state = { - 'interfaces': {}, - 'routes': [], - 'dns': { - 'nameservers': [], - 'search': [], - } - } + self.network_state = copy.deepcopy(self.initial_network_state) def dump(self): state = { @@ -97,19 +132,26 @@ class NetworkState(object): def dump_network_state(self): return dump_config(self.network_state) - def parse_config(self): + def parse_config(self, skip_broken=True): # rebuild network state for command in self.config: - handler = self.command_handlers.get(command['type']) - handler(command) - - def valid_command(self, command, required_keys): - if not required_keys: - return False - - found_keys = [key for key in command.keys() if key in required_keys] - return len(found_keys) == len(required_keys) - + command_type = command['type'] + try: + handler = self.command_handlers[command_type] + except KeyError: + raise RuntimeError("No handler found for" + " command '%s'" % command_type) + try: + handler(command) + except InvalidCommand: + if not skip_broken: + raise + else: + LOG.warn("Skipping invalid command: %s", command, + exc_info=True) + LOG.debug(self.dump_network_state()) + + @ensure_command_keys(['name']) def handle_physical(self, command): ''' command = { @@ -121,13 +163,6 @@ class NetworkState(object): ] } ''' - required_keys = [ - 'name', - ] - if not self.valid_command(command, required_keys): - LOG.warn('Skipping Invalid command: {}'.format(command)) - LOG.debug(self.dump_network_state()) - return interfaces = self.network_state.get('interfaces') iface = interfaces.get(command['name'], {}) @@ -158,6 +193,7 @@ class NetworkState(object): self.network_state['interfaces'].update({command.get('name'): iface}) self.dump_network_state() + @ensure_command_keys(['name', 'vlan_id', 'vlan_link']) def handle_vlan(self, command): ''' auto eth0.222 @@ -167,16 +203,6 @@ class NetworkState(object): hwaddress ether BC:76:4E:06:96:B3 vlan-raw-device eth0 ''' - required_keys = [ - 'name', - 'vlan_link', - 'vlan_id', - ] - if not self.valid_command(command, required_keys): - print('Skipping Invalid command: {}'.format(command)) - print(self.dump_network_state()) - return - interfaces = self.network_state.get('interfaces') self.handle_physical(command) iface = interfaces.get(command.get('name'), {}) @@ -184,6 +210,7 @@ class NetworkState(object): iface['vlan_id'] = command.get('vlan_id') interfaces.update({iface['name']: iface}) + @ensure_command_keys(['name', 'bond_interfaces', 'params']) def handle_bond(self, command): ''' #/etc/network/interfaces @@ -209,15 +236,6 @@ class NetworkState(object): bond-updelay 200 bond-lacp-rate 4 ''' - required_keys = [ - 'name', - 'bond_interfaces', - 'params', - ] - if not self.valid_command(command, required_keys): - print('Skipping Invalid command: {}'.format(command)) - print(self.dump_network_state()) - return self.handle_physical(command) interfaces = self.network_state.get('interfaces') @@ -245,6 +263,7 @@ class NetworkState(object): bond_if.update({param: val}) self.network_state['interfaces'].update({ifname: bond_if}) + @ensure_command_keys(['name', 'bridge_interfaces', 'params']) def handle_bridge(self, command): ''' auto br0 @@ -272,15 +291,6 @@ class NetworkState(object): "bridge_waitport", ] ''' - required_keys = [ - 'name', - 'bridge_interfaces', - 'params', - ] - if not self.valid_command(command, required_keys): - print('Skipping Invalid command: {}'.format(command)) - print(self.dump_network_state()) - return # find one of the bridge port ifaces to get mac_addr # handle bridge_slaves @@ -304,15 +314,8 @@ class NetworkState(object): interfaces.update({iface['name']: iface}) + @ensure_command_keys(['address']) def handle_nameserver(self, command): - required_keys = [ - 'address', - ] - if not self.valid_command(command, required_keys): - print('Skipping Invalid command: {}'.format(command)) - print(self.dump_network_state()) - return - dns = self.network_state.get('dns') if 'address' in command: addrs = command['address'] @@ -327,15 +330,8 @@ class NetworkState(object): for path in paths: dns['search'].append(path) + @ensure_command_keys(['destination']) def handle_route(self, command): - required_keys = [ - 'destination', - ] - if not self.valid_command(command, required_keys): - print('Skipping Invalid command: {}'.format(command)) - print(self.dump_network_state()) - return - routes = self.network_state.get('routes') network, cidr = command['destination'].split("/") netmask = cidr2mask(int(cidr)) -- cgit v1.2.3 From 4b0d2430e7674d5abb8fb27ac9ddb129d2bc0715 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 10 May 2016 14:16:07 -0700 Subject: Fix removal of validate_command --- cloudinit/net/network_state.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) (limited to 'cloudinit/net/network_state.py') diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 73be84e1..2530a601 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -39,26 +39,25 @@ def from_state_file(state_file): return network_state +def diff_keys(expected, actual): + missing = set(expected) + for key in actual: + missing.discard(key) + return missing + + class InvalidCommand(Exception): pass def ensure_command_keys(required_keys): - required_keys = frozenset(required_keys) - - def extract_missing(command): - missing_keys = set() - for key in required_keys: - if key not in command: - missing_keys.add(key) - return missing_keys def wrapper(func): @six.wraps(func) def decorator(self, command, *args, **kwargs): if required_keys: - missing_keys = extract_missing(command) + missing_keys = diff_keys(required_keys, command) if missing_keys: raise InvalidCommand("Command missing %s of required" " keys %s" % (missing_keys, @@ -120,10 +119,11 @@ class NetworkState(object): raise Exception('Invalid state, missing version field') required_keys = NETWORK_STATE_REQUIRED_KEYS[state['version']] - if not self.valid_command(state, required_keys): - msg = 'Invalid state, missing keys: {}'.format(required_keys) + missing_keys = diff_keys(required_keys, state) + if missing_keys: + msg = 'Invalid state, missing keys: %s'.format(missing_keys) LOG.error(msg) - raise Exception(msg) + raise ValueError(msg) # v1 - direct attr mapping, except version for key in [k for k in required_keys if k not in ['version']]: -- cgit v1.2.3 From c5eb65ed705475640fce1025c74a54052c6e9731 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 10 May 2016 15:12:44 -0700 Subject: Add some basic eni rendering tests --- cloudinit/net/__init__.py | 11 ++++--- cloudinit/net/network_state.py | 2 +- tests/unittests/test_net.py | 68 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 7 deletions(-) (limited to 'cloudinit/net/network_state.py') diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index e911ed0c..0202cbd8 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -107,7 +107,7 @@ class ParserError(Exception): """Raised when parser has issue parsing the interfaces file.""" -def parse_net_config_data(net_config): +def parse_net_config_data(net_config, skip_broken=True): """Parses the config, returns NetworkState object :param net_config: curtin network config dict @@ -116,20 +116,19 @@ def parse_net_config_data(net_config): if 'version' in net_config and 'config' in net_config: ns = network_state.NetworkState(version=net_config.get('version'), config=net_config.get('config')) - ns.parse_config() + ns.parse_config(skip_broken=skip_broken) state = ns.network_state - return state -def parse_net_config(path): +def parse_net_config(path, skip_broken=True): """Parses a curtin network configuration file and return network state""" ns = None net_config = util.read_conf(path) if 'network' in net_config: - ns = parse_net_config_data(net_config.get('network')) - + ns = parse_net_config_data(net_config.get('network'), + skip_broken=skip_broken) return ns diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 2530a601..2feffa71 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -142,7 +142,7 @@ class NetworkState(object): raise RuntimeError("No handler found for" " command '%s'" % command_type) try: - handler(command) + handler(self, command) except InvalidCommand: if not skip_broken: raise diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 6daf9601..37b48efb 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -1,7 +1,9 @@ from cloudinit import util from cloudinit import net from cloudinit.net import cmdline +from cloudinit.net import eni from .helpers import TestCase +from .helpers import mock import base64 import copy @@ -9,6 +11,8 @@ import io import gzip import json import os +import shutil +import tempfile DHCP_CONTENT_1 = """ DEVICE='eth0' @@ -69,6 +73,70 @@ STATIC_EXPECTED_1 = { } +class TestEniNetRendering(TestCase): + + @mock.patch("cloudinit.net.sys_dev_path") + @mock.patch("cloudinit.net.sys_netdev_info") + @mock.patch("cloudinit.net.get_devicelist") + def test_generation(self, mock_get_devicelist, mock_sys_netdev_info, + mock_sys_dev_path): + mock_get_devicelist.return_value = ['eth1000', 'lo'] + + dev_characteristics = { + 'eth1000': { + "bridge": False, + "carrier": False, + "dormant": False, + "operstate": "down", + "address": "07-1C-C6-75-A4-BE", + } + } + + def netdev_info(name, field): + return dev_characteristics[name][field] + + mock_sys_netdev_info.side_effect = netdev_info + + tmp_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp_dir) + + def sys_dev_path(devname, path=""): + return tmp_dir + devname + "/" + path + + for dev in dev_characteristics: + os.makedirs(os.path.join(tmp_dir, dev)) + with open(os.path.join(tmp_dir, dev, 'operstate'), 'w') as fh: + fh.write("down") + + mock_sys_dev_path.side_effect = sys_dev_path + + network_cfg = net.generate_fallback_config() + network_state = net.parse_net_config_data(network_cfg, + skip_broken=False) + + render_dir = os.path.join(tmp_dir, "render") + os.makedirs(render_dir) + + renderer = eni.Renderer() + renderer.render_network_state(render_dir, network_state, + eni="interfaces", + links_prefix=None, + netrules=None) + + self.assertTrue(os.path.exists(os.path.join(render_dir, + 'interfaces'))) + with open(os.path.join(render_dir, 'interfaces')) as fh: + contents = fh.read() + + expected = """auto lo +iface lo inet loopback + +auto eth1000 +iface eth1000 inet dhcp +""" + self.assertEqual(expected, contents) + + class TestNetConfigParsing(TestCase): simple_cfg = { 'config': [{"type": "physical", "name": "eth0", -- cgit v1.2.3 From 880d9fc2f9c62abf19b1506595aa81e5417dea45 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 19 May 2016 14:13:07 -0700 Subject: Adjust net module to be more isolated This allows it to be used outside of cloudinit more easily in the future. --- cloudinit/net/__init__.py | 108 ++++++++++++++++++--------------- cloudinit/net/cmdline.py | 15 ++++- cloudinit/net/eni.py | 35 +++++------ cloudinit/net/network_state.py | 38 +++++++++--- cloudinit/sources/helpers/openstack.py | 4 ++ cloudinit/util.py | 9 --- tests/unittests/test_net.py | 7 ++- 7 files changed, 129 insertions(+), 87 deletions(-) (limited to 'cloudinit/net/network_state.py') diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 0202cbd8..07e7307e 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -17,44 +17,81 @@ # along with Curtin. If not, see . import errno +import logging import os - -from cloudinit import log as logging -from cloudinit.net import network_state -from cloudinit import util - +import six +import yaml LOG = logging.getLogger(__name__) SYS_CLASS_NET = "/sys/class/net/" LINKS_FNAME_PREFIX = "etc/systemd/network/50-cloud-init-" DEFAULT_PRIMARY_INTERFACE = 'eth0' +# NOTE(harlowja): some of these are similar to what is in cloudinit main +# source or utils tree/module but the reason that is done is so that this +# whole module can be easily extracted and placed into other +# code-bases (curtin for example). -def sys_dev_path(devname, path=""): - return SYS_CLASS_NET + devname + "/" + path +def write_file(path, content): + """Simple writing a file helper.""" + base_path = os.path.dirname(path) + if not os.path.isdir(base_path): + os.makedirs(base_path) + with open(path, "wb+") as fh: + if isinstance(content, six.text_type): + content = content.encode("utf8") + fh.write(content) -def read_sys_net(devname, path, translate=None, enoent=None, keyerror=None): +def read_file(path, decode='utf8', enoent=None): try: - contents = "" - with open(sys_dev_path(devname, path), "r") as fp: - contents = fp.read().strip() - if translate is None: - return contents - - try: - return translate.get(contents) - except KeyError: - LOG.debug("found unexpected value '%s' in '%s/%s'", contents, - devname, path) - if keyerror is not None: - return keyerror - raise + with open(path, "rb") as fh: + contents = fh.load() except OSError as e: if e.errno == errno.ENOENT and enoent is not None: return enoent raise + if decode: + return contents.decode(decode) + return contents + + +def dump_yaml(obj): + return yaml.safe_dump(obj, + line_break="\n", + indent=4, + explicit_start=True, + explicit_end=True, + default_flow_style=False) + + +def read_yaml_file(path): + val = yaml.safe_load(read_file(path)) + if not isinstance(val, dict): + gotten_type_name = type(val).__name__ + raise TypeError("Expected dict to be loaded from %s, got" + " '%s' instead" % (path, gotten_type_name)) + return val + + +def sys_dev_path(devname, path=""): + return SYS_CLASS_NET + devname + "/" + path + + +def read_sys_net(devname, path, translate=None, enoent=None, keyerror=None): + contents = read_file(sys_dev_path(devname, path), enoent=enoent) + contents = contents.strip() + if translate is None: + return contents + try: + return translate.get(contents) + except KeyError: + LOG.debug("found unexpected value '%s' in '%s/%s'", contents, + devname, path) + if keyerror is not None: + return keyerror + raise def is_up(devname): @@ -107,31 +144,6 @@ class ParserError(Exception): """Raised when parser has issue parsing the interfaces file.""" -def parse_net_config_data(net_config, skip_broken=True): - """Parses the config, returns NetworkState object - - :param net_config: curtin network config dict - """ - state = None - if 'version' in net_config and 'config' in net_config: - ns = network_state.NetworkState(version=net_config.get('version'), - config=net_config.get('config')) - ns.parse_config(skip_broken=skip_broken) - state = ns.network_state - return state - - -def parse_net_config(path, skip_broken=True): - """Parses a curtin network configuration file and - return network state""" - ns = None - net_config = util.read_conf(path) - if 'network' in net_config: - ns = parse_net_config_data(net_config.get('network'), - skip_broken=skip_broken) - return ns - - def is_disabled_cfg(cfg): if not cfg or not isinstance(cfg, dict): return False @@ -146,7 +158,7 @@ def sys_netdev_info(name, field): fname = os.path.join(SYS_CLASS_NET, name, field) if not os.path.exists(fname): raise OSError("%s: could not find sysfs entry: %s" % (name, fname)) - data = util.load_file(fname) + data = read_file(fname) if data[-1] == '\n': data = data[:-1] return data diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py index f5712533..b85d4b0a 100644 --- a/cloudinit/net/cmdline.py +++ b/cloudinit/net/cmdline.py @@ -20,12 +20,25 @@ import base64 import glob import gzip import io +import shlex +import sys + +import six from cloudinit.net import get_devicelist from cloudinit.net import sys_netdev_info from cloudinit import util +PY26 = sys.version_info[0:2] == (2, 6) + + +def _shlex_split(blob): + if PY26 and isinstance(blob, six.text_type): + # Older versions don't support unicode input + blob = blob.encode("utf8") + return shlex.split(blob) + def _load_shell_content(content, add_empty=False, empty_val=None): """Given shell like syntax (key=value\nkey2=value2\n) in content @@ -33,7 +46,7 @@ def _load_shell_content(content, add_empty=False, empty_val=None): then add entries in to the returned dictionary for 'VAR=' variables. Set their value to empty_val.""" data = {} - for line in util.shlex_split(content): + for line in _shlex_split(content): try: key, value = line.split("=", 1) except ValueError: diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index b1bdac24..adb31c22 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -16,10 +16,11 @@ import glob import os import re +from cloudinit import net + from cloudinit.net import LINKS_FNAME_PREFIX from cloudinit.net import ParserError from cloudinit.net.udev import generate_udev_rule -from cloudinit import util NET_CONFIG_COMMANDS = [ @@ -363,16 +364,13 @@ class Renderer(object): links_prefix=LINKS_FNAME_PREFIX, netrules='etc/udev/rules.d/70-persistent-net.rules'): - fpeni = os.path.sep.join((target, eni,)) - util.ensure_dir(os.path.dirname(fpeni)) - with open(fpeni, 'w+') as f: - f.write(self._render_interfaces(network_state)) + fpeni = os.path.join(target, eni) + net.write_file(fpeni, self._render_interfaces(network_state)) if netrules: - netrules = os.path.sep.join((target, netrules,)) - util.ensure_dir(os.path.dirname(netrules)) - with open(netrules, 'w+') as f: - f.write(self._render_persistent_net(network_state)) + netrules = os.path.join(target, netrules) + net.write_file(netrules, + self._render_persistent_net(network_state)) if links_prefix: self._render_systemd_links(target, network_state, links_prefix) @@ -382,18 +380,17 @@ class Renderer(object): fp_prefix = os.path.sep.join((target, links_prefix)) for f in glob.glob(fp_prefix + "*"): os.unlink(f) - interfaces = network_state.get('interfaces') for iface in interfaces.values(): if (iface['type'] == 'physical' and 'name' in iface and iface.get('mac_address')): fname = fp_prefix + iface['name'] + ".link" - with open(fname, "w") as fp: - fp.write("\n".join([ - "[Match]", - "MACAddress=" + iface['mac_address'], - "", - "[Link]", - "Name=" + iface['name'], - "" - ])) + content = "\n".join([ + "[Match]", + "MACAddress=" + iface['mac_address'], + "", + "[Link]", + "Name=" + iface['name'], + "" + ]) + net.write_file(fname, content) diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 2feffa71..c5aeadb5 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -16,12 +16,11 @@ # along with Curtin. If not, see . import copy +import logging import six -from cloudinit import log as logging -from cloudinit import util -from cloudinit.util import yaml_dumps as dump_config +from cloudinit import net LOG = logging.getLogger(__name__) @@ -31,9 +30,34 @@ NETWORK_STATE_REQUIRED_KEYS = { } +def parse_net_config_data(net_config, skip_broken=True): + """Parses the config, returns NetworkState object + + :param net_config: curtin network config dict + """ + state = None + if 'version' in net_config and 'config' in net_config: + ns = NetworkState(version=net_config.get('version'), + config=net_config.get('config')) + ns.parse_config(skip_broken=skip_broken) + state = ns.network_state + return state + + +def parse_net_config(path, skip_broken=True): + """Parses a curtin network configuration file and + return network state""" + ns = None + net_config = net.read_yaml_file(path) + if 'network' in net_config: + ns = parse_net_config_data(net_config.get('network'), + skip_broken=skip_broken) + return ns + + def from_state_file(state_file): network_state = None - state = util.read_conf(state_file) + state = net.read_yaml_file(state_file) network_state = NetworkState() network_state.load(state) return network_state @@ -111,7 +135,7 @@ class NetworkState(object): 'config': self.config, 'network_state': self.network_state, } - return dump_config(state) + return net.dump_yaml(state) def load(self, state): if 'version' not in state: @@ -121,7 +145,7 @@ class NetworkState(object): required_keys = NETWORK_STATE_REQUIRED_KEYS[state['version']] missing_keys = diff_keys(required_keys, state) if missing_keys: - msg = 'Invalid state, missing keys: %s'.format(missing_keys) + msg = 'Invalid state, missing keys: %s' % (missing_keys) LOG.error(msg) raise ValueError(msg) @@ -130,7 +154,7 @@ class NetworkState(object): setattr(self, key, state[key]) def dump_network_state(self): - return dump_config(self.network_state) + return net.dump_yaml(self.network_state) def parse_config(self, skip_broken=True): # rebuild network state diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index b2acc648..f85fd864 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -551,6 +551,10 @@ def convert_net_json(network_json): 'type': 'static', 'address': network.get('ip_address'), }) + if network['type'] == 'ipv6': + subnet['ipv6'] = True + else: + subnet['ipv4'] = True subnets.append(subnet) cfg.update({'subnets': subnets}) if link['type'] in ['ethernet', 'vif', 'ovs', 'phy']: diff --git a/cloudinit/util.py b/cloudinit/util.py index 7562b97a..2bec476e 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -37,7 +37,6 @@ import pwd import random import re import shutil -import shlex import socket import stat import string @@ -82,7 +81,6 @@ CONTAINER_TESTS = (['systemd-detect-virt', '--quiet', '--container'], ['lxc-is-container']) PROC_CMDLINE = None -PY26 = sys.version_info[0:2] == (2, 6) def decode_binary(blob, encoding='utf-8'): @@ -1202,13 +1200,6 @@ def load_file(fname, read_cb=None, quiet=False, decode=True): return contents -def shlex_split(blob): - if PY26 and isinstance(blob, six.text_type): - # Older versions don't support unicode input - blob = blob.encode("utf8") - return shlex.split(blob) - - def get_cmdline(): if 'DEBUG_PROC_CMDLINE' in os.environ: return os.environ["DEBUG_PROC_CMDLINE"] diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index ed2c6d0f..75c433f6 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -2,6 +2,7 @@ from cloudinit import util from cloudinit import net from cloudinit.net import cmdline from cloudinit.net import eni +from cloudinit.net import network_state from .helpers import TestCase from .helpers import mock @@ -112,14 +113,14 @@ class TestEniNetRendering(TestCase): mock_sys_dev_path.side_effect = sys_dev_path network_cfg = net.generate_fallback_config() - network_state = net.parse_net_config_data(network_cfg, - skip_broken=False) + ns = network_state.parse_net_config_data(network_cfg, + skip_broken=False) render_dir = os.path.join(tmp_dir, "render") os.makedirs(render_dir) renderer = eni.Renderer() - renderer.render_network_state(render_dir, network_state, + renderer.render_network_state(render_dir, ns, eni="interfaces", links_prefix=None, netrules=None) -- cgit v1.2.3 From 85a53d66ad0241b2d6453d902487bb2edc1512b8 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 24 May 2016 13:48:50 -0700 Subject: Fix up some of the net usage and restore imports and add a mini compat module --- cloudinit/net/__init__.py | 5 +++-- cloudinit/net/cmdline.py | 15 +++++-------- cloudinit/net/compat.py | 51 ++++++++++++++++++++++++++++++++++++++++++ cloudinit/net/eni.py | 15 ++++++------- cloudinit/net/network_state.py | 23 ++++++++++--------- 5 files changed, 79 insertions(+), 30 deletions(-) create mode 100644 cloudinit/net/compat.py (limited to 'cloudinit/net/network_state.py') diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 776af0d3..f8df58f0 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -20,7 +20,8 @@ import errno import logging import os -import six +from .import compat + import yaml LOG = logging.getLogger(__name__) @@ -55,7 +56,7 @@ def write_file(path, content): if not os.path.isdir(base_path): os.makedirs(base_path) with open(path, "wb+") as fh: - if isinstance(content, six.text_type): + if isinstance(content, compat.text_type): content = content.encode("utf8") fh.write(content) diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py index b85d4b0a..41cba893 100644 --- a/cloudinit/net/cmdline.py +++ b/cloudinit/net/cmdline.py @@ -21,20 +21,17 @@ import glob import gzip import io import shlex -import sys -import six - -from cloudinit.net import get_devicelist -from cloudinit.net import sys_netdev_info +from . import compat +from . import get_devicelist +from . import read_file +from . import sys_netdev_info from cloudinit import util -PY26 = sys.version_info[0:2] == (2, 6) - def _shlex_split(blob): - if PY26 and isinstance(blob, six.text_type): + if compat.PY26 and isinstance(blob, compat.text_type): # Older versions don't support unicode input blob = blob.encode("utf8") return shlex.split(blob) @@ -143,7 +140,7 @@ def config_from_klibc_net_cfg(files=None, mac_addrs=None): entries = [] names = {} for cfg_file in files: - name, entry = _klibc_to_config_entry(util.load_file(cfg_file), + name, entry = _klibc_to_config_entry(read_file(cfg_file), mac_addrs=mac_addrs) if name in names: raise ValueError( diff --git a/cloudinit/net/compat.py b/cloudinit/net/compat.py new file mode 100644 index 00000000..8bf92ef5 --- /dev/null +++ b/cloudinit/net/compat.py @@ -0,0 +1,51 @@ +# Curtin is free software: you can redistribute it and/or modify it under +# the terms of the GNU Affero General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# Curtin is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for +# more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Curtin. If not, see . + +# Mini six like module (so that this code can be more easily extracted). + +import sys + +PY26 = sys.version_info[0:2] == (2, 6) +PY27 = sys.version_info[0:2] == (2, 7) +PY2 = PY26 or PY27 +PY3 = sys.version_info[0:2] >= (3, 0) + +if PY3: + text_type = str + binary_type = bytes + string_types = (text_type, text_type) + import io + StringIO = io.StringIO +else: + text_type = unicode + binary_type = bytes + string_types = (binary_type, text_type) + from StringIO import StringIO # noqa + + +# Taken from six (remove when we can actually directly use six) + +def add_metaclass(metaclass): + """Class decorator for creating a class with a metaclass.""" + def wrapper(cls): + orig_vars = cls.__dict__.copy() + slots = orig_vars.get('__slots__') + if slots is not None: + if isinstance(slots, str): + slots = [slots] + for slots_var in slots: + orig_vars.pop(slots_var) + orig_vars.pop('__dict__', None) + orig_vars.pop('__weakref__', None) + return metaclass(cls.__name__, cls.__bases__, orig_vars) + return wrapper diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index 18bae97a..f82c7f54 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -16,11 +16,11 @@ import glob import os import re -from cloudinit import net +from . import LINKS_FNAME_PREFIX +from . import ParserError +from . import write_file -from cloudinit.net import LINKS_FNAME_PREFIX -from cloudinit.net import ParserError -from cloudinit.net.udev import generate_udev_rule +from .udev import generate_udev_rule NET_CONFIG_COMMANDS = [ @@ -365,12 +365,11 @@ class Renderer(object): netrules='etc/udev/rules.d/70-persistent-net.rules'): fpeni = os.path.join(target, eni) - net.write_file(fpeni, self._render_interfaces(network_state)) + write_file(fpeni, self._render_interfaces(network_state)) if netrules: netrules = os.path.join(target, netrules) - net.write_file(netrules, - self._render_persistent_net(network_state)) + write_file(netrules, self._render_persistent_net(network_state)) if links_prefix: self._render_systemd_links(target, network_state, links_prefix) @@ -393,4 +392,4 @@ class Renderer(object): "Name=" + iface['name'], "" ]) - net.write_file(fname, content) + write_file(fname, content) diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index c5aeadb5..32c48229 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -16,11 +16,12 @@ # along with Curtin. If not, see . import copy +import functools import logging -import six - -from cloudinit import net +from . import compat +from . import dump_yaml +from . import read_yaml_file LOG = logging.getLogger(__name__) @@ -48,7 +49,7 @@ def parse_net_config(path, skip_broken=True): """Parses a curtin network configuration file and return network state""" ns = None - net_config = net.read_yaml_file(path) + net_config = read_yaml_file(path) if 'network' in net_config: ns = parse_net_config_data(net_config.get('network'), skip_broken=skip_broken) @@ -57,7 +58,7 @@ def parse_net_config(path, skip_broken=True): def from_state_file(state_file): network_state = None - state = net.read_yaml_file(state_file) + state = read_yaml_file(state_file) network_state = NetworkState() network_state.load(state) return network_state @@ -78,7 +79,7 @@ def ensure_command_keys(required_keys): def wrapper(func): - @six.wraps(func) + @functools.wraps(func) def decorator(self, command, *args, **kwargs): if required_keys: missing_keys = diff_keys(required_keys, command) @@ -102,8 +103,8 @@ class CommandHandlerMeta(type): """ def __new__(cls, name, parents, dct): command_handlers = {} - for attr_name, attr in six.iteritems(dct): - if six.callable(attr) and attr_name.startswith('handle_'): + for attr_name, attr in dct.items(): + if callable(attr) and attr_name.startswith('handle_'): handles_what = attr_name[len('handle_'):] if handles_what: command_handlers[handles_what] = attr @@ -112,7 +113,7 @@ class CommandHandlerMeta(type): parents, dct) -@six.add_metaclass(CommandHandlerMeta) +@compat.add_metaclass(CommandHandlerMeta) class NetworkState(object): initial_network_state = { @@ -135,7 +136,7 @@ class NetworkState(object): 'config': self.config, 'network_state': self.network_state, } - return net.dump_yaml(state) + return dump_yaml(state) def load(self, state): if 'version' not in state: @@ -154,7 +155,7 @@ class NetworkState(object): setattr(self, key, state[key]) def dump_network_state(self): - return net.dump_yaml(self.network_state) + return dump_yaml(self.network_state) def parse_config(self, skip_broken=True): # rebuild network state -- cgit v1.2.3 From 6d0e6068f2692fd34c3c5cc93af7ec8ef5be4711 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 7 Jun 2016 10:59:27 -0700 Subject: For now just remove compat.py module Let's reduce the size of this change for now. --- cloudinit/net/cmdline.py | 10 ++++++--- cloudinit/net/compat.py | 51 ------------------------------------------ cloudinit/net/network_state.py | 4 ++-- 3 files changed, 9 insertions(+), 56 deletions(-) delete mode 100644 cloudinit/net/compat.py (limited to 'cloudinit/net/network_state.py') diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py index 187f7026..822a020b 100644 --- a/cloudinit/net/cmdline.py +++ b/cloudinit/net/cmdline.py @@ -21,16 +21,20 @@ import glob import gzip import io import shlex +import sys + +import six -from . import compat from . import get_devicelist from . import sys_netdev_info from cloudinit import util +PY26 = sys.version_info[0:2] == (2, 6) + def _shlex_split(blob): - if compat.PY26 and isinstance(blob, compat.text_type): + if PY26 and isinstance(blob, six.text_type): # Older versions don't support unicode input blob = blob.encode("utf8") return shlex.split(blob) @@ -42,7 +46,7 @@ def _load_shell_content(content, add_empty=False, empty_val=None): then add entries in to the returned dictionary for 'VAR=' variables. Set their value to empty_val.""" data = {} - for line in shlex.split(content): + for line in _shlex_split(content): key, value = line.split("=", 1) if not value: value = empty_val diff --git a/cloudinit/net/compat.py b/cloudinit/net/compat.py deleted file mode 100644 index 8bf92ef5..00000000 --- a/cloudinit/net/compat.py +++ /dev/null @@ -1,51 +0,0 @@ -# Curtin is free software: you can redistribute it and/or modify it under -# the terms of the GNU Affero General Public License as published by the -# Free Software Foundation, either version 3 of the License, or (at your -# option) any later version. -# -# Curtin is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for -# more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Curtin. If not, see . - -# Mini six like module (so that this code can be more easily extracted). - -import sys - -PY26 = sys.version_info[0:2] == (2, 6) -PY27 = sys.version_info[0:2] == (2, 7) -PY2 = PY26 or PY27 -PY3 = sys.version_info[0:2] >= (3, 0) - -if PY3: - text_type = str - binary_type = bytes - string_types = (text_type, text_type) - import io - StringIO = io.StringIO -else: - text_type = unicode - binary_type = bytes - string_types = (binary_type, text_type) - from StringIO import StringIO # noqa - - -# Taken from six (remove when we can actually directly use six) - -def add_metaclass(metaclass): - """Class decorator for creating a class with a metaclass.""" - def wrapper(cls): - orig_vars = cls.__dict__.copy() - slots = orig_vars.get('__slots__') - if slots is not None: - if isinstance(slots, str): - slots = [slots] - for slots_var in slots: - orig_vars.pop(slots_var) - orig_vars.pop('__dict__', None) - orig_vars.pop('__weakref__', None) - return metaclass(cls.__name__, cls.__bases__, orig_vars) - return wrapper diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 1e82e75d..a8be5e26 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -19,7 +19,7 @@ import copy import functools import logging -from . import compat +import six from cloudinit import util @@ -113,7 +113,7 @@ class CommandHandlerMeta(type): parents, dct) -@compat.add_metaclass(CommandHandlerMeta) +@six.add_metaclass(CommandHandlerMeta) class NetworkState(object): initial_network_state = { -- cgit v1.2.3 From 6e19efb17609cd5c49941ea4f1608900b8459ca8 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 13 Jun 2016 13:10:07 -0700 Subject: Refactor some of sysconfig changes -> network_state module --- cloudinit/net/eni.py | 22 +++++---- cloudinit/net/network_state.py | 103 +++++++++++++++++++++++++++++------------ cloudinit/net/sysconfig.py | 26 +---------- 3 files changed, 87 insertions(+), 64 deletions(-) (limited to 'cloudinit/net/network_state.py') diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index a695f5ed..df0df9b2 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -360,7 +360,15 @@ class Renderer(object): '''Given state, emit etc/network/interfaces content.''' content = "" - interfaces = network_state.get('interfaces') + content += "auto lo\niface lo inet loopback\n" + + nameservers = network_state.dns_nameservers + if nameservers: + content += " dns-nameservers %s\n" % (" ".join(nameservers)) + searchdomains = network_state.dns_searchdomains + if searchdomains: + content += " dns-search %s\n" % (" ".join(searchdomains)) + ''' Apply a sort order to ensure that we write out the physical interfaces first; this is critical for bonding @@ -371,12 +379,7 @@ class Renderer(object): 'bridge': 2, 'vlan': 3, } - content += "auto lo\niface lo inet loopback\n" - for dnskey, value in network_state.get('dns', {}).items(): - if len(value): - content += " dns-{} {}\n".format(dnskey, " ".join(value)) - - for iface in sorted(interfaces.values(), + for iface in sorted(network_state.iter_interfaces(), key=lambda k: (order[k['type']], k['name'])): if content[-2:] != "\n\n": @@ -409,7 +412,7 @@ class Renderer(object): content += "iface {name} {inet} {mode}\n".format(**iface) content += _iface_add_attrs(iface) - for route in network_state.get('routes'): + for route in network_state.iter_routes(): content += self._render_route(route) # global replacements until v2 format @@ -441,8 +444,7 @@ class Renderer(object): fp_prefix = os.path.sep.join((target, links_prefix)) for f in glob.glob(fp_prefix + "*"): os.unlink(f) - interfaces = network_state.get('interfaces') - for iface in interfaces.values(): + for iface in network_state.iter_interfaces(): if (iface['type'] == 'physical' and 'name' in iface and iface.get('mac_address')): fname = fp_prefix + iface['name'] + ".link" diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index a8be5e26..d75fc852 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -38,10 +38,10 @@ def parse_net_config_data(net_config, skip_broken=True): """ state = None if 'version' in net_config and 'config' in net_config: - ns = NetworkState(version=net_config.get('version'), - config=net_config.get('config')) - ns.parse_config(skip_broken=skip_broken) - state = ns.network_state + nsi = NetworkStateInterpreter(version=net_config.get('version'), + config=net_config.get('config')) + nsi.parse_config(skip_broken=skip_broken) + state = nsi.network_state return state @@ -57,11 +57,10 @@ def parse_net_config(path, skip_broken=True): def from_state_file(state_file): - network_state = None state = util.read_conf(state_file) - network_state = NetworkState() - network_state.load(state) - return network_state + nsi = NetworkStateInterpreter() + nsi.load(state) + return nsi def diff_keys(expected, actual): @@ -113,9 +112,51 @@ class CommandHandlerMeta(type): parents, dct) -@six.add_metaclass(CommandHandlerMeta) class NetworkState(object): + def __init__(self, network_state, version=NETWORK_STATE_VERSION): + self._network_state = copy.deepcopy(network_state) + self._version = version + + @property + def version(self): + return self._version + + def iter_routes(self, filter_func=None): + for route in self._network_state.get('routes', []): + if filter_func is not None: + if filter_func(route): + yield route + else: + yield route + + @property + def dns_nameservers(self): + try: + return self._network_state['dns']['nameservers'] + except KeyError: + return [] + + @property + def dns_searchdomains(self): + try: + return self._network_state['dns']['search'] + except KeyError: + return [] + + def iter_interfaces(self, filter_func=None): + ifaces = self._network_state.get('interfaces', {}) + for iface in six.itervalues(ifaces): + if filter_func is None: + yield iface + else: + if filter_func(iface): + yield iface + + +@six.add_metaclass(CommandHandlerMeta) +class NetworkStateInterpreter(object): + initial_network_state = { 'interfaces': {}, 'routes': [], @@ -126,22 +167,26 @@ class NetworkState(object): } def __init__(self, version=NETWORK_STATE_VERSION, config=None): - self.version = version - self.config = config - self.network_state = copy.deepcopy(self.initial_network_state) + self._version = version + self._config = config + self._network_state = copy.deepcopy(self.initial_network_state) + + @property + def network_state(self): + return NetworkState(self._network_state, version=self._version) def dump(self): state = { - 'version': self.version, - 'config': self.config, - 'network_state': self.network_state, + 'version': self._version, + 'config': self._config, + 'network_state': self._network_state, } return util.yaml_dumps(state) def load(self, state): if 'version' not in state: LOG.error('Invalid state, missing version field') - raise Exception('Invalid state, missing version field') + raise ValueError('Invalid state, missing version field') required_keys = NETWORK_STATE_REQUIRED_KEYS[state['version']] missing_keys = diff_keys(required_keys, state) @@ -155,11 +200,11 @@ class NetworkState(object): setattr(self, key, state[key]) def dump_network_state(self): - return util.yaml_dumps(self.network_state) + return util.yaml_dumps(self._network_state) def parse_config(self, skip_broken=True): # rebuild network state - for command in self.config: + for command in self._config: command_type = command['type'] try: handler = self.command_handlers[command_type] @@ -189,7 +234,7 @@ class NetworkState(object): } ''' - interfaces = self.network_state.get('interfaces') + interfaces = self._network_state.get('interfaces', {}) iface = interfaces.get(command['name'], {}) for param, val in command.get('params', {}).items(): iface.update({param: val}) @@ -215,7 +260,7 @@ class NetworkState(object): 'gateway': None, 'subnets': subnets, }) - self.network_state['interfaces'].update({command.get('name'): iface}) + self._network_state['interfaces'].update({command.get('name'): iface}) self.dump_network_state() @ensure_command_keys(['name', 'vlan_id', 'vlan_link']) @@ -228,7 +273,7 @@ class NetworkState(object): hwaddress ether BC:76:4E:06:96:B3 vlan-raw-device eth0 ''' - interfaces = self.network_state.get('interfaces') + interfaces = self._network_state.get('interfaces', {}) self.handle_physical(command) iface = interfaces.get(command.get('name'), {}) iface['vlan-raw-device'] = command.get('vlan_link') @@ -263,12 +308,12 @@ class NetworkState(object): ''' self.handle_physical(command) - interfaces = self.network_state.get('interfaces') + interfaces = self._network_state.get('interfaces') iface = interfaces.get(command.get('name'), {}) for param, val in command.get('params').items(): iface.update({param: val}) iface.update({'bond-slaves': 'none'}) - self.network_state['interfaces'].update({iface['name']: iface}) + self._network_state['interfaces'].update({iface['name']: iface}) # handle bond slaves for ifname in command.get('bond_interfaces'): @@ -280,13 +325,13 @@ class NetworkState(object): # inject placeholder self.handle_physical(cmd) - interfaces = self.network_state.get('interfaces') + interfaces = self._network_state.get('interfaces', {}) bond_if = interfaces.get(ifname) bond_if['bond-master'] = command.get('name') # copy in bond config into slave for param, val in command.get('params').items(): bond_if.update({param: val}) - self.network_state['interfaces'].update({ifname: bond_if}) + self._network_state['interfaces'].update({ifname: bond_if}) @ensure_command_keys(['name', 'bridge_interfaces', 'params']) def handle_bridge(self, command): @@ -319,7 +364,7 @@ class NetworkState(object): # find one of the bridge port ifaces to get mac_addr # handle bridge_slaves - interfaces = self.network_state.get('interfaces') + interfaces = self._network_state.get('interfaces', {}) for ifname in command.get('bridge_interfaces'): if ifname in interfaces: continue @@ -330,7 +375,7 @@ class NetworkState(object): # inject placeholder self.handle_physical(cmd) - interfaces = self.network_state.get('interfaces') + interfaces = self._network_state.get('interfaces', {}) self.handle_physical(command) iface = interfaces.get(command.get('name'), {}) iface['bridge_ports'] = command['bridge_interfaces'] @@ -341,7 +386,7 @@ class NetworkState(object): @ensure_command_keys(['address']) def handle_nameserver(self, command): - dns = self.network_state.get('dns') + dns = self._network_state.get('dns') if 'address' in command: addrs = command['address'] if not type(addrs) == list: @@ -357,7 +402,7 @@ class NetworkState(object): @ensure_command_keys(['destination']) def handle_route(self, command): - routes = self.network_state.get('routes') + routes = self._network_state.get('routes', []) network, cidr = command['destination'].split("/") netmask = cidr2mask(int(cidr)) route = { diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index a4bcaaad..647ca3ed 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -67,29 +67,6 @@ def _quote_value(value): return value -class NetworkStateHelper(object): - def __init__(self, network_state): - self._network_state = network_state.copy() - - @property - def dns_nameservers(self): - return self._network_state['dns']['nameservers'] - - @property - def dns_searchdomains(self): - return self._network_state['dns']['search'] - - def iter_interfaces(self, filter_func=None): - ifaces = self._network_state.get('interfaces') - if ifaces: - for iface in ifaces.values(): - if filter_func is None: - yield iface - else: - if filter_func(iface): - yield iface - - class ConfigMap(object): """Sysconfig like dictionary object.""" @@ -424,7 +401,6 @@ class Renderer(object): self, target, network_state, sysconf_dir="etc/sysconfig/", netrules='etc/udev/rules.d/70-persistent-net.rules', dns='etc/resolv.conf'): - network_state = NetworkStateHelper(network_state) if target: base_sysconf_dir = os.path.join(target, sysconf_dir) else: @@ -480,7 +456,7 @@ def main(): net_state = network_state.parse_net_config_data( openstack.convert_net_json(network_json), skip_broken=False) r = Renderer() - r.render_network_state(args.dir, NetworkStateHelper(net_state)) + r.render_network_state(args.dir, net_state) if __name__ == '__main__': -- cgit v1.2.3 From e0b1e3a8c925446ee9e6d15aae1783b49fb870f8 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 14 Jun 2016 10:58:18 -0700 Subject: Make the os samples easier to extend (for new samples) --- cloudinit/net/network_state.py | 1 + tests/unittests/test_net.py | 153 +++++++++++++++++++++-------------------- 2 files changed, 79 insertions(+), 75 deletions(-) (limited to 'cloudinit/net/network_state.py') diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index d75fc852..8ca5106f 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -170,6 +170,7 @@ class NetworkStateInterpreter(object): self._version = version self._config = config self._network_state = copy.deepcopy(self.initial_network_state) + self._parsed = False @property def network_state(self): diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 4f4515a9..43602030 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -76,6 +76,71 @@ STATIC_EXPECTED_1 = { 'dns_nameservers': ['10.0.1.1']}], } +# Examples (and expected outputs for various renderers). +OS_SAMPLES = [ + { + 'in_data': { + "services": [{"type": "dns", "address": "172.19.0.12"}], + "networks": [{ + "network_id": "dacd568d-5be6-4786-91fe-750c374b78b4", + "type": "ipv4", "netmask": "255.255.252.0", + "link": "tap1a81968a-79", + "routes": [{ + "netmask": "0.0.0.0", + "network": "0.0.0.0", + "gateway": "172.19.3.254", + }], + "ip_address": "172.19.1.34", "id": "network0" + }], + "links": [ + { + "ethernet_mac_address": "fa:16:3e:ed:9a:59", + "mtu": None, "type": "bridge", "id": + "tap1a81968a-79", + "vif_id": "1a81968a-797a-400f-8a80-567f997eb93f" + }, + ], + }, + 'in_macs': { + 'fa:16:3e:ed:9a:59': 'eth0', + }, + 'out_sysconfig': [ + ('etc/sysconfig/network-scripts/ifcfg-eth0', + """ +# Created by cloud-init on instance boot automatically, do not edit. +# +BOOTPROTO=static +DEFROUTE=yes +DEVICE=eth0 +GATEWAY=172.19.3.254 +HWADDR=fa:16:3e:ed:9a:59 +IPADDR=172.19.1.34 +NETMASK=255.255.252.0 +NM_CONTROLLED=no +ONBOOT=yes +TYPE=Ethernet +USERCTL=no +""".lstrip()), + ('etc/sysconfig/network-scripts/route-eth0', + """ +# Created by cloud-init on instance boot automatically, do not edit. +# +ADDRESS0=0.0.0.0 +GATEWAY0=172.19.3.254 +NETMASK0=0.0.0.0 +""".lstrip()), + ('etc/resolv.conf', + """ +; Created by cloud-init on instance boot automatically, do not edit. +; +nameserver 172.19.0.12 +""".lstrip()), + ('etc/udev/rules.d/70-persistent-net.rules', + "".join(['SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ', + 'ATTR{address}=="fa:16:3e:ed:9a:59", NAME="eth0"\n']))] + } +] + def _setup_test(tmp_dir, mock_get_devicelist, mock_sys_netdev_info, mock_sys_dev_path): @@ -145,84 +210,22 @@ USERCTL=no """.lstrip() self.assertEqual(expected_content, content) - def test_openstack_rendering_sample(self): - ex_input = { - "services": [{"type": "dns", "address": "172.19.0.12"}], - "networks": [ - {"network_id": "dacd568d-5be6-4786-91fe-750c374b78b4", - "type": "ipv4", "netmask": "255.255.252.0", - "link": "tap1a81968a-79", - "routes": [{"netmask": "0.0.0.0", - "network": "0.0.0.0", - "gateway": "172.19.3.254"}], - "ip_address": "172.19.1.34", "id": "network0"}], - "links": [{"ethernet_mac_address": "fa:16:3e:ed:9a:59", - "mtu": None, "type": "bridge", "id": - "tap1a81968a-79", - "vif_id": "1a81968a-797a-400f-8a80-567f997eb93f"}]} - ex_mac_addrs = { - 'fa:16:3e:ed:9a:59': 'eth0', - } - network_cfg = openstack.convert_net_json( - ex_input, known_macs=ex_mac_addrs) - ns = network_state.parse_net_config_data(network_cfg, - skip_broken=False) - + def test_openstack_rendering_samples(self): tmp_dir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, tmp_dir) render_dir = os.path.join(tmp_dir, "render") - renderer = sysconfig.Renderer() - renderer.render_network_state(render_dir, ns) - - render_file = 'etc/sysconfig/network-scripts/ifcfg-eth0' - with open(os.path.join(render_dir, render_file)) as fh: - content = fh.read() - expected_content = """ -# Created by cloud-init on instance boot automatically, do not edit. -# -BOOTPROTO=static -DEFROUTE=yes -DEVICE=eth0 -GATEWAY=172.19.3.254 -HWADDR=fa:16:3e:ed:9a:59 -IPADDR=172.19.1.34 -NETMASK=255.255.252.0 -NM_CONTROLLED=no -ONBOOT=yes -TYPE=Ethernet -USERCTL=no -""".lstrip() - self.assertEqual(expected_content, content) - - route_render_file = 'etc/sysconfig/network-scripts/route-eth0' - with open(os.path.join(render_dir, route_render_file)) as fh: - content = fh.read() - expected_content = """ -# Created by cloud-init on instance boot automatically, do not edit. -# -ADDRESS0=0.0.0.0 -GATEWAY0=172.19.3.254 -NETMASK0=0.0.0.0 -""".lstrip() - self.assertEqual(expected_content, content) - - resolv_render_file = 'etc/resolv.conf' - with open(os.path.join(render_dir, resolv_render_file)) as fh: - content = fh.read() - expected_content = """ -; Created by cloud-init on instance boot automatically, do not edit. -; -nameserver 172.19.0.12 -""".lstrip() - self.assertEqual(expected_content, content) - - rules_render_file = 'etc/udev/rules.d/70-persistent-net.rules' - with open(os.path.join(render_dir, rules_render_file)) as fh: - content = fh.read() - expected_content = "".join([ - 'SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ', - 'ATTR{address}=="fa:16:3e:ed:9a:59", NAME="eth0"\n']) - self.assertEqual(expected_content, content) + for os_sample in OS_SAMPLES: + ex_input = os_sample['in_data'] + ex_mac_addrs = os_sample['in_macs'] + network_cfg = openstack.convert_net_json( + ex_input, known_macs=ex_mac_addrs) + ns = network_state.parse_net_config_data(network_cfg, + skip_broken=False) + renderer = sysconfig.Renderer() + renderer.render_network_state(render_dir, ns) + for fn, expected_content in os_sample.get('out_sysconfig', []): + with open(os.path.join(render_dir, fn)) as fh: + self.assertEqual(expected_content, fh.read()) class TestEniNetRendering(TestCase): -- cgit v1.2.3