From f9180c9b0951420b052871bd7bb21082ce1e660e Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Wed, 16 Mar 2016 13:39:47 -0500 Subject: In distros.debian call net.merge_from_cmdline_config to ensure that the results from any ip= parameters passed on the kernel cmdline are merged into network state --- cloudinit/distros/debian.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 909d6deb..c7a4ba07 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -79,6 +79,7 @@ class Distro(distros.Distro): def _write_network_config(self, netconfig): ns = net.parse_net_config_data(netconfig) + ns = net.merge_from_cmdline_config(ns) net.render_network_state(network_state=ns, target="/") return [] -- cgit v1.2.3 From 16a44056eaa5cc36fd9f6b08fe3a6bb4700fe1e7 Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Thu, 17 Mar 2016 21:54:57 -0500 Subject: Check for and merge in configuration caused by the 'ip' parameter on the kernel's cmdline during network configuration parsing. - Search for .conf files in /run with names starting with 'net', as these are created during early boot if the ip parameter is present - If any are present and valid they are merged with network configuration from the current data source - If the devices affected by the 'ip' parameter are already present in network configuration, then a subnet entry will be added to the device's configuration unless an identical entry is already present - If any of the devices affected are not present then a mostly blank configuration will be generated for the device and the appropriate subnet specified --- cloudinit/net/__init__.py | 56 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 3cf99604..e455040d 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -280,6 +280,62 @@ def parse_net_config(path): return ns +def load_key_value_pair_net_cfg(data_mapping): + """Process key value pairs from files written because of the ip parameter + on the kernel cmdline""" + subnets = [] + entry_ns = { + 'mtu': None, 'name': data_mapping['DEVICE'], 'type': 'physical', + 'mode': 'manual', 'inet': 'inet', 'gateway': None, 'address': None + } + + if data_mapping.get('PROTO') == 'dhcp': + if 'IPV4ADDR' in data_mapping: + subnets.append({'type': 'dhcp4'}) + if 'IPV6ADDR' in data_mapping: + subnets.append({'type': 'dhcp6'}) + entry_ns['subnets'] = subnets + + return entry_ns + + +def merge_from_cmdline_config(ns): + """If ip parameter passed on kernel cmdline then some initial network + configuration may have been done in initramfs. Files from the result of + this may have been written into /run. If any are present they should be + merged into network state""" + + if 'interfaces' not in ns: + ns['interfaces'] = {} + + for cfg_file in glob.glob('/run/net*.conf'): + with open(cfg_file, 'r') as fp: + data = [l.replace("'", "") for l in fp.readlines()] + try: + parsed = dict([l.strip().split('=') for l in data]) + except: + # if split did not work then this is likely not a netcfg file + continue + + dev_name = parsed.get('DEVICE') + if not dev_name: + # Not a net cfg file + continue + + loaded_ns = load_key_value_pair_net_cfg(parsed) + + if dev_name in ns['interfaces']: + if 'subnets' not in ns['interfaces'][dev_name]: + ns['interfaces'][dev_name]['subnets'] = [] + for newsubnet in loaded_ns['subnets']: + if newsubnet not in ns['interfaces'][dev_name]['subnets']: + ns['interfaces'][dev_name]['subnets'].append(newsubnet) + else: + ns['interfaces'][dev_name] = loaded_ns + + return ns + + def render_persistent_net(network_state): ''' Given state, emit udev rules to map mac to ifname -- cgit v1.2.3 From c03915a454cfa947c1d905ba5b55a5207ad85cda Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Fri, 18 Mar 2016 02:30:59 -0500 Subject: Fully parse files at /run/net-dev.conf, loading parameters for address, broadcast, netmask, gateway and hostname if present --- cloudinit/net/__init__.py | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index e455040d..cce0773f 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -280,21 +280,39 @@ def parse_net_config(path): return ns -def load_key_value_pair_net_cfg(data_mapping): +def load_klibc_net_cfg(data_mapping): """Process key value pairs from files written because of the ip parameter on the kernel cmdline""" - subnets = [] entry_ns = { 'mtu': None, 'name': data_mapping['DEVICE'], 'type': 'physical', - 'mode': 'manual', 'inet': 'inet', 'gateway': None, 'address': None + 'mode': 'manual', 'inet': 'inet', 'gateway': None, 'address': None, + 'subnets': [] } if data_mapping.get('PROTO') == 'dhcp': - if 'IPV4ADDR' in data_mapping: - subnets.append({'type': 'dhcp4'}) - if 'IPV6ADDR' in data_mapping: - subnets.append({'type': 'dhcp6'}) - entry_ns['subnets'] = subnets + if data_mapping.get('IPV4ADDR'): + entry_ns['subnets'].append({'type': 'dhcp4'}) + if data_mapping.get('IPV6ADDR'): + entry_ns['subnets'].append({'type': 'dhcp6'}) + + if data_mapping.get('IPV4ADDR'): + entry_ns['address'] = data_mapping['IPV4ADDR'] + if data_mapping.get('IPV6ADDR'): + entry_ns['address'] = data_mapping['IPV6ADDR'] + if data_mapping.get('IPV4BROADCAST'): + entry_ns['broadcast'] = data_mapping['IPV4BROADCAST'] + if data_mapping.get('IPV6BROADCAST'): + entry_ns['broadcast'] = data_mapping['IPV6BROADCAST'] + if data_mapping.get('IPV4NETMASK'): + entry_ns['netmask'] = data_mapping['IPV4NETMASK'] + if data_mapping.get('IPV6NETMASK'): + entry_ns['netmask'] = data_mapping['IPV6NETMASK'] + if data_mapping.get('IPV4GATEWAY'): + entry_ns['gateway'] = data_mapping['IPV4GATEWAY'] + if data_mapping.get('IPV6GATEWAY'): + entry_ns['gateway'] = data_mapping['IPV6GATEWAY'] + if data_mapping.get('HOSTNAME'): + entry_ns['hostname'] = data_mapping['HOSTNAME'] return entry_ns @@ -322,7 +340,7 @@ def merge_from_cmdline_config(ns): # Not a net cfg file continue - loaded_ns = load_key_value_pair_net_cfg(parsed) + loaded_ns = load_klibc_net_cfg(parsed) if dev_name in ns['interfaces']: if 'subnets' not in ns['interfaces'][dev_name]: -- cgit v1.2.3 From b123a912448b74a1d51a50baed8d3d7edd4875d7 Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Fri, 18 Mar 2016 02:40:13 -0500 Subject: If proto not specified, determine it using logic from: lp:cloud-initramfs-tools/dyn-netconf/scripts/init-bottom/cloud-initramfs-dyn-netconf --- cloudinit/net/__init__.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index cce0773f..2bfaf149 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -289,6 +289,15 @@ def load_klibc_net_cfg(data_mapping): 'subnets': [] } + # ipconfig on precise does not write PROTO + # (lp:cloud-initramfs-tools/dyn-netconf/scripts/init-bottom/ + # cloud-initramfs-dyn-netconf) + if not data_mapping.get('PROTO'): + if data_mapping.get('filename'): + data_mapping['PROTO'] = 'dhcp' + else: + data_mapping['PROTO'] = 'static' + if data_mapping.get('PROTO') == 'dhcp': if data_mapping.get('IPV4ADDR'): entry_ns['subnets'].append({'type': 'dhcp4'}) -- cgit v1.2.3 From 266717afcc01c9ae2f9b4e8cfc2db08aad83de97 Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Fri, 18 Mar 2016 12:17:15 -0500 Subject: Handle static ip= entries by appending a static subnet to the device --- cloudinit/net/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 2bfaf149..88d0061c 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -303,6 +303,9 @@ def load_klibc_net_cfg(data_mapping): entry_ns['subnets'].append({'type': 'dhcp4'}) if data_mapping.get('IPV6ADDR'): entry_ns['subnets'].append({'type': 'dhcp6'}) + elif data_mapping.get('PROTO') in ['static', 'none']: + entry_ns['subnets'].append( + {'type': 'static', 'address': data_mapping.get('IPV4ADDR')}) if data_mapping.get('IPV4ADDR'): entry_ns['address'] = data_mapping['IPV4ADDR'] -- cgit v1.2.3 From bb9cb8df25eaeca8740f5e1bcbc9cd2feafcba24 Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Fri, 18 Mar 2016 14:41:44 -0500 Subject: Added comments --- cloudinit/net/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 88d0061c..e2dcaee7 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -282,7 +282,9 @@ def parse_net_config(path): def load_klibc_net_cfg(data_mapping): """Process key value pairs from files written because of the ip parameter - on the kernel cmdline""" + on the kernel cmdline, note that mode: manual is used because the + interface should already have been brought up by the kernel and + cloud-initramfs-tools""" entry_ns = { 'mtu': None, 'name': data_mapping['DEVICE'], 'type': 'physical', 'mode': 'manual', 'inet': 'inet', 'gateway': None, 'address': None, @@ -304,6 +306,8 @@ def load_klibc_net_cfg(data_mapping): if data_mapping.get('IPV6ADDR'): entry_ns['subnets'].append({'type': 'dhcp6'}) elif data_mapping.get('PROTO') in ['static', 'none']: + # It appears that specifying ipv6 static addrs does not work, so only + # check for ipv4 addr entry_ns['subnets'].append( {'type': 'static', 'address': data_mapping.get('IPV4ADDR')}) -- cgit v1.2.3 From 0f4811e307af08cb8f0dd552346f1148ea2f9c10 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 23 Mar 2016 13:52:14 -0400 Subject: add config_from_klibc_net_cfg and helper functions Wesley's loader returned network state, so that got me updating it, and i implemented as such. Then realized that actually ipconfig (klibc) has no support for ipv6. So even though i painfully generalized that, its pointless. next commit will drop it. --- cloudinit/net/__init__.py | 170 ++++++++++++++++++++++++---------------------- 1 file changed, 88 insertions(+), 82 deletions(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 0560fa45..3362d172 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -20,6 +20,7 @@ import errno import glob import os import re +import shlex from cloudinit import log as logging from cloudinit import util @@ -283,94 +284,99 @@ def parse_net_config(path): return ns -def load_klibc_net_cfg(data_mapping): - """Process key value pairs from files written because of the ip parameter - on the kernel cmdline, note that mode: manual is used because the - interface should already have been brought up by the kernel and - cloud-initramfs-tools""" - entry_ns = { - 'mtu': None, 'name': data_mapping['DEVICE'], 'type': 'physical', - 'mode': 'manual', 'inet': 'inet', 'gateway': None, 'address': None, - 'subnets': [] - } +def _load_shell_content(content, add_empty=False, empty_val=None): + """Given the content of a klibc created /run/net*.conf file, return + its data in dictionary form.""" + data = {} + for line in shlex.split(content): + key, value = line.split("=", maxsplit=1) + if not value: + value = empty_val + if add_empty or value: + data[key] = value - # ipconfig on precise does not write PROTO - # (lp:cloud-initramfs-tools/dyn-netconf/scripts/init-bottom/ - # cloud-initramfs-dyn-netconf) - if not data_mapping.get('PROTO'): - if data_mapping.get('filename'): - data_mapping['PROTO'] = 'dhcp' - else: - data_mapping['PROTO'] = 'static' - - if data_mapping.get('PROTO') == 'dhcp': - if data_mapping.get('IPV4ADDR'): - entry_ns['subnets'].append({'type': 'dhcp4'}) - if data_mapping.get('IPV6ADDR'): - entry_ns['subnets'].append({'type': 'dhcp6'}) - elif data_mapping.get('PROTO') in ['static', 'none']: - # It appears that specifying ipv6 static addrs does not work, so only - # check for ipv4 addr - entry_ns['subnets'].append( - {'type': 'static', 'address': data_mapping.get('IPV4ADDR')}) - - if data_mapping.get('IPV4ADDR'): - entry_ns['address'] = data_mapping['IPV4ADDR'] - if data_mapping.get('IPV6ADDR'): - entry_ns['address'] = data_mapping['IPV6ADDR'] - if data_mapping.get('IPV4BROADCAST'): - entry_ns['broadcast'] = data_mapping['IPV4BROADCAST'] - if data_mapping.get('IPV6BROADCAST'): - entry_ns['broadcast'] = data_mapping['IPV6BROADCAST'] - if data_mapping.get('IPV4NETMASK'): - entry_ns['netmask'] = data_mapping['IPV4NETMASK'] - if data_mapping.get('IPV6NETMASK'): - entry_ns['netmask'] = data_mapping['IPV6NETMASK'] - if data_mapping.get('IPV4GATEWAY'): - entry_ns['gateway'] = data_mapping['IPV4GATEWAY'] - if data_mapping.get('IPV6GATEWAY'): - entry_ns['gateway'] = data_mapping['IPV6GATEWAY'] - if data_mapping.get('HOSTNAME'): - entry_ns['hostname'] = data_mapping['HOSTNAME'] - - return entry_ns - - -def merge_from_cmdline_config(ns): - """If ip parameter passed on kernel cmdline then some initial network - configuration may have been done in initramfs. Files from the result of - this may have been written into /run. If any are present they should be - merged into network state""" - - if 'interfaces' not in ns: - ns['interfaces'] = {} - - for cfg_file in glob.glob('/run/net*.conf'): - with open(cfg_file, 'r') as fp: - data = [l.replace("'", "") for l in fp.readlines()] - try: - parsed = dict([l.strip().split('=') for l in data]) - except: - # if split did not work then this is likely not a netcfg file - continue + return data - dev_name = parsed.get('DEVICE') - if not dev_name: - # Not a net cfg file - continue - loaded_ns = load_klibc_net_cfg(parsed) +def _klibc_to_config_entry(content): + data = _load_shell_content(content) + try: + name = data['DEVICE'] + except KeyError: + raise ValueError("no 'DEVICE' entry in data") - if dev_name in ns['interfaces']: - if 'subnets' not in ns['interfaces'][dev_name]: - ns['interfaces'][dev_name]['subnets'] = [] - for newsubnet in loaded_ns['subnets']: - if newsubnet not in ns['interfaces'][dev_name]['subnets']: - ns['interfaces'][dev_name]['subnets'].append(newsubnet) + # ipconfig on precise does not write PROTO + proto = data.get('PROTO') + if not proto: + if data.get('filename'): + proto = 'dhcp' else: - ns['interfaces'][dev_name] = loaded_ns + proto = 'static' - return ns + if proto not in ('static', 'dhcp'): + raise ValueError("Unexpected value for PROTO: %s" % proto) + + iface = { + 'type': 'physical', + 'name': name, + 'subnets': [], + } + subnets = {} + + for v, pre in (('ipv4', 'IPV4'), ('ipv6', 'IPV6')): + # if no IPV4ADDR or IPV6ADDR, then go on. + if pre + "ADDR" not in data: + continue + subnet = {'type': proto} + + # these fields go right on the subnet + for key in ('NETMASK', 'BROADCAST', 'GATEWAY'): + if pre + key in data: + subnet[key.lower()] = data[pre + key] + + dns = [] + # handle IPV4DNS0 or IPV6DNS0 + for nskey in ('DNS0', 'DNS1'): + ns = data.get(pre + nskey) + # verify it has something other than 0.0.0.0 (or ipv6) + if ns and len(ns.strip(":.0")): + dns.append(data[pre + nskey]) + if dns: + subnet['dns_nameservers'] = dns + # add search to both ipv4 and ipv6, as it has no namespace + search = data.get('DOMAINSEARCH') + if search: + if ',' in search: + subnet['dns_search'] = search.split(",") + else: + subnet['dns_search'] = search.split() + + iface['subnets'].append(subnet) + + for subnet in subnets: + iface[subnet].append(subnet) + + return name, iface + + +def config_from_klibc_net_cfg(files=None): + if files is None: + files = glob.glob('/run/net*.conf') + + devs = {} + entries = [] + names = {} + for cfg_file in files: + name, entry = _klibc_to_config_entry(util.load_file(cfg_file)) + print("name: %s file: %s" % (name, cfg_file)) + if name in names: + raise ValueError( + "device '%s' defined multiple times: %s and %s" % ( + name, names[name], cfg_file + )) + names[name] = cfg_file + entries.append(entry) + return {'config': entries, 'version': 1} def render_persistent_net(network_state): -- cgit v1.2.3 From 3791d25694444fe49e026a575b556117a1ea99c3 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 23 Mar 2016 14:23:54 -0400 Subject: add sys_netdev_info helper, support reading macs in. --- cloudinit/net/__init__.py | 55 ++++++++++++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 3362d172..a167c0a1 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -298,7 +298,10 @@ def _load_shell_content(content, add_empty=False, empty_val=None): return data -def _klibc_to_config_entry(content): +def _klibc_to_config_entry(content, mac_addrs=None): + if mac_addrs is None: + mac_addrs = {} + data = _load_shell_content(content) try: name = data['DEVICE'] @@ -321,14 +324,17 @@ def _klibc_to_config_entry(content): 'name': name, 'subnets': [], } - subnets = {} - for v, pre in (('ipv4', 'IPV4'), ('ipv6', 'IPV6')): + if name in mac_addrs: + iface['mac_address'] = mac_addrs[name] + + # originally believed there might be IPV6* values + for v, pre in (('ipv4', 'IPV4'),): # if no IPV4ADDR or IPV6ADDR, then go on. if pre + "ADDR" not in data: continue subnet = {'type': proto} - + # these fields go right on the subnet for key in ('NETMASK', 'BROADCAST', 'GATEWAY'): if pre + key in data: @@ -353,13 +359,10 @@ def _klibc_to_config_entry(content): iface['subnets'].append(subnet) - for subnet in subnets: - iface[subnet].append(subnet) - return name, iface -def config_from_klibc_net_cfg(files=None): +def config_from_klibc_net_cfg(files=None, mac_addrs=None): if files is None: files = glob.glob('/run/net*.conf') @@ -367,13 +370,13 @@ def config_from_klibc_net_cfg(files=None): entries = [] names = {} for cfg_file in files: - name, entry = _klibc_to_config_entry(util.load_file(cfg_file)) - print("name: %s file: %s" % (name, cfg_file)) + name, entry = _klibc_to_config_entry(util.load_file(cfg_file), + mac_addrs=mac_addrs) if name in names: raise ValueError( "device '%s' defined multiple times: %s and %s" % ( - name, names[name], cfg_file - )) + name, names[name], cfg_file)) + names[name] = cfg_file entries.append(entry) return {'config': entries, 'version': 1} @@ -566,6 +569,19 @@ def is_disabled_cfg(cfg): return cfg.get('config') == "disabled" +def sys_netdev_info(name, field): + if not os.path.exists(os.path.join(SYS_CLASS_NET, name)): + raise OSError("%s: interface does not exist in /sys" % name) + + fname = os.path.join(SYS_CLASS_NET, name, field) + if not os.path.exists(fname): + raise OSError("%s: %s does not exist in /sys" % (name, fname)) + data = util.load_file(fname) + if data[-1] == '\n': + data = data[:-1] + return data + + def generate_fallback_config(): """Determine which attached net dev is most likely to have a connection and generate network state to run dhcp on that interface""" @@ -574,7 +590,7 @@ def generate_fallback_config(): # get list of interfaces that could have connections invalid_interfaces = set(['lo']) - potential_interfaces = set(os.listdir(SYS_CLASS_NET)) + potential_interfaces = set(get_devicelist()) potential_interfaces = potential_interfaces.difference(invalid_interfaces) # sort into interfaces with carrier, interfaces which could have carrier, # and ignore interfaces that are definitely disconnected @@ -582,8 +598,7 @@ def generate_fallback_config(): possibly_connected = [] for interface in potential_interfaces: try: - sysfs_carrier = os.path.join(SYS_CLASS_NET, interface, 'carrier') - carrier = int(util.load_file(sysfs_carrier).strip()) + carrier = int(sys_netdev_info(interface, 'carrier')) if carrier: connected.append(interface) continue @@ -593,17 +608,14 @@ def generate_fallback_config(): # not have a carrier even though it could acquire one when brought # online by dhclient try: - sysfs_dormant = os.path.join(SYS_CLASS_NET, interface, 'dormant') - dormant = int(util.load_file(sysfs_dormant).strip()) + dormant = int(sys_netdev_info(interface, 'dormant')) if dormant: possibly_connected.append(interface) continue except OSError: pass try: - sysfs_operstate = os.path.join(SYS_CLASS_NET, interface, - 'operstate') - operstate = util.load_file(sysfs_operstate).strip() + operstate = sys_netdev_info(interface, 'operstate') if operstate in ['dormant', 'down', 'lowerlayerdown', 'unknown']: possibly_connected.append(interface) continue @@ -626,8 +638,7 @@ def generate_fallback_config(): else: name = sorted(potential_interfaces)[0] - sysfs_mac = os.path.join(SYS_CLASS_NET, name, 'address') - mac = util.load_file(sysfs_mac).strip() + mac = sys_netdev_info(name, 'address') target_name = name nconf['config'].append( -- cgit v1.2.3 From 5cf44d7847b085c5fc881c9eb39bcf6bc891e0d9 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 23 Mar 2016 14:29:22 -0400 Subject: add the implementation for read_kernel_cmdline_config --- cloudinit/net/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index a167c0a1..9f5a7fd7 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -647,9 +647,11 @@ def generate_fallback_config(): return nconf -def read_kernel_cmdline_config(): - # FIXME: add implementation here - return None +def read_kernel_cmdline_config(files=None, mac_addrs=None): + if mac_addrs is None: + mac_addrs = {k: sys_netdev_info(k, 'address') + for k in get_devicelist()} + return config_from_klibc_net_cfg(files=files, mac_addrs=mac_addrs) # vi: ts=4 expandtab syntax=python -- cgit v1.2.3 From 44f71ff4ccb72c89f6cdebc8a7b4e7a0d7029818 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 23 Mar 2016 15:32:51 -0400 Subject: add unit test --- tests/unittests/test_net.py | 94 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 tests/unittests/test_net.py diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py new file mode 100644 index 00000000..06a55643 --- /dev/null +++ b/tests/unittests/test_net.py @@ -0,0 +1,94 @@ +from cloudinit import util +from cloudinit import net +from .helpers import TestCase +import copy +import os + +DHCP_CONTENT_1 = """ +DEVICE='eth0' +PROTO='dhcp' +IPV4ADDR='192.168.122.89' +IPV4BROADCAST='192.168.122.255' +IPV4NETMASK='255.255.255.0' +IPV4GATEWAY='192.168.122.1' +IPV4DNS0='192.168.122.1' +IPV4DNS1='0.0.0.0' +HOSTNAME='foohost' +DNSDOMAIN='' +NISDOMAIN='' +ROOTSERVER='192.168.122.1' +ROOTPATH='' +filename='' +UPTIME='21' +DHCPLEASETIME='3600' +DOMAINSEARCH='foo.com' +""" + +DHCP_EXPECTED_1 = { + 'name': 'eth0', + 'type': 'physical', + 'subnets': [{'broadcast': '192.168.122.255', + 'gateway': '192.168.122.1', + 'dns_search': ['foo.com'], + 'type': 'dhcp', + 'netmask': '255.255.255.0', + 'dns_nameservers': ['192.168.122.1']}], +} + + +STATIC_CONTENT_1 = """ +DEVICE='eth1' +PROTO='static' +IPV4ADDR='10.0.0.2' +IPV4BROADCAST='10.0.0.255' +IPV4NETMASK='255.255.255.0' +IPV4GATEWAY='10.0.0.1' +IPV4DNS0='10.0.1.1' +IPV4DNS1='0.0.0.0' +HOSTNAME='foohost' +UPTIME='21' +DHCPLEASETIME='3600' +DOMAINSEARCH='foo.com' +""" + +STATIC_EXPECTED_1 = { + 'name': 'eth1', + 'type': 'physical', + 'subnets': [{'broadcast': '10.0.0.255', 'gateway': '10.0.0.1', + 'dns_search': ['foo.com'], 'type': 'static', + 'netmask': '255.255.255.0', + 'dns_nameservers': ['10.0.1.1']}], +} + +class TestNetConfigParsing(TestCase): + def test_klibc_convert_dhcp(self): + found = net._klibc_to_config_entry(DHCP_CONTENT_1) + self.assertEqual(found, ('eth0', DHCP_EXPECTED_1)) + + def test_klibc_convert_static(self): + found = net._klibc_to_config_entry(STATIC_CONTENT_1) + self.assertEqual(found, ('eth1', STATIC_EXPECTED_1)) + + def test_config_from_klibc_net_cfg(self): + files = [] + pairs = (('net-eth0.cfg', DHCP_CONTENT_1), + ('net-eth1.cfg', STATIC_CONTENT_1)) + + macs = {'eth1': 'b8:ae:ed:75:ff:2b', + 'eth0': 'b8:ae:ed:75:ff:2a'} + + dhcp = copy.deepcopy(DHCP_EXPECTED_1) + dhcp['mac_address'] = macs['eth0'] + + static = copy.deepcopy(STATIC_EXPECTED_1) + static['mac_address'] = macs['eth1'] + + expected = {'version': 1, 'config': [dhcp, static]} + with util.tempdir() as tmpd: + for fname, content in pairs: + fp = os.path.join(tmpd, fname) + files.append(fp) + util.write_file(fp, content) + + found = net.config_from_klibc_net_cfg(files=files, mac_addrs=macs) + self.assertEqual(found, expected) -- cgit v1.2.3 From 1471ddd3210a4ac5753330f97c12bf8960fedbf7 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 23 Mar 2016 15:37:11 -0400 Subject: fix tox issues --- cloudinit/net/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 9f5a7fd7..4f6b4dfc 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -289,7 +289,7 @@ def _load_shell_content(content, add_empty=False, empty_val=None): its data in dictionary form.""" data = {} for line in shlex.split(content): - key, value = line.split("=", maxsplit=1) + key, value = line.split("=", 1) if not value: value = empty_val if add_empty or value: @@ -366,7 +366,6 @@ def config_from_klibc_net_cfg(files=None, mac_addrs=None): if files is None: files = glob.glob('/run/net*.conf') - devs = {} entries = [] names = {} for cfg_file in files: -- cgit v1.2.3 From 51fa67a88dc0dc631c19770142b16c0b56c21384 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 23 Mar 2016 15:38:32 -0400 Subject: one more tox --- tests/unittests/test_net.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 06a55643..11c0b1eb 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -60,6 +60,7 @@ STATIC_EXPECTED_1 = { 'dns_nameservers': ['10.0.1.1']}], } + class TestNetConfigParsing(TestCase): def test_klibc_convert_dhcp(self): found = net._klibc_to_config_entry(DHCP_CONTENT_1) -- cgit v1.2.3 From 740facb79c70cd8e3380e08acb4f5c5a285bb1ae Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 23 Mar 2016 15:53:37 -0400 Subject: support [untested] network-config= on kernel command line --- cloudinit/net/__init__.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 4f6b4dfc..3a208e43 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -16,6 +16,7 @@ # You should have received a copy of the GNU Affero General Public License # along with Curtin. If not, see . +import base64 import errno import glob import os @@ -646,10 +647,25 @@ def generate_fallback_config(): return nconf -def read_kernel_cmdline_config(files=None, mac_addrs=None): +def read_kernel_cmdline_config(files=None, mac_addrs=None, cmdline=None): + if cmdline is None: + cmdline = util.get_cmdline() + + if 'network-config=' in cmdline: + data64 = None + for tok in cmdline.split(): + if tok.startswith("network-config="): + data64 = tok.split("=", 1)[1] + if data64: + return util.load_yaml(base64.b64decode(data64)) + + if 'ip=' not in cmdline: + return None + if mac_addrs is None: mac_addrs = {k: sys_netdev_info(k, 'address') for k in get_devicelist()} + return config_from_klibc_net_cfg(files=files, mac_addrs=mac_addrs) -- cgit v1.2.3 From f9cae62c2f70c582a6b12a98911dc82180881364 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 24 Mar 2016 11:19:41 -0400 Subject: add suport for base64 encoded gzipped text on command line add tests to show this functional. --- cloudinit/net/__init__.py | 34 +++++++++++++++++++++++++++++++++- tests/unittests/test_net.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 3a208e43..7d7f274d 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -19,6 +19,8 @@ import base64 import errno import glob +import gzip +import io import os import re import shlex @@ -647,6 +649,36 @@ def generate_fallback_config(): return nconf +def _decomp_gzip(blob, strict=True): + # decompress blob. raise exception if not compressed unless strict=False. + with io.BytesIO(blob) as iobuf: + gzfp = None + try: + gzfp = gzip.GzipFile(mode="rb", fileobj=iobuf) + return gzfp.read() + except IOError: + if strict: + raise + return blob + finally: + if gzfp: + gzfp.close() + + +def _b64dgz(b64str, gzipped="try"): + # decode a base64 string. If gzipped is true, transparently uncompresss + # if gzipped is 'try', then try gunzip, returning the original on fail. + try: + blob = base64.b64decode(b64str) + except TypeError: + raise ValueError("Invalid base64 text: %s" % b64str) + + if not gzipped: + return blob + + return _decomp_gzip(blob, strict=gzipped != "try") + + def read_kernel_cmdline_config(files=None, mac_addrs=None, cmdline=None): if cmdline is None: cmdline = util.get_cmdline() @@ -657,7 +689,7 @@ def read_kernel_cmdline_config(files=None, mac_addrs=None, cmdline=None): if tok.startswith("network-config="): data64 = tok.split("=", 1)[1] if data64: - return util.load_yaml(base64.b64decode(data64)) + return util.load_yaml(_b64dgz(data64)) if 'ip=' not in cmdline: return None diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 11c0b1eb..16c44588 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -1,7 +1,12 @@ from cloudinit import util from cloudinit import net from .helpers import TestCase + +import base64 import copy +import io +import gzip +import json import os DHCP_CONTENT_1 = """ @@ -62,6 +67,11 @@ STATIC_EXPECTED_1 = { class TestNetConfigParsing(TestCase): + simple_cfg = { + 'config': [{"type": "physical", "name": "eth0", + "mac_address": "c0:d6:9f:2c:e8:80", + "subnets": [{"type": "dhcp4"}]}]} + def test_klibc_convert_dhcp(self): found = net._klibc_to_config_entry(DHCP_CONTENT_1) self.assertEqual(found, ('eth0', DHCP_EXPECTED_1)) @@ -93,3 +103,25 @@ class TestNetConfigParsing(TestCase): found = net.config_from_klibc_net_cfg(files=files, mac_addrs=macs) self.assertEqual(found, expected) + + def test_cmdline_with_b64(self): + data = base64.b64encode(json.dumps(self.simple_cfg).encode()) + encoded_text = data.decode() + cmdline = 'ro network-config=' + encoded_text + ' root=foo' + found = net.read_kernel_cmdline_config(cmdline=cmdline) + self.assertEqual(found, self.simple_cfg) + + def test_cmdline_with_b64_gz(self): + data = _gzip_data(json.dumps(self.simple_cfg).encode()) + encoded_text = base64.b64encode(data).decode() + cmdline = 'ro network-config=' + encoded_text + ' root=foo' + found = net.read_kernel_cmdline_config(cmdline=cmdline) + self.assertEqual(found, self.simple_cfg) + + +def _gzip_data(data): + with io.BytesIO() as iobuf: + gzfp = gzip.GzipFile(mode="wb", fileobj=iobuf) + gzfp.write(data) + gzfp.close() + return iobuf.getvalue() -- cgit v1.2.3 From 841a773fd36968419354507fa45f44afa6eb8470 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 24 Mar 2016 11:50:49 -0400 Subject: improve comment --- cloudinit/net/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 7d7f274d..e13ca470 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -288,8 +288,10 @@ def parse_net_config(path): def _load_shell_content(content, add_empty=False, empty_val=None): - """Given the content of a klibc created /run/net*.conf file, return - its data in dictionary form.""" + """Given shell like syntax (key=value\nkey2=value2\n) in content + return the data in dictionary form. If 'add_empty' is True + then add entries in to the returned dictionary for 'VAR=' + variables. Set their value to empty_val.""" data = {} for line in shlex.split(content): key, value = line.split("=", 1) -- cgit v1.2.3 From 3ad9929efcab614a6ffc170c75c1c6c81b57a2b8 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 24 Mar 2016 15:30:03 -0400 Subject: make get_cmdline read /proc/1/cmdline if inside a container This follows behavior of systemd/cloud-init-generator. This way you can feed a command line into lxc container. --- cloudinit/util.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/cloudinit/util.py b/cloudinit/util.py index 20916e53..0d21e11b 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -80,6 +80,8 @@ CONTAINER_TESTS = (['systemd-detect-virt', '--quiet', '--container'], ['running-in-container'], ['lxc-is-container']) +PROC_CMDLINE = None + def decode_binary(blob, encoding='utf-8'): # Converts a binary type into a text type using given encoding. @@ -1191,12 +1193,27 @@ def load_file(fname, read_cb=None, quiet=False, decode=True): def get_cmdline(): if 'DEBUG_PROC_CMDLINE' in os.environ: - cmdline = os.environ["DEBUG_PROC_CMDLINE"] + return os.environ["DEBUG_PROC_CMDLINE"] + + global PROC_CMDLINE + if PROC_CMDLINE is not None: + return PROC_CMDLINE + + if is_container(): + try: + contents = load_file("/proc/1/cmdline") + # replace nulls with space and drop trailing null + cmdline = contents.replace("\x00", " ")[:-1] + except Exception as e: + LOG.warn("failed reading /proc/1/cmdline: %s", e) + cmdline = "" else: try: cmdline = load_file("/proc/cmdline").strip() except: cmdline = "" + + PROC_CMDLINE = cmdline return cmdline @@ -1569,7 +1586,7 @@ def uptime(): try: if os.path.exists("/proc/uptime"): method = '/proc/uptime' - contents = load_file("/proc/uptime").strip() + contents = load_file("/proc/uptime") if contents: uptime_str = contents.split()[0] else: -- cgit v1.2.3 From 557650728d3c1b1c1bbd29f9292d43133c00cdd4 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 24 Mar 2016 15:46:52 -0400 Subject: add comments and improve error messages --- cloudinit/net/__init__.py | 20 +++++++++++++++++--- tests/unittests/test_net.py | 2 +- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index e13ca470..7af9b03a 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -304,6 +304,19 @@ def _load_shell_content(content, add_empty=False, empty_val=None): def _klibc_to_config_entry(content, mac_addrs=None): + """Convert a klibc writtent shell content file to a 'config' entry + When ip= is seen on the kernel command line in debian initramfs + and networking is brought up, ipconfig will populate + /run/net-.cfg. + + The files are shell style syntax, and examples are in the tests + provided here. There is no good documentation on this unfortunately. + + DEVICE= is expected/required and PROTO should indicate if + this is 'static' or 'dhcp'. + """ + + if mac_addrs is None: mac_addrs = {} @@ -575,11 +588,12 @@ def is_disabled_cfg(cfg): def sys_netdev_info(name, field): if not os.path.exists(os.path.join(SYS_CLASS_NET, name)): - raise OSError("%s: interface does not exist in /sys" % name) + raise OSError("%s: interface does not exist in %s" % + (name, SYS_CLASS_NET)) fname = os.path.join(SYS_CLASS_NET, name, field) if not os.path.exists(fname): - raise OSError("%s: %s does not exist in /sys" % (name, fname)) + raise OSError("%s: could not find sysfs entry: %s" % (name, fname)) data = util.load_file(fname) if data[-1] == '\n': data = data[:-1] @@ -647,7 +661,7 @@ def generate_fallback_config(): nconf['config'].append( {'type': 'physical', 'name': target_name, - 'mac_address': mac, 'subnets': [{'type': 'dhcp4'}]}) + 'mac_address': mac, 'subnets': [{'type': 'dhcp'}]}) return nconf diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 16c44588..dfb31710 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -70,7 +70,7 @@ class TestNetConfigParsing(TestCase): simple_cfg = { 'config': [{"type": "physical", "name": "eth0", "mac_address": "c0:d6:9f:2c:e8:80", - "subnets": [{"type": "dhcp4"}]}]} + "subnets": [{"type": "dhcp"}]}]} def test_klibc_convert_dhcp(self): found = net._klibc_to_config_entry(DHCP_CONTENT_1) -- cgit v1.2.3