From 1dd9102afda920d486a144b3153d6c9951f45cf9 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 16 Mar 2016 21:06:28 -0400 Subject: fix regression when command line (ds=nocloud) is present parsing the command line parameters returned a dictionary but _merge_new_seed was expecting a string to be yaml loaded. Change is to make _merge_new_seed take either string or dict. --- cloudinit/sources/DataSourceNoCloud.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index a3532463..64853385 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -58,7 +58,7 @@ class DataSourceNoCloud(sources.DataSource): md = {} if parse_cmdline_data(self.cmdline_id, md): found.append("cmdline") - mydata = _merge_new_seed({'meta-data': md}) + mydata = _merge_new_seed(mydata, {'meta-data': md}) except: util.logexc(LOG, "Unable to parse command line data") return False @@ -256,8 +256,12 @@ def parse_cmdline_data(ds_id, fill, cmdline=None): def _merge_new_seed(cur, seeded): ret = cur.copy() - ret['meta-data'] = util.mergemanydict([cur['meta-data'], - util.load_yaml(seeded['meta-data'])]) + + newmd = seeded.get('meta-data', {}) + if not isinstance(seeded['meta-data'], dict): + newmd = util.load_yaml(seeded['meta-data']) + ret['meta-data'] = util.mergemanydict([cur['meta-data'], newmd]) + if seeded.get('network-config'): ret['network-config'] = util.load_yaml(seeded['network-config']) -- cgit v1.2.3 From 5916180c5ecb2ef74c1c993dcd32f95cb4581172 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 18 Mar 2016 19:55:53 -0400 Subject: add atomic_write_file and use it from atomic_write_json atomic_write_file just does less and easily utilized for the same purpose that atomic_write_json served. --- bin/cloud-init | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/bin/cloud-init b/bin/cloud-init index 7f665e7e..42166580 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -435,20 +435,24 @@ def main_single(name, args): return 0 -def atomic_write_json(path, data): +def atomic_write_file(path, content, mode='w'): tf = None try: tf = tempfile.NamedTemporaryFile(dir=os.path.dirname(path), - delete=False) - tf.write(util.encode_text(json.dumps(data, indent=1) + "\n")) + delete=False, mode=mode) + tf.write(content)u tf.close() os.rename(tf.name, path) except Exception as e: if tf is not None: - util.del_file(tf.name) + os.unlink(tf.name) raise e +def atomic_write_json(path, data): + return atomic_write_file(path, json.dumps(data, indent=1) + "\n") + + def status_wrapper(name, args, data_d=None, link_d=None): if data_d is None: data_d = os.path.normpath("/var/lib/cloud/data") -- cgit v1.2.3 From 72b56d0f59f321519f25b039937a24b0ce338295 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 18 Mar 2016 20:35:36 -0400 Subject: add this, its getting moved, but i wanted some of the content stored --- udev/79-cloud-init-net-setup-link.rules | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 udev/79-cloud-init-net-setup-link.rules diff --git a/udev/79-cloud-init-net-setup-link.rules b/udev/79-cloud-init-net-setup-link.rules new file mode 100644 index 00000000..03dba382 --- /dev/null +++ b/udev/79-cloud-init-net-setup-link.rules @@ -0,0 +1,18 @@ +# cloud-init rules to apply + +SUBSYSTEM!="net", GOTO="cloudinit_naming_end" + +IMPORT{builtin}="path_id" + +ACTION!="add", GOTO="cloudinit_naming_end" + +# net_setup_link provides us with systemd names for reference +IMPORT{builtin}="net_setup_link" +ATTR{address}!="", ENV{MAC_ADDRESS}="$attr{address}" +IMPORT{program}="/lib/udev/cloud-init-name-device" + +ENV{CLOUDINIT_NET_NAME}!="", NAME="$env{CLOUDINIT_NET_NAME}" + +LABEL="cloudinit_naming_end" + +# vi: ts=4 expandtab syntax=udevrules -- cgit v1.2.3 From 519c0936e3e80fc14225e500fbb61d0d12d28c35 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 18 Mar 2016 20:40:54 -0400 Subject: commit the systemd waiting mechanism Note, still broken as cloud-init local is not going to ever touch the CI_NET_READY file (/run/cloud-init/network-config-ready). So as this is , it will actually just block for 60 seconds and go on. --- setup.py | 3 +- systemd/cloud-init-generator | 3 ++ udev/79-cloud-init-net-setup-link.rules | 18 -------- udev/79-cloud-init-net-wait.rules | 10 ++++ udev/cloud-init-wait | 82 +++++++++++++++++++++++++++++++++ 5 files changed, 97 insertions(+), 19 deletions(-) delete mode 100644 udev/79-cloud-init-net-setup-link.rules create mode 100644 udev/79-cloud-init-net-wait.rules create mode 100755 udev/cloud-init-wait diff --git a/setup.py b/setup.py index 0b261dfe..f86727b2 100755 --- a/setup.py +++ b/setup.py @@ -183,7 +183,8 @@ else: [f for f in glob('doc/examples/*') if is_f(f)]), (USR + '/share/doc/cloud-init/examples/seed', [f for f in glob('doc/examples/seed/*') if is_f(f)]), - (LIB + '/udev/rules.d', ['udev/66-azure-ephemeral.rules']), + (LIB + '/udev/rules.d', [f for f in glob('udev/*.rules')]), + (LIB + '/udev', ['udev/cloud-init-wait']), ] # Use a subclass for install that handles # adding on the right init system configuration files diff --git a/systemd/cloud-init-generator b/systemd/cloud-init-generator index 2d319695..ae286d58 100755 --- a/systemd/cloud-init-generator +++ b/systemd/cloud-init-generator @@ -107,6 +107,9 @@ main() { "ln $CLOUD_SYSTEM_TARGET $link_path" fi fi + # this touches /run/cloud-init/enabled, which is read by + # udev/cloud-init-wait. If not present, it will exit quickly. + touch "$LOG_D/$ENABLE" elif [ "$result" = "$DISABLE" ]; then if [ -f "$link_path" ]; then if rm -f "$link_path"; then diff --git a/udev/79-cloud-init-net-setup-link.rules b/udev/79-cloud-init-net-setup-link.rules deleted file mode 100644 index 03dba382..00000000 --- a/udev/79-cloud-init-net-setup-link.rules +++ /dev/null @@ -1,18 +0,0 @@ -# cloud-init rules to apply - -SUBSYSTEM!="net", GOTO="cloudinit_naming_end" - -IMPORT{builtin}="path_id" - -ACTION!="add", GOTO="cloudinit_naming_end" - -# net_setup_link provides us with systemd names for reference -IMPORT{builtin}="net_setup_link" -ATTR{address}!="", ENV{MAC_ADDRESS}="$attr{address}" -IMPORT{program}="/lib/udev/cloud-init-name-device" - -ENV{CLOUDINIT_NET_NAME}!="", NAME="$env{CLOUDINIT_NET_NAME}" - -LABEL="cloudinit_naming_end" - -# vi: ts=4 expandtab syntax=udevrules diff --git a/udev/79-cloud-init-net-wait.rules b/udev/79-cloud-init-net-wait.rules new file mode 100644 index 00000000..8344222a --- /dev/null +++ b/udev/79-cloud-init-net-wait.rules @@ -0,0 +1,10 @@ +# cloud-init cold/hot-plug blocking mechanism +# this file blocks further processing of network events +# until cloud-init local has had a chance to read and apply network +SUBSYSTEM!="net", GOTO="cloudinit_naming_end" +ACTION!="add", GOTO="cloudinit_naming_end" + +IMPORT{program}="/lib/udev/cloud-init-wait" + +LABEL="cloudinit_naming_end" +# vi: ts=4 expandtab syntax=udevrules diff --git a/udev/cloud-init-wait b/udev/cloud-init-wait new file mode 100755 index 00000000..345333f9 --- /dev/null +++ b/udev/cloud-init-wait @@ -0,0 +1,82 @@ +#!/bin/sh + +CI_NET_READY="/run/cloud-init/network-config-ready" +LOG="/run/cloud-init/${0##*/}.log" +LOG_INIT=0 +DEBUG=0 + +find_name() { + local match="" name="" none="_UNSET" pound="#" + while read match name; do + [ "${match#${pound}}" = "$match" ] || continue + case "$match" in + ID_NET_NAME=${ID_NET_NAME:-$none}) _RET="$name"; return 0;; + ID_NET_NAME_PATH=${ID_NET_NAME_PATH:-$none}) _RET="$name"; return 0;; + MAC_ADDRESS=${MAC_ADDRESS:-$none}) _RET="$name"; return 0;; + INTERFACE=${INTERFACE:-$none}) _RET="$name"; return 0;; + esac + done + return 0 +} + +block_until_ready() { + local fname="$1" + local naplen="$2" max="$3" n=0 + while ! [ -f "$fname" ]; do + n=$(($n+1)) + [ "$n" -ge "$max" ] && return 1 + sleep $naplen + done +} + +log() { + [ -n "${LOG}" ] || return + [ "${DEBUG:-0}" = "0" ] && return + + if [ $LOG_INIT = 0 ]; then + if [ -d "${LOG%/*}" ] || mkdir -p "${LOG%/*}"; then + LOG_INIT=1 + else + echo "${0##*/}: WARN: log init to ${LOG%/*}" 1>&2 + return + fi + elif [ "$LOG_INIT" = "-1" ]; then + return + fi + local info="$$ $INTERFACE" + if [ "$DEBUG" -gt 1 ]; then + local up idle + read up idle < /proc/uptime + info="$$ $INTERFACE $up" + fi + echo "[$info]" "$@" >> "$LOG" +} + +main() { + local name="" readyfile="$CI_NET_READY" + local info="INTERFACE=${INTERFACE} ID_NET_NAME=${ID_NET_NAME}" + info="$info ID_NET_NAME_PATH=${ID_NET_NAME_PATH}" + info="$info MAC_ADDRESS=${MAC_ADDRESS}" + log "$info" + + ## Check to see if cloud-init.target is set. If cloud-init is + ## disabled we do not want to do anything. + if [ ! -f "/run/cloud-init/enabled" ]; then + log "cloud-init disabled" + return 0 + fi + + block_until_ready "$readyfile" .1 600 || + { log "failed waiting for ready on $INTERFACE"; return 1; } + + #find_name < "$CI_NET_RULES" && name="$_RET" || + # { log "failed to find match for $INTERFACE"; return 0; } + + log "net config ready" + #[ -z "$name" ] || echo "CLOUDINIT_NET_NAME=$name" +} + +main "$@" +exit + +# vi: ts=4 expandtab -- cgit v1.2.3 From db54b59b90c8db2fc4a637ae09d3f0df14e77acb Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 18 Mar 2016 20:42:49 -0400 Subject: remove the 'find_name' function that was here. I had left this in to commit it, it was my first pass at cloud-init doing the naming itself. That design was then replaced with the idea for cloud-init to instead write systemd.rules files. --- udev/cloud-init-wait | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/udev/cloud-init-wait b/udev/cloud-init-wait index 345333f9..f27309e3 100755 --- a/udev/cloud-init-wait +++ b/udev/cloud-init-wait @@ -5,20 +5,6 @@ LOG="/run/cloud-init/${0##*/}.log" LOG_INIT=0 DEBUG=0 -find_name() { - local match="" name="" none="_UNSET" pound="#" - while read match name; do - [ "${match#${pound}}" = "$match" ] || continue - case "$match" in - ID_NET_NAME=${ID_NET_NAME:-$none}) _RET="$name"; return 0;; - ID_NET_NAME_PATH=${ID_NET_NAME_PATH:-$none}) _RET="$name"; return 0;; - MAC_ADDRESS=${MAC_ADDRESS:-$none}) _RET="$name"; return 0;; - INTERFACE=${INTERFACE:-$none}) _RET="$name"; return 0;; - esac - done - return 0 -} - block_until_ready() { local fname="$1" local naplen="$2" max="$3" n=0 @@ -69,11 +55,7 @@ main() { block_until_ready "$readyfile" .1 600 || { log "failed waiting for ready on $INTERFACE"; return 1; } - #find_name < "$CI_NET_RULES" && name="$_RET" || - # { log "failed to find match for $INTERFACE"; return 0; } - log "net config ready" - #[ -z "$name" ] || echo "CLOUDINIT_NET_NAME=$name" } main "$@" -- cgit v1.2.3 From 1c8e0d93bb48338777e689e6303702bf84fed0d1 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 18 Mar 2016 20:47:06 -0400 Subject: cloud-init-local.service: touch file that cloud-init-wait will wait for this might work. And if it does means we could generally test this as the file that the cloud-init-wait will wait for will actually get created. --- systemd/cloud-init-local.service | 3 +++ 1 file changed, 3 insertions(+) diff --git a/systemd/cloud-init-local.service b/systemd/cloud-init-local.service index 475a2e11..f3a92e2f 100644 --- a/systemd/cloud-init-local.service +++ b/systemd/cloud-init-local.service @@ -10,6 +10,9 @@ Before=shutdown.target [Service] Type=oneshot ExecStart=/usr/bin/cloud-init init --local +## FIXME: remove this when cloud-initn local does it itself +## or otherwise better signals any blocking udev events +ExecStopPost=touch /run/cloud-init/network-config-ready RemainAfterExit=yes TimeoutSec=0 -- cgit v1.2.3 From 5d1d36f617f9ce342930a78183a13877b5a619cd Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Sun, 20 Mar 2016 06:26:11 -0400 Subject: fix syntax --- bin/cloud-init | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/cloud-init b/bin/cloud-init index 42166580..f101a713 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -440,7 +440,7 @@ def atomic_write_file(path, content, mode='w'): try: tf = tempfile.NamedTemporaryFile(dir=os.path.dirname(path), delete=False, mode=mode) - tf.write(content)u + tf.write(content) tf.close() os.rename(tf.name, path) except Exception as e: -- cgit v1.2.3 From 16751f75a51814e4873199eddec15040dd221561 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Sun, 20 Mar 2016 22:31:21 -0400 Subject: fix creation of network-config-ready and dont bother waiting on lo --- systemd/cloud-init-local.service | 4 +--- udev/cloud-init-wait | 4 ++++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/systemd/cloud-init-local.service b/systemd/cloud-init-local.service index f3a92e2f..dd737644 100644 --- a/systemd/cloud-init-local.service +++ b/systemd/cloud-init-local.service @@ -10,9 +10,7 @@ Before=shutdown.target [Service] Type=oneshot ExecStart=/usr/bin/cloud-init init --local -## FIXME: remove this when cloud-initn local does it itself -## or otherwise better signals any blocking udev events -ExecStopPost=touch /run/cloud-init/network-config-ready +ExecStart=/bin/touch /run/cloud-init/network-config-ready RemainAfterExit=yes TimeoutSec=0 diff --git a/udev/cloud-init-wait b/udev/cloud-init-wait index f27309e3..7d53dee4 100755 --- a/udev/cloud-init-wait +++ b/udev/cloud-init-wait @@ -52,6 +52,10 @@ main() { return 0 fi + if [ "${INTERFACE#lo}" != "$INTERFACE" ]; then + return 0 + fi + block_until_ready "$readyfile" .1 600 || { log "failed waiting for ready on $INTERFACE"; return 1; } -- cgit v1.2.3 From 7a22e352b2f87636554d9787f60cd3168f3d77bc Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 21 Mar 2016 08:59:55 -0400 Subject: cloud-init-local needs to want network-pre or it isnt guaranteed to start --- systemd/cloud-init-local.service | 1 + 1 file changed, 1 insertion(+) diff --git a/systemd/cloud-init-local.service b/systemd/cloud-init-local.service index dd737644..b19eeaee 100644 --- a/systemd/cloud-init-local.service +++ b/systemd/cloud-init-local.service @@ -2,6 +2,7 @@ Description=Initial cloud-init job (pre-networking) DefaultDependencies=no Wants=local-fs.target +Wants=network-pre.target After=local-fs.target Conflicts=shutdown.target Before=network-pre.target -- cgit v1.2.3 From bb58463474e334b8c8d1769101bd3afc48ebfef4 Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Mon, 21 Mar 2016 20:42:26 -0500 Subject: Added net.find_fallback_network_device() to find an appropriate device to dhcp on in the event that no network configuration was provided to cloud-init - Devices in /sys/class/net aside from loopback devices are scanned - Each device is tested to determine if it has a carrier using /sys/class/net/DEV/carrier, devices which do are preferred as they are most likely connected to the outside world - Devices which do not have a carrier but which might still be connected due to being in a dormant or down state are used as fallbacks in case no devices are found which have a carrier - A network state dictionary is generated to be passed to render_network_state to write ENI - A systemd link file is generated that will rename the chosen device to eth0 --- cloudinit/net/__init__.py | 83 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 3cf99604..800ffe61 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -20,6 +20,8 @@ import errno import glob import os import re +import string +import textwrap from cloudinit import log as logging from cloudinit import util @@ -280,6 +282,87 @@ def parse_net_config(path): return ns +def find_fallback_network_device(): + """Determine which attached net dev is most likely to have a connection and + generate network state to run dhcp on that interface""" + ns = {'interfaces': {}, 'dns': {'search': [], 'nameservers': []}, + 'routes': []} + default_link_file = textwrap.dedent(""" + #cloud-init + [Match] + MACAddress={mac} + + [Link] + Name={name} + """) + + # get list of interfaces that could have connections + invalid_interfaces = set(['lo']) + potential_interfaces = set(os.listdir(SYS_CLASS_NET)) + potential_interfaces = potential_interfaces.difference(invalid_interfaces) + + # sort into interfaces with carrier, interfaces which could have carrier, + # and ignore interfaces that are definitely disconnected + connected = [] + possibly_connected = [] + for interface in potential_interfaces: + sysfs_carrier = os.path.join(SYS_CLASS_NET, interface, 'carrier') + carrier = int(util.load_file(sysfs_carrier).strip()) + if carrier: + connected.append(interface) + continue + # check if nic is dormant or down, as this may make a nick appear to + # not have a carrier even though it could acquire one when brought + # online by dhclient + sysfs_dormant = os.path.join(SYS_CLASS_NET, interface, 'dormant') + dormant = int(util.load_file(sysfs_dormant).strip()) + if dormant: + possibly_connected.append(interface) + continue + sysfs_operstate = os.path.join(SYS_CLASS_NET, interface, 'operstate') + operstate = util.load_file(sysfs_operstate).strip() + if operstate in ['dormant', 'down', 'lowerlayerdown', 'unknown']: + possibly_connected.append(interface) + continue + + # don't bother with interfaces that might not be connected if there are + # some that definitely are + if connected: + potential_interfaces = connected + else: + potential_interfaces = possibly_connected + + # if there are no interfaces, give up + if not potential_interfaces: + return + + # if eth0 exists use it above anything else, otherwise get the interface + # that looks 'first' + if 'eth0' in potential_interfaces: + name = 'eth0' + else: + name = potential_interfaces.sort( + key=lambda x: int(x.strip(string.ascii_letters)))[0] + + sysfs_mac = os.path.join(SYS_CLASS_NET, name, 'address') + mac = util.load_file(sysfs_mac).strip() + + # generate net config for interface, rename interface to eth0 for backwards + # compatibility, and attempt both dhcp4 and dhcp6 + ns['interfaces']['eth0'] = { + 'mac_address': mac, 'name': 'eth0', 'type': 'physical', + 'mode': 'manual', 'inet': 'inet', + 'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}] + } + + # insert params into link file + link_file = default_link_file.format(name=name, mac=mac) + + syslink_name = "/etc/systemd/network/50-cloud-init-{}.link".format(name) + + return (ns, link_file, syslink_name) + + def render_persistent_net(network_state): ''' Given state, emit udev rules to map mac to ifname -- cgit v1.2.3 From 9a146e83189ef3128a04c9e0c1d21c6181f554f1 Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Mon, 21 Mar 2016 21:10:20 -0500 Subject: Added _write_network_fallback function to distros.debian and abstract to distros base, and apply_fallback_network to distros to call _write_network_fallback. Note that since _write_network_fallback is only implemented for debian and ubuntu a check is needed to ensure that it does not break behaviour for other distros. Added function to disable .cfg files to util, since it may be useful elsewhere --- cloudinit/distros/__init__.py | 12 ++++++++++++ cloudinit/distros/debian.py | 10 ++++++++++ cloudinit/util.py | 9 +++++++++ 3 files changed, 31 insertions(+) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 74b484a7..e32ddd57 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -78,6 +78,10 @@ class Distro(object): def _write_network_config(self, settings): raise NotImplementedError() + @abc.abstractmethod + def _write_network_fallback(self): + raise NotImplementedError() + def _find_tz_file(self, tz): tz_file = os.path.join(self.tz_zone_dir, str(tz)) if not os.path.isfile(tz_file): @@ -143,6 +147,14 @@ class Distro(object): return self._bring_up_interfaces(dev_names) return False + def apply_fallback_network(self, bring_up=True): + # Write it out + dev_names = self._write_network_fallback() + # Now try to bring them up + if bring_up: + return self._bring_up_interfaces(dev_names) + return False + @abc.abstractmethod def apply_locale(self, locale, out_fn=None): raise NotImplementedError() diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 909d6deb..18d5d124 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -82,6 +82,16 @@ class Distro(distros.Distro): net.render_network_state(network_state=ns, target="/") return [] + def _write_network_fallback(self): + # old fallback configuration is obsolete, disable it + util.disable_cfg_file('/etc/network/interfaces.d/eth0.cfg') + (ns, link_file, syslink_name) = net.find_fallback_network_device() + if link_file is not None: + util.write_file(syslink_name, link_file) + if ns is not None: + net.render_network_stat(network_state=ns, target="/") + return [] + def _bring_up_interfaces(self, device_names): use_all = False for d in device_names: diff --git a/cloudinit/util.py b/cloudinit/util.py index 20916e53..fa3a6163 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -849,6 +849,15 @@ def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0): return (md, ud) +def disable_conf_file(conf): + # disable .cfg file by renaming it if it exists + if not os.path.exists(conf): + return None + target_path = os.path.join(conf, '.disabled') + rename(conf, target_path) + return target_path + + def read_conf_d(confd): # Get reverse sorted list (later trumps newer) confs = sorted(os.listdir(confd), reverse=True) -- cgit v1.2.3 From 4c3468985d93929df4e9486b2e68938806fbfa1b Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Mon, 21 Mar 2016 23:41:47 -0500 Subject: Fix typo in disable_conf_file and mistake in call --- cloudinit/distros/debian.py | 2 +- cloudinit/util.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 18d5d124..38d22d85 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -84,7 +84,7 @@ class Distro(distros.Distro): def _write_network_fallback(self): # old fallback configuration is obsolete, disable it - util.disable_cfg_file('/etc/network/interfaces.d/eth0.cfg') + util.disable_conf_file('/etc/network/interfaces.d/eth0.cfg') (ns, link_file, syslink_name) = net.find_fallback_network_device() if link_file is not None: util.write_file(syslink_name, link_file) diff --git a/cloudinit/util.py b/cloudinit/util.py index fa3a6163..58ab3c75 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -853,7 +853,7 @@ def disable_conf_file(conf): # disable .cfg file by renaming it if it exists if not os.path.exists(conf): return None - target_path = os.path.join(conf, '.disabled') + target_path = conf + '.disabled' rename(conf, target_path) return target_path -- cgit v1.2.3 From 2aacb06be37e7e8aa84d11ae8c566a26f9df27e4 Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Tue, 22 Mar 2016 00:33:35 -0500 Subject: Wrap read calls to /sys/class/net/DEV/{carrier, dormant, operstate} in try/except blocks because there are sometimes read errors on the files and this should not cause a stacktrace --- cloudinit/net/__init__.py | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 800ffe61..e5b45926 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -306,24 +306,34 @@ def find_fallback_network_device(): connected = [] possibly_connected = [] for interface in potential_interfaces: - sysfs_carrier = os.path.join(SYS_CLASS_NET, interface, 'carrier') - carrier = int(util.load_file(sysfs_carrier).strip()) - if carrier: - connected.append(interface) - continue + try: + sysfs_carrier = os.path.join(SYS_CLASS_NET, interface, 'carrier') + carrier = int(util.load_file(sysfs_carrier).strip()) + if carrier: + connected.append(interface) + continue + except OSError: + pass # check if nic is dormant or down, as this may make a nick appear to # not have a carrier even though it could acquire one when brought # online by dhclient - sysfs_dormant = os.path.join(SYS_CLASS_NET, interface, 'dormant') - dormant = int(util.load_file(sysfs_dormant).strip()) - if dormant: - possibly_connected.append(interface) - continue - sysfs_operstate = os.path.join(SYS_CLASS_NET, interface, 'operstate') - operstate = util.load_file(sysfs_operstate).strip() - if operstate in ['dormant', 'down', 'lowerlayerdown', 'unknown']: - possibly_connected.append(interface) - continue + try: + sysfs_dormant = os.path.join(SYS_CLASS_NET, interface, 'dormant') + dormant = int(util.load_file(sysfs_dormant).strip()) + if dormant: + possibly_connected.append(interface) + continue + except OSError: + pass + try: + sysfs_operstate = os.path.join(SYS_CLASS_NET, interface, + 'operstate') + operstate = util.load_file(sysfs_operstate).strip() + if operstate in ['dormant', 'down', 'lowerlayerdown', 'unknown']: + possibly_connected.append(interface) + continue + except OSError: + pass # don't bother with interfaces that might not be connected if there are # some that definitely are -- cgit v1.2.3 From 217d92372ca5a4e994f1e9bc9580363dbba59032 Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Tue, 22 Mar 2016 00:43:31 -0500 Subject: Fix typo --- cloudinit/net/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index e5b45926..4641e54d 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -351,8 +351,9 @@ def find_fallback_network_device(): if 'eth0' in potential_interfaces: name = 'eth0' else: - name = potential_interfaces.sort( - key=lambda x: int(x.strip(string.ascii_letters)))[0] + potential_interfaces.sort( + key=lambda x: int(x.strip(string.ascii_letters))) + name = potential_interfaces[0] sysfs_mac = os.path.join(SYS_CLASS_NET, name, 'address') mac = util.load_file(sysfs_mac).strip() -- cgit v1.2.3 From aab9331089abcf3f5074f3cf7659502ca0752114 Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Tue, 22 Mar 2016 00:48:17 -0500 Subject: Typo fix --- cloudinit/distros/debian.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 38d22d85..8e57f70e 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -89,7 +89,7 @@ class Distro(distros.Distro): if link_file is not None: util.write_file(syslink_name, link_file) if ns is not None: - net.render_network_stat(network_state=ns, target="/") + net.render_network_state(network_state=ns, target="/") return [] def _bring_up_interfaces(self, device_names): -- cgit v1.2.3 From 7e399773a95e21e4c825dab61847d6abcd2aa511 Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Tue, 22 Mar 2016 01:17:12 -0500 Subject: For find_fallback_network_device, kwarg rename_to_default specifies whether or not to attempt renaming the network interface to the default interface. Default interface is controleld by net.DEFAULT_PRIMARY_INTERFACE and is currently set to eth0 for legacy reasons. By default cloud-init will not attempt to rename the device as this does not work in some situtations depending on the backing driver of the device. --- cloudinit/net/__init__.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 4641e54d..e2e50441 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -48,6 +48,8 @@ NET_CONFIG_BRIDGE_OPTIONS = [ "bridge_hello", "bridge_maxage", "bridge_maxwait", "bridge_stp", ] +DEFAULT_PRIMARY_INTERFACE = 'eth0' + def sys_dev_path(devname, path=""): return SYS_CLASS_NET + devname + "/" + path @@ -282,9 +284,10 @@ def parse_net_config(path): return ns -def find_fallback_network_device(): +def find_fallback_network_device(rename_to_default=False): """Determine which attached net dev is most likely to have a connection and generate network state to run dhcp on that interface""" + # by default use eth0 as primary interface ns = {'interfaces': {}, 'dns': {'search': [], 'nameservers': []}, 'routes': []} default_link_file = textwrap.dedent(""" @@ -348,8 +351,8 @@ def find_fallback_network_device(): # if eth0 exists use it above anything else, otherwise get the interface # that looks 'first' - if 'eth0' in potential_interfaces: - name = 'eth0' + if DEFAULT_PRIMARY_INTERFACE in potential_interfaces: + name = DEFAULT_PRIMARY_INTERFACE else: potential_interfaces.sort( key=lambda x: int(x.strip(string.ascii_letters))) @@ -358,18 +361,23 @@ def find_fallback_network_device(): sysfs_mac = os.path.join(SYS_CLASS_NET, name, 'address') mac = util.load_file(sysfs_mac).strip() + target_name = name + if rename_to_default: + target_name = DEFAULT_PRIMARY_INTERFACE + # generate net config for interface, rename interface to eth0 for backwards # compatibility, and attempt both dhcp4 and dhcp6 - ns['interfaces']['eth0'] = { - 'mac_address': mac, 'name': 'eth0', 'type': 'physical', + ns['interfaces'][target_name] = { + 'mac_address': mac, 'name': target_name, 'type': 'physical', 'mode': 'manual', 'inet': 'inet', 'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}] } # insert params into link file - link_file = default_link_file.format(name=name, mac=mac) + link_file = default_link_file.format(name=target_name, mac=mac) - syslink_name = "/etc/systemd/network/50-cloud-init-{}.link".format(name) + syslink_name = "/etc/systemd/network/50-cloud-init-{}.link".format( + target_name) return (ns, link_file, syslink_name) -- cgit v1.2.3 From 88bfbe22a2f1128f358501bc10f5a2cbd1f7facf Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Tue, 22 Mar 2016 01:24:01 -0500 Subject: Basic code added to kick off network configuration and cause fallback network configuration to be run if there is no network configuration provided by the datasource. NOTE: the code added here will not behave correctly if a net datasource has network configuration. this code is temporary and should be reverted once support for network configuration for net datasources after retrieving config is in place. based on: http://paste.ubuntu.com/15443576/ With this in place cloud-init properly chooses a fallback interface, configures it and brings it online --- bin/cloud-init | 2 ++ cloudinit/sources/__init__.py | 4 ++++ cloudinit/stages.py | 11 +++++++++++ 3 files changed, 17 insertions(+) diff --git a/bin/cloud-init b/bin/cloud-init index f101a713..2dfa8ec7 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -251,6 +251,7 @@ def main_init(name, args): # Stage 5 try: init.fetch() + init.apply_networking() except sources.DataSourceNotFoundException: # In the case of 'cloud-init init' without '--local' it is a bit # more likely that the user would consider it failure if nothing was @@ -261,6 +262,7 @@ def main_init(name, args): else: util.logexc(LOG, ("No instance datasource found!" " Likely bad things to come!")) + init.apply_networking() if not args.force: if args.local: return (None, []) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index d3cfa560..08058762 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -217,6 +217,10 @@ class DataSource(object): def get_package_mirror_info(self): return self.distro.get_package_mirror_info(data_source=self) + @property + def network_config(self): + return self.metadata.network_config + def normalize_pubkey_data(pubkey_data): keys = [] diff --git a/cloudinit/stages.py b/cloudinit/stages.py index dbcf3d55..d508897b 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -587,6 +587,17 @@ class Init(object): # Run the handlers self._do_handlers(user_data_msg, c_handlers_list, frequency) + def apply_networking(self): + """Attempt to apply network configuration, either using network + configuration from datasource or fallback configuration if that is + not available""" + if self.datasource and self.datasource.network_config: + ds_net_conf = self.datasource.network_config + res = self.distro.apply_network_config(ds_net_conf, bring_up=True) + else: + res = self.distro.apply_fallback_network(bring_up=True) + return res + class Modules(object): def __init__(self, init, cfg_files=None, reporter=None): -- cgit v1.2.3 From 3a7e3d198172f26b7b97707520d06fe5303fadbc Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Tue, 22 Mar 2016 01:33:20 -0500 Subject: Got rid of blank lines in net.find_fallback_network_device --- cloudinit/net/__init__.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index e2e50441..2596b4f5 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -303,7 +303,6 @@ def find_fallback_network_device(rename_to_default=False): invalid_interfaces = set(['lo']) potential_interfaces = set(os.listdir(SYS_CLASS_NET)) potential_interfaces = potential_interfaces.difference(invalid_interfaces) - # sort into interfaces with carrier, interfaces which could have carrier, # and ignore interfaces that are definitely disconnected connected = [] @@ -344,11 +343,9 @@ def find_fallback_network_device(rename_to_default=False): potential_interfaces = connected else: potential_interfaces = possibly_connected - # if there are no interfaces, give up if not potential_interfaces: return - # if eth0 exists use it above anything else, otherwise get the interface # that looks 'first' if DEFAULT_PRIMARY_INTERFACE in potential_interfaces: @@ -360,11 +357,9 @@ def find_fallback_network_device(rename_to_default=False): sysfs_mac = os.path.join(SYS_CLASS_NET, name, 'address') mac = util.load_file(sysfs_mac).strip() - target_name = name if rename_to_default: target_name = DEFAULT_PRIMARY_INTERFACE - # generate net config for interface, rename interface to eth0 for backwards # compatibility, and attempt both dhcp4 and dhcp6 ns['interfaces'][target_name] = { @@ -372,10 +367,8 @@ def find_fallback_network_device(rename_to_default=False): 'mode': 'manual', 'inet': 'inet', 'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}] } - # insert params into link file link_file = default_link_file.format(name=target_name, mac=mac) - syslink_name = "/etc/systemd/network/50-cloud-init-{}.link".format( target_name) -- cgit v1.2.3 From e66bcc8b2ea7648c15476cdd43d3753ee6c27ff1 Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Tue, 22 Mar 2016 01:48:39 -0500 Subject: - Rename find_fallback_network_device to generate_fallback_config - Removed systemd .link file generation, as it is not needed right now - Changed return of generate_fallback_config to be just ns dict - In distros.debian don't attempt to write .link file --- cloudinit/distros/debian.py | 4 +--- cloudinit/net/__init__.py | 23 ++++------------------- 2 files changed, 5 insertions(+), 22 deletions(-) diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 8e57f70e..0fa47274 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -85,9 +85,7 @@ class Distro(distros.Distro): def _write_network_fallback(self): # old fallback configuration is obsolete, disable it util.disable_conf_file('/etc/network/interfaces.d/eth0.cfg') - (ns, link_file, syslink_name) = net.find_fallback_network_device() - if link_file is not None: - util.write_file(syslink_name, link_file) + ns = net.generate_fallback_config() if ns is not None: net.render_network_state(network_state=ns, target="/") return [] diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 2596b4f5..48b82a2c 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -21,7 +21,6 @@ import glob import os import re import string -import textwrap from cloudinit import log as logging from cloudinit import util @@ -284,20 +283,12 @@ def parse_net_config(path): return ns -def find_fallback_network_device(rename_to_default=False): +def generate_fallback_config(): """Determine which attached net dev is most likely to have a connection and generate network state to run dhcp on that interface""" # by default use eth0 as primary interface ns = {'interfaces': {}, 'dns': {'search': [], 'nameservers': []}, 'routes': []} - default_link_file = textwrap.dedent(""" - #cloud-init - [Match] - MACAddress={mac} - - [Link] - Name={name} - """) # get list of interfaces that could have connections invalid_interfaces = set(['lo']) @@ -358,21 +349,15 @@ def find_fallback_network_device(rename_to_default=False): sysfs_mac = os.path.join(SYS_CLASS_NET, name, 'address') mac = util.load_file(sysfs_mac).strip() target_name = name - if rename_to_default: - target_name = DEFAULT_PRIMARY_INTERFACE - # generate net config for interface, rename interface to eth0 for backwards - # compatibility, and attempt both dhcp4 and dhcp6 + + # generate net config for interface ns['interfaces'][target_name] = { 'mac_address': mac, 'name': target_name, 'type': 'physical', 'mode': 'manual', 'inet': 'inet', 'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}] } - # insert params into link file - link_file = default_link_file.format(name=target_name, mac=mac) - syslink_name = "/etc/systemd/network/50-cloud-init-{}.link".format( - target_name) - return (ns, link_file, syslink_name) + return ns def render_persistent_net(network_state): -- cgit v1.2.3 From 7eccb0f0f3693662b3f288e7a74cb5bd6d7814ea Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Tue, 22 Mar 2016 02:02:22 -0500 Subject: In generate_fallback_config return full netconfig dict with 'config' and 'version' keys --- cloudinit/distros/debian.py | 5 +++-- cloudinit/net/__init__.py | 11 +++++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 0fa47274..de8c4c6c 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -85,8 +85,9 @@ class Distro(distros.Distro): def _write_network_fallback(self): # old fallback configuration is obsolete, disable it util.disable_conf_file('/etc/network/interfaces.d/eth0.cfg') - ns = net.generate_fallback_config() - if ns is not None: + nconf = net.generate_fallback_config() + if nconf is not None: + ns = nconf['config'] net.render_network_state(network_state=ns, target="/") return [] diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 48b82a2c..389c2afb 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -287,8 +287,11 @@ def generate_fallback_config(): """Determine which attached net dev is most likely to have a connection and generate network state to run dhcp on that interface""" # by default use eth0 as primary interface - ns = {'interfaces': {}, 'dns': {'search': [], 'nameservers': []}, - 'routes': []} + nconf = {'config': {'interfaces': {}, + 'dns': {'search': [], 'nameservers': []}, 'routes': [] + }, + 'version': 1 + } # get list of interfaces that could have connections invalid_interfaces = set(['lo']) @@ -351,13 +354,13 @@ def generate_fallback_config(): target_name = name # generate net config for interface - ns['interfaces'][target_name] = { + nconf['config']['interfaces'][target_name] = { 'mac_address': mac, 'name': target_name, 'type': 'physical', 'mode': 'manual', 'inet': 'inet', 'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}] } - return ns + return nconf def render_persistent_net(network_state): -- cgit v1.2.3 From 6ce134c1868478345471ba9166f1523f7d9bf19d Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 22 Mar 2016 03:02:31 -0400 Subject: move some of the pickle loading out of Init, into private methods I plan to re-use these methods later. They stand alone even if they dont end up getting used, though. --- cloudinit/stages.py | 65 ++++++++++++++++++++++++++++------------------------- 1 file changed, 35 insertions(+), 30 deletions(-) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index edad6450..c230ec0d 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -193,40 +193,12 @@ class Init(object): # We try to restore from a current link and static path # by using the instance link, if purge_cache was called # the file wont exist. - pickled_fn = self.paths.get_ipath_cur('obj_pkl') - pickle_contents = None - try: - pickle_contents = util.load_file(pickled_fn, decode=False) - except Exception as e: - if os.path.isfile(pickled_fn): - LOG.warn("failed loading pickle in %s: %s" % (pickled_fn, e)) - pass - - # This is expected so just return nothing - # successfully loaded... - if not pickle_contents: - return None - try: - return pickle.loads(pickle_contents) - except Exception: - util.logexc(LOG, "Failed loading pickled blob from %s", pickled_fn) - return None + return _pkl_load(self.paths.get_ipath_cur('obj_pkl')) def _write_to_cache(self): if self.datasource is NULL_DATA_SOURCE: return False - pickled_fn = self.paths.get_ipath_cur("obj_pkl") - try: - pk_contents = pickle.dumps(self.datasource) - except Exception: - util.logexc(LOG, "Failed pickling datasource %s", self.datasource) - return False - try: - util.write_file(pickled_fn, pk_contents, omode="wb", mode=0o400) - except Exception: - util.logexc(LOG, "Failed pickling datasource to %s", pickled_fn) - return False - return True + return _pkl_store(self.datasource, self.paths.get_ipath_cur("obj_pkl")) def _get_datasources(self): # Any config provided??? @@ -796,3 +768,36 @@ def fetch_base_config(): base_cfgs.append(default_cfg) return util.mergemanydict(base_cfgs) + + +def _pkl_store(obj, fname): + try: + pk_contents = pickle.dumps(obj) + except Exception: + util.logexc(LOG, "Failed pickling datasource %s", obj) + return False + try: + util.write_file(fname, pk_contents, omode="wb", mode=0o400) + except Exception: + util.logexc(LOG, "Failed pickling datasource to %s", fname) + return False + return True + + +def _pkl_load(fname): + pickle_contents = None + try: + pickle_contents = util.load_file(fname, decode=False) + except Exception as e: + if os.path.isfile(fname): + LOG.warn("failed loading pickle in %s: %s" % (fname, e)) + pass + + # This is allowed so just return nothing successfully loaded... + if not pickle_contents: + return None + try: + return pickle.loads(pickle_contents) + except Exception: + util.logexc(LOG, "Failed loading pickled blob from %s", fname) + return None -- cgit v1.2.3 From 3a3f960d5cfee60766e7de9e1fced537cac72106 Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Tue, 22 Mar 2016 02:20:06 -0500 Subject: In generate_fallback_config() fix function to sort potential interfaces to work on interfaces with characters between their numbers --- cloudinit/net/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 389c2afb..36f07a02 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -346,7 +346,7 @@ def generate_fallback_config(): name = DEFAULT_PRIMARY_INTERFACE else: potential_interfaces.sort( - key=lambda x: int(x.strip(string.ascii_letters))) + key=lambda x: int(''.join(i for i in x if i in string.digits))) name = potential_interfaces[0] sysfs_mac = os.path.join(SYS_CLASS_NET, name, 'address') -- cgit v1.2.3 From 78c99ef3faecde46b3e460dffa1af69654e8bbff Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 22 Mar 2016 03:33:05 -0400 Subject: drop changes other than generate_fallback_config --- cloudinit/distros/__init__.py | 12 ------------ cloudinit/distros/debian.py | 9 --------- cloudinit/stages.py | 11 ----------- cloudinit/util.py | 9 --------- 4 files changed, 41 deletions(-) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index e32ddd57..74b484a7 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -78,10 +78,6 @@ class Distro(object): def _write_network_config(self, settings): raise NotImplementedError() - @abc.abstractmethod - def _write_network_fallback(self): - raise NotImplementedError() - def _find_tz_file(self, tz): tz_file = os.path.join(self.tz_zone_dir, str(tz)) if not os.path.isfile(tz_file): @@ -147,14 +143,6 @@ class Distro(object): return self._bring_up_interfaces(dev_names) return False - def apply_fallback_network(self, bring_up=True): - # Write it out - dev_names = self._write_network_fallback() - # Now try to bring them up - if bring_up: - return self._bring_up_interfaces(dev_names) - return False - @abc.abstractmethod def apply_locale(self, locale, out_fn=None): raise NotImplementedError() diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index de8c4c6c..909d6deb 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -82,15 +82,6 @@ class Distro(distros.Distro): net.render_network_state(network_state=ns, target="/") return [] - def _write_network_fallback(self): - # old fallback configuration is obsolete, disable it - util.disable_conf_file('/etc/network/interfaces.d/eth0.cfg') - nconf = net.generate_fallback_config() - if nconf is not None: - ns = nconf['config'] - net.render_network_state(network_state=ns, target="/") - return [] - def _bring_up_interfaces(self, device_names): use_all = False for d in device_names: diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 64da3b5b..c230ec0d 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -567,17 +567,6 @@ class Init(object): # Run the handlers self._do_handlers(user_data_msg, c_handlers_list, frequency) - def apply_networking(self): - """Attempt to apply network configuration, either using network - configuration from datasource or fallback configuration if that is - not available""" - if self.datasource and self.datasource.network_config: - ds_net_conf = self.datasource.network_config - res = self.distro.apply_network_config(ds_net_conf, bring_up=True) - else: - res = self.distro.apply_fallback_network(bring_up=True) - return res - class Modules(object): def __init__(self, init, cfg_files=None, reporter=None): diff --git a/cloudinit/util.py b/cloudinit/util.py index 58ab3c75..20916e53 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -849,15 +849,6 @@ def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0): return (md, ud) -def disable_conf_file(conf): - # disable .cfg file by renaming it if it exists - if not os.path.exists(conf): - return None - target_path = conf + '.disabled' - rename(conf, target_path) - return target_path - - def read_conf_d(confd): # Get reverse sorted list (later trumps newer) confs = sorted(os.listdir(confd), reverse=True) -- cgit v1.2.3 From 9c0a2abc8d2c0e390745ddb163f5eae07b20d61d Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 22 Mar 2016 03:50:28 -0400 Subject: add code to invoke networking config there is no data source that has a populated network_config() so at this point this doesn't do anything. --- bin/cloud-init | 4 ++++ cloudinit/distros/__init__.py | 2 +- cloudinit/net/__init__.py | 17 +++++++++++++++++ cloudinit/sources/__init__.py | 4 ++++ cloudinit/stages.py | 24 ++++++++++++++++++++++++ 5 files changed, 50 insertions(+), 1 deletion(-) diff --git a/bin/cloud-init b/bin/cloud-init index 63aa765b..8875d2f6 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -263,6 +263,10 @@ def main_init(name, args): return (None, []) else: return (None, ["No instance datasource found."]) + + if args.local: + init.apply_network_config() + # Stage 6 iid = init.instancify() LOG.debug("%s will now be targeting instance id: %s", name, iid) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 74b484a7..418421b9 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -135,7 +135,7 @@ class Distro(object): return self._bring_up_interfaces(dev_names) return False - def apply_network_config(self, netconfig, bring_up=True): + def apply_network_config(self, netconfig, bring_up=False): # Write it out dev_names = self._write_network_config(netconfig) # Now try to bring them up diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 3cf99604..799cb97e 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -434,4 +434,21 @@ def render_network_state(target, network_state): with open(netrules, 'w+') as f: f.write(render_persistent_net(network_state)) + +def is_disabled_cfg(cfg): + if not cfg or not isinstance(cfg, dict): + return False + return cfg.get('config') == "disabled" + + +def generate_fallback_config(): + # FIXME: add implementation here + return None + + +def read_kernel_cmdline_config(): + # FIXME: add implementation here + return None + + # vi: ts=4 expandtab syntax=python diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 28540a7b..c63464b2 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -221,6 +221,10 @@ class DataSource(object): # quickly (local check only) if self.instance_id is still return False + @property + def network_config(self): + return None + def normalize_pubkey_data(pubkey_data): keys = [] diff --git a/cloudinit/stages.py b/cloudinit/stages.py index c230ec0d..8e681e29 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -43,6 +43,7 @@ from cloudinit import distros from cloudinit import helpers from cloudinit import importer from cloudinit import log as logging +from cloudinit import net from cloudinit import sources from cloudinit import type_utils from cloudinit import util @@ -567,6 +568,29 @@ class Init(object): # Run the handlers self._do_handlers(user_data_msg, c_handlers_list, frequency) + def _find_networking_config(self): + cmdline_cfg = ('cmdline', net.read_kernel_cmdline_config()) + dscfg = ('ds', None) + if self.datasource and hasattr(self.datasource, 'network_config'): + dscfg = ('ds', self.datasource.network_config) + sys_cfg = ('system_cfg', self.cfg.get('network')) + + for loc, ncfg in (cmdline_cfg, dscfg, sys_cfg): + if net.is_disabled_cfg(ncfg): + LOG.debug("network config disabled by %s", loc) + return None + if ncfg: + return ncfg + return net.generate_fallback_config() + + def apply_network_config(self): + netcfg = self._find_networking_config() + if netcfg is None: + LOG.info("network config is disabled") + return + + return self.distro.apply_network_config(netcfg) + class Modules(object): def __init__(self, init, cfg_files=None, reporter=None): -- cgit v1.2.3 From ca00b0f1f8c8a40409328c595d44234bb61c24c4 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 22 Mar 2016 04:49:34 -0400 Subject: make NoCloud work for seeding network. Tested now with the generated fallback config in an lxc container. Had to change to return a config rather than a network state. Also this makes nocloud look in nocloud-net's seed dir. This way it will read the seed and clame the datasource but not do anything other than apply networking and the init_modules early. It is a change in behavior of the time that boothooks woudl run to do this. May need to change that back. --- cloudinit/net/__init__.py | 20 +++++--------------- cloudinit/sources/DataSourceNoCloud.py | 34 +++++++++++++++++++--------------- cloudinit/stages.py | 1 + 3 files changed, 25 insertions(+), 30 deletions(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index b45153f4..63fad2fa 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -448,11 +448,7 @@ def generate_fallback_config(): """Determine which attached net dev is most likely to have a connection and generate network state to run dhcp on that interface""" # by default use eth0 as primary interface - nconf = {'config': {'interfaces': {}, - 'dns': {'search': [], 'nameservers': []}, 'routes': [] - }, - 'version': 1 - } + nconf = {'config': [], 'version': 1} # get list of interfaces that could have connections invalid_interfaces = set(['lo']) @@ -506,21 +502,15 @@ def generate_fallback_config(): if DEFAULT_PRIMARY_INTERFACE in potential_interfaces: name = DEFAULT_PRIMARY_INTERFACE else: - potential_interfaces.sort( - key=lambda x: int(''.join(i for i in x if i in string.digits))) - name = potential_interfaces[0] + name = sorted(potential_interfaces)[0] sysfs_mac = os.path.join(SYS_CLASS_NET, name, 'address') mac = util.load_file(sysfs_mac).strip() target_name = name - # generate net config for interface - nconf['config']['interfaces'][target_name] = { - 'mac_address': mac, 'name': target_name, 'type': 'physical', - 'mode': 'manual', 'inet': 'inet', - 'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}] - } - + nconf['config'].append( + {'type': 'physical', 'name': target_name, + 'mac_address': mac, 'subnets': [{'type': 'dhcp4'}]}) return nconf diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 538df7d9..bd04a6fe 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -36,7 +36,9 @@ class DataSourceNoCloud(sources.DataSource): self.dsmode = 'local' self.seed = None self.cmdline_id = "ds=nocloud" - self.seed_dir = os.path.join(paths.seed_dir, 'nocloud') + self.seed_dirs = [os.path.join(paths.seed_dir, 'nocloud'), + os.path.join(paths.seed_dir, 'nocloud-net')] + self.seed_dir = None self.supported_seed_starts = ("/", "file://") def __str__(self): @@ -67,15 +69,15 @@ class DataSourceNoCloud(sources.DataSource): pp2d_kwargs = {'required': ['user-data', 'meta-data'], 'optional': ['vendor-data', 'network-config']} - try: - seeded = util.pathprefix2dict(self.seed_dir, **pp2d_kwargs) - found.append(self.seed_dir) - LOG.debug("Using seeded data from %s", self.seed_dir) - except ValueError as e: - pass - - if self.seed_dir in found: - mydata = _merge_new_seed(mydata, seeded) + for path in self.seed_dirs: + try: + seeded = util.pathprefix2dict(path, **pp2d_kwargs) + found.append(path) + LOG.debug("Using seeded data from %s", path) + mydata = _merge_new_seed(mydata, seeded) + break + except ValueError as e: + pass # If the datasource config had a 'seedfrom' entry, then that takes # precedence over a 'seedfrom' that was found in a filesystem @@ -188,21 +190,19 @@ class DataSourceNoCloud(sources.DataSource): # if this is the local datasource or 'seedfrom' was used # and the source of the seed was self.dsmode. # Then see if there is network config to apply. + # note this is obsolete network-interfaces style seeding. if self.dsmode in ("local", seeded_network): if mydata['meta-data'].get('network-interfaces'): LOG.debug("Updating network interfaces from %s", self) self.distro.apply_network( mydata['meta-data']['network-interfaces']) - elif mydata.get('network-config'): - LOG.debug("Updating network config from %s", self) - self.distro.apply_network_config(mydata['network-config'], - bring_up=False) if mydata['meta-data']['dsmode'] == self.dsmode: self.seed = ",".join(found) self.metadata = mydata['meta-data'] self.userdata_raw = mydata['user-data'] self.vendordata_raw = mydata['vendor-data'] + self._network_config = mydata['network-config'] return True LOG.debug("%s: not claiming datasource, dsmode=%s", self, @@ -222,6 +222,10 @@ class DataSourceNoCloud(sources.DataSource): return None return quick_id == current + @property + def network_config(self): + return self._network_config + def _quick_read_instance_id(cmdline_id, dirs=None): if dirs is None: @@ -312,7 +316,7 @@ class DataSourceNoCloudNet(DataSourceNoCloud): DataSourceNoCloud.__init__(self, sys_cfg, distro, paths) self.cmdline_id = "ds=nocloud-net" self.supported_seed_starts = ("http://", "https://", "ftp://") - self.seed_dir = os.path.join(paths.seed_dir, 'nocloud-net') + self.seed_dirs = [os.path.join(paths.seed_dir, 'nocloud-net')] self.dsmode = "net" diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 8e681e29..73090025 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -589,6 +589,7 @@ class Init(object): LOG.info("network config is disabled") return + LOG.info("Applying configuration: %s", netcfg) return self.distro.apply_network_config(netcfg) -- cgit v1.2.3 From 4445b881380a39a56490d8a8f9e07bba4540ec62 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 22 Mar 2016 05:39:58 -0400 Subject: fix quick_read_instance_id in nocloud for seed_dirs change --- cloudinit/sources/DataSourceNoCloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index bd04a6fe..afd08935 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -217,7 +217,7 @@ class DataSourceNoCloud(sources.DataSource): return None quick_id = _quick_read_instance_id(cmdline_id=self.cmdline_id, - dirs=[self.seed_dir]) + dirs=self.seed_dirs) if not quick_id: return None return quick_id == current -- cgit v1.2.3 From 7f0871dc5b141ff4bf601b6d96021eba8a3bb0ec Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 22 Mar 2016 05:40:05 -0400 Subject: write to 50-cloud-init.cfg and write systemd.link rules. --- cloudinit/distros/debian.py | 8 +++-- cloudinit/net/__init__.py | 48 ++++++++++++++++++++------ tests/unittests/test_distros/test_netconfig.py | 5 +-- 3 files changed, 46 insertions(+), 15 deletions(-) diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 909d6deb..b14fa3e2 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -46,7 +46,8 @@ APT_GET_WRAPPER = { class Distro(distros.Distro): hostname_conf_fn = "/etc/hostname" locale_conf_fn = "/etc/default/locale" - network_conf_fn = "/etc/network/interfaces" + network_conf_fn = "/etc/network/interfaces.d/50-cloud-init.cfg" + links_prefix = "/etc/systemd/network/50-cloud-init-" def __init__(self, name, cfg, paths): distros.Distro.__init__(self, name, cfg, paths) @@ -79,7 +80,10 @@ class Distro(distros.Distro): def _write_network_config(self, netconfig): ns = net.parse_net_config_data(netconfig) - net.render_network_state(network_state=ns, target="/") + net.render_network_state(target="/", network_state=ns, + eni=self.network_conf_fn, + links_prefix=self.links_prefix) + util.del_file("/etc/network/interfaces.d/eth0.cfg") return [] def _bring_up_interfaces(self, device_names): diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 63fad2fa..ae7b1c04 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -20,7 +20,6 @@ import errno import glob import os import re -import string from cloudinit import log as logging from cloudinit import util @@ -30,6 +29,7 @@ from . import network_state LOG = logging.getLogger(__name__) SYS_CLASS_NET = "/sys/class/net/" +LINKS_FNAME_PREFIX = "etc/systemd/network/50-cloud-init-" NET_CONFIG_OPTIONS = [ "address", "netmask", "broadcast", "network", "metric", "gateway", @@ -423,19 +423,45 @@ def render_interfaces(network_state): return content -def render_network_state(target, network_state): - eni = 'etc/network/interfaces' - netrules = 'etc/udev/rules.d/70-persistent-net.rules' +def render_network_state(target, network_state, eni="etc/network/interfaces", + links_prefix=LINKS_FNAME_PREFIX, + netrules='etc/udev/rules.d/70-persistent-net.rules'): - eni = os.path.sep.join((target, eni,)) - util.ensure_dir(os.path.dirname(eni)) - with open(eni, 'w+') as f: + fpeni = os.path.sep.join((target, eni,)) + util.ensure_dir(os.path.dirname(fpeni)) + with open(fpeni, 'w+') as f: f.write(render_interfaces(network_state)) - netrules = os.path.sep.join((target, netrules,)) - util.ensure_dir(os.path.dirname(netrules)) - with open(netrules, 'w+') as f: - f.write(render_persistent_net(network_state)) + if netrules: + netrules = os.path.sep.join((target, netrules,)) + util.ensure_dir(os.path.dirname(netrules)) + with open(netrules, 'w+') as f: + f.write(render_persistent_net(network_state)) + + if links_prefix: + render_systemd_links(target, network_state, links_prefix) + + +def render_systemd_links(target, network_state, + links_prefix=LINKS_FNAME_PREFIX): + fp_prefix = os.path.sep.join((target, links_prefix)) + for f in glob.glob(fp_prefix + "*"): + os.unlink(f) + + interfaces = network_state.get('interfaces') + for iface in interfaces.values(): + if (iface['type'] == 'physical' and 'name' in iface and + 'mac_address' in iface): + fname = fp_prefix + iface['name'] + ".link" + with open(fname, "w") as fp: + fp.write("\n".join([ + "[Match]", + "MACAddress=" + iface['mac_address'], + "", + "[Link]", + "Name=" + iface['name'], + "" + ])) def is_disabled_cfg(cfg): diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py index 6d30c5b8..2c2a424d 100644 --- a/tests/unittests/test_distros/test_netconfig.py +++ b/tests/unittests/test_distros/test_netconfig.py @@ -109,8 +109,9 @@ class TestNetCfgDistro(TestCase): ub_distro.apply_network(BASE_NET_CFG, False) self.assertEquals(len(write_bufs), 1) - self.assertIn('/etc/network/interfaces', write_bufs) - write_buf = write_bufs['/etc/network/interfaces'] + eni_name = '/etc/network/interfaces.d/50-cloud-init.cfg' + self.assertIn(eni_name, write_bufs) + write_buf = write_bufs[eni_name] self.assertEquals(str(write_buf).strip(), BASE_NET_CFG.strip()) self.assertEquals(write_buf.mode, 0o644) -- cgit v1.2.3 From ed55d6bac52c53b9473b9644ce50f61404bfd438 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 22 Mar 2016 20:02:29 -0400 Subject: better log message about applying networking. --- cloudinit/stages.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 73090025..8ebbe6a9 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -578,18 +578,18 @@ class Init(object): for loc, ncfg in (cmdline_cfg, dscfg, sys_cfg): if net.is_disabled_cfg(ncfg): LOG.debug("network config disabled by %s", loc) - return None + return (None, loc) if ncfg: - return ncfg - return net.generate_fallback_config() + return (ncfg, loc) + return (net.generate_fallback_config(), "fallback") def apply_network_config(self): - netcfg = self._find_networking_config() + netcfg, src = self._find_networking_config() if netcfg is None: - LOG.info("network config is disabled") + LOG.info("network config is disabled by %s", src) return - LOG.info("Applying configuration: %s", netcfg) + LOG.info("Applying network configuration from %s: %s", src, netcfg) return self.distro.apply_network_config(netcfg) -- cgit v1.2.3 From 5b3cad36be8981cd12cffdf5c5e539b522404000 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 23 Mar 2016 10:31:11 -0400 Subject: trust existing datasource in modules or single This fixes a bug where modules mode was not passing a 'existing' flag to fetch. fetch had existing default to 'check'. The DataSourceNoCloud when fed with data from a disk will return False to check() as it is not a guarantee'd hit. That caused fetch to go looking for a new datasource. That would have actually worked, but modules and single create the Init with deps=[]. So it went looking for Datasources that matched those deps, and only found DataSourceNone. I'm going to keep having modules and single specify deps=[] as that will prevent them from going to look for a DS and further making things worse. --- bin/cloud-init | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/cloud-init b/bin/cloud-init index 8875d2f6..341359e3 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -329,7 +329,7 @@ def main_modules(action_name, args): init.read_cfg(extract_fns(args)) # Stage 2 try: - init.fetch() + init.fetch(existing="trust") except sources.DataSourceNotFoundException: # There was no datasource found, theres nothing to do msg = ('Can not apply stage %s, no datasource found! Likely bad ' @@ -383,7 +383,7 @@ def main_single(name, args): init.read_cfg(extract_fns(args)) # Stage 2 try: - init.fetch() + init.fetch(existing="trust") except sources.DataSourceNotFoundException: # There was no datasource found, # that might be bad (or ok) depending on -- cgit v1.2.3