From ef18b8ac4cf7e3dfd98830fbdb298380a192a0fc Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Sun, 19 Mar 2017 08:39:01 -0500 Subject: cloudinit.net: add network config v2 parsing and rendering Network configuration version 2 format is implemented in a package called netplan (nplan)[1] which allows consolidated network config for multiple network controllers. - Add a new netplan renderer - Update default policy, placing eni and sysconfig first This requires explicit policy to enable netplan over eni on systems which have both (Yakkety, Zesty, UC16) - Allow any network state (parsed from any format cloud-init supports) to render to v2 if system supports netplan. - Move eni's _subnet_is_ipv6 to common code for use by other renderers - Make sysconfig renderer always emit /etc/syconfig/network configuration - Update cloud-init.service systemd unit to also wait on systemd-networkd-wait-online.service 1. https://lists.ubuntu.com/archives/ubuntu-devel/2016-July/039464.html --- tools/net-convert.py | 84 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100755 tools/net-convert.py (limited to 'tools') diff --git a/tools/net-convert.py b/tools/net-convert.py new file mode 100755 index 00000000..870da639 --- /dev/null +++ b/tools/net-convert.py @@ -0,0 +1,84 @@ +#!/usr/bin/python3 +# This file is part of cloud-init. See LICENSE file for license information. + +import argparse +import json +import os +import yaml + +from cloudinit.sources.helpers import openstack + +from cloudinit.net import eni +from cloudinit.net import network_state +from cloudinit.net import netplan +from cloudinit.net import sysconfig + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--network-data", "-p", type=open, + metavar="PATH", required=True) + parser.add_argument("--kind", "-k", + choices=['eni', 'network_data.json', 'yaml'], + required=True) + parser.add_argument("-d", "--directory", + metavar="PATH", + help="directory to place output in", + required=True) + parser.add_argument("-m", "--mac", + metavar="name,mac", + action='append', + help="interface name to mac mapping") + parser.add_argument("--output-kind", "-ok", + choices=['eni', 'netplan', 'sysconfig'], + required=True) + args = parser.parse_args() + + if not os.path.isdir(args.directory): + os.makedirs(args.directory) + + if args.mac: + known_macs = {} + for item in args.mac: + iface_name, iface_mac = item.split(",", 1) + known_macs[iface_mac] = iface_name + else: + known_macs = None + + net_data = args.network_data.read() + if args.kind == "eni": + pre_ns = eni.convert_eni_data(net_data) + ns = network_state.parse_net_config_data(pre_ns) + elif args.kind == "yaml": + pre_ns = yaml.load(net_data) + if 'network' in pre_ns: + pre_ns = pre_ns.get('network') + print("Input YAML") + print(yaml.dump(pre_ns, default_flow_style=False, indent=4)) + ns = network_state.parse_net_config_data(pre_ns) + else: + pre_ns = openstack.convert_net_json( + json.loads(net_data), known_macs=known_macs) + ns = network_state.parse_net_config_data(pre_ns) + + if not ns: + raise RuntimeError("No valid network_state object created from" + "input data") + + print("\nInternal State") + print(yaml.dump(ns, default_flow_style=False, indent=4)) + if args.output_kind == "eni": + r_cls = eni.Renderer + elif args.output_kind == "netplan": + r_cls = netplan.Renderer + else: + r_cls = sysconfig.Renderer + + r = r_cls() + r.render_network_state(ns, target=args.directory) + + +if __name__ == '__main__': + main() + +# vi: ts=4 expandtab -- cgit v1.2.3 From 2e879da890a4287dd52eab17938d227da7af253a Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 22 Mar 2017 11:15:49 -0400 Subject: ds-identify: fix bug where filename expansion was left on. The script is written to have the protection of disabling filename expansion (set -f) and explicitly enabling expansion when needed. However, the check_config function failed to disable it after enabling. --- tools/ds-identify | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tools') diff --git a/tools/ds-identify b/tools/ds-identify index e138d780..6f1e983d 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -473,7 +473,7 @@ check_config() { files="$*" fi shift - set +f; set -- $files; set +f; + set +f; set -- $files; set -f; if [ "$1" = "$files" -a ! -f "$1" ]; then return 1 fi -- cgit v1.2.3 From 20a628c122583e159782c88547128fec1983376c Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 23 Mar 2017 16:32:32 -0400 Subject: ds-identify: fix detection of Bigstep datasource. The path for checking presence of Bigstep datasource was simply wrong. Set the correct path. LP: #1674766 --- tools/ds-identify | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'tools') diff --git a/tools/ds-identify b/tools/ds-identify index 6f1e983d..b3220c1f 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -633,7 +633,8 @@ dscheck_Azure() { dscheck_Bigstep() { # bigstep is activated by presense of seed file 'url' - check_seed_dir "bigstep" url && return ${DS_FOUND} + [ -f "${PATH_VAR_LIB_CLOUD}/data/seed/bigstep/url" ] && + return ${DS_FOUND} return ${DS_NOT_FOUND} } -- cgit v1.2.3 From 443095f4d4b6feba30c7011b7ab48adb2a40fcf5 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 16 Mar 2017 11:20:02 -0400 Subject: ConfigDrive: support reading config drive data from /config-drive. This is thie cloud-init part of a fix to allow nova-lxd to provide config drive data. The other part will be done in nova-lxd. The agreement here is that nova-lxd will copy the contents of the config drive to /config-drive in the container. LP: #1673411 --- cloudinit/sources/DataSourceConfigDrive.py | 13 ++++++++----- tools/ds-identify | 13 +++++++++++++ 2 files changed, 21 insertions(+), 5 deletions(-) (limited to 'tools') diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 8a448dc9..46dd89e0 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -54,13 +54,16 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): found = None md = {} results = {} - if os.path.isdir(self.seed_dir): + for sdir in (self.seed_dir, "/config-drive"): + if not os.path.isdir(sdir): + continue try: - results = read_config_drive(self.seed_dir) - found = self.seed_dir + results = read_config_drive(sdir) + found = sdir + break except openstack.NonReadable: - util.logexc(LOG, "Failed reading config drive from %s", - self.seed_dir) + util.logexc(LOG, "Failed reading config drive from %s", sdir) + if not found: for dev in find_candidate_devs(): try: diff --git a/tools/ds-identify b/tools/ds-identify index b3220c1f..bf09a3ad 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -538,6 +538,19 @@ check_configdrive_v2() { if has_fs_with_label "config-2"; then return ${DS_FOUND} fi + # look in /config-drive /seed/config_drive for a directory + # openstack/YYYY-MM-DD format with a file meta_data.json + local d="" + for d in /config-drive "${PATH_VAR_LIB_CLOUD}/seed/config_drive"; do + set +f; set -- "$d/openstack/"2???-??-??/meta_data.json; set -f; + [ -f "$1" ] && return ${DS_FOUND} + done + # at least one cloud (softlayer) seeds config drive with only 'latest'. + local lpath="openstack/latest/meta_data.json" + if [ -e "${PATH_VAR_LIB_CLOUD}/$lpath" ]; then + debug 1 "config drive seeded directory had only 'latest'" + return ${DS_FOUND} + fi return ${DS_NOT_FOUND} } -- cgit v1.2.3 From 328fe5ab399b1f5b48d1985f41fc2ef66e368922 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 27 Mar 2017 12:43:15 -0400 Subject: GCE: Search GCE in ds-identify, consider serial number in check. While documentation indicates that the smbios product name should contain 'Google Compute Engine', experimentation and bug reports indicate that is not always the case. The change here is to change the check for GCE to also consider a serial number that starts with 'GoogleCompute-'. Also, ds-identify was not currently searching for GCE if no config of datasource_list was found. Most images have a datasource_list defined. So update the list to include GCE. LP: #1674861 --- cloudinit/sources/DataSourceGCE.py | 18 ++++++++++++++++++ tests/unittests/test_datasource/test_gce.py | 14 +++++++++++++- tools/ds-identify | 14 +++++++++++++- 3 files changed, 44 insertions(+), 2 deletions(-) (limited to 'tools') diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index b1a1c8f2..637c9505 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -62,6 +62,9 @@ class DataSourceGCE(sources.DataSource): return public_key def get_data(self): + if not platform_reports_gce(): + return False + # url_map: (our-key, path, required, is_text) url_map = [ ('instance-id', ('instance/id',), True, True), @@ -144,6 +147,21 @@ class DataSourceGCE(sources.DataSource): return self.availability_zone.rsplit('-', 1)[0] +def platform_reports_gce(): + pname = util.read_dmi_data('system-product-name') or "N/A" + if pname == "Google Compute Engine": + return True + + # system-product-name is not always guaranteed (LP: #1674861) + serial = util.read_dmi_data('system-serial-number') or "N/A" + if serial.startswith("GoogleCloud-"): + return True + + LOG.debug("Not running on google cloud. product-name=%s serial=%s", + pname, serial) + return False + + # Used to match classes to dependencies datasources = [ (DataSourceGCE, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py index 4f83454e..3eaa58e3 100644 --- a/tests/unittests/test_datasource/test_gce.py +++ b/tests/unittests/test_datasource/test_gce.py @@ -5,6 +5,7 @@ # This file is part of cloud-init. See LICENSE file for license information. import httpretty +import mock import re from base64 import b64encode, b64decode @@ -71,6 +72,11 @@ class TestDataSourceGCE(test_helpers.HttprettyTestCase): self.ds = DataSourceGCE.DataSourceGCE( settings.CFG_BUILTIN, None, helpers.Paths({})) + self.m_platform_reports_gce = mock.patch( + 'cloudinit.sources.DataSourceGCE.platform_reports_gce', + return_value=True) + self.m_platform_reports_gce.start() + self.addCleanup(self.m_platform_reports_gce.stop) super(TestDataSourceGCE, self).setUp() def test_connection(self): @@ -153,7 +159,13 @@ class TestDataSourceGCE(test_helpers.HttprettyTestCase): def test_only_last_part_of_zone_used_for_availability_zone(self): _set_mock_metadata() - self.ds.get_data() + r = self.ds.get_data() + self.assertEqual(True, r) self.assertEqual('bar', self.ds.availability_zone) + def test_get_data_returns_false_if_not_on_gce(self): + self.m_platform_reports_gce.return_value = False + self.assertEqual(False, self.ds.get_data()) + + # vi: ts=4 expandtab diff --git a/tools/ds-identify b/tools/ds-identify index bf09a3ad..15d6600e 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -108,7 +108,7 @@ DI_DSNAME="" # this has to match the builtin list in cloud-init, it is what will # be searched if there is no setting found in config. DI_DSLIST_DEFAULT="MAAS ConfigDrive NoCloud AltCloud Azure Bigstep \ -CloudSigma CloudStack DigitalOcean Ec2 OpenNebula OpenStack OVF SmartOS" +CloudSigma CloudStack DigitalOcean Ec2 GCE OpenNebula OpenStack OVF SmartOS" DI_DSLIST="" DI_MODE="" DI_ON_FOUND="" @@ -383,6 +383,14 @@ dmi_product_name_matches() { return 1 } +dmi_product_serial_matches() { + is_container && return 1 + case "${DI_DMI_PRODUCT_SERIAL}" in + $1) return 0;; + esac + return 1 +} + dmi_product_name_is() { is_container && return 1 [ "${DI_DMI_PRODUCT_NAME}" = "$1" ] @@ -770,6 +778,10 @@ dscheck_GCE() { if dmi_product_name_is "Google Compute Engine"; then return ${DS_FOUND} fi + # product name is not guaranteed (LP: #1674861) + if dmi_product_serial_matches "GoogleCloud-*"; then + return ${DS_FOUND} + fi return ${DS_NOT_FOUND} } -- cgit v1.2.3 From dab9d6e0e92c7c933d86a0f696504fad2cebbda7 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 27 Mar 2017 14:19:17 -0400 Subject: OpenStack: identify OpenStack by product 'OpenStack Compute'. OpenStack clouds installed with RedHat RDO have the nova product configured in /etc/nova/release to be 'OpenStack Compute' rather than upstream nova default of 'OpenStack Nova'. This was first reported on Finnish provider Nebula (http://nebula.fi). LP: #1675349 --- tools/ds-identify | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'tools') diff --git a/tools/ds-identify b/tools/ds-identify index 15d6600e..30106347 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -795,10 +795,15 @@ dscheck_OpenStack() { if [ $? -eq ${DS_FOUND} ]; then return ${DS_NOT_FOUND} fi - if dmi_product_name_is "OpenStack Nova"; then + local nova="OpenStack Nova" compute="OpenStack Compute" + if dmi_product_name_is "$nova"; then return ${DS_FOUND} fi - if [ "${DI_PID_1_PLATFORM}" = "OpenStack Nova" ]; then + if dmi_product_name_is "$compute"; then + # RDO installed nova (LP: #1675349). + return ${DS_FOUND} + fi + if [ "${DI_PID_1_PLATFORM}" = "$nova" ]; then return ${DS_FOUND} fi -- cgit v1.2.3 From 32b21ee7245bf00d55f875506ffc149aad9bb546 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 28 Mar 2017 14:27:33 -0400 Subject: ds-identify: do not write None twice to datasource_list. If the only the None datasource was listed in datasource_list, then ds-identify would write a cloud.cfg witih: datasource_list: [None, None] The fix is to just append None if the list only has None. --- tools/ds-identify | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'tools') diff --git a/tools/ds-identify b/tools/ds-identify index 30106347..54bd9999 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -954,7 +954,9 @@ found() { shift fi # always write the None datasource last. - list="${list:+${list}, }None" + if [ "$list" != "None" ]; then + list="${list:+${list}, }None" + fi write_result "datasource_list: [ $list ]" "$@" return } -- cgit v1.2.3 From a68e7d50d25e774018588a5312c7698c38ec4de4 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 30 Mar 2017 14:42:19 -0400 Subject: ds-identify: fix detecting of maas datasource. The reading of MAAS datasource configuration was simply broken. it was looking in /etc/cloud/*maas*.cfg rather than /etc/cloud/cloud.cfg.d/*maas*.cfg. along side here there is also: * doc improvement on check_config * remove the path restrictions when searching for values in both maas and ovf_vmware_guest_customization. that was done to improve performance as check_config's parsing is slow. * change to maas to search all config files rather than restricting to a subset as it tried before. that was done for * better variable names. - rename path_cloud_confd to path_etc_cloud - PATH_ETC_CLOUD: /etc/cloud - PATH_ETC_CI_CFG: /etc/cloud/cloud.cfg - PATH_ETC_CI_CFG_D: /etc/cloud/cloud.cfg.d LP: #1677710 --- tools/ds-identify | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) (limited to 'tools') diff --git a/tools/ds-identify b/tools/ds-identify index 54bd9999..5d390ef7 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -70,7 +70,9 @@ PATH_PROC_CMDLINE="${PATH_PROC_CMDLINE:-${PATH_ROOT}/proc/cmdline}" PATH_PROC_1_CMDLINE="${PATH_PROC_1_CMDLINE:-${PATH_ROOT}/proc/1/cmdline}" PATH_PROC_1_ENVIRON="${PATH_PROC_1_ENVIRON:-${PATH_ROOT}/proc/1/environ}" PATH_PROC_UPTIME=${PATH_PROC_UPTIME:-${PATH_ROOT}/proc/uptime} -PATH_CLOUD_CONFD="${PATH_CLOUD_CONFD:-${PATH_ROOT}/etc/cloud}" +PATH_ETC_CLOUD="${PATH_ETC_CLOUD:-${PATH_ROOT}/etc/cloud}" +PATH_ETC_CI_CFG="${PATH_ETC_CI_CFG:-${PATH_ETC_CLOUD}/cloud.cfg}" +PATH_ETC_CI_CFG_D="${PATH_ETC_CI_CFG_D:-${PATH_ETC_CI_CFG}.d}" PATH_RUN_CI="${PATH_RUN_CI:-${PATH_RUN}/cloud-init}" PATH_RUN_CI_CFG=${PATH_RUN_CI_CFG:-${PATH_RUN_CI}/cloud.cfg} PATH_RUN_DI_RESULT=${PATH_RUN_DI_RESULT:-${PATH_RUN_CI}/.ds-identify.result} @@ -472,15 +474,18 @@ dscheck_CloudSigma() { } check_config() { - # somewhat hackily read config for 'key' in files matching 'files' - # currently does not respect any hierarchy. - local key="$1" files="" bp="${PATH_CLOUD_CONFD}/cloud.cfg" - if [ $# -eq 1 ]; then - files="$bp ${bp}.d/*.cfg" + # check_config(key [,file_globs]) + # somewhat hackily read through file_globs for 'key' + # file_globs are expanded via path expansion and + # default to /etc/cloud/cloud.cfg /etc/cloud/cloud.cfg.d/*.cfg + # currently does not respect any hierarchy in searching for key. + local key="$1" files="" + shift + if [ $# -eq 0 ]; then + files="${PATH_ETC_CI_CFG} ${PATH_ETC_CI_CFG_D}/*.cfg" else files="$*" fi - shift set +f; set -- $files; set -f; if [ "$1" = "$files" -a ! -f "$1" ]; then return 1 @@ -520,9 +525,7 @@ dscheck_MAAS() { esac # check config files written by maas for installed system. - local confd="${PATH_CLOUD_CONFD}" - local fnmatch="$confd/*maas*.cfg $confd/*kernel_cmdline*.cfg" - if check_config "MAAS" "$fnmatch"; then + if check_config "MAAS"; then return "${DS_FOUND}" fi return ${DS_NOT_FOUND} @@ -607,9 +610,7 @@ ovf_vmware_guest_customization() { # (disable_vmware_customization=true). If it is set to false, then # user has requested customization. local key="disable_vmware_customization" - local match="" bp="${PATH_CLOUD_CONFD}/cloud.cfg" - match="$bp $bp.d/*[Oo][Vv][Ff]*.cfg" - if check_config "$key" "$match"; then + if check_config "$key"; then debug 2 "${_RET_fname} set $key to $_RET" case "$_RET" in 0|false|False) return 0;; @@ -680,9 +681,9 @@ ec2_read_strict_setting() { esac # 3. look for the key 'strict_id' (datasource/Ec2/strict_id) - local match="" bp="${PATH_CLOUD_CONFD}/cloud.cfg" - match="$bp $bp.d/*[Ee][Cc]2*.cfg" - if check_config strict_id "$match"; then + # only in cloud.cfg or cloud.cfg.d/EC2.cfg (case insensitive) + local cfg="${PATH_ETC_CI_CFG}" cfg_d="${PATH_ETC_CI_CFG_D}" + if check_config strict_id $cfg "$cfg_d/*[Ee][Cc]2*.cfg"; then debug 2 "${_RET_fname} set strict_id to $_RET" return 0 fi -- cgit v1.2.3