From 8abbeae7ce15a6fb7a08adc697205d614f868a98 Mon Sep 17 00:00:00 2001 From: Gerard Dethier Date: Wed, 19 Dec 2012 09:27:33 -0500 Subject: DataSourceCloudStack: use virtual router rather than default route In CloudStack's documentation, it is stated that meta/user-data can be retrieved from CloudStack's Virtual Router [1]. However, cloud-init retrieves these information from default gateway. VR and default gateway may be the same machine (i.e. have the same address) in some cases, but that is not be always true (actually, in my case, it is not). This change searches the lease files in /var/lib/dhclient to pick out the dhcp-server-identifier. It admittedly does make this specific to dhclient. -- [1] http://incubator.apache.org/cloudstack/docs/en-US/Apache_CloudStack/4.0.0-incubating/html/Admin_Guide/user-data-and-meta-data.html). LP: #1089989 --- cloudinit/sources/DataSourceCloudStack.py | 51 ++++++++++++++++++------------- 1 file changed, 30 insertions(+), 21 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 076dba5a..82e1e130 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -3,10 +3,12 @@ # Copyright (C) 2012 Canonical Ltd. # Copyright (C) 2012 Cosmin Luta # Copyright (C) 2012 Yahoo! Inc. +# Copyright (C) 2012 Gerard Dethier # # Author: Cosmin Luta # Author: Scott Moser # Author: Joshua Harlow +# Author: Gerard Dethier # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -20,9 +22,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from socket import inet_ntoa -from struct import pack - import os import time @@ -40,24 +39,12 @@ class DataSourceCloudStack(sources.DataSource): sources.DataSource.__init__(self, sys_cfg, distro, paths) self.seed_dir = os.path.join(paths.seed_dir, 'cs') # Cloudstack has its metadata/userdata URLs located at - # http:///latest/ + # http:///latest/ self.api_ver = 'latest' - gw_addr = self.get_default_gateway() - if not gw_addr: - raise RuntimeError("No default gateway found!") - self.metadata_address = "http://%s/" % (gw_addr) - - def get_default_gateway(self): - """Returns the default gateway ip address in the dotted format.""" - lines = util.load_file("/proc/net/route").splitlines() - for line in lines: - items = line.split("\t") - if items[1] == "00000000": - # Found the default route, get the gateway - gw = inet_ntoa(pack(" 2: + dhcp = words[2] + LOG.debug("Found DHCP identifier %s", dhcp) + addresses.add(dhcp) + if len(addresses) != 1: + # No unique virtual router found + return None + return addresses.pop() + + # Used to match classes to dependencies datasources = [ (DataSourceCloudStack, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), -- cgit v1.2.3 From 4fde399a38765fa9641b3177b966ad6c8ec9750f Mon Sep 17 00:00:00 2001 From: Gerard Dethier Date: Mon, 7 Jan 2013 12:20:58 -0500 Subject: DataSourceCloudStack: fallback to default route if no virtual router found Changes in revision 753 broke cloud-init on ubuntu, as it has a different dhclient directory than Fedora where the change was developed and tested. This change does 2 things: * searches multiple directories (including /var/lib/dhcp) for the lease files. * adds a fallback to the old code path of choosing the default route as the virtual router if there were no virtual routers found in the lease files. LP: #1089989 --- ChangeLog | 6 ++- cloudinit/sources/DataSourceCloudStack.py | 82 ++++++++++++++++++++++++------- 2 files changed, 68 insertions(+), 20 deletions(-) (limited to 'cloudinit/sources') diff --git a/ChangeLog b/ChangeLog index 9534be26..18e25725 100644 --- a/ChangeLog +++ b/ChangeLog @@ -11,8 +11,10 @@ - support omnibus installer for chef [Anatoliy Dobrosynets] - fix bug where cloud-config in user-data could not modify system_info settings (LP: #1090482) - - fix CloudStack DataSource to use Virtual Router as found in - /var/lib/dhcpclient rather than default gateway (LP: #1089989) + - fix CloudStack DataSource to use Virtual Router as described by + CloudStack documentation if it is available by searching through dhclient + lease files. If it is not available, then fall back to the default + gateway. (LP: #1089989) - fix redaction of password field in log (LP: #1096417) - fix to cloud-config user setup. Previously, lock_passwd was broken and all accounts would be locked unless 'system' was given (LP: #1096423). diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 82e1e130..275caf0d 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -30,6 +30,8 @@ from cloudinit import log as logging from cloudinit import sources from cloudinit import url_helper as uhelp from cloudinit import util +from socket import inet_ntoa +from struct import pack LOG = logging.getLogger(__name__) @@ -122,26 +124,70 @@ class DataSourceCloudStack(sources.DataSource): return self.metadata['availability-zone'] +def get_default_gateway(): + # Returns the default gateway ip address in the dotted format. + lines = util.load_file("/proc/net/route").splitlines() + for line in lines: + items = line.split("\t") + if items[1] == "00000000": + # Found the default route, get the gateway + gw = inet_ntoa(pack(" latest_mtime: + latest_mtime = mtime + latest_file = abs_path + return latest_file + + def get_vr_address(): - # get the address of the virtual router via dhcp responses + # Get the address of the virtual router via dhcp leases # see http://bit.ly/T76eKC for documentation on the virtual router. - dhclient_d = "/var/lib/dhclient" - addresses = set() - dhclient_files = os.listdir(dhclient_d) - for file_name in dhclient_files: - if file_name.endswith(".lease") or file_name.endswith(".leases"): - with open(os.path.join(dhclient_d, file_name), "r") as fd: - for line in fd: - if "dhcp-server-identifier" in line: - words = line.strip(" ;\r\n").split(" ") - if len(words) > 2: - dhcp = words[2] - LOG.debug("Found DHCP identifier %s", dhcp) - addresses.add(dhcp) - if len(addresses) != 1: - # No unique virtual router found - return None - return addresses.pop() + # If no virtual router is detected, fallback on default gateway. + lease_file = get_latest_lease() + if not lease_file: + LOG.debug("No lease file found, using default gateway") + return get_default_gateway() + + latest_address = None + with open(lease_file, "r") as fd: + for line in fd: + if "dhcp-server-identifier" in line: + words = line.strip(" ;\r\n").split(" ") + if len(words) > 2: + dhcp = words[2] + LOG.debug("Found DHCP identifier %s", dhcp) + latest_address = dhcp + if not latest_address: + # No virtual router found, fallback on default gateway + LOG.debug("No DHCP found, using default gateway") + return get_default_gateway() + return latest_address # Used to match classes to dependencies -- cgit v1.2.3 From e561742aeab1e8090467f0fa304ee06e82e85f2c Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 16 Jan 2013 19:46:30 -0500 Subject: DataSourceConfigDrive: consider CD rom as valid config-drive source. previously, there was an attempt in the config drive source to limit the source device to a "full block device" rather than a partition. This was done by a simplistic approach of checking that the last character of the name was not a number. That was filtering out CD-rom devices (sr0). Now, we have a bit more sophisticated approach to that same problem. We filter out block devices that have a 'partition' entry in /sys/class/block/DEVICE_NAME/partition . LP: #1100545 --- ChangeLog | 2 ++ cloudinit/sources/DataSourceConfigDrive.py | 2 +- cloudinit/util.py | 7 +++++++ tests/unittests/test_datasource/test_configdrive.py | 17 ++++++++++++----- 4 files changed, 22 insertions(+), 6 deletions(-) (limited to 'cloudinit/sources') diff --git a/ChangeLog b/ChangeLog index 544032a2..f076a27f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -20,6 +20,8 @@ all accounts would be locked unless 'system' was given (LP: #1096423). - Allow 'sr0' (or sr[0-9]) to be specified without /dev/ as a source for mounts. [Vlastimil Holer] + - allow config-drive-data to come from a CD device by more correctly + filtering out partitions. (LP: #1100545) 0.7.1: - sysvinit: fix missing dependency in cloud-init job for RHEL 5.6 - config-drive: map hostname to local-hostname (LP: #1061964) diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index c7826851..ec016a1d 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -270,7 +270,7 @@ def find_candidate_devs(): combined = (by_label + [d for d in by_fstype if d not in by_label]) # We are looking for block device (sda, not sda1), ignore partitions - combined = [d for d in combined if d[-1] not in "0123456789"] + combined = [d for d in combined if not util.is_partition(d)] return combined diff --git a/cloudinit/util.py b/cloudinit/util.py index ab918433..c0ea8d91 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1553,3 +1553,10 @@ def keyval_str_to_dict(kvstring): val = True ret[key] = val return ret + + +def is_partition(device): + if device.startswith("/dev/"): + device = device[5:] + + return os.path.isfile("/sys/class/block/%s/partition" % device) diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index 6751a679..930086db 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -257,19 +257,25 @@ class TestConfigDriveDataSource(MockerTestCase): ds.read_config_drive_dir, my_d) def test_find_candidates(self): - devs_with_answers = { - "TYPE=vfat": [], - "TYPE=iso9660": ["/dev/vdb"], - "LABEL=config-2": ["/dev/vdb"], - } + devs_with_answers = {} def my_devs_with(criteria): return devs_with_answers[criteria] + def my_is_partition(dev): + return dev[-1] in "0123456789" and not dev.startswith("sr") + try: orig_find_devs_with = util.find_devs_with util.find_devs_with = my_devs_with + orig_is_partition = util.is_partition + util.is_partition = my_is_partition + + devs_with_answers = {"TYPE=vfat": [], + "TYPE=iso9660": ["/dev/vdb"], + "LABEL=config-2": ["/dev/vdb"], + } self.assertEqual(["/dev/vdb"], ds.find_candidate_devs()) # add a vfat item @@ -285,6 +291,7 @@ class TestConfigDriveDataSource(MockerTestCase): finally: util.find_devs_with = orig_find_devs_with + util.is_partition = orig_is_partition def test_pubkeys_v2(self): """Verify that public-keys work in config-drive-v2.""" -- cgit v1.2.3 From f3b25e68ac6d28cdacf7408c91da6e9a215ad1e6 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 4 Feb 2013 22:23:42 -0500 Subject: make config of nocloud datasource able to specify meta-data and user-data LP: #1115833 --- cloudinit/sources/DataSourceNoCloud.py | 72 ++++++++++++++++++------------- doc/examples/cloud-config-datasources.txt | 11 +++++ 2 files changed, 52 insertions(+), 31 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index bed500a2..d8484437 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -77,37 +77,47 @@ class DataSourceNoCloud(sources.DataSource): found.append("ds_config") md["seedfrom"] = self.ds_cfg['seedfrom'] - fslist = util.find_devs_with("TYPE=vfat") - fslist.extend(util.find_devs_with("TYPE=iso9660")) - - label_list = util.find_devs_with("LABEL=cidata") - devlist = list(set(fslist) & set(label_list)) - devlist.sort(reverse=True) - - for dev in devlist: - try: - LOG.debug("Attempting to use data from %s", dev) - - (newmd, newud) = util.mount_cb(dev, util.read_seeded) - md = util.mergedict(newmd, md) - ud = newud - - # For seed from a device, the default mode is 'net'. - # that is more likely to be what is desired. - # If they want dsmode of local, then they must - # specify that. - if 'dsmode' not in md: - md['dsmode'] = "net" - - LOG.debug("Using data from %s", dev) - found.append(dev) - break - except OSError as e: - if e.errno != errno.ENOENT: - raise - except util.MountFailedError: - util.logexc(LOG, ("Failed to mount %s" - " when looking for data"), dev) + # if ds_cfg has 'user-data' and 'meta-data' + if 'user-data' in self.ds_cfg and 'meta-data' in self.ds_cfg: + if self.ds_cfg['user-data']: + ud = util.mergedict(md, self.ds_cfg['user-data']) + if self.ds_cfg['meta-data'] is not False: + md = util.mergedict(md, self.ds_cfg['meta-data']) + if 'ds_config' not in found: + found.append("ds_config") + + if self.ds_cfg.get('fs_label', "cidata"): + fslist = util.find_devs_with("TYPE=vfat") + fslist.extend(util.find_devs_with("TYPE=iso9660")) + + label = self.ds_cfg.get('fs_label') + label_list = util.find_devs_with("LABEL=%s" % label) + devlist = list(set(fslist) & set(label_list)) + devlist.sort(reverse=True) + + for dev in devlist: + try: + LOG.debug("Attempting to use data from %s", dev) + + (newmd, newud) = util.mount_cb(dev, util.read_seeded) + md = util.mergedict(newmd, md) + ud = newud + + # For seed from a device, the default mode is 'net'. + # that is more likely to be what is desired. If they want + # dsmode of local, then they must specify that. + if 'dsmode' not in md: + md['dsmode'] = "net" + + LOG.debug("Using data from %s", dev) + found.append(dev) + break + except OSError as e: + if e.errno != errno.ENOENT: + raise + except util.MountFailedError: + util.logexc(LOG, ("Failed to mount %s" + " when looking for data"), dev) # There was no indication on kernel cmdline or data # in the seeddir suggesting this handler should be used. diff --git a/doc/examples/cloud-config-datasources.txt b/doc/examples/cloud-config-datasources.txt index d10dde05..fc8c22d4 100644 --- a/doc/examples/cloud-config-datasources.txt +++ b/doc/examples/cloud-config-datasources.txt @@ -31,3 +31,14 @@ datasource: # /user-data and /meta-data # seedfrom: http://my.example.com/i-abcde seedfrom: None + + # fs_label: the label on filesystems to be searched for NoCloud source + fs_label: cidata + + # these are optional, but allow you to basically provide a datasource + # right here + user-data: | + # This is the user-data verbatum + meta-data: + instance-id: i-87018aed + local-hostname: myhost.internal -- cgit v1.2.3 From bf0db8d0793c9c18871cbbafbbad9c127b0bd8ee Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 7 Feb 2013 08:33:48 -0500 Subject: user-data doesn't merge in from meta-data (typo) --- cloudinit/sources/DataSourceNoCloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index d8484437..5ccd6b99 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -80,7 +80,7 @@ class DataSourceNoCloud(sources.DataSource): # if ds_cfg has 'user-data' and 'meta-data' if 'user-data' in self.ds_cfg and 'meta-data' in self.ds_cfg: if self.ds_cfg['user-data']: - ud = util.mergedict(md, self.ds_cfg['user-data']) + ud = self.ds_cfg['user-data'] if self.ds_cfg['meta-data'] is not False: md = util.mergedict(md, self.ds_cfg['meta-data']) if 'ds_config' not in found: -- cgit v1.2.3 From 88a369c5324e74a2d1bb8dd0bdf8fdc9a95393c8 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 7 Feb 2013 09:12:36 -0500 Subject: add test_nocloud unit tests, fix one issue found --- cloudinit/sources/DataSourceNoCloud.py | 2 + tests/unittests/test_datasource/test_nocloud.py | 56 +++++++++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 tests/unittests/test_datasource/test_nocloud.py (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 5ccd6b99..097bbc52 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -205,6 +205,8 @@ def parse_cmdline_data(ds_id, fill, cmdline=None): # short2long mapping to save cmdline typing s2l = {"h": "local-hostname", "i": "instance-id", "s": "seedfrom"} for item in kvpairs: + if item == "": + continue try: (k, v) = item.split("=", 1) except: diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py new file mode 100644 index 00000000..850d3214 --- /dev/null +++ b/tests/unittests/test_datasource/test_nocloud.py @@ -0,0 +1,56 @@ +from cloudinit.sources import DataSourceNoCloud + +from mocker import MockerTestCase + + +class TestNoCloudDataSource(MockerTestCase): + + def setUp(self): + super(TestNoCloudDataSource, self).setUp() + + def test_parse_cmdline_data_valid(self): + parse = DataSourceNoCloud.parse_cmdline_data + + ds_id = "ds=nocloud" + pairs = ( + ("root=/dev/sda1 %(ds_id)s", {}), + ("%(ds_id)s; root=/dev/foo", {}), + ("%(ds_id)s", {}), + ("%(ds_id)s;", {}), + ("%(ds_id)s;s=SEED", {'seedfrom': 'SEED'}), + ("%(ds_id)s;seedfrom=SEED;local-hostname=xhost", + {'seedfrom': 'SEED', 'local-hostname': 'xhost'}), + ("%(ds_id)s;h=xhost", + {'local-hostname': 'xhost'}), + ("%(ds_id)s;h=xhost;i=IID", + {'local-hostname': 'xhost', 'instance-id': 'IID'}), + ) + + for (fmt, expected) in pairs: + fill = {} + cmdline = fmt % {'ds_id': ds_id} + ret = parse(ds_id=ds_id, fill=fill, cmdline=cmdline) + self.assertEqual(expected, fill) + self.assertTrue(ret) + + def test_parse_cmdline_data_none(self): + parse = DataSourceNoCloud.parse_cmdline_data + + ds_id = "ds=foo" + cmdlines = ( + "root=/dev/sda1 ro", + "console=/dev/ttyS0 root=/dev/foo", + "", + "ds=foocloud", + "ds=foo-net", + "ds=nocloud;s=SEED", + ) + + for cmdline in cmdlines: + fill = {} + ret = parse(ds_id=ds_id, fill=fill, cmdline=cmdline) + self.assertEqual(fill, {}) + self.assertFalse(ret) + + +# vi: ts=4 expandtab -- cgit v1.2.3 From ac83536339d2622f4a896f50681497a388e7e26f Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 1 Mar 2013 16:03:02 -0500 Subject: fix regression on expected label of filesystem for DataSourceNone Last addition to DataSourceNoCloud left it looking for a filesystem named 'None'. --- cloudinit/sources/DataSourceNoCloud.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 097bbc52..603f0155 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -86,11 +86,11 @@ class DataSourceNoCloud(sources.DataSource): if 'ds_config' not in found: found.append("ds_config") - if self.ds_cfg.get('fs_label', "cidata"): + label = self.ds_cfg.get('fs_label', "cidata") + if label is not None: fslist = util.find_devs_with("TYPE=vfat") fslist.extend(util.find_devs_with("TYPE=iso9660")) - label = self.ds_cfg.get('fs_label') label_list = util.find_devs_with("LABEL=%s" % label) devlist = list(set(fslist) & set(label_list)) devlist.sort(reverse=True) -- cgit v1.2.3