From 758e152721891c707573757fe7a7ff410ec446e2 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 24 Oct 2012 20:31:19 -0700 Subject: Handle the case where newer versions of boto are used that lazily load the metadata from the ec2 metadata service. 1. Add a ec2_utils module that checks which version of boto is being used and under the right versions the metadata dictionary will be expanded. 2. Use this new ec2_utils module in the cloudstack and ec2 datasources as there entrypoints into boto. --- cloudinit/sources/DataSourceCloudStack.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'cloudinit/sources/DataSourceCloudStack.py') diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index f7ffa7cb..78cf24d7 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -26,8 +26,7 @@ from struct import pack import os import time -import boto.utils as boto_utils - +from cloudinit import ec2_utils as ec2 from cloudinit import log as logging from cloudinit import sources from cloudinit import url_helper as uhelp @@ -116,10 +115,10 @@ class DataSourceCloudStack(sources.DataSource): if not self.wait_for_metadata_service(): return False start_time = time.time() - self.userdata_raw = boto_utils.get_instance_userdata(self.api_ver, - None, self.metadata_address) - self.metadata = boto_utils.get_instance_metadata(self.api_ver, - self.metadata_address) + self.userdata_raw = ec2.get_instance_userdata(self.api_ver, + self.metadata_address) + self.metadata = ec2.get_instance_metadata(self.api_ver, + self.metadata_address) LOG.debug("Crawl of metadata service took %s seconds", int(time.time() - start_time)) return True -- cgit v1.2.3 From 8730e143ec07372107d794abe9f4857ead6d4718 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 12 Nov 2012 12:23:44 -0500 Subject: pep8 and pylint fixups --- cloudinit/ec2_utils.py | 9 +++------ cloudinit/sources/DataSourceCloudStack.py | 2 +- cloudinit/sources/DataSourceEc2.py | 2 +- 3 files changed, 5 insertions(+), 8 deletions(-) (limited to 'cloudinit/sources/DataSourceCloudStack.py') diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index a278ef04..32bf3968 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -17,10 +17,7 @@ # along with this program. If not, see . import pkg_resources -from pkg_resources import parse_version - -import cloudinit.util as util -import cloudinit.url_helper as uh +from pkg_resources import parse_version as pver import boto.utils as boto_utils @@ -36,7 +33,7 @@ import boto.utils as boto_utils BOTO_LAZY = False try: _boto_lib = pkg_resources.get_distribution('boto') - if _boto_lib.parsed_version > parse_version("2.5.2"): + if _boto_lib.parsed_version > pver("2.5.2"): # pylint: disable=E1103 BOTO_LAZY = True except pkg_resources.DistributionNotFound: pass @@ -47,7 +44,7 @@ def _unlazy_dict(mp): return mp if not BOTO_LAZY: return mp - for (k, v) in mp.items(): + for (_k, v) in mp.items(): _unlazy_dict(v) return mp diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 78cf24d7..076dba5a 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -116,7 +116,7 @@ class DataSourceCloudStack(sources.DataSource): return False start_time = time.time() self.userdata_raw = ec2.get_instance_userdata(self.api_ver, - self.metadata_address) + self.metadata_address) self.metadata = ec2.get_instance_metadata(self.api_ver, self.metadata_address) LOG.debug("Crawl of metadata service took %s seconds", diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 6f51dfae..2db53446 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -65,7 +65,7 @@ class DataSourceEc2(sources.DataSource): return False start_time = time.time() self.userdata_raw = ec2.get_instance_userdata(self.api_ver, - self.metadata_address) + self.metadata_address) self.metadata = ec2.get_instance_metadata(self.api_ver, self.metadata_address) LOG.debug("Crawl of metadata service took %s seconds", -- cgit v1.2.3 From 8abbeae7ce15a6fb7a08adc697205d614f868a98 Mon Sep 17 00:00:00 2001 From: Gerard Dethier Date: Wed, 19 Dec 2012 09:27:33 -0500 Subject: DataSourceCloudStack: use virtual router rather than default route In CloudStack's documentation, it is stated that meta/user-data can be retrieved from CloudStack's Virtual Router [1]. However, cloud-init retrieves these information from default gateway. VR and default gateway may be the same machine (i.e. have the same address) in some cases, but that is not be always true (actually, in my case, it is not). This change searches the lease files in /var/lib/dhclient to pick out the dhcp-server-identifier. It admittedly does make this specific to dhclient. -- [1] http://incubator.apache.org/cloudstack/docs/en-US/Apache_CloudStack/4.0.0-incubating/html/Admin_Guide/user-data-and-meta-data.html). LP: #1089989 --- cloudinit/sources/DataSourceCloudStack.py | 51 ++++++++++++++++++------------- 1 file changed, 30 insertions(+), 21 deletions(-) (limited to 'cloudinit/sources/DataSourceCloudStack.py') diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 076dba5a..82e1e130 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -3,10 +3,12 @@ # Copyright (C) 2012 Canonical Ltd. # Copyright (C) 2012 Cosmin Luta # Copyright (C) 2012 Yahoo! Inc. +# Copyright (C) 2012 Gerard Dethier # # Author: Cosmin Luta # Author: Scott Moser # Author: Joshua Harlow +# Author: Gerard Dethier # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -20,9 +22,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from socket import inet_ntoa -from struct import pack - import os import time @@ -40,24 +39,12 @@ class DataSourceCloudStack(sources.DataSource): sources.DataSource.__init__(self, sys_cfg, distro, paths) self.seed_dir = os.path.join(paths.seed_dir, 'cs') # Cloudstack has its metadata/userdata URLs located at - # http:///latest/ + # http:///latest/ self.api_ver = 'latest' - gw_addr = self.get_default_gateway() - if not gw_addr: - raise RuntimeError("No default gateway found!") - self.metadata_address = "http://%s/" % (gw_addr) - - def get_default_gateway(self): - """Returns the default gateway ip address in the dotted format.""" - lines = util.load_file("/proc/net/route").splitlines() - for line in lines: - items = line.split("\t") - if items[1] == "00000000": - # Found the default route, get the gateway - gw = inet_ntoa(pack(" 2: + dhcp = words[2] + LOG.debug("Found DHCP identifier %s", dhcp) + addresses.add(dhcp) + if len(addresses) != 1: + # No unique virtual router found + return None + return addresses.pop() + + # Used to match classes to dependencies datasources = [ (DataSourceCloudStack, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), -- cgit v1.2.3 From 4fde399a38765fa9641b3177b966ad6c8ec9750f Mon Sep 17 00:00:00 2001 From: Gerard Dethier Date: Mon, 7 Jan 2013 12:20:58 -0500 Subject: DataSourceCloudStack: fallback to default route if no virtual router found Changes in revision 753 broke cloud-init on ubuntu, as it has a different dhclient directory than Fedora where the change was developed and tested. This change does 2 things: * searches multiple directories (including /var/lib/dhcp) for the lease files. * adds a fallback to the old code path of choosing the default route as the virtual router if there were no virtual routers found in the lease files. LP: #1089989 --- ChangeLog | 6 ++- cloudinit/sources/DataSourceCloudStack.py | 82 ++++++++++++++++++++++++------- 2 files changed, 68 insertions(+), 20 deletions(-) (limited to 'cloudinit/sources/DataSourceCloudStack.py') diff --git a/ChangeLog b/ChangeLog index 9534be26..18e25725 100644 --- a/ChangeLog +++ b/ChangeLog @@ -11,8 +11,10 @@ - support omnibus installer for chef [Anatoliy Dobrosynets] - fix bug where cloud-config in user-data could not modify system_info settings (LP: #1090482) - - fix CloudStack DataSource to use Virtual Router as found in - /var/lib/dhcpclient rather than default gateway (LP: #1089989) + - fix CloudStack DataSource to use Virtual Router as described by + CloudStack documentation if it is available by searching through dhclient + lease files. If it is not available, then fall back to the default + gateway. (LP: #1089989) - fix redaction of password field in log (LP: #1096417) - fix to cloud-config user setup. Previously, lock_passwd was broken and all accounts would be locked unless 'system' was given (LP: #1096423). diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 82e1e130..275caf0d 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -30,6 +30,8 @@ from cloudinit import log as logging from cloudinit import sources from cloudinit import url_helper as uhelp from cloudinit import util +from socket import inet_ntoa +from struct import pack LOG = logging.getLogger(__name__) @@ -122,26 +124,70 @@ class DataSourceCloudStack(sources.DataSource): return self.metadata['availability-zone'] +def get_default_gateway(): + # Returns the default gateway ip address in the dotted format. + lines = util.load_file("/proc/net/route").splitlines() + for line in lines: + items = line.split("\t") + if items[1] == "00000000": + # Found the default route, get the gateway + gw = inet_ntoa(pack(" latest_mtime: + latest_mtime = mtime + latest_file = abs_path + return latest_file + + def get_vr_address(): - # get the address of the virtual router via dhcp responses + # Get the address of the virtual router via dhcp leases # see http://bit.ly/T76eKC for documentation on the virtual router. - dhclient_d = "/var/lib/dhclient" - addresses = set() - dhclient_files = os.listdir(dhclient_d) - for file_name in dhclient_files: - if file_name.endswith(".lease") or file_name.endswith(".leases"): - with open(os.path.join(dhclient_d, file_name), "r") as fd: - for line in fd: - if "dhcp-server-identifier" in line: - words = line.strip(" ;\r\n").split(" ") - if len(words) > 2: - dhcp = words[2] - LOG.debug("Found DHCP identifier %s", dhcp) - addresses.add(dhcp) - if len(addresses) != 1: - # No unique virtual router found - return None - return addresses.pop() + # If no virtual router is detected, fallback on default gateway. + lease_file = get_latest_lease() + if not lease_file: + LOG.debug("No lease file found, using default gateway") + return get_default_gateway() + + latest_address = None + with open(lease_file, "r") as fd: + for line in fd: + if "dhcp-server-identifier" in line: + words = line.strip(" ;\r\n").split(" ") + if len(words) > 2: + dhcp = words[2] + LOG.debug("Found DHCP identifier %s", dhcp) + latest_address = dhcp + if not latest_address: + # No virtual router found, fallback on default gateway + LOG.debug("No DHCP found, using default gateway") + return get_default_gateway() + return latest_address # Used to match classes to dependencies -- cgit v1.2.3