From 7b65c24f389471bbaa9aa38e164c6440b18f22f4 Mon Sep 17 00:00:00 2001 From: Robert Jennings Date: Wed, 27 Apr 2016 16:33:13 -0500 Subject: Ensure instance path is a child of cloud_dir A cloud has an instance-id metadata value in the form: /Compute-$TENANT/$CLOUDUSERNAME/$UUID The leading '/' causes /var/lib/cloud/instance to link to /Compute-$TENANT/$CLOUDUSERNAME/$UUID rather than /var/lib/cloud/instances/Compute-$TENANT/$CLOUDUSERNAME/$UUID This patch strips the leading path separator from the instance-id. LP: #1575938 --- cloudinit/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index 0cf982f3..24a3d5b5 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -375,7 +375,7 @@ class Paths(object): def _get_ipath(self, name=None): if not self.datasource: return None - iid = self.datasource.get_instance_id() + iid = self.datasource.get_instance_id().lstrip(os.sep) if iid is None: return None ipath = os.path.join(self.cloud_dir, 'instances', str(iid)) -- cgit v1.2.3 From aee0edd93cb4d78b5e0d1aec71e977aabf31cdd0 Mon Sep 17 00:00:00 2001 From: Wido den Hollander Date: Thu, 28 Apr 2016 17:33:43 +0200 Subject: cloudstack: Only use DHCPv4 lease files as a datasource It could be that there are also 'dhclient6.leases' files in /var/lib/dhcp when DHCPv6 is used next to DHCPv4. This patch makes sure we only read from DHCPv4 lease files --- cloudinit/sources/DataSourceCloudStack.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 455a4652..4ba019df 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -206,7 +206,8 @@ def get_latest_lease(): latest_mtime = -1 latest_file = None for file_name in lease_files: - if file_name.endswith(".lease") or file_name.endswith(".leases"): + if file_name.startswith("dhclient.") and \ + (file_name.endswith(".lease") or file_name.endswith(".leases")): abs_path = os.path.join(lease_d, file_name) mtime = os.path.getmtime(abs_path) if mtime > latest_mtime: -- cgit v1.2.3 From 89fe4242aa3d825ee1b439cc8d184ddf06382086 Mon Sep 17 00:00:00 2001 From: Robert Jennings Date: Thu, 28 Apr 2016 12:10:33 -0500 Subject: Tweak instance path name based on review feedback r1213 (Ensure instance path is a child of cloud_dir) stripped the leading path separator. This patch goes further by replacing all path seperators with '_' which will avoid a deep directory structure under /var/lib/cloud/instances. LP: #1575938 --- cloudinit/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index 24a3d5b5..e9fec628 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -375,7 +375,7 @@ class Paths(object): def _get_ipath(self, name=None): if not self.datasource: return None - iid = self.datasource.get_instance_id().lstrip(os.sep) + iid = self.datasource.get_instance_id().replace(os.sep, '_') if iid is None: return None ipath = os.path.join(self.cloud_dir, 'instances', str(iid)) -- cgit v1.2.3 From c92f02037afc6b0434c9498904f7d888e00cd55b Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 29 Apr 2016 09:04:36 -0400 Subject: Config Drive: fix check_instance_id signature. After reboot cloud-init would fail as the previously pickled object would have a check_instance_id signature but it didn't match expected LP: #1575055 --- cloudinit/sources/DataSourceConfigDrive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 3fa62ef3..52a9f543 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -155,7 +155,7 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): return True - def check_instance_id(self): + def check_instance_id(self, sys_cfg): # quickly (local check only) if self.instance_id is still valid return sources.instance_id_matches_system_uuid(self.get_instance_id()) -- cgit v1.2.3 From 0b3e3f898f70cc98c6e694c7b7a11e654fff9967 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Sun, 1 May 2016 13:03:23 -0400 Subject: initial commit of rework --- cloudinit/sources/DataSourceSmartOS.py | 450 +++++++++++++++++++++------------ 1 file changed, 286 insertions(+), 164 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 6cbd8dfa..46cf117a 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -32,8 +32,10 @@ # http://us-east.manta.joyent.com/jmc/public/mdata/datadict.html # Comments with "@datadictionary" are snippets of the definition +import base64 import binascii import contextlib +import json import os import random import re @@ -66,12 +68,26 @@ SMARTOS_ATTRIB_MAP = { DS_NAME = 'SmartOS' DS_CFG_PATH = ['datasource', DS_NAME] +NO_BASE64_DECODE = [ + 'iptables_disable', + 'motd_sys_info', + 'root_authorized_keys', + 'sdc:datacenter_name', + 'sdc:uuid' + 'user-data', + 'user-script', +] + +METADATA_SOCKFILE = '/native/.zonecontrol/metadata.sock' +SERIAL_DEVICE = '/dev/ttyS1' +SERIAL_TIMEOUT = 60 + # BUILT-IN DATASOURCE CONFIGURATION # The following is the built-in configuration. If the values # are not set via the system configuration, then these default # will be used: # serial_device: which serial device to use for the meta-data -# seed_timeout: how long to wait on the device +# serial_timeout: how long to wait on the device # no_base64_decode: values which are not base64 encoded and # are fetched directly from SmartOS, not meta-data values # base64_keys: meta-data keys that are delivered in base64 @@ -81,16 +97,10 @@ DS_CFG_PATH = ['datasource', DS_NAME] # fs_setup: describes how to format the ephemeral drive # BUILTIN_DS_CONFIG = { - 'serial_device': '/dev/ttyS1', - 'metadata_sockfile': '/native/.zonecontrol/metadata.sock', - 'seed_timeout': 60, - 'no_base64_decode': ['root_authorized_keys', - 'motd_sys_info', - 'iptables_disable', - 'user-data', - 'user-script', - 'sdc:datacenter_name', - 'sdc:uuid'], + 'serial_device': SERIAL_DEVICE, + 'serial_timeout': SERIAL_TIMEOUT, + 'metadata_sockfile': METADATA_SOCKFILE, + 'no_base64_decode': NO_BASE64_DECODE, 'base64_keys': [], 'base64_all': False, 'disk_aliases': {'ephemeral0': '/dev/vdb'}, @@ -154,9 +164,12 @@ LEGACY_USER_D = "/var/db" class DataSourceSmartOS(sources.DataSource): + _unset = "_unset" + smartos_environ = _unset + md_client = _unset + def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) - self.is_smartdc = None self.ds_cfg = util.mergemanydict([ self.ds_cfg, util.get_cfg_by_path(sys_cfg, DS_CFG_PATH, {}), @@ -164,49 +177,24 @@ class DataSourceSmartOS(sources.DataSource): self.metadata = {} - # SDC LX-Brand Zones lack dmidecode (no /dev/mem) but - # report 'BrandZ virtual linux' as the kernel version - if os.uname()[3].lower() == 'brandz virtual linux': - LOG.debug("Host is SmartOS, guest in Zone") - self.is_smartdc = True - self.smartos_type = 'lx-brand' - self.cfg = {} - self.seed = self.ds_cfg.get("metadata_sockfile") - else: - self.is_smartdc = True - self.smartos_type = 'kvm' - self.seed = self.ds_cfg.get("serial_device") - self.cfg = BUILTIN_CLOUD_CONFIG - self.seed_timeout = self.ds_cfg.get("serial_timeout") - self.smartos_no_base64 = self.ds_cfg.get('no_base64_decode') - self.b64_keys = self.ds_cfg.get('base64_keys') - self.b64_all = self.ds_cfg.get('base64_all') self.script_base_d = os.path.join(self.paths.get_cpath("scripts")) + self._init() + def __str__(self): root = sources.DataSource.__str__(self) - return "%s [seed=%s]" % (root, self.seed) - - def _get_seed_file_object(self): - if not self.seed: - raise AttributeError("seed device is not set") - - if self.smartos_type == 'lx-brand': - if not stat.S_ISSOCK(os.stat(self.seed).st_mode): - LOG.debug("Seed %s is not a socket", self.seed) - return None - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - sock.connect(self.seed) - return sock.makefile('rwb') - else: - if not stat.S_ISCHR(os.stat(self.seed).st_mode): - LOG.debug("Seed %s is not a character device") - return None - ser = serial.Serial(self.seed, timeout=self.seed_timeout) - if not ser.isOpen(): - raise SystemError("Unable to open %s" % self.seed) - return ser - return None + return "%s [client=%s]" % (root, self.md_client) + + def _init(self): + if self.smartos_environ == self._unset: + self.smartos_env = get_smartos_environ() + + if self.md_client == self._unset: + self.md_client = jmc_client_factory( + metadata_sockfile=self.ds_cfg['metadata_sockfile'], + serial_device=self.ds_cfg['serial_device'], + serial_timeout=self.ds_cfg['serial_timeout']) + def _set_provisioned(self): '''Mark the instance provisioning state as successful. @@ -225,50 +213,23 @@ class DataSourceSmartOS(sources.DataSource): '/'.join([svc_path, 'provision_success'])) def get_data(self): + self._init() + md = {} ud = "" - if not device_exists(self.seed): - LOG.debug("No metadata device '%s' found for SmartOS datasource", - self.seed) - return False - - uname_arch = os.uname()[4] - if uname_arch.startswith("arm") or uname_arch == "aarch64": - # Disabling because dmidcode in dmi_data() crashes kvm process - LOG.debug("Disabling SmartOS datasource on arm (LP: #1243287)") + if not self.smartos_env: + LOG.debug("Not running on smartos") return False - - # SDC KVM instances will provide dmi data, LX-brand does not - if self.smartos_type == 'kvm': - dmi_info = dmi_data() - if dmi_info is None: - LOG.debug("No dmidata utility found") - return False - - system_type = dmi_info - if 'smartdc' not in system_type.lower(): - LOG.debug("Host is not on SmartOS. system_type=%s", - system_type) - return False - LOG.debug("Host is SmartOS, guest in KVM") - - seed_obj = self._get_seed_file_object() - if seed_obj is None: - LOG.debug('Seed file object not found.') + + if not self.md_client.exists(): + LOG.debug("No metadata device '%r' found for SmartOS datasource", + self.md_client) return False - with contextlib.closing(seed_obj) as seed: - b64_keys = self.query('base64_keys', seed, strip=True, b64=False) - if b64_keys is not None: - self.b64_keys = [k.strip() for k in str(b64_keys).split(',')] - b64_all = self.query('base64_all', seed, strip=True, b64=False) - if b64_all is not None: - self.b64_all = util.is_true(b64_all) - - for ci_noun, attribute in SMARTOS_ATTRIB_MAP.items(): - smartos_noun, strip = attribute - md[ci_noun] = self.query(smartos_noun, seed, strip=strip) + for ci_noun, attribute in SMARTOS_ATTRIB_MAP.items(): + smartos_noun, strip = attribute + md[ci_noun] = self.md_client.get(smartos_noun, strip=strip) # @datadictionary: This key may contain a program that is written # to a file in the filesystem of the guest on each boot and then @@ -331,65 +292,6 @@ class DataSourceSmartOS(sources.DataSource): def get_instance_id(self): return self.metadata['instance-id'] - def query(self, noun, seed_file, strip=False, default=None, b64=None): - if b64 is None: - if noun in self.smartos_no_base64: - b64 = False - elif self.b64_all or noun in self.b64_keys: - b64 = True - - return self._query_data(noun, seed_file, strip=strip, - default=default, b64=b64) - - def _query_data(self, noun, seed_file, strip=False, - default=None, b64=None): - """Makes a request via "GET " - - In the response, the first line is the status, while subsequent - lines are is the value. A blank line with a "." is used to - indicate end of response. - - If the response is expected to be base64 encoded, then set - b64encoded to true. Unfortantely, there is no way to know if - something is 100% encoded, so this method relies on being told - if the data is base64 or not. - """ - - if not noun: - return False - - response = JoyentMetadataClient(seed_file).get_metadata(noun) - - if response is None: - return default - - if b64 is None: - b64 = self._query_data('b64-%s' % noun, seed_file, b64=False, - default=False, strip=True) - b64 = util.is_true(b64) - - resp = None - if b64 or strip: - resp = "".join(response).rstrip() - else: - resp = "".join(response) - - if b64: - try: - return util.b64d(resp) - # Bogus input produces different errors in Python 2 and 3; - # catch both. - except (TypeError, binascii.Error): - LOG.warn("Failed base64 decoding key '%s'", noun) - return resp - - return resp - - -def device_exists(device): - """Symplistic method to determine if the device exists or not""" - return os.path.exists(device) - class JoyentMetadataFetchException(Exception): pass @@ -407,8 +309,11 @@ class JoyentMetadataClient(object): r' (?P(?P[0-9a-f]+) (?PSUCCESS|NOTFOUND)' r'( (?P.+))?)') - def __init__(self, metasource): - self.metasource = metasource + def __init__(self, smartos_type=None): + if smartos_type is None: + smartos_type = get_smartos_environ() + self.smartos_type = smartos_type + self.fp = None def _checksum(self, body): return '{0:08x}'.format( @@ -436,37 +341,227 @@ class JoyentMetadataClient(object): LOG.debug('Value "%s" found.', value) return value - def get_metadata(self, metadata_key): - LOG.debug('Fetching metadata key "%s"...', metadata_key) + def request(self, rtype, param=None): request_id = '{0:08x}'.format(random.randint(0, 0xffffffff)) - message_body = '{0} GET {1}'.format(request_id, - util.b64e(metadata_key)) + message_body = ' '.join((request_id, rtype,)) + if param: + message_body += ' ' + base64.b64encode(param.encode()).decode() msg = 'V2 {0} {1} {2}\n'.format( len(message_body), self._checksum(message_body), message_body) LOG.debug('Writing "%s" to metadata transport.', msg) - self.metasource.write(msg.encode('ascii')) - self.metasource.flush() + + need_close = False + if not self.fp: + self.open_transport() + need_close = True + + self.fp.write(msg.encode('ascii')) + self.fp.flush() response = bytearray() - response.extend(self.metasource.read(1)) + response.extend(self.fp.read(1)) while response[-1:] != b'\n': - response.extend(self.metasource.read(1)) + response.extend(self.fp.read(1)) + + if need_close: + self.close_transport() + response = response.rstrip().decode('ascii') LOG.debug('Read "%s" from metadata transport.', response) if 'SUCCESS' not in response: return None - return self._get_value_from_frame(request_id, response) + value = self._get_value_from_frame(request_id, response) + if value is None: + return default + return value -def dmi_data(): - sys_type = util.read_dmi_data("system-product-name") + def get(self, key, default=None, strip=False): + result = self.request(rtype='GET', param=key) + if result is None: + return default + if result and strip: + result = result.strip() + return result + + def get_json(self, key, default=None): + result = self.get(key) + if result is None: + return default + return json.loads(result) + + def list(self): + result = self.request(rtype='KEYS') + if result: + result = result.split('\n') + return result + + def put(self, key, val): + param = b' '.join([base64.b64encode(i.encode()) + for i in (key, val)]).decode() + return self.request(rtype='PUT', param=param) + + def delete(self, key): + return self.request(rtype='DELETE', param=key) + + def close_transport(self): + if self.fp: + self.fp.close() + self.fp = None + + def __enter__(self): + if self.fp: + return self + self.open_transport() + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.close_transport() + return - if not sys_type: - return None - return sys_type +class JoyentMetadataSocketClient(JoyentMetadataClient): + def __init__(self, socketpath): + self.socketpath = metadata_socketfile + + def open_transport(self): + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.connect(path) + self.fp = sock.makefile('rwb') + + def exists(self): + return os.path.exists(self.socketpath) + + def __repr__(self): + return "%s(socketpath=%s)" % (self.__class__.__name__, self.socketpath) + + +class JoyentMetadataSerialClient(JoyentMetadataClient): + def __init__(self, device, timeout=10, smartos_type=None): + super(JoyentMetadataSerialClient, self).__init__(smartos_type) + self.device = device + self.timeout = timeout + + def exists(self): + return os.path.exists(self.device) + + def open_transport(self): + ser = serial.Serial(self.device, timeout=self.timeout) + if not ser.isOpen(): + raise SystemError("Unable to open %s" % self.device) + self.fp = ser + + def __repr__(self): + return "%s(device=%s, timeout=%s)" % ( + self.__class__.__name__, self.device, self.timeout) + + +class JoyentMetadataLegacySerialClient(JoyentMetadataSerialClient): + """V1 of the protocol was not safe for all values. + Thus, we allowed the user to pass values in as base64 encoded. + Users may still reasonably expect to be able to send base64 data + and have it transparently decoded. So even though the V2 format is + now used, and is safe (using base64 itself), we keep legacy support. + + The way for a user to do this was: + a.) specify 'base64_keys' key whose value is a comma delimited + list of keys that were base64 encoded. + b.) base64_all: string interpreted as a boolean that indicates + if all keys are base64 encoded. + c.) set a key named b64- with a boolean indicating that + is base64 encoded.""" + + def __init__(self, device, timeout=10, smartos_type=None): + s = super(JoyentMetadataLegacySerialClient, self) + s.__init__(device, timeout, smartos_type) + self.base64_keys = None + self.base64_all = None + + def _init_base64_keys(self, reset=False): + if reset: + self.base64_keys = None + self.base64_all = None + + keys = None + if self.base64_all is None: + keys = self.list() + if 'base64_all' in keys: + self.base64_all = util.is_true(self._get("base64_all")) + else: + self.base64_all = False + + if self.base64_all: + # short circuit if base64_all is true + return + + if self.base64_keys is None: + if keys is None: + keys = self.list() + b64_keys = set() + if 'base64_keys' in keys: + b64_keys = set(self._get("base64_keys").split(",")) + + # now add any b64- that has a true value + for key in [k[3:] for k in keys if k.startswith("b64-")]: + if util.is_true(self._get(key)): + b64_keys.append(key) + else: + if key in b64_keys: + b64_keys.remove(key) + + self.base64_keys = b64_keys + + def _get(self, key, default=None, strip=False): + return (super(JoyentMetadataLegacySerialClient, self). + get(key, default=default, strip=strip)) + + def is_b64_encoded(self, key, reset=False): + if key in NO_BASE64_DECODE: + return False + + self._init_base64_keys(reset=reset) + if self.base64_all: + return True + + return key in self.base64_keys + + def get(self, key, default=None, strip=False): + mdefault = object() + val = self._get(key, strip=False, default=mdefault) + if val is mdefault: + return default + + if self.is_b64_encoded(key): + try: + val = base64.b64decode(val.encode()).decode() + # Bogus input produces different errors in Python 2 and 3 + except (TypeError, binascii.Error): + LOG.warn("Failed base64 decoding key '%s': %s", key, val) + + if strip: + val = val.strip() + + return val + + +def jmc_client_factory( + smartos_type=None, metadata_sockfile=METADATA_SOCKFILE, + serial_device=SERIAL_DEVICE, serial_timeout=SERIAL_TIMEOUT, + uname_version=None): + + if smartos_type is None: + smartos_type = get_smartos_environ(uname_version) + + if smartos_type == 'kvm': + return JoyentMetadataLegacySerialClient( + device=serial_device, timeout=serial_timeout, + smartos_type=smartos_type) + elif smartos_type == 'lx-brand': + return JoyentMetadataSerialClient(socketpath=metadata_socketfile) + + raise ValueError("Unknown value for smartos_type: %s" % smartos_type) def write_boot_content(content, content_f, link=None, shebang=False, @@ -525,6 +620,29 @@ def write_boot_content(content, content_f, link=None, shebang=False, util.logexc(LOG, "failed establishing content link", e) +def get_smartos_environ(uname_version=None, product_name=None, + uname_arch=None): + uname = os.uname() + if uname_arch is None: + uname_arch = uname[4] + + if uname_arch.startswith("arm") or uname_arch == "aarch64": + return None + + # SDC LX-Brand Zones lack dmidecode (no /dev/mem) but + # report 'BrandZ virtual linux' as the kernel version + if uname_version is None: + uname_version = uname[3] + if uname_version.lower() == 'brandz virtual linux': + return 'lx-brand' + + system_type = util.read_dmi_data("system-product-name") + if system_type and 'smartdc' in system_type.lower(): + return 'kvm' + + return None + + # Used to match classes to dependencies datasources = [ (DataSourceSmartOS, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), @@ -534,3 +652,7 @@ datasources = [ # Return a list of data sources that match this set of dependencies def get_datasource_list(depends): return sources.list_from_depends(depends, datasources) + + +if __name__ == "__main__": + jmc = JoyentMetadataClient(seed_file).get_metadata(noun) -- cgit v1.2.3 From a2249942522d140a063acdc007aced991d4f0588 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 5 May 2016 14:57:48 -0700 Subject: Work on refactoring (and adding) network conversion tests --- cloudinit/net/network_state.py | 68 ----------- cloudinit/sources/DataSourceConfigDrive.py | 127 +-------------------- cloudinit/sources/helpers/openstack.py | 116 +++++++++++++++++++ .../unittests/test_datasource/test_configdrive.py | 104 ++++++++++++++--- tests/unittests/test_net.py | 1 + 5 files changed, 209 insertions(+), 207 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index e32d2cdf..d08e94fe 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -376,71 +376,3 @@ def mask2cidr(mask): return ipv4mask2cidr(mask) else: return mask - - -if __name__ == '__main__': - import sys - import random - from cloudinit import net - - def load_config(nc): - version = nc.get('version') - config = nc.get('config') - return (version, config) - - def test_parse(network_config): - (version, config) = load_config(network_config) - ns1 = NetworkState(version=version, config=config) - ns1.parse_config() - random.shuffle(config) - ns2 = NetworkState(version=version, config=config) - ns2.parse_config() - print("----NS1-----") - print(ns1.dump_network_state()) - print() - print("----NS2-----") - print(ns2.dump_network_state()) - print("NS1 == NS2 ?=> {}".format( - ns1.network_state == ns2.network_state)) - eni = net.render_interfaces(ns2.network_state) - print(eni) - udev_rules = net.render_persistent_net(ns2.network_state) - print(udev_rules) - - def test_dump_and_load(network_config): - print("Loading network_config into NetworkState") - (version, config) = load_config(network_config) - ns1 = NetworkState(version=version, config=config) - ns1.parse_config() - print("Dumping state to file") - ns1_dump = ns1.dump() - ns1_state = "/tmp/ns1.state" - with open(ns1_state, "w+") as f: - f.write(ns1_dump) - - print("Loading state from file") - ns2 = from_state_file(ns1_state) - print("NS1 == NS2 ?=> {}".format( - ns1.network_state == ns2.network_state)) - - def test_output(network_config): - (version, config) = load_config(network_config) - ns1 = NetworkState(version=version, config=config) - ns1.parse_config() - random.shuffle(config) - ns2 = NetworkState(version=version, config=config) - ns2.parse_config() - print("NS1 == NS2 ?=> {}".format( - ns1.network_state == ns2.network_state)) - eni_1 = net.render_interfaces(ns1.network_state) - eni_2 = net.render_interfaces(ns2.network_state) - print(eni_1) - print(eni_2) - print("eni_1 == eni_2 ?=> {}".format( - eni_1 == eni_2)) - - y = util.read_conf(sys.argv[1]) - network_config = y.get('network') - test_parse(network_config) - test_dump_and_load(network_config) - test_output(network_config) diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 52a9f543..70373b43 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -61,7 +61,7 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): mstr += "[source=%s]" % (self.source) return mstr - def get_data(self): + def get_data(self, skip_first_boot=False): found = None md = {} results = {} @@ -119,7 +119,8 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): # instance-id prev_iid = get_previous_iid(self.paths) cur_iid = md['instance-id'] - if prev_iid != cur_iid and self.dsmode == "local": + if prev_iid != cur_iid and \ + self.dsmode == "local" and not skip_first_boot: on_first_boot(results, distro=self.distro) # dsmode != self.dsmode here if: @@ -163,7 +164,8 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): def network_config(self): if self._network_config is None: if self.network_json is not None: - self._network_config = convert_network_data(self.network_json) + self._network_config = openstack.convert_net_json( + self.network_json) return self._network_config @@ -303,122 +305,3 @@ datasources = [ # Return a list of data sources that match this set of dependencies def get_datasource_list(depends): return sources.list_from_depends(depends, datasources) - - -# Convert OpenStack ConfigDrive NetworkData json to network_config yaml -def convert_network_data(network_json=None): - """Return a dictionary of network_config by parsing provided - OpenStack ConfigDrive NetworkData json format - - OpenStack network_data.json provides a 3 element dictionary - - "links" (links are network devices, physical or virtual) - - "networks" (networks are ip network configurations for one or more - links) - - services (non-ip services, like dns) - - networks and links are combined via network items referencing specific - links via a 'link_id' which maps to a links 'id' field. - - To convert this format to network_config yaml, we first iterate over the - links and then walk the network list to determine if any of the networks - utilize the current link; if so we generate a subnet entry for the device - - We also need to map network_data.json fields to network_config fields. For - example, the network_data links 'id' field is equivalent to network_config - 'name' field for devices. We apply more of this mapping to the various - link types that we encounter. - - There are additional fields that are populated in the network_data.json - from OpenStack that are not relevant to network_config yaml, so we - enumerate a dictionary of valid keys for network_yaml and apply filtering - to drop these superflous keys from the network_config yaml. - """ - if network_json is None: - return None - - # dict of network_config key for filtering network_json - valid_keys = { - 'physical': [ - 'name', - 'type', - 'mac_address', - 'subnets', - 'params', - ], - 'subnet': [ - 'type', - 'address', - 'netmask', - 'broadcast', - 'metric', - 'gateway', - 'pointopoint', - 'mtu', - 'scope', - 'dns_nameservers', - 'dns_search', - 'routes', - ], - } - - links = network_json.get('links', []) - networks = network_json.get('networks', []) - services = network_json.get('services', []) - - config = [] - for link in links: - subnets = [] - cfg = {k: v for k, v in link.items() - if k in valid_keys['physical']} - cfg.update({'name': link['id']}) - for network in [net for net in networks - if net['link'] == link['id']]: - subnet = {k: v for k, v in network.items() - if k in valid_keys['subnet']} - if 'dhcp' in network['type']: - t = 'dhcp6' if network['type'].startswith('ipv6') else 'dhcp4' - subnet.update({ - 'type': t, - }) - else: - subnet.update({ - 'type': 'static', - 'address': network.get('ip_address'), - }) - subnets.append(subnet) - cfg.update({'subnets': subnets}) - if link['type'] in ['ethernet', 'vif', 'ovs', 'phy']: - cfg.update({ - 'type': 'physical', - 'mac_address': link['ethernet_mac_address']}) - elif link['type'] in ['bond']: - params = {} - for k, v in link.items(): - if k == 'bond_links': - continue - elif k.startswith('bond'): - params.update({k: v}) - cfg.update({ - 'bond_interfaces': copy.deepcopy(link['bond_links']), - 'params': params, - }) - elif link['type'] in ['vlan']: - cfg.update({ - 'name': "%s.%s" % (link['vlan_link'], - link['vlan_id']), - 'vlan_link': link['vlan_link'], - 'vlan_id': link['vlan_id'], - 'mac_address': link['vlan_mac_address'], - }) - else: - raise ValueError( - 'Unknown network_data link type: %s' % link['type']) - - config.append(cfg) - - for service in services: - cfg = service - cfg.update({'type': 'nameserver'}) - config.append(cfg) - - return {'version': 1, 'config': config} diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 1aa6bbae..475ccab3 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -474,6 +474,122 @@ class MetadataReader(BaseReader): retries=self.retries) +def convert_net_json(network_json): + """Return a dictionary of network_config by parsing provided + OpenStack ConfigDrive NetworkData json format + + OpenStack network_data.json provides a 3 element dictionary + - "links" (links are network devices, physical or virtual) + - "networks" (networks are ip network configurations for one or more + links) + - services (non-ip services, like dns) + + networks and links are combined via network items referencing specific + links via a 'link_id' which maps to a links 'id' field. + + To convert this format to network_config yaml, we first iterate over the + links and then walk the network list to determine if any of the networks + utilize the current link; if so we generate a subnet entry for the device + + We also need to map network_data.json fields to network_config fields. For + example, the network_data links 'id' field is equivalent to network_config + 'name' field for devices. We apply more of this mapping to the various + link types that we encounter. + + There are additional fields that are populated in the network_data.json + from OpenStack that are not relevant to network_config yaml, so we + enumerate a dictionary of valid keys for network_yaml and apply filtering + to drop these superflous keys from the network_config yaml. + """ + + # Dict of network_config key for filtering network_json + valid_keys = { + 'physical': [ + 'name', + 'type', + 'mac_address', + 'subnets', + 'params', + ], + 'subnet': [ + 'type', + 'address', + 'netmask', + 'broadcast', + 'metric', + 'gateway', + 'pointopoint', + 'mtu', + 'scope', + 'dns_nameservers', + 'dns_search', + 'routes', + ], + } + + links = network_json.get('links', []) + networks = network_json.get('networks', []) + services = network_json.get('services', []) + + config = [] + for link in links: + subnets = [] + cfg = {k: v for k, v in link.items() + if k in valid_keys['physical']} + cfg.update({'name': link['id']}) + for network in [net for net in networks + if net['link'] == link['id']]: + subnet = {k: v for k, v in network.items() + if k in valid_keys['subnet']} + if 'dhcp' in network['type']: + t = 'dhcp6' if network['type'].startswith('ipv6') else 'dhcp4' + subnet.update({ + 'type': t, + }) + else: + subnet.update({ + 'type': 'static', + 'address': network.get('ip_address'), + }) + subnets.append(subnet) + cfg.update({'subnets': subnets}) + if link['type'] in ['ethernet', 'vif', 'ovs', 'phy']: + cfg.update({ + 'type': 'physical', + 'mac_address': link['ethernet_mac_address']}) + elif link['type'] in ['bond']: + params = {} + for k, v in link.items(): + if k == 'bond_links': + continue + elif k.startswith('bond'): + params.update({k: v}) + cfg.update({ + 'bond_interfaces': copy.deepcopy(link['bond_links']), + 'params': params, + }) + elif link['type'] in ['vlan']: + cfg.update({ + 'name': "%s.%s" % (link['vlan_link'], + link['vlan_id']), + 'vlan_link': link['vlan_link'], + 'vlan_id': link['vlan_id'], + 'mac_address': link['vlan_mac_address'], + }) + else: + raise ValueError( + 'Unknown network_data link type: %s' % link['type']) + + config.append(cfg) + + for service in services: + cfg = copy.deepcopy(service) + cfg.update({'type': 'nameserver'}) + config.append(cfg) + + return {'version': 1, 'config': config} + + def convert_vendordata_json(data, recurse=True): """ data: a loaded json *object* (strings, arrays, dicts). return something suitable for cloudinit vendordata_raw. diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index 89b15f54..06fba202 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -355,6 +355,14 @@ class TestConfigDriveDataSource(TestCase): self.assertEqual(myds.get_public_ssh_keys(), [OSTACK_META['public_keys']['mykey']]) + +class TestNetJson(TestCase): + def setUp(self): + super(TestNetJson, self).setUp() + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp) + self.maxDiff = None + def test_network_data_is_found(self): """Verify that network_data is present in ds in config-drive-v2.""" populate_dir(self.tmp, CFG_DRIVE_FILES_V2) @@ -365,31 +373,94 @@ class TestConfigDriveDataSource(TestCase): """Verify that network_data is converted and present on ds object.""" populate_dir(self.tmp, CFG_DRIVE_FILES_V2) myds = cfg_ds_from_dir(self.tmp) - network_config = ds.convert_network_data(NETWORK_DATA) + network_config = openstack.convert_net_json(NETWORK_DATA) self.assertEqual(myds.network_config, network_config) + def test_network_config_conversions(self): + """Tests a bunch of input network json and checks the expected conversions.""" + in_datas = [ + NETWORK_DATA, + { + 'services': [{'type': 'dns', 'address': '172.19.0.12'}], + 'networks': [ + {'network_id': 'dacd568d-5be6-4786-91fe-750c374b78b4', + 'type': 'ipv4', 'netmask': '255.255.252.0', + 'link': 'tap1a81968a-79', + 'routes': [ + { + 'netmask': '0.0.0.0', + 'network': '0.0.0.0', + 'gateway': '172.19.3.254' + }, + ], + 'ip_address': '172.19.1.34', + 'id': 'network0', + }], + 'links': [ + {'type': 'bridge', + 'vif_id': '1a81968a-797a-400f-8a80-567f997eb93f', + 'ethernet_mac_address': 'fa:16:3e:ed:9a:59', + 'id': 'tap1a81968a-79', 'mtu': None}] + }, + ] + out_datas = [ + { + 'version': 1, + 'config': [ + { + 'subnets': [{'type': 'dhcp4'}], + 'type': 'physical', + 'mac_address': 'fa:16:3e:69:b0:58', + 'name': 'tap2ecc7709-b3', + }, + { + 'subnets': [{'type': 'dhcp4'}], + 'type': 'physical', + 'mac_address': 'fa:16:3e:d4:57:ad', + 'name': 'tap2f88d109-5b', + }, + { + 'subnets': [{'type': 'dhcp4'}], + 'type': 'physical', + 'mac_address': 'fa:16:3e:05:30:fe', + 'name': 'tap1a5382f8-04', + }, + { + 'type': 'nameserver', + 'address': '199.204.44.24', + }, + { + 'type': 'nameserver', + 'address': '199.204.47.54', + } + ], + + }, + { + 'version': 1, + 'config': [ + { + 'name': 'tap1a81968a-79', + + } + ], + }, + ] + for in_data, out_data in zip(in_datas, out_datas): + self.assertEqual(openstack.convert_net_json(in_data), + out_data) + def cfg_ds_from_dir(seed_d): - found = ds.read_config_drive(seed_d) cfg_ds = ds.DataSourceConfigDrive(settings.CFG_BUILTIN, None, helpers.Paths({})) - populate_ds_from_read_config(cfg_ds, seed_d, found) + cfg_ds.seed_dir = seed_d + if not cfg_ds.get_data(skip_first_boot=True): + raise RuntimeError("Data source did not extract itself from" + " seed directory %s" % seed_d) return cfg_ds -def populate_ds_from_read_config(cfg_ds, source, results): - """Patch the DataSourceConfigDrive from the results of - read_config_drive_dir hopefully in line with what it would have - if cfg_ds.get_data had been successfully called""" - cfg_ds.source = source - cfg_ds.metadata = results.get('metadata') - cfg_ds.ec2_metadata = results.get('ec2-metadata') - cfg_ds.userdata_raw = results.get('userdata') - cfg_ds.version = results.get('version') - cfg_ds.network_json = results.get('networkdata') - cfg_ds._network_config = ds.convert_network_data(cfg_ds.network_json) - - def populate_dir(seed_dir, files): for (name, content) in files.items(): path = os.path.join(seed_dir, name) @@ -400,7 +471,6 @@ def populate_dir(seed_dir, files): mode = "w" else: mode = "wb" - with open(path, mode) as fp: fp.write(content) diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 09235c4d..a0cdc493 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -121,6 +121,7 @@ class TestNetConfigParsing(TestCase): self.assertEqual(found, self.simple_cfg) + def _gzip_data(data): with io.BytesIO() as iobuf: gzfp = gzip.GzipFile(mode="wb", fileobj=iobuf) -- cgit v1.2.3 From db9d958a0e76c2c59298041a1355aba97fda000f Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 5 May 2016 16:38:53 -0700 Subject: Add the bridge net type --- cloudinit/net/__init__.py | 2 +- cloudinit/net/network_state.py | 39 +++++++++++++++++++++------------- cloudinit/sources/helpers/openstack.py | 5 +++++ 3 files changed, 30 insertions(+), 16 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 31544fd8..cc154c57 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -262,7 +262,7 @@ def parse_deb_config(path): def parse_net_config_data(net_config): - """Parses the config, returns NetworkState dictionary + """Parses the config, returns NetworkState object :param net_config: curtin network config dict """ diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index d08e94fe..27d35256 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -15,6 +15,8 @@ # You should have received a copy of the GNU Affero General Public License # along with Curtin. If not, see . +import six + from cloudinit import log as logging from cloudinit import util from cloudinit.util import yaml_dumps as dump_config @@ -32,11 +34,31 @@ def from_state_file(state_file): state = util.read_conf(state_file) network_state = NetworkState() network_state.load(state) - return network_state -class NetworkState: +class CommandHandlerMeta(type): + """Metaclass that dynamically creates a 'command_handlers' attribute. + + This will scan the to-be-created class for methods that start with + 'handle_' and on finding those will populate a class attribute mapping + so that those methods can be quickly located and called. + """ + def __new__(cls, name, parents, dct): + command_handlers = {} + for attr_name, attr in six.iteritems(dct): + if six.callable(attr) and attr_name.startswith('handle_'): + handles_what = attr_name[len('handle_'):] + if handles_what: + command_handlers[handles_what] = attr + dct['command_handlers'] = command_handlers + return super(CommandHandlerMeta, cls).__new__(cls, name, + parents, dct) + + +@six.add_metaclass(CommandHandlerMeta) +class NetworkState(object): + def __init__(self, version=NETWORK_STATE_VERSION, config=None): self.version = version self.config = config @@ -48,18 +70,6 @@ class NetworkState: 'search': [], } } - self.command_handlers = self.get_command_handlers() - - def get_command_handlers(self): - METHOD_PREFIX = 'handle_' - methods = filter(lambda x: callable(getattr(self, x)) and - x.startswith(METHOD_PREFIX), dir(self)) - handlers = {} - for m in methods: - key = m.replace(METHOD_PREFIX, '') - handlers[key] = getattr(self, m) - - return handlers def dump(self): state = { @@ -83,7 +93,6 @@ class NetworkState: # v1 - direct attr mapping, except version for key in [k for k in required_keys if k not in ['version']]: setattr(self, key, state[key]) - self.command_handlers = self.get_command_handlers() def dump_network_state(self): return dump_config(self.network_state) diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 475ccab3..845ea971 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -576,6 +576,11 @@ def convert_net_json(network_json): 'vlan_id': link['vlan_id'], 'mac_address': link['vlan_mac_address'], }) + elif link['type'] in ['bridge']: + cfg.update({ + 'type': 'bridge', + 'mac_address': link['ethernet_mac_address'], + 'mtu': link['mtu']}) else: raise ValueError( 'Unknown network_data link type: %s' % link['type']) -- cgit v1.2.3 From 2c32c855278c22b77b6ac3bfca23fa2fa6bc6330 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 5 May 2016 17:03:08 -0700 Subject: Use a decorator vs repeated key checks. --- cloudinit/net/network_state.py | 136 ++++++++++++++++++++--------------------- 1 file changed, 66 insertions(+), 70 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 27d35256..73be84e1 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -15,6 +15,8 @@ # You should have received a copy of the GNU Affero General Public License # along with Curtin. If not, see . +import copy + import six from cloudinit import log as logging @@ -37,6 +39,37 @@ def from_state_file(state_file): return network_state +class InvalidCommand(Exception): + pass + + +def ensure_command_keys(required_keys): + required_keys = frozenset(required_keys) + + def extract_missing(command): + missing_keys = set() + for key in required_keys: + if key not in command: + missing_keys.add(key) + return missing_keys + + def wrapper(func): + + @six.wraps(func) + def decorator(self, command, *args, **kwargs): + if required_keys: + missing_keys = extract_missing(command) + if missing_keys: + raise InvalidCommand("Command missing %s of required" + " keys %s" % (missing_keys, + required_keys)) + return func(self, command, *args, **kwargs) + + return decorator + + return wrapper + + class CommandHandlerMeta(type): """Metaclass that dynamically creates a 'command_handlers' attribute. @@ -59,17 +92,19 @@ class CommandHandlerMeta(type): @six.add_metaclass(CommandHandlerMeta) class NetworkState(object): + initial_network_state = { + 'interfaces': {}, + 'routes': [], + 'dns': { + 'nameservers': [], + 'search': [], + } + } + def __init__(self, version=NETWORK_STATE_VERSION, config=None): self.version = version self.config = config - self.network_state = { - 'interfaces': {}, - 'routes': [], - 'dns': { - 'nameservers': [], - 'search': [], - } - } + self.network_state = copy.deepcopy(self.initial_network_state) def dump(self): state = { @@ -97,19 +132,26 @@ class NetworkState(object): def dump_network_state(self): return dump_config(self.network_state) - def parse_config(self): + def parse_config(self, skip_broken=True): # rebuild network state for command in self.config: - handler = self.command_handlers.get(command['type']) - handler(command) - - def valid_command(self, command, required_keys): - if not required_keys: - return False - - found_keys = [key for key in command.keys() if key in required_keys] - return len(found_keys) == len(required_keys) - + command_type = command['type'] + try: + handler = self.command_handlers[command_type] + except KeyError: + raise RuntimeError("No handler found for" + " command '%s'" % command_type) + try: + handler(command) + except InvalidCommand: + if not skip_broken: + raise + else: + LOG.warn("Skipping invalid command: %s", command, + exc_info=True) + LOG.debug(self.dump_network_state()) + + @ensure_command_keys(['name']) def handle_physical(self, command): ''' command = { @@ -121,13 +163,6 @@ class NetworkState(object): ] } ''' - required_keys = [ - 'name', - ] - if not self.valid_command(command, required_keys): - LOG.warn('Skipping Invalid command: {}'.format(command)) - LOG.debug(self.dump_network_state()) - return interfaces = self.network_state.get('interfaces') iface = interfaces.get(command['name'], {}) @@ -158,6 +193,7 @@ class NetworkState(object): self.network_state['interfaces'].update({command.get('name'): iface}) self.dump_network_state() + @ensure_command_keys(['name', 'vlan_id', 'vlan_link']) def handle_vlan(self, command): ''' auto eth0.222 @@ -167,16 +203,6 @@ class NetworkState(object): hwaddress ether BC:76:4E:06:96:B3 vlan-raw-device eth0 ''' - required_keys = [ - 'name', - 'vlan_link', - 'vlan_id', - ] - if not self.valid_command(command, required_keys): - print('Skipping Invalid command: {}'.format(command)) - print(self.dump_network_state()) - return - interfaces = self.network_state.get('interfaces') self.handle_physical(command) iface = interfaces.get(command.get('name'), {}) @@ -184,6 +210,7 @@ class NetworkState(object): iface['vlan_id'] = command.get('vlan_id') interfaces.update({iface['name']: iface}) + @ensure_command_keys(['name', 'bond_interfaces', 'params']) def handle_bond(self, command): ''' #/etc/network/interfaces @@ -209,15 +236,6 @@ class NetworkState(object): bond-updelay 200 bond-lacp-rate 4 ''' - required_keys = [ - 'name', - 'bond_interfaces', - 'params', - ] - if not self.valid_command(command, required_keys): - print('Skipping Invalid command: {}'.format(command)) - print(self.dump_network_state()) - return self.handle_physical(command) interfaces = self.network_state.get('interfaces') @@ -245,6 +263,7 @@ class NetworkState(object): bond_if.update({param: val}) self.network_state['interfaces'].update({ifname: bond_if}) + @ensure_command_keys(['name', 'bridge_interfaces', 'params']) def handle_bridge(self, command): ''' auto br0 @@ -272,15 +291,6 @@ class NetworkState(object): "bridge_waitport", ] ''' - required_keys = [ - 'name', - 'bridge_interfaces', - 'params', - ] - if not self.valid_command(command, required_keys): - print('Skipping Invalid command: {}'.format(command)) - print(self.dump_network_state()) - return # find one of the bridge port ifaces to get mac_addr # handle bridge_slaves @@ -304,15 +314,8 @@ class NetworkState(object): interfaces.update({iface['name']: iface}) + @ensure_command_keys(['address']) def handle_nameserver(self, command): - required_keys = [ - 'address', - ] - if not self.valid_command(command, required_keys): - print('Skipping Invalid command: {}'.format(command)) - print(self.dump_network_state()) - return - dns = self.network_state.get('dns') if 'address' in command: addrs = command['address'] @@ -327,15 +330,8 @@ class NetworkState(object): for path in paths: dns['search'].append(path) + @ensure_command_keys(['destination']) def handle_route(self, command): - required_keys = [ - 'destination', - ] - if not self.valid_command(command, required_keys): - print('Skipping Invalid command: {}'.format(command)) - print(self.dump_network_state()) - return - routes = self.network_state.get('routes') network, cidr = command['destination'].split("/") netmask = cidr2mask(int(cidr)) -- cgit v1.2.3 From 690bdce9a73ec6d00b4eebbbca31cc527e7b2fce Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 6 May 2016 12:19:24 -0500 Subject: fix timestamp in reporting events. If no timestamp was passed into a ReportingEvent, then the default was used. That default was 'time.time()' which was evaluated once only at import time. --- cloudinit/reporting/events.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/reporting/events.py b/cloudinit/reporting/events.py index 2f767f64..df2b9b4a 100644 --- a/cloudinit/reporting/events.py +++ b/cloudinit/reporting/events.py @@ -33,11 +33,13 @@ class ReportingEvent(object): """Encapsulation of event formatting.""" def __init__(self, event_type, name, description, - origin=DEFAULT_EVENT_ORIGIN, timestamp=time.time()): + origin=DEFAULT_EVENT_ORIGIN, timestamp=None): self.event_type = event_type self.name = name self.description = description self.origin = origin + if timestamp is None: + timestamp = time.time() self.timestamp = timestamp def as_string(self): -- cgit v1.2.3 From 247fbc9ce9ac6f47d670a19f073bda0a1f746669 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 9 May 2016 16:27:59 -0700 Subject: Move the current rendering to a debian distro file This format allows for rendering to work in other distros and clearly separates the API needed to do this (it also moves the klibc parsing into its own module so that the leftover code in net/__init__.py is smaller and only focused on util code). --- cloudinit/net/__init__.py | 553 +------------------------------------- cloudinit/net/distros/__init__.py | 0 cloudinit/net/distros/debian.py | 401 +++++++++++++++++++++++++++ cloudinit/net/klibc.py | 191 +++++++++++++ 4 files changed, 595 insertions(+), 550 deletions(-) create mode 100644 cloudinit/net/distros/__init__.py create mode 100644 cloudinit/net/distros/debian.py create mode 100644 cloudinit/net/klibc.py (limited to 'cloudinit') diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index cc154c57..e911ed0c 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -16,41 +16,18 @@ # You should have received a copy of the GNU Affero General Public License # along with Curtin. If not, see . -import base64 import errno -import glob -import gzip -import io import os -import re -import shlex + from cloudinit import log as logging +from cloudinit.net import network_state from cloudinit import util -from .udev import generate_udev_rule -from . import network_state -LOG = logging.getLogger(__name__) +LOG = logging.getLogger(__name__) SYS_CLASS_NET = "/sys/class/net/" LINKS_FNAME_PREFIX = "etc/systemd/network/50-cloud-init-" - -NET_CONFIG_OPTIONS = [ - "address", "netmask", "broadcast", "network", "metric", "gateway", - "pointtopoint", "media", "mtu", "hostname", "leasehours", "leasetime", - "vendor", "client", "bootfile", "server", "hwaddr", "provider", "frame", - "netnum", "endpoint", "local", "ttl", - ] - -NET_CONFIG_COMMANDS = [ - "pre-up", "up", "post-up", "down", "pre-down", "post-down", - ] - -NET_CONFIG_BRIDGE_OPTIONS = [ - "bridge_ageing", "bridge_bridgeprio", "bridge_fd", "bridge_gcinit", - "bridge_hello", "bridge_maxage", "bridge_maxwait", "bridge_stp", - ] - DEFAULT_PRIMARY_INTERFACE = 'eth0' @@ -130,137 +107,6 @@ class ParserError(Exception): """Raised when parser has issue parsing the interfaces file.""" -def parse_deb_config_data(ifaces, contents, src_dir, src_path): - """Parses the file contents, placing result into ifaces. - - '_source_path' is added to every dictionary entry to define which file - the configration information came from. - - :param ifaces: interface dictionary - :param contents: contents of interfaces file - :param src_dir: directory interfaces file was located - :param src_path: file path the `contents` was read - """ - currif = None - for line in contents.splitlines(): - line = line.strip() - if line.startswith('#'): - continue - split = line.split(' ') - option = split[0] - if option == "source-directory": - parsed_src_dir = split[1] - if not parsed_src_dir.startswith("/"): - parsed_src_dir = os.path.join(src_dir, parsed_src_dir) - for expanded_path in glob.glob(parsed_src_dir): - dir_contents = os.listdir(expanded_path) - dir_contents = [ - os.path.join(expanded_path, path) - for path in dir_contents - if (os.path.isfile(os.path.join(expanded_path, path)) and - re.match("^[a-zA-Z0-9_-]+$", path) is not None) - ] - for entry in dir_contents: - with open(entry, "r") as fp: - src_data = fp.read().strip() - abs_entry = os.path.abspath(entry) - parse_deb_config_data( - ifaces, src_data, - os.path.dirname(abs_entry), abs_entry) - elif option == "source": - new_src_path = split[1] - if not new_src_path.startswith("/"): - new_src_path = os.path.join(src_dir, new_src_path) - for expanded_path in glob.glob(new_src_path): - with open(expanded_path, "r") as fp: - src_data = fp.read().strip() - abs_path = os.path.abspath(expanded_path) - parse_deb_config_data( - ifaces, src_data, - os.path.dirname(abs_path), abs_path) - elif option == "auto": - for iface in split[1:]: - if iface not in ifaces: - ifaces[iface] = { - # Include the source path this interface was found in. - "_source_path": src_path - } - ifaces[iface]['auto'] = True - elif option == "iface": - iface, family, method = split[1:4] - if iface not in ifaces: - ifaces[iface] = { - # Include the source path this interface was found in. - "_source_path": src_path - } - elif 'family' in ifaces[iface]: - raise ParserError( - "Interface %s can only be defined once. " - "Re-defined in '%s'." % (iface, src_path)) - ifaces[iface]['family'] = family - ifaces[iface]['method'] = method - currif = iface - elif option == "hwaddress": - ifaces[currif]['hwaddress'] = split[1] - elif option in NET_CONFIG_OPTIONS: - ifaces[currif][option] = split[1] - elif option in NET_CONFIG_COMMANDS: - if option not in ifaces[currif]: - ifaces[currif][option] = [] - ifaces[currif][option].append(' '.join(split[1:])) - elif option.startswith('dns-'): - if 'dns' not in ifaces[currif]: - ifaces[currif]['dns'] = {} - if option == 'dns-search': - ifaces[currif]['dns']['search'] = [] - for domain in split[1:]: - ifaces[currif]['dns']['search'].append(domain) - elif option == 'dns-nameservers': - ifaces[currif]['dns']['nameservers'] = [] - for server in split[1:]: - ifaces[currif]['dns']['nameservers'].append(server) - elif option.startswith('bridge_'): - if 'bridge' not in ifaces[currif]: - ifaces[currif]['bridge'] = {} - if option in NET_CONFIG_BRIDGE_OPTIONS: - bridge_option = option.replace('bridge_', '', 1) - ifaces[currif]['bridge'][bridge_option] = split[1] - elif option == "bridge_ports": - ifaces[currif]['bridge']['ports'] = [] - for iface in split[1:]: - ifaces[currif]['bridge']['ports'].append(iface) - elif option == "bridge_hw" and split[1].lower() == "mac": - ifaces[currif]['bridge']['mac'] = split[2] - elif option == "bridge_pathcost": - if 'pathcost' not in ifaces[currif]['bridge']: - ifaces[currif]['bridge']['pathcost'] = {} - ifaces[currif]['bridge']['pathcost'][split[1]] = split[2] - elif option == "bridge_portprio": - if 'portprio' not in ifaces[currif]['bridge']: - ifaces[currif]['bridge']['portprio'] = {} - ifaces[currif]['bridge']['portprio'][split[1]] = split[2] - elif option.startswith('bond-'): - if 'bond' not in ifaces[currif]: - ifaces[currif]['bond'] = {} - bond_option = option.replace('bond-', '', 1) - ifaces[currif]['bond'][bond_option] = split[1] - for iface in ifaces.keys(): - if 'auto' not in ifaces[iface]: - ifaces[iface]['auto'] = False - - -def parse_deb_config(path): - """Parses a debian network configuration file.""" - ifaces = {} - with open(path, "r") as fp: - contents = fp.read().strip() - abs_path = os.path.abspath(path) - parse_deb_config_data( - ifaces, contents, - os.path.dirname(abs_path), abs_path) - return ifaces - - def parse_net_config_data(net_config): """Parses the config, returns NetworkState object @@ -287,347 +133,6 @@ def parse_net_config(path): return ns -def _load_shell_content(content, add_empty=False, empty_val=None): - """Given shell like syntax (key=value\nkey2=value2\n) in content - return the data in dictionary form. If 'add_empty' is True - then add entries in to the returned dictionary for 'VAR=' - variables. Set their value to empty_val.""" - data = {} - for line in shlex.split(content): - key, value = line.split("=", 1) - if not value: - value = empty_val - if add_empty or value: - data[key] = value - - return data - - -def _klibc_to_config_entry(content, mac_addrs=None): - """Convert a klibc writtent shell content file to a 'config' entry - When ip= is seen on the kernel command line in debian initramfs - and networking is brought up, ipconfig will populate - /run/net-.cfg. - - The files are shell style syntax, and examples are in the tests - provided here. There is no good documentation on this unfortunately. - - DEVICE= is expected/required and PROTO should indicate if - this is 'static' or 'dhcp'. - """ - - if mac_addrs is None: - mac_addrs = {} - - data = _load_shell_content(content) - try: - name = data['DEVICE'] - except KeyError: - raise ValueError("no 'DEVICE' entry in data") - - # ipconfig on precise does not write PROTO - proto = data.get('PROTO') - if not proto: - if data.get('filename'): - proto = 'dhcp' - else: - proto = 'static' - - if proto not in ('static', 'dhcp'): - raise ValueError("Unexpected value for PROTO: %s" % proto) - - iface = { - 'type': 'physical', - 'name': name, - 'subnets': [], - } - - if name in mac_addrs: - iface['mac_address'] = mac_addrs[name] - - # originally believed there might be IPV6* values - for v, pre in (('ipv4', 'IPV4'),): - # if no IPV4ADDR or IPV6ADDR, then go on. - if pre + "ADDR" not in data: - continue - subnet = {'type': proto, 'control': 'manual'} - - # these fields go right on the subnet - for key in ('NETMASK', 'BROADCAST', 'GATEWAY'): - if pre + key in data: - subnet[key.lower()] = data[pre + key] - - dns = [] - # handle IPV4DNS0 or IPV6DNS0 - for nskey in ('DNS0', 'DNS1'): - ns = data.get(pre + nskey) - # verify it has something other than 0.0.0.0 (or ipv6) - if ns and len(ns.strip(":.0")): - dns.append(data[pre + nskey]) - if dns: - subnet['dns_nameservers'] = dns - # add search to both ipv4 and ipv6, as it has no namespace - search = data.get('DOMAINSEARCH') - if search: - if ',' in search: - subnet['dns_search'] = search.split(",") - else: - subnet['dns_search'] = search.split() - - iface['subnets'].append(subnet) - - return name, iface - - -def config_from_klibc_net_cfg(files=None, mac_addrs=None): - if files is None: - files = glob.glob('/run/net*.conf') - - entries = [] - names = {} - for cfg_file in files: - name, entry = _klibc_to_config_entry(util.load_file(cfg_file), - mac_addrs=mac_addrs) - if name in names: - raise ValueError( - "device '%s' defined multiple times: %s and %s" % ( - name, names[name], cfg_file)) - - names[name] = cfg_file - entries.append(entry) - return {'config': entries, 'version': 1} - - -def render_persistent_net(network_state): - ''' Given state, emit udev rules to map - mac to ifname - ''' - content = "" - interfaces = network_state.get('interfaces') - for iface in interfaces.values(): - # for physical interfaces write out a persist net udev rule - if iface['type'] == 'physical' and \ - 'name' in iface and iface.get('mac_address'): - content += generate_udev_rule(iface['name'], - iface['mac_address']) - - return content - - -# TODO: switch valid_map based on mode inet/inet6 -def iface_add_subnet(iface, subnet): - content = "" - valid_map = [ - 'address', - 'netmask', - 'broadcast', - 'metric', - 'gateway', - 'pointopoint', - 'mtu', - 'scope', - 'dns_search', - 'dns_nameservers', - ] - for key, value in subnet.items(): - if value and key in valid_map: - if type(value) == list: - value = " ".join(value) - if '_' in key: - key = key.replace('_', '-') - content += " {} {}\n".format(key, value) - - return content - - -# TODO: switch to valid_map for attrs -def iface_add_attrs(iface): - content = "" - ignore_map = [ - 'control', - 'index', - 'inet', - 'mode', - 'name', - 'subnets', - 'type', - ] - if iface['type'] not in ['bond', 'bridge', 'vlan']: - ignore_map.append('mac_address') - - for key, value in iface.items(): - if value and key not in ignore_map: - if type(value) == list: - value = " ".join(value) - content += " {} {}\n".format(key, value) - - return content - - -def render_route(route, indent=""): - """ When rendering routes for an iface, in some cases applying a route - may result in the route command returning non-zero which produces - some confusing output for users manually using ifup/ifdown[1]. To - that end, we will optionally include an '|| true' postfix to each - route line allowing users to work with ifup/ifdown without using - --force option. - - We may at somepoint not want to emit this additional postfix, and - add a 'strict' flag to this function. When called with strict=True, - then we will not append the postfix. - - 1. http://askubuntu.com/questions/168033/ - how-to-set-static-routes-in-ubuntu-server - """ - content = "" - up = indent + "post-up route add" - down = indent + "pre-down route del" - eol = " || true\n" - mapping = { - 'network': '-net', - 'netmask': 'netmask', - 'gateway': 'gw', - 'metric': 'metric', - } - if route['network'] == '0.0.0.0' and route['netmask'] == '0.0.0.0': - default_gw = " default gw %s" % route['gateway'] - content += up + default_gw + eol - content += down + default_gw + eol - elif route['network'] == '::' and route['netmask'] == 0: - # ipv6! - default_gw = " -A inet6 default gw %s" % route['gateway'] - content += up + default_gw + eol - content += down + default_gw + eol - else: - route_line = "" - for k in ['network', 'netmask', 'gateway', 'metric']: - if k in route: - route_line += " %s %s" % (mapping[k], route[k]) - content += up + route_line + eol - content += down + route_line + eol - - return content - - -def iface_start_entry(iface, index): - fullname = iface['name'] - if index != 0: - fullname += ":%s" % index - - control = iface['control'] - if control == "auto": - cverb = "auto" - elif control in ("hotplug",): - cverb = "allow-" + control - else: - cverb = "# control-" + control - - subst = iface.copy() - subst.update({'fullname': fullname, 'cverb': cverb}) - - return ("{cverb} {fullname}\n" - "iface {fullname} {inet} {mode}\n").format(**subst) - - -def render_interfaces(network_state): - ''' Given state, emit etc/network/interfaces content ''' - - content = "" - interfaces = network_state.get('interfaces') - ''' Apply a sort order to ensure that we write out - the physical interfaces first; this is critical for - bonding - ''' - order = { - 'physical': 0, - 'bond': 1, - 'bridge': 2, - 'vlan': 3, - } - content += "auto lo\niface lo inet loopback\n" - for dnskey, value in network_state.get('dns', {}).items(): - if len(value): - content += " dns-{} {}\n".format(dnskey, " ".join(value)) - - for iface in sorted(interfaces.values(), - key=lambda k: (order[k['type']], k['name'])): - - if content[-2:] != "\n\n": - content += "\n" - subnets = iface.get('subnets', {}) - if subnets: - for index, subnet in zip(range(0, len(subnets)), subnets): - if content[-2:] != "\n\n": - content += "\n" - iface['index'] = index - iface['mode'] = subnet['type'] - iface['control'] = subnet.get('control', 'auto') - if iface['mode'].endswith('6'): - iface['inet'] += '6' - elif iface['mode'] == 'static' and ":" in subnet['address']: - iface['inet'] += '6' - if iface['mode'].startswith('dhcp'): - iface['mode'] = 'dhcp' - - content += iface_start_entry(iface, index) - content += iface_add_subnet(iface, subnet) - content += iface_add_attrs(iface) - else: - # ifenslave docs say to auto the slave devices - if 'bond-master' in iface: - content += "auto {name}\n".format(**iface) - content += "iface {name} {inet} {mode}\n".format(**iface) - content += iface_add_attrs(iface) - - for route in network_state.get('routes'): - content += render_route(route) - - # global replacements until v2 format - content = content.replace('mac_address', 'hwaddress') - return content - - -def render_network_state(target, network_state, eni="etc/network/interfaces", - links_prefix=LINKS_FNAME_PREFIX, - netrules='etc/udev/rules.d/70-persistent-net.rules'): - - fpeni = os.path.sep.join((target, eni,)) - util.ensure_dir(os.path.dirname(fpeni)) - with open(fpeni, 'w+') as f: - f.write(render_interfaces(network_state)) - - if netrules: - netrules = os.path.sep.join((target, netrules,)) - util.ensure_dir(os.path.dirname(netrules)) - with open(netrules, 'w+') as f: - f.write(render_persistent_net(network_state)) - - if links_prefix: - render_systemd_links(target, network_state, links_prefix) - - -def render_systemd_links(target, network_state, - links_prefix=LINKS_FNAME_PREFIX): - fp_prefix = os.path.sep.join((target, links_prefix)) - for f in glob.glob(fp_prefix + "*"): - os.unlink(f) - - interfaces = network_state.get('interfaces') - for iface in interfaces.values(): - if (iface['type'] == 'physical' and 'name' in iface and - iface.get('mac_address')): - fname = fp_prefix + iface['name'] + ".link" - with open(fname, "w") as fp: - fp.write("\n".join([ - "[Match]", - "MACAddress=" + iface['mac_address'], - "", - "[Link]", - "Name=" + iface['name'], - "" - ])) - - def is_disabled_cfg(cfg): if not cfg or not isinstance(cfg, dict): return False @@ -718,56 +223,4 @@ def generate_fallback_config(): return nconf -def _decomp_gzip(blob, strict=True): - # decompress blob. raise exception if not compressed unless strict=False. - with io.BytesIO(blob) as iobuf: - gzfp = None - try: - gzfp = gzip.GzipFile(mode="rb", fileobj=iobuf) - return gzfp.read() - except IOError: - if strict: - raise - return blob - finally: - if gzfp: - gzfp.close() - - -def _b64dgz(b64str, gzipped="try"): - # decode a base64 string. If gzipped is true, transparently uncompresss - # if gzipped is 'try', then try gunzip, returning the original on fail. - try: - blob = base64.b64decode(b64str) - except TypeError: - raise ValueError("Invalid base64 text: %s" % b64str) - - if not gzipped: - return blob - - return _decomp_gzip(blob, strict=gzipped != "try") - - -def read_kernel_cmdline_config(files=None, mac_addrs=None, cmdline=None): - if cmdline is None: - cmdline = util.get_cmdline() - - if 'network-config=' in cmdline: - data64 = None - for tok in cmdline.split(): - if tok.startswith("network-config="): - data64 = tok.split("=", 1)[1] - if data64: - return util.load_yaml(_b64dgz(data64)) - - if 'ip=' not in cmdline: - return None - - if mac_addrs is None: - mac_addrs = {k: sys_netdev_info(k, 'address') - for k in get_devicelist()} - - return config_from_klibc_net_cfg(files=files, mac_addrs=mac_addrs) - - # vi: ts=4 expandtab syntax=python diff --git a/cloudinit/net/distros/__init__.py b/cloudinit/net/distros/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cloudinit/net/distros/debian.py b/cloudinit/net/distros/debian.py new file mode 100644 index 00000000..3ab0483e --- /dev/null +++ b/cloudinit/net/distros/debian.py @@ -0,0 +1,401 @@ +# vi: ts=4 expandtab +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import glob +import os +import re + +from cloudinit.net import LINKS_FNAME_PREFIX +from cloudinit.net import ParserError +from cloudinit.net.udev import generate_udev_rule +from cloudinit import util + + +NET_CONFIG_COMMANDS = [ + "pre-up", "up", "post-up", "down", "pre-down", "post-down", +] + +NET_CONFIG_BRIDGE_OPTIONS = [ + "bridge_ageing", "bridge_bridgeprio", "bridge_fd", "bridge_gcinit", + "bridge_hello", "bridge_maxage", "bridge_maxwait", "bridge_stp", +] + +NET_CONFIG_OPTIONS = [ + "address", "netmask", "broadcast", "network", "metric", "gateway", + "pointtopoint", "media", "mtu", "hostname", "leasehours", "leasetime", + "vendor", "client", "bootfile", "server", "hwaddr", "provider", "frame", + "netnum", "endpoint", "local", "ttl", +] + + +# TODO: switch valid_map based on mode inet/inet6 +def _iface_add_subnet(iface, subnet): + content = "" + valid_map = [ + 'address', + 'netmask', + 'broadcast', + 'metric', + 'gateway', + 'pointopoint', + 'mtu', + 'scope', + 'dns_search', + 'dns_nameservers', + ] + for key, value in subnet.items(): + if value and key in valid_map: + if type(value) == list: + value = " ".join(value) + if '_' in key: + key = key.replace('_', '-') + content += " {} {}\n".format(key, value) + + return content + + +# TODO: switch to valid_map for attrs +def _iface_add_attrs(iface): + content = "" + ignore_map = [ + 'control', + 'index', + 'inet', + 'mode', + 'name', + 'subnets', + 'type', + ] + if iface['type'] not in ['bond', 'bridge', 'vlan']: + ignore_map.append('mac_address') + + for key, value in iface.items(): + if value and key not in ignore_map: + if type(value) == list: + value = " ".join(value) + content += " {} {}\n".format(key, value) + + return content + + +def _iface_start_entry(iface, index): + fullname = iface['name'] + if index != 0: + fullname += ":%s" % index + + control = iface['control'] + if control == "auto": + cverb = "auto" + elif control in ("hotplug",): + cverb = "allow-" + control + else: + cverb = "# control-" + control + + subst = iface.copy() + subst.update({'fullname': fullname, 'cverb': cverb}) + + return ("{cverb} {fullname}\n" + "iface {fullname} {inet} {mode}\n").format(**subst) + + +def _parse_deb_config_data(ifaces, contents, src_dir, src_path): + """Parses the file contents, placing result into ifaces. + + '_source_path' is added to every dictionary entry to define which file + the configration information came from. + + :param ifaces: interface dictionary + :param contents: contents of interfaces file + :param src_dir: directory interfaces file was located + :param src_path: file path the `contents` was read + """ + currif = None + for line in contents.splitlines(): + line = line.strip() + if line.startswith('#'): + continue + split = line.split(' ') + option = split[0] + if option == "source-directory": + parsed_src_dir = split[1] + if not parsed_src_dir.startswith("/"): + parsed_src_dir = os.path.join(src_dir, parsed_src_dir) + for expanded_path in glob.glob(parsed_src_dir): + dir_contents = os.listdir(expanded_path) + dir_contents = [ + os.path.join(expanded_path, path) + for path in dir_contents + if (os.path.isfile(os.path.join(expanded_path, path)) and + re.match("^[a-zA-Z0-9_-]+$", path) is not None) + ] + for entry in dir_contents: + with open(entry, "r") as fp: + src_data = fp.read().strip() + abs_entry = os.path.abspath(entry) + _parse_deb_config_data( + ifaces, src_data, + os.path.dirname(abs_entry), abs_entry) + elif option == "source": + new_src_path = split[1] + if not new_src_path.startswith("/"): + new_src_path = os.path.join(src_dir, new_src_path) + for expanded_path in glob.glob(new_src_path): + with open(expanded_path, "r") as fp: + src_data = fp.read().strip() + abs_path = os.path.abspath(expanded_path) + _parse_deb_config_data( + ifaces, src_data, + os.path.dirname(abs_path), abs_path) + elif option == "auto": + for iface in split[1:]: + if iface not in ifaces: + ifaces[iface] = { + # Include the source path this interface was found in. + "_source_path": src_path + } + ifaces[iface]['auto'] = True + elif option == "iface": + iface, family, method = split[1:4] + if iface not in ifaces: + ifaces[iface] = { + # Include the source path this interface was found in. + "_source_path": src_path + } + elif 'family' in ifaces[iface]: + raise ParserError( + "Interface %s can only be defined once. " + "Re-defined in '%s'." % (iface, src_path)) + ifaces[iface]['family'] = family + ifaces[iface]['method'] = method + currif = iface + elif option == "hwaddress": + ifaces[currif]['hwaddress'] = split[1] + elif option in NET_CONFIG_OPTIONS: + ifaces[currif][option] = split[1] + elif option in NET_CONFIG_COMMANDS: + if option not in ifaces[currif]: + ifaces[currif][option] = [] + ifaces[currif][option].append(' '.join(split[1:])) + elif option.startswith('dns-'): + if 'dns' not in ifaces[currif]: + ifaces[currif]['dns'] = {} + if option == 'dns-search': + ifaces[currif]['dns']['search'] = [] + for domain in split[1:]: + ifaces[currif]['dns']['search'].append(domain) + elif option == 'dns-nameservers': + ifaces[currif]['dns']['nameservers'] = [] + for server in split[1:]: + ifaces[currif]['dns']['nameservers'].append(server) + elif option.startswith('bridge_'): + if 'bridge' not in ifaces[currif]: + ifaces[currif]['bridge'] = {} + if option in NET_CONFIG_BRIDGE_OPTIONS: + bridge_option = option.replace('bridge_', '', 1) + ifaces[currif]['bridge'][bridge_option] = split[1] + elif option == "bridge_ports": + ifaces[currif]['bridge']['ports'] = [] + for iface in split[1:]: + ifaces[currif]['bridge']['ports'].append(iface) + elif option == "bridge_hw" and split[1].lower() == "mac": + ifaces[currif]['bridge']['mac'] = split[2] + elif option == "bridge_pathcost": + if 'pathcost' not in ifaces[currif]['bridge']: + ifaces[currif]['bridge']['pathcost'] = {} + ifaces[currif]['bridge']['pathcost'][split[1]] = split[2] + elif option == "bridge_portprio": + if 'portprio' not in ifaces[currif]['bridge']: + ifaces[currif]['bridge']['portprio'] = {} + ifaces[currif]['bridge']['portprio'][split[1]] = split[2] + elif option.startswith('bond-'): + if 'bond' not in ifaces[currif]: + ifaces[currif]['bond'] = {} + bond_option = option.replace('bond-', '', 1) + ifaces[currif]['bond'][bond_option] = split[1] + for iface in ifaces.keys(): + if 'auto' not in ifaces[iface]: + ifaces[iface]['auto'] = False + + +def _parse_deb_config(path): + """Parses a debian network configuration file.""" + ifaces = {} + with open(path, "r") as fp: + contents = fp.read().strip() + abs_path = os.path.abspath(path) + _parse_deb_config_data( + ifaces, contents, + os.path.dirname(abs_path), abs_path) + return ifaces + + +class Renderer(object): + """Renders network information in a debian format.""" + + def render_persistent_net(self, network_state): + ''' Given state, emit udev rules to map + mac to ifname + ''' + content = "" + interfaces = network_state.get('interfaces') + for iface in interfaces.values(): + # for physical interfaces write out a persist net udev rule + if iface['type'] == 'physical' and \ + 'name' in iface and iface.get('mac_address'): + content += generate_udev_rule(iface['name'], + iface['mac_address']) + + return content + + def render_routes(self, route, indent=""): + """ When rendering routes for an iface, in some cases applying a route + may result in the route command returning non-zero which produces + some confusing output for users manually using ifup/ifdown[1]. To + that end, we will optionally include an '|| true' postfix to each + route line allowing users to work with ifup/ifdown without using + --force option. + + We may at somepoint not want to emit this additional postfix, and + add a 'strict' flag to this function. When called with strict=True, + then we will not append the postfix. + + 1. http://askubuntu.com/questions/168033/ + how-to-set-static-routes-in-ubuntu-server + """ + content = "" + up = indent + "post-up route add" + down = indent + "pre-down route del" + eol = " || true\n" + mapping = { + 'network': '-net', + 'netmask': 'netmask', + 'gateway': 'gw', + 'metric': 'metric', + } + if route['network'] == '0.0.0.0' and route['netmask'] == '0.0.0.0': + default_gw = " default gw %s" % route['gateway'] + content += up + default_gw + eol + content += down + default_gw + eol + elif route['network'] == '::' and route['netmask'] == 0: + # ipv6! + default_gw = " -A inet6 default gw %s" % route['gateway'] + content += up + default_gw + eol + content += down + default_gw + eol + else: + route_line = "" + for k in ['network', 'netmask', 'gateway', 'metric']: + if k in route: + route_line += " %s %s" % (mapping[k], route[k]) + content += up + route_line + eol + content += down + route_line + eol + + return content + + def render_interfaces(self, network_state): + ''' Given state, emit etc/network/interfaces content ''' + + content = "" + interfaces = network_state.get('interfaces') + ''' Apply a sort order to ensure that we write out + the physical interfaces first; this is critical for + bonding + ''' + order = { + 'physical': 0, + 'bond': 1, + 'bridge': 2, + 'vlan': 3, + } + content += "auto lo\niface lo inet loopback\n" + for dnskey, value in network_state.get('dns', {}).items(): + if len(value): + content += " dns-{} {}\n".format(dnskey, " ".join(value)) + + for iface in sorted(interfaces.values(), + key=lambda k: (order[k['type']], k['name'])): + + if content[-2:] != "\n\n": + content += "\n" + subnets = iface.get('subnets', {}) + if subnets: + for index, subnet in zip(range(0, len(subnets)), subnets): + if content[-2:] != "\n\n": + content += "\n" + iface['index'] = index + iface['mode'] = subnet['type'] + iface['control'] = subnet.get('control', 'auto') + if iface['mode'].endswith('6'): + iface['inet'] += '6' + elif iface['mode'] == 'static' \ + and ":" in subnet['address']: + iface['inet'] += '6' + if iface['mode'].startswith('dhcp'): + iface['mode'] = 'dhcp' + + content += _iface_start_entry(iface, index) + content += _iface_add_subnet(iface, subnet) + content += _iface_add_attrs(iface) + else: + # ifenslave docs say to auto the slave devices + if 'bond-master' in iface: + content += "auto {name}\n".format(**iface) + content += "iface {name} {inet} {mode}\n".format(**iface) + content += _iface_add_attrs(iface) + + for route in network_state.get('routes'): + content += self.render_route(route) + + # global replacements until v2 format + content = content.replace('mac_address', 'hwaddress') + return content + + def render_network_state(self, + target, network_state, eni="etc/network/interfaces", + links_prefix=LINKS_FNAME_PREFIX, + netrules='etc/udev/rules.d/70-persistent-net.rules'): + + fpeni = os.path.sep.join((target, eni,)) + util.ensure_dir(os.path.dirname(fpeni)) + with open(fpeni, 'w+') as f: + f.write(self.render_interfaces(network_state)) + + if netrules: + netrules = os.path.sep.join((target, netrules,)) + util.ensure_dir(os.path.dirname(netrules)) + with open(netrules, 'w+') as f: + f.write(self.render_persistent_net(network_state)) + + if links_prefix: + self.render_systemd_links(target, network_state, links_prefix) + + def render_systemd_links(self, target, network_state, + links_prefix=LINKS_FNAME_PREFIX): + fp_prefix = os.path.sep.join((target, links_prefix)) + for f in glob.glob(fp_prefix + "*"): + os.unlink(f) + + interfaces = network_state.get('interfaces') + for iface in interfaces.values(): + if (iface['type'] == 'physical' and 'name' in iface and + iface.get('mac_address')): + fname = fp_prefix + iface['name'] + ".link" + with open(fname, "w") as fp: + fp.write("\n".join([ + "[Match]", + "MACAddress=" + iface['mac_address'], + "", + "[Link]", + "Name=" + iface['name'], + "" + ])) diff --git a/cloudinit/net/klibc.py b/cloudinit/net/klibc.py new file mode 100644 index 00000000..958c264b --- /dev/null +++ b/cloudinit/net/klibc.py @@ -0,0 +1,191 @@ +# Copyright (C) 2013-2014 Canonical Ltd. +# +# Author: Scott Moser +# Author: Blake Rouse +# +# Curtin is free software: you can redistribute it and/or modify it under +# the terms of the GNU Affero General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# Curtin is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for +# more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Curtin. If not, see . + +import base64 +import glob +import gzip +import io +import shlex + +from cloudinit.net import get_devicelist +from cloudinit.net import sys_netdev_info + +from cloudinit import util + + +def _load_shell_content(content, add_empty=False, empty_val=None): + """Given shell like syntax (key=value\nkey2=value2\n) in content + return the data in dictionary form. If 'add_empty' is True + then add entries in to the returned dictionary for 'VAR=' + variables. Set their value to empty_val.""" + data = {} + for line in shlex.split(content): + key, value = line.split("=", 1) + if not value: + value = empty_val + if add_empty or value: + data[key] = value + + return data + + +def _klibc_to_config_entry(content, mac_addrs=None): + """Convert a klibc writtent shell content file to a 'config' entry + When ip= is seen on the kernel command line in debian initramfs + and networking is brought up, ipconfig will populate + /run/net-.cfg. + + The files are shell style syntax, and examples are in the tests + provided here. There is no good documentation on this unfortunately. + + DEVICE= is expected/required and PROTO should indicate if + this is 'static' or 'dhcp'. + """ + + if mac_addrs is None: + mac_addrs = {} + + data = _load_shell_content(content) + try: + name = data['DEVICE'] + except KeyError: + raise ValueError("no 'DEVICE' entry in data") + + # ipconfig on precise does not write PROTO + proto = data.get('PROTO') + if not proto: + if data.get('filename'): + proto = 'dhcp' + else: + proto = 'static' + + if proto not in ('static', 'dhcp'): + raise ValueError("Unexpected value for PROTO: %s" % proto) + + iface = { + 'type': 'physical', + 'name': name, + 'subnets': [], + } + + if name in mac_addrs: + iface['mac_address'] = mac_addrs[name] + + # originally believed there might be IPV6* values + for v, pre in (('ipv4', 'IPV4'),): + # if no IPV4ADDR or IPV6ADDR, then go on. + if pre + "ADDR" not in data: + continue + subnet = {'type': proto, 'control': 'manual'} + + # these fields go right on the subnet + for key in ('NETMASK', 'BROADCAST', 'GATEWAY'): + if pre + key in data: + subnet[key.lower()] = data[pre + key] + + dns = [] + # handle IPV4DNS0 or IPV6DNS0 + for nskey in ('DNS0', 'DNS1'): + ns = data.get(pre + nskey) + # verify it has something other than 0.0.0.0 (or ipv6) + if ns and len(ns.strip(":.0")): + dns.append(data[pre + nskey]) + if dns: + subnet['dns_nameservers'] = dns + # add search to both ipv4 and ipv6, as it has no namespace + search = data.get('DOMAINSEARCH') + if search: + if ',' in search: + subnet['dns_search'] = search.split(",") + else: + subnet['dns_search'] = search.split() + + iface['subnets'].append(subnet) + + return name, iface + + +def config_from_klibc_net_cfg(files=None, mac_addrs=None): + if files is None: + files = glob.glob('/run/net*.conf') + + entries = [] + names = {} + for cfg_file in files: + name, entry = _klibc_to_config_entry(util.load_file(cfg_file), + mac_addrs=mac_addrs) + if name in names: + raise ValueError( + "device '%s' defined multiple times: %s and %s" % ( + name, names[name], cfg_file)) + + names[name] = cfg_file + entries.append(entry) + return {'config': entries, 'version': 1} + + +def _decomp_gzip(blob, strict=True): + # decompress blob. raise exception if not compressed unless strict=False. + with io.BytesIO(blob) as iobuf: + gzfp = None + try: + gzfp = gzip.GzipFile(mode="rb", fileobj=iobuf) + return gzfp.read() + except IOError: + if strict: + raise + return blob + finally: + if gzfp: + gzfp.close() + + +def _b64dgz(b64str, gzipped="try"): + # decode a base64 string. If gzipped is true, transparently uncompresss + # if gzipped is 'try', then try gunzip, returning the original on fail. + try: + blob = base64.b64decode(b64str) + except TypeError: + raise ValueError("Invalid base64 text: %s" % b64str) + + if not gzipped: + return blob + + return _decomp_gzip(blob, strict=gzipped != "try") + + +def read_kernel_cmdline_config(files=None, mac_addrs=None, cmdline=None): + if cmdline is None: + cmdline = util.get_cmdline() + + if 'network-config=' in cmdline: + data64 = None + for tok in cmdline.split(): + if tok.startswith("network-config="): + data64 = tok.split("=", 1)[1] + if data64: + return util.load_yaml(_b64dgz(data64)) + + if 'ip=' not in cmdline: + return None + + if mac_addrs is None: + mac_addrs = {k: sys_netdev_info(k, 'address') + for k in get_devicelist()} + + return config_from_klibc_net_cfg(files=files, mac_addrs=mac_addrs) -- cgit v1.2.3 From a800da2371b85b2ece1a30de00f988035819c958 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 9 May 2016 16:36:04 -0700 Subject: Use the new renderer in the debian and stages files --- cloudinit/distros/debian.py | 10 ++++++---- cloudinit/net/distros/debian.py | 2 +- cloudinit/stages.py | 4 ++-- 3 files changed, 9 insertions(+), 7 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 75ab340f..e8fc1df4 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -27,6 +27,7 @@ from cloudinit import helpers from cloudinit import log as logging from cloudinit import util from cloudinit import net +from cloudinit.net.distros import debian from cloudinit.distros.parsers.hostname import HostnameConf @@ -56,6 +57,7 @@ class Distro(distros.Distro): # should only happen say once per instance...) self._runner = helpers.Runners(paths) self.osfamily = 'debian' + self.renderer = debian.Renderer() def apply_locale(self, locale, out_fn=None): if not out_fn: @@ -80,10 +82,10 @@ class Distro(distros.Distro): def _write_network_config(self, netconfig): ns = net.parse_net_config_data(netconfig) - net.render_network_state(target="/", network_state=ns, - eni=self.network_conf_fn, - links_prefix=self.links_prefix, - netrules=None) + self.renderer.render_network_state( + target="/", network_state=ns, + eni=self.network_conf_fn, links_prefix=self.links_prefix, + netrules=None) _maybe_remove_legacy_eth0() return [] diff --git a/cloudinit/net/distros/debian.py b/cloudinit/net/distros/debian.py index 3ab0483e..4bf34fd7 100644 --- a/cloudinit/net/distros/debian.py +++ b/cloudinit/net/distros/debian.py @@ -258,7 +258,7 @@ class Renderer(object): return content - def render_routes(self, route, indent=""): + def render_route(self, route, indent=""): """ When rendering routes for an iface, in some cases applying a route may result in the route command returning non-zero which produces some confusing output for users manually using ifup/ifdown[1]. To diff --git a/cloudinit/stages.py b/cloudinit/stages.py index ffb15165..e6bd34fe 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -43,7 +43,7 @@ from cloudinit import distros from cloudinit import helpers from cloudinit import importer from cloudinit import log as logging -from cloudinit import net +from cloudinit.net import klibc from cloudinit import sources from cloudinit import type_utils from cloudinit import util @@ -579,7 +579,7 @@ class Init(object): if os.path.exists(disable_file): return (None, disable_file) - cmdline_cfg = ('cmdline', net.read_kernel_cmdline_config()) + cmdline_cfg = ('cmdline', klibc.read_kernel_cmdline_config()) dscfg = ('ds', None) if self.datasource and hasattr(self.datasource, 'network_config'): dscfg = ('ds', self.datasource.network_config) -- cgit v1.2.3 From e459e3525cf8321245a2f29b869e9978eb15f2c9 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 9 May 2016 16:50:17 -0700 Subject: Rename renderer attribute to _net_renderer --- cloudinit/distros/debian.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index e8fc1df4..21a4f805 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -57,7 +57,7 @@ class Distro(distros.Distro): # should only happen say once per instance...) self._runner = helpers.Runners(paths) self.osfamily = 'debian' - self.renderer = debian.Renderer() + self._net_renderer = debian.Renderer() def apply_locale(self, locale, out_fn=None): if not out_fn: @@ -82,7 +82,7 @@ class Distro(distros.Distro): def _write_network_config(self, netconfig): ns = net.parse_net_config_data(netconfig) - self.renderer.render_network_state( + self._net_renderer.render_network_state( target="/", network_state=ns, eni=self.network_conf_fn, links_prefix=self.links_prefix, netrules=None) -- cgit v1.2.3 From 483545957dca362452b94fc49064e298fc07dd71 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 10 May 2016 13:02:29 -0700 Subject: Rename to net/renderers and klibc -> cmdline.py --- cloudinit/net/cmdline.py | 191 +++++++++++++++++ cloudinit/net/distros/__init__.py | 0 cloudinit/net/distros/debian.py | 401 ------------------------------------ cloudinit/net/klibc.py | 191 ----------------- cloudinit/net/renderers/__init__.py | 0 cloudinit/net/renderers/eni.py | 399 +++++++++++++++++++++++++++++++++++ cloudinit/stages.py | 4 +- 7 files changed, 592 insertions(+), 594 deletions(-) create mode 100644 cloudinit/net/cmdline.py delete mode 100644 cloudinit/net/distros/__init__.py delete mode 100644 cloudinit/net/distros/debian.py delete mode 100644 cloudinit/net/klibc.py create mode 100644 cloudinit/net/renderers/__init__.py create mode 100644 cloudinit/net/renderers/eni.py (limited to 'cloudinit') diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py new file mode 100644 index 00000000..958c264b --- /dev/null +++ b/cloudinit/net/cmdline.py @@ -0,0 +1,191 @@ +# Copyright (C) 2013-2014 Canonical Ltd. +# +# Author: Scott Moser +# Author: Blake Rouse +# +# Curtin is free software: you can redistribute it and/or modify it under +# the terms of the GNU Affero General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# Curtin is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for +# more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Curtin. If not, see . + +import base64 +import glob +import gzip +import io +import shlex + +from cloudinit.net import get_devicelist +from cloudinit.net import sys_netdev_info + +from cloudinit import util + + +def _load_shell_content(content, add_empty=False, empty_val=None): + """Given shell like syntax (key=value\nkey2=value2\n) in content + return the data in dictionary form. If 'add_empty' is True + then add entries in to the returned dictionary for 'VAR=' + variables. Set their value to empty_val.""" + data = {} + for line in shlex.split(content): + key, value = line.split("=", 1) + if not value: + value = empty_val + if add_empty or value: + data[key] = value + + return data + + +def _klibc_to_config_entry(content, mac_addrs=None): + """Convert a klibc writtent shell content file to a 'config' entry + When ip= is seen on the kernel command line in debian initramfs + and networking is brought up, ipconfig will populate + /run/net-.cfg. + + The files are shell style syntax, and examples are in the tests + provided here. There is no good documentation on this unfortunately. + + DEVICE= is expected/required and PROTO should indicate if + this is 'static' or 'dhcp'. + """ + + if mac_addrs is None: + mac_addrs = {} + + data = _load_shell_content(content) + try: + name = data['DEVICE'] + except KeyError: + raise ValueError("no 'DEVICE' entry in data") + + # ipconfig on precise does not write PROTO + proto = data.get('PROTO') + if not proto: + if data.get('filename'): + proto = 'dhcp' + else: + proto = 'static' + + if proto not in ('static', 'dhcp'): + raise ValueError("Unexpected value for PROTO: %s" % proto) + + iface = { + 'type': 'physical', + 'name': name, + 'subnets': [], + } + + if name in mac_addrs: + iface['mac_address'] = mac_addrs[name] + + # originally believed there might be IPV6* values + for v, pre in (('ipv4', 'IPV4'),): + # if no IPV4ADDR or IPV6ADDR, then go on. + if pre + "ADDR" not in data: + continue + subnet = {'type': proto, 'control': 'manual'} + + # these fields go right on the subnet + for key in ('NETMASK', 'BROADCAST', 'GATEWAY'): + if pre + key in data: + subnet[key.lower()] = data[pre + key] + + dns = [] + # handle IPV4DNS0 or IPV6DNS0 + for nskey in ('DNS0', 'DNS1'): + ns = data.get(pre + nskey) + # verify it has something other than 0.0.0.0 (or ipv6) + if ns and len(ns.strip(":.0")): + dns.append(data[pre + nskey]) + if dns: + subnet['dns_nameservers'] = dns + # add search to both ipv4 and ipv6, as it has no namespace + search = data.get('DOMAINSEARCH') + if search: + if ',' in search: + subnet['dns_search'] = search.split(",") + else: + subnet['dns_search'] = search.split() + + iface['subnets'].append(subnet) + + return name, iface + + +def config_from_klibc_net_cfg(files=None, mac_addrs=None): + if files is None: + files = glob.glob('/run/net*.conf') + + entries = [] + names = {} + for cfg_file in files: + name, entry = _klibc_to_config_entry(util.load_file(cfg_file), + mac_addrs=mac_addrs) + if name in names: + raise ValueError( + "device '%s' defined multiple times: %s and %s" % ( + name, names[name], cfg_file)) + + names[name] = cfg_file + entries.append(entry) + return {'config': entries, 'version': 1} + + +def _decomp_gzip(blob, strict=True): + # decompress blob. raise exception if not compressed unless strict=False. + with io.BytesIO(blob) as iobuf: + gzfp = None + try: + gzfp = gzip.GzipFile(mode="rb", fileobj=iobuf) + return gzfp.read() + except IOError: + if strict: + raise + return blob + finally: + if gzfp: + gzfp.close() + + +def _b64dgz(b64str, gzipped="try"): + # decode a base64 string. If gzipped is true, transparently uncompresss + # if gzipped is 'try', then try gunzip, returning the original on fail. + try: + blob = base64.b64decode(b64str) + except TypeError: + raise ValueError("Invalid base64 text: %s" % b64str) + + if not gzipped: + return blob + + return _decomp_gzip(blob, strict=gzipped != "try") + + +def read_kernel_cmdline_config(files=None, mac_addrs=None, cmdline=None): + if cmdline is None: + cmdline = util.get_cmdline() + + if 'network-config=' in cmdline: + data64 = None + for tok in cmdline.split(): + if tok.startswith("network-config="): + data64 = tok.split("=", 1)[1] + if data64: + return util.load_yaml(_b64dgz(data64)) + + if 'ip=' not in cmdline: + return None + + if mac_addrs is None: + mac_addrs = {k: sys_netdev_info(k, 'address') + for k in get_devicelist()} + + return config_from_klibc_net_cfg(files=files, mac_addrs=mac_addrs) diff --git a/cloudinit/net/distros/__init__.py b/cloudinit/net/distros/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/cloudinit/net/distros/debian.py b/cloudinit/net/distros/debian.py deleted file mode 100644 index 4bf34fd7..00000000 --- a/cloudinit/net/distros/debian.py +++ /dev/null @@ -1,401 +0,0 @@ -# vi: ts=4 expandtab -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import glob -import os -import re - -from cloudinit.net import LINKS_FNAME_PREFIX -from cloudinit.net import ParserError -from cloudinit.net.udev import generate_udev_rule -from cloudinit import util - - -NET_CONFIG_COMMANDS = [ - "pre-up", "up", "post-up", "down", "pre-down", "post-down", -] - -NET_CONFIG_BRIDGE_OPTIONS = [ - "bridge_ageing", "bridge_bridgeprio", "bridge_fd", "bridge_gcinit", - "bridge_hello", "bridge_maxage", "bridge_maxwait", "bridge_stp", -] - -NET_CONFIG_OPTIONS = [ - "address", "netmask", "broadcast", "network", "metric", "gateway", - "pointtopoint", "media", "mtu", "hostname", "leasehours", "leasetime", - "vendor", "client", "bootfile", "server", "hwaddr", "provider", "frame", - "netnum", "endpoint", "local", "ttl", -] - - -# TODO: switch valid_map based on mode inet/inet6 -def _iface_add_subnet(iface, subnet): - content = "" - valid_map = [ - 'address', - 'netmask', - 'broadcast', - 'metric', - 'gateway', - 'pointopoint', - 'mtu', - 'scope', - 'dns_search', - 'dns_nameservers', - ] - for key, value in subnet.items(): - if value and key in valid_map: - if type(value) == list: - value = " ".join(value) - if '_' in key: - key = key.replace('_', '-') - content += " {} {}\n".format(key, value) - - return content - - -# TODO: switch to valid_map for attrs -def _iface_add_attrs(iface): - content = "" - ignore_map = [ - 'control', - 'index', - 'inet', - 'mode', - 'name', - 'subnets', - 'type', - ] - if iface['type'] not in ['bond', 'bridge', 'vlan']: - ignore_map.append('mac_address') - - for key, value in iface.items(): - if value and key not in ignore_map: - if type(value) == list: - value = " ".join(value) - content += " {} {}\n".format(key, value) - - return content - - -def _iface_start_entry(iface, index): - fullname = iface['name'] - if index != 0: - fullname += ":%s" % index - - control = iface['control'] - if control == "auto": - cverb = "auto" - elif control in ("hotplug",): - cverb = "allow-" + control - else: - cverb = "# control-" + control - - subst = iface.copy() - subst.update({'fullname': fullname, 'cverb': cverb}) - - return ("{cverb} {fullname}\n" - "iface {fullname} {inet} {mode}\n").format(**subst) - - -def _parse_deb_config_data(ifaces, contents, src_dir, src_path): - """Parses the file contents, placing result into ifaces. - - '_source_path' is added to every dictionary entry to define which file - the configration information came from. - - :param ifaces: interface dictionary - :param contents: contents of interfaces file - :param src_dir: directory interfaces file was located - :param src_path: file path the `contents` was read - """ - currif = None - for line in contents.splitlines(): - line = line.strip() - if line.startswith('#'): - continue - split = line.split(' ') - option = split[0] - if option == "source-directory": - parsed_src_dir = split[1] - if not parsed_src_dir.startswith("/"): - parsed_src_dir = os.path.join(src_dir, parsed_src_dir) - for expanded_path in glob.glob(parsed_src_dir): - dir_contents = os.listdir(expanded_path) - dir_contents = [ - os.path.join(expanded_path, path) - for path in dir_contents - if (os.path.isfile(os.path.join(expanded_path, path)) and - re.match("^[a-zA-Z0-9_-]+$", path) is not None) - ] - for entry in dir_contents: - with open(entry, "r") as fp: - src_data = fp.read().strip() - abs_entry = os.path.abspath(entry) - _parse_deb_config_data( - ifaces, src_data, - os.path.dirname(abs_entry), abs_entry) - elif option == "source": - new_src_path = split[1] - if not new_src_path.startswith("/"): - new_src_path = os.path.join(src_dir, new_src_path) - for expanded_path in glob.glob(new_src_path): - with open(expanded_path, "r") as fp: - src_data = fp.read().strip() - abs_path = os.path.abspath(expanded_path) - _parse_deb_config_data( - ifaces, src_data, - os.path.dirname(abs_path), abs_path) - elif option == "auto": - for iface in split[1:]: - if iface not in ifaces: - ifaces[iface] = { - # Include the source path this interface was found in. - "_source_path": src_path - } - ifaces[iface]['auto'] = True - elif option == "iface": - iface, family, method = split[1:4] - if iface not in ifaces: - ifaces[iface] = { - # Include the source path this interface was found in. - "_source_path": src_path - } - elif 'family' in ifaces[iface]: - raise ParserError( - "Interface %s can only be defined once. " - "Re-defined in '%s'." % (iface, src_path)) - ifaces[iface]['family'] = family - ifaces[iface]['method'] = method - currif = iface - elif option == "hwaddress": - ifaces[currif]['hwaddress'] = split[1] - elif option in NET_CONFIG_OPTIONS: - ifaces[currif][option] = split[1] - elif option in NET_CONFIG_COMMANDS: - if option not in ifaces[currif]: - ifaces[currif][option] = [] - ifaces[currif][option].append(' '.join(split[1:])) - elif option.startswith('dns-'): - if 'dns' not in ifaces[currif]: - ifaces[currif]['dns'] = {} - if option == 'dns-search': - ifaces[currif]['dns']['search'] = [] - for domain in split[1:]: - ifaces[currif]['dns']['search'].append(domain) - elif option == 'dns-nameservers': - ifaces[currif]['dns']['nameservers'] = [] - for server in split[1:]: - ifaces[currif]['dns']['nameservers'].append(server) - elif option.startswith('bridge_'): - if 'bridge' not in ifaces[currif]: - ifaces[currif]['bridge'] = {} - if option in NET_CONFIG_BRIDGE_OPTIONS: - bridge_option = option.replace('bridge_', '', 1) - ifaces[currif]['bridge'][bridge_option] = split[1] - elif option == "bridge_ports": - ifaces[currif]['bridge']['ports'] = [] - for iface in split[1:]: - ifaces[currif]['bridge']['ports'].append(iface) - elif option == "bridge_hw" and split[1].lower() == "mac": - ifaces[currif]['bridge']['mac'] = split[2] - elif option == "bridge_pathcost": - if 'pathcost' not in ifaces[currif]['bridge']: - ifaces[currif]['bridge']['pathcost'] = {} - ifaces[currif]['bridge']['pathcost'][split[1]] = split[2] - elif option == "bridge_portprio": - if 'portprio' not in ifaces[currif]['bridge']: - ifaces[currif]['bridge']['portprio'] = {} - ifaces[currif]['bridge']['portprio'][split[1]] = split[2] - elif option.startswith('bond-'): - if 'bond' not in ifaces[currif]: - ifaces[currif]['bond'] = {} - bond_option = option.replace('bond-', '', 1) - ifaces[currif]['bond'][bond_option] = split[1] - for iface in ifaces.keys(): - if 'auto' not in ifaces[iface]: - ifaces[iface]['auto'] = False - - -def _parse_deb_config(path): - """Parses a debian network configuration file.""" - ifaces = {} - with open(path, "r") as fp: - contents = fp.read().strip() - abs_path = os.path.abspath(path) - _parse_deb_config_data( - ifaces, contents, - os.path.dirname(abs_path), abs_path) - return ifaces - - -class Renderer(object): - """Renders network information in a debian format.""" - - def render_persistent_net(self, network_state): - ''' Given state, emit udev rules to map - mac to ifname - ''' - content = "" - interfaces = network_state.get('interfaces') - for iface in interfaces.values(): - # for physical interfaces write out a persist net udev rule - if iface['type'] == 'physical' and \ - 'name' in iface and iface.get('mac_address'): - content += generate_udev_rule(iface['name'], - iface['mac_address']) - - return content - - def render_route(self, route, indent=""): - """ When rendering routes for an iface, in some cases applying a route - may result in the route command returning non-zero which produces - some confusing output for users manually using ifup/ifdown[1]. To - that end, we will optionally include an '|| true' postfix to each - route line allowing users to work with ifup/ifdown without using - --force option. - - We may at somepoint not want to emit this additional postfix, and - add a 'strict' flag to this function. When called with strict=True, - then we will not append the postfix. - - 1. http://askubuntu.com/questions/168033/ - how-to-set-static-routes-in-ubuntu-server - """ - content = "" - up = indent + "post-up route add" - down = indent + "pre-down route del" - eol = " || true\n" - mapping = { - 'network': '-net', - 'netmask': 'netmask', - 'gateway': 'gw', - 'metric': 'metric', - } - if route['network'] == '0.0.0.0' and route['netmask'] == '0.0.0.0': - default_gw = " default gw %s" % route['gateway'] - content += up + default_gw + eol - content += down + default_gw + eol - elif route['network'] == '::' and route['netmask'] == 0: - # ipv6! - default_gw = " -A inet6 default gw %s" % route['gateway'] - content += up + default_gw + eol - content += down + default_gw + eol - else: - route_line = "" - for k in ['network', 'netmask', 'gateway', 'metric']: - if k in route: - route_line += " %s %s" % (mapping[k], route[k]) - content += up + route_line + eol - content += down + route_line + eol - - return content - - def render_interfaces(self, network_state): - ''' Given state, emit etc/network/interfaces content ''' - - content = "" - interfaces = network_state.get('interfaces') - ''' Apply a sort order to ensure that we write out - the physical interfaces first; this is critical for - bonding - ''' - order = { - 'physical': 0, - 'bond': 1, - 'bridge': 2, - 'vlan': 3, - } - content += "auto lo\niface lo inet loopback\n" - for dnskey, value in network_state.get('dns', {}).items(): - if len(value): - content += " dns-{} {}\n".format(dnskey, " ".join(value)) - - for iface in sorted(interfaces.values(), - key=lambda k: (order[k['type']], k['name'])): - - if content[-2:] != "\n\n": - content += "\n" - subnets = iface.get('subnets', {}) - if subnets: - for index, subnet in zip(range(0, len(subnets)), subnets): - if content[-2:] != "\n\n": - content += "\n" - iface['index'] = index - iface['mode'] = subnet['type'] - iface['control'] = subnet.get('control', 'auto') - if iface['mode'].endswith('6'): - iface['inet'] += '6' - elif iface['mode'] == 'static' \ - and ":" in subnet['address']: - iface['inet'] += '6' - if iface['mode'].startswith('dhcp'): - iface['mode'] = 'dhcp' - - content += _iface_start_entry(iface, index) - content += _iface_add_subnet(iface, subnet) - content += _iface_add_attrs(iface) - else: - # ifenslave docs say to auto the slave devices - if 'bond-master' in iface: - content += "auto {name}\n".format(**iface) - content += "iface {name} {inet} {mode}\n".format(**iface) - content += _iface_add_attrs(iface) - - for route in network_state.get('routes'): - content += self.render_route(route) - - # global replacements until v2 format - content = content.replace('mac_address', 'hwaddress') - return content - - def render_network_state(self, - target, network_state, eni="etc/network/interfaces", - links_prefix=LINKS_FNAME_PREFIX, - netrules='etc/udev/rules.d/70-persistent-net.rules'): - - fpeni = os.path.sep.join((target, eni,)) - util.ensure_dir(os.path.dirname(fpeni)) - with open(fpeni, 'w+') as f: - f.write(self.render_interfaces(network_state)) - - if netrules: - netrules = os.path.sep.join((target, netrules,)) - util.ensure_dir(os.path.dirname(netrules)) - with open(netrules, 'w+') as f: - f.write(self.render_persistent_net(network_state)) - - if links_prefix: - self.render_systemd_links(target, network_state, links_prefix) - - def render_systemd_links(self, target, network_state, - links_prefix=LINKS_FNAME_PREFIX): - fp_prefix = os.path.sep.join((target, links_prefix)) - for f in glob.glob(fp_prefix + "*"): - os.unlink(f) - - interfaces = network_state.get('interfaces') - for iface in interfaces.values(): - if (iface['type'] == 'physical' and 'name' in iface and - iface.get('mac_address')): - fname = fp_prefix + iface['name'] + ".link" - with open(fname, "w") as fp: - fp.write("\n".join([ - "[Match]", - "MACAddress=" + iface['mac_address'], - "", - "[Link]", - "Name=" + iface['name'], - "" - ])) diff --git a/cloudinit/net/klibc.py b/cloudinit/net/klibc.py deleted file mode 100644 index 958c264b..00000000 --- a/cloudinit/net/klibc.py +++ /dev/null @@ -1,191 +0,0 @@ -# Copyright (C) 2013-2014 Canonical Ltd. -# -# Author: Scott Moser -# Author: Blake Rouse -# -# Curtin is free software: you can redistribute it and/or modify it under -# the terms of the GNU Affero General Public License as published by the -# Free Software Foundation, either version 3 of the License, or (at your -# option) any later version. -# -# Curtin is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for -# more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Curtin. If not, see . - -import base64 -import glob -import gzip -import io -import shlex - -from cloudinit.net import get_devicelist -from cloudinit.net import sys_netdev_info - -from cloudinit import util - - -def _load_shell_content(content, add_empty=False, empty_val=None): - """Given shell like syntax (key=value\nkey2=value2\n) in content - return the data in dictionary form. If 'add_empty' is True - then add entries in to the returned dictionary for 'VAR=' - variables. Set their value to empty_val.""" - data = {} - for line in shlex.split(content): - key, value = line.split("=", 1) - if not value: - value = empty_val - if add_empty or value: - data[key] = value - - return data - - -def _klibc_to_config_entry(content, mac_addrs=None): - """Convert a klibc writtent shell content file to a 'config' entry - When ip= is seen on the kernel command line in debian initramfs - and networking is brought up, ipconfig will populate - /run/net-.cfg. - - The files are shell style syntax, and examples are in the tests - provided here. There is no good documentation on this unfortunately. - - DEVICE= is expected/required and PROTO should indicate if - this is 'static' or 'dhcp'. - """ - - if mac_addrs is None: - mac_addrs = {} - - data = _load_shell_content(content) - try: - name = data['DEVICE'] - except KeyError: - raise ValueError("no 'DEVICE' entry in data") - - # ipconfig on precise does not write PROTO - proto = data.get('PROTO') - if not proto: - if data.get('filename'): - proto = 'dhcp' - else: - proto = 'static' - - if proto not in ('static', 'dhcp'): - raise ValueError("Unexpected value for PROTO: %s" % proto) - - iface = { - 'type': 'physical', - 'name': name, - 'subnets': [], - } - - if name in mac_addrs: - iface['mac_address'] = mac_addrs[name] - - # originally believed there might be IPV6* values - for v, pre in (('ipv4', 'IPV4'),): - # if no IPV4ADDR or IPV6ADDR, then go on. - if pre + "ADDR" not in data: - continue - subnet = {'type': proto, 'control': 'manual'} - - # these fields go right on the subnet - for key in ('NETMASK', 'BROADCAST', 'GATEWAY'): - if pre + key in data: - subnet[key.lower()] = data[pre + key] - - dns = [] - # handle IPV4DNS0 or IPV6DNS0 - for nskey in ('DNS0', 'DNS1'): - ns = data.get(pre + nskey) - # verify it has something other than 0.0.0.0 (or ipv6) - if ns and len(ns.strip(":.0")): - dns.append(data[pre + nskey]) - if dns: - subnet['dns_nameservers'] = dns - # add search to both ipv4 and ipv6, as it has no namespace - search = data.get('DOMAINSEARCH') - if search: - if ',' in search: - subnet['dns_search'] = search.split(",") - else: - subnet['dns_search'] = search.split() - - iface['subnets'].append(subnet) - - return name, iface - - -def config_from_klibc_net_cfg(files=None, mac_addrs=None): - if files is None: - files = glob.glob('/run/net*.conf') - - entries = [] - names = {} - for cfg_file in files: - name, entry = _klibc_to_config_entry(util.load_file(cfg_file), - mac_addrs=mac_addrs) - if name in names: - raise ValueError( - "device '%s' defined multiple times: %s and %s" % ( - name, names[name], cfg_file)) - - names[name] = cfg_file - entries.append(entry) - return {'config': entries, 'version': 1} - - -def _decomp_gzip(blob, strict=True): - # decompress blob. raise exception if not compressed unless strict=False. - with io.BytesIO(blob) as iobuf: - gzfp = None - try: - gzfp = gzip.GzipFile(mode="rb", fileobj=iobuf) - return gzfp.read() - except IOError: - if strict: - raise - return blob - finally: - if gzfp: - gzfp.close() - - -def _b64dgz(b64str, gzipped="try"): - # decode a base64 string. If gzipped is true, transparently uncompresss - # if gzipped is 'try', then try gunzip, returning the original on fail. - try: - blob = base64.b64decode(b64str) - except TypeError: - raise ValueError("Invalid base64 text: %s" % b64str) - - if not gzipped: - return blob - - return _decomp_gzip(blob, strict=gzipped != "try") - - -def read_kernel_cmdline_config(files=None, mac_addrs=None, cmdline=None): - if cmdline is None: - cmdline = util.get_cmdline() - - if 'network-config=' in cmdline: - data64 = None - for tok in cmdline.split(): - if tok.startswith("network-config="): - data64 = tok.split("=", 1)[1] - if data64: - return util.load_yaml(_b64dgz(data64)) - - if 'ip=' not in cmdline: - return None - - if mac_addrs is None: - mac_addrs = {k: sys_netdev_info(k, 'address') - for k in get_devicelist()} - - return config_from_klibc_net_cfg(files=files, mac_addrs=mac_addrs) diff --git a/cloudinit/net/renderers/__init__.py b/cloudinit/net/renderers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cloudinit/net/renderers/eni.py b/cloudinit/net/renderers/eni.py new file mode 100644 index 00000000..b427012e --- /dev/null +++ b/cloudinit/net/renderers/eni.py @@ -0,0 +1,399 @@ +# vi: ts=4 expandtab +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import glob +import os +import re + +from cloudinit.net import LINKS_FNAME_PREFIX +from cloudinit.net import ParserError +from cloudinit.net.udev import generate_udev_rule +from cloudinit import util + + +NET_CONFIG_COMMANDS = [ + "pre-up", "up", "post-up", "down", "pre-down", "post-down", +] + +NET_CONFIG_BRIDGE_OPTIONS = [ + "bridge_ageing", "bridge_bridgeprio", "bridge_fd", "bridge_gcinit", + "bridge_hello", "bridge_maxage", "bridge_maxwait", "bridge_stp", +] + +NET_CONFIG_OPTIONS = [ + "address", "netmask", "broadcast", "network", "metric", "gateway", + "pointtopoint", "media", "mtu", "hostname", "leasehours", "leasetime", + "vendor", "client", "bootfile", "server", "hwaddr", "provider", "frame", + "netnum", "endpoint", "local", "ttl", +] + + +# TODO: switch valid_map based on mode inet/inet6 +def _iface_add_subnet(iface, subnet): + content = "" + valid_map = [ + 'address', + 'netmask', + 'broadcast', + 'metric', + 'gateway', + 'pointopoint', + 'mtu', + 'scope', + 'dns_search', + 'dns_nameservers', + ] + for key, value in subnet.items(): + if value and key in valid_map: + if type(value) == list: + value = " ".join(value) + if '_' in key: + key = key.replace('_', '-') + content += " {} {}\n".format(key, value) + + return content + + +# TODO: switch to valid_map for attrs +def _iface_add_attrs(iface): + content = "" + ignore_map = [ + 'control', + 'index', + 'inet', + 'mode', + 'name', + 'subnets', + 'type', + ] + if iface['type'] not in ['bond', 'bridge', 'vlan']: + ignore_map.append('mac_address') + + for key, value in iface.items(): + if value and key not in ignore_map: + if type(value) == list: + value = " ".join(value) + content += " {} {}\n".format(key, value) + + return content + + +def _iface_start_entry(iface, index): + fullname = iface['name'] + if index != 0: + fullname += ":%s" % index + + control = iface['control'] + if control == "auto": + cverb = "auto" + elif control in ("hotplug",): + cverb = "allow-" + control + else: + cverb = "# control-" + control + + subst = iface.copy() + subst.update({'fullname': fullname, 'cverb': cverb}) + + return ("{cverb} {fullname}\n" + "iface {fullname} {inet} {mode}\n").format(**subst) + + +def _parse_deb_config_data(ifaces, contents, src_dir, src_path): + """Parses the file contents, placing result into ifaces. + + '_source_path' is added to every dictionary entry to define which file + the configration information came from. + + :param ifaces: interface dictionary + :param contents: contents of interfaces file + :param src_dir: directory interfaces file was located + :param src_path: file path the `contents` was read + """ + currif = None + for line in contents.splitlines(): + line = line.strip() + if line.startswith('#'): + continue + split = line.split(' ') + option = split[0] + if option == "source-directory": + parsed_src_dir = split[1] + if not parsed_src_dir.startswith("/"): + parsed_src_dir = os.path.join(src_dir, parsed_src_dir) + for expanded_path in glob.glob(parsed_src_dir): + dir_contents = os.listdir(expanded_path) + dir_contents = [ + os.path.join(expanded_path, path) + for path in dir_contents + if (os.path.isfile(os.path.join(expanded_path, path)) and + re.match("^[a-zA-Z0-9_-]+$", path) is not None) + ] + for entry in dir_contents: + with open(entry, "r") as fp: + src_data = fp.read().strip() + abs_entry = os.path.abspath(entry) + _parse_deb_config_data( + ifaces, src_data, + os.path.dirname(abs_entry), abs_entry) + elif option == "source": + new_src_path = split[1] + if not new_src_path.startswith("/"): + new_src_path = os.path.join(src_dir, new_src_path) + for expanded_path in glob.glob(new_src_path): + with open(expanded_path, "r") as fp: + src_data = fp.read().strip() + abs_path = os.path.abspath(expanded_path) + _parse_deb_config_data( + ifaces, src_data, + os.path.dirname(abs_path), abs_path) + elif option == "auto": + for iface in split[1:]: + if iface not in ifaces: + ifaces[iface] = { + # Include the source path this interface was found in. + "_source_path": src_path + } + ifaces[iface]['auto'] = True + elif option == "iface": + iface, family, method = split[1:4] + if iface not in ifaces: + ifaces[iface] = { + # Include the source path this interface was found in. + "_source_path": src_path + } + elif 'family' in ifaces[iface]: + raise ParserError( + "Interface %s can only be defined once. " + "Re-defined in '%s'." % (iface, src_path)) + ifaces[iface]['family'] = family + ifaces[iface]['method'] = method + currif = iface + elif option == "hwaddress": + ifaces[currif]['hwaddress'] = split[1] + elif option in NET_CONFIG_OPTIONS: + ifaces[currif][option] = split[1] + elif option in NET_CONFIG_COMMANDS: + if option not in ifaces[currif]: + ifaces[currif][option] = [] + ifaces[currif][option].append(' '.join(split[1:])) + elif option.startswith('dns-'): + if 'dns' not in ifaces[currif]: + ifaces[currif]['dns'] = {} + if option == 'dns-search': + ifaces[currif]['dns']['search'] = [] + for domain in split[1:]: + ifaces[currif]['dns']['search'].append(domain) + elif option == 'dns-nameservers': + ifaces[currif]['dns']['nameservers'] = [] + for server in split[1:]: + ifaces[currif]['dns']['nameservers'].append(server) + elif option.startswith('bridge_'): + if 'bridge' not in ifaces[currif]: + ifaces[currif]['bridge'] = {} + if option in NET_CONFIG_BRIDGE_OPTIONS: + bridge_option = option.replace('bridge_', '', 1) + ifaces[currif]['bridge'][bridge_option] = split[1] + elif option == "bridge_ports": + ifaces[currif]['bridge']['ports'] = [] + for iface in split[1:]: + ifaces[currif]['bridge']['ports'].append(iface) + elif option == "bridge_hw" and split[1].lower() == "mac": + ifaces[currif]['bridge']['mac'] = split[2] + elif option == "bridge_pathcost": + if 'pathcost' not in ifaces[currif]['bridge']: + ifaces[currif]['bridge']['pathcost'] = {} + ifaces[currif]['bridge']['pathcost'][split[1]] = split[2] + elif option == "bridge_portprio": + if 'portprio' not in ifaces[currif]['bridge']: + ifaces[currif]['bridge']['portprio'] = {} + ifaces[currif]['bridge']['portprio'][split[1]] = split[2] + elif option.startswith('bond-'): + if 'bond' not in ifaces[currif]: + ifaces[currif]['bond'] = {} + bond_option = option.replace('bond-', '', 1) + ifaces[currif]['bond'][bond_option] = split[1] + for iface in ifaces.keys(): + if 'auto' not in ifaces[iface]: + ifaces[iface]['auto'] = False + + +def _parse_deb_config(path): + """Parses a debian network configuration file.""" + ifaces = {} + with open(path, "r") as fp: + contents = fp.read().strip() + abs_path = os.path.abspath(path) + _parse_deb_config_data( + ifaces, contents, + os.path.dirname(abs_path), abs_path) + return ifaces + + +class Renderer(object): + """Renders network information in a /etc/network/interfaces format.""" + + def render_persistent_net(self, network_state): + """Given state, emit udev rules to map mac to ifname.""" + content = "" + interfaces = network_state.get('interfaces') + for iface in interfaces.values(): + # for physical interfaces write out a persist net udev rule + if iface['type'] == 'physical' and \ + 'name' in iface and iface.get('mac_address'): + content += generate_udev_rule(iface['name'], + iface['mac_address']) + + return content + + def render_route(self, route, indent=""): + """ When rendering routes for an iface, in some cases applying a route + may result in the route command returning non-zero which produces + some confusing output for users manually using ifup/ifdown[1]. To + that end, we will optionally include an '|| true' postfix to each + route line allowing users to work with ifup/ifdown without using + --force option. + + We may at somepoint not want to emit this additional postfix, and + add a 'strict' flag to this function. When called with strict=True, + then we will not append the postfix. + + 1. http://askubuntu.com/questions/168033/ + how-to-set-static-routes-in-ubuntu-server + """ + content = "" + up = indent + "post-up route add" + down = indent + "pre-down route del" + eol = " || true\n" + mapping = { + 'network': '-net', + 'netmask': 'netmask', + 'gateway': 'gw', + 'metric': 'metric', + } + if route['network'] == '0.0.0.0' and route['netmask'] == '0.0.0.0': + default_gw = " default gw %s" % route['gateway'] + content += up + default_gw + eol + content += down + default_gw + eol + elif route['network'] == '::' and route['netmask'] == 0: + # ipv6! + default_gw = " -A inet6 default gw %s" % route['gateway'] + content += up + default_gw + eol + content += down + default_gw + eol + else: + route_line = "" + for k in ['network', 'netmask', 'gateway', 'metric']: + if k in route: + route_line += " %s %s" % (mapping[k], route[k]) + content += up + route_line + eol + content += down + route_line + eol + + return content + + def render_interfaces(self, network_state): + ''' Given state, emit etc/network/interfaces content ''' + + content = "" + interfaces = network_state.get('interfaces') + ''' Apply a sort order to ensure that we write out + the physical interfaces first; this is critical for + bonding + ''' + order = { + 'physical': 0, + 'bond': 1, + 'bridge': 2, + 'vlan': 3, + } + content += "auto lo\niface lo inet loopback\n" + for dnskey, value in network_state.get('dns', {}).items(): + if len(value): + content += " dns-{} {}\n".format(dnskey, " ".join(value)) + + for iface in sorted(interfaces.values(), + key=lambda k: (order[k['type']], k['name'])): + + if content[-2:] != "\n\n": + content += "\n" + subnets = iface.get('subnets', {}) + if subnets: + for index, subnet in zip(range(0, len(subnets)), subnets): + if content[-2:] != "\n\n": + content += "\n" + iface['index'] = index + iface['mode'] = subnet['type'] + iface['control'] = subnet.get('control', 'auto') + if iface['mode'].endswith('6'): + iface['inet'] += '6' + elif iface['mode'] == 'static' \ + and ":" in subnet['address']: + iface['inet'] += '6' + if iface['mode'].startswith('dhcp'): + iface['mode'] = 'dhcp' + + content += _iface_start_entry(iface, index) + content += _iface_add_subnet(iface, subnet) + content += _iface_add_attrs(iface) + else: + # ifenslave docs say to auto the slave devices + if 'bond-master' in iface: + content += "auto {name}\n".format(**iface) + content += "iface {name} {inet} {mode}\n".format(**iface) + content += _iface_add_attrs(iface) + + for route in network_state.get('routes'): + content += self.render_route(route) + + # global replacements until v2 format + content = content.replace('mac_address', 'hwaddress') + return content + + def render_network_state(self, + target, network_state, eni="etc/network/interfaces", + links_prefix=LINKS_FNAME_PREFIX, + netrules='etc/udev/rules.d/70-persistent-net.rules'): + + fpeni = os.path.sep.join((target, eni,)) + util.ensure_dir(os.path.dirname(fpeni)) + with open(fpeni, 'w+') as f: + f.write(self.render_interfaces(network_state)) + + if netrules: + netrules = os.path.sep.join((target, netrules,)) + util.ensure_dir(os.path.dirname(netrules)) + with open(netrules, 'w+') as f: + f.write(self.render_persistent_net(network_state)) + + if links_prefix: + self.render_systemd_links(target, network_state, links_prefix) + + def render_systemd_links(self, target, network_state, + links_prefix=LINKS_FNAME_PREFIX): + fp_prefix = os.path.sep.join((target, links_prefix)) + for f in glob.glob(fp_prefix + "*"): + os.unlink(f) + + interfaces = network_state.get('interfaces') + for iface in interfaces.values(): + if (iface['type'] == 'physical' and 'name' in iface and + iface.get('mac_address')): + fname = fp_prefix + iface['name'] + ".link" + with open(fname, "w") as fp: + fp.write("\n".join([ + "[Match]", + "MACAddress=" + iface['mac_address'], + "", + "[Link]", + "Name=" + iface['name'], + "" + ])) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index e6bd34fe..1112bf0d 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -43,7 +43,7 @@ from cloudinit import distros from cloudinit import helpers from cloudinit import importer from cloudinit import log as logging -from cloudinit.net import klibc +from cloudinit.net import cmdline from cloudinit import sources from cloudinit import type_utils from cloudinit import util @@ -579,7 +579,7 @@ class Init(object): if os.path.exists(disable_file): return (None, disable_file) - cmdline_cfg = ('cmdline', klibc.read_kernel_cmdline_config()) + cmdline_cfg = ('cmdline', cmdline.read_kernel_cmdline_config()) dscfg = ('ds', None) if self.datasource and hasattr(self.datasource, 'network_config'): dscfg = ('ds', self.datasource.network_config) -- cgit v1.2.3 From cc56ef479a4cfa4520dfcc7cc27c35bb6ac86bd2 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 10 May 2016 13:18:53 -0700 Subject: Fix up tests and debian distro --- cloudinit/distros/debian.py | 4 ++-- tests/unittests/test_net.py | 23 ++++++++++++----------- 2 files changed, 14 insertions(+), 13 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 21a4f805..465413a8 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -27,7 +27,7 @@ from cloudinit import helpers from cloudinit import log as logging from cloudinit import util from cloudinit import net -from cloudinit.net.distros import debian +from cloudinit.net.renderers import eni from cloudinit.distros.parsers.hostname import HostnameConf @@ -57,7 +57,7 @@ class Distro(distros.Distro): # should only happen say once per instance...) self._runner = helpers.Runners(paths) self.osfamily = 'debian' - self._net_renderer = debian.Renderer() + self._net_renderer = eni.Renderer() def apply_locale(self, locale, out_fn=None): if not out_fn: diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index a0cdc493..6daf9601 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -1,5 +1,6 @@ from cloudinit import util from cloudinit import net +from cloudinit.net import cmdline from .helpers import TestCase import base64 @@ -74,15 +75,15 @@ class TestNetConfigParsing(TestCase): "mac_address": "c0:d6:9f:2c:e8:80", "subnets": [{"type": "dhcp"}]}]} - def test_klibc_convert_dhcp(self): - found = net._klibc_to_config_entry(DHCP_CONTENT_1) + def test_cmdline_convert_dhcp(self): + found = cmdline._klibc_to_config_entry(DHCP_CONTENT_1) self.assertEqual(found, ('eth0', DHCP_EXPECTED_1)) - def test_klibc_convert_static(self): - found = net._klibc_to_config_entry(STATIC_CONTENT_1) + def test_cmdline_convert_static(self): + found = cmdline._klibc_to_config_entry(STATIC_CONTENT_1) self.assertEqual(found, ('eth1', STATIC_EXPECTED_1)) - def test_config_from_klibc_net_cfg(self): + def test_config_from_cmdline_net_cfg(self): files = [] pairs = (('net-eth0.cfg', DHCP_CONTENT_1), ('net-eth1.cfg', STATIC_CONTENT_1)) @@ -103,25 +104,25 @@ class TestNetConfigParsing(TestCase): files.append(fp) util.write_file(fp, content) - found = net.config_from_klibc_net_cfg(files=files, mac_addrs=macs) + found = cmdline.config_from_klibc_net_cfg(files=files, + mac_addrs=macs) self.assertEqual(found, expected) def test_cmdline_with_b64(self): data = base64.b64encode(json.dumps(self.simple_cfg).encode()) encoded_text = data.decode() - cmdline = 'ro network-config=' + encoded_text + ' root=foo' - found = net.read_kernel_cmdline_config(cmdline=cmdline) + raw_cmdline = 'ro network-config=' + encoded_text + ' root=foo' + found = cmdline.read_kernel_cmdline_config(cmdline=raw_cmdline) self.assertEqual(found, self.simple_cfg) def test_cmdline_with_b64_gz(self): data = _gzip_data(json.dumps(self.simple_cfg).encode()) encoded_text = base64.b64encode(data).decode() - cmdline = 'ro network-config=' + encoded_text + ' root=foo' - found = net.read_kernel_cmdline_config(cmdline=cmdline) + raw_cmdline = 'ro network-config=' + encoded_text + ' root=foo' + found = cmdline.read_kernel_cmdline_config(cmdline=raw_cmdline) self.assertEqual(found, self.simple_cfg) - def _gzip_data(data): with io.BytesIO() as iobuf: gzfp = gzip.GzipFile(mode="wb", fileobj=iobuf) -- cgit v1.2.3 From 281551d4125b40836686793b6a0f8d2c34c3357f Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 10 May 2016 14:10:54 -0700 Subject: Move net/renderers -> net --- cloudinit/distros/debian.py | 2 +- cloudinit/net/eni.py | 399 ++++++++++++++++++++++++++++++++++++ cloudinit/net/renderers/__init__.py | 0 cloudinit/net/renderers/eni.py | 399 ------------------------------------ 4 files changed, 400 insertions(+), 400 deletions(-) create mode 100644 cloudinit/net/eni.py delete mode 100644 cloudinit/net/renderers/__init__.py delete mode 100644 cloudinit/net/renderers/eni.py (limited to 'cloudinit') diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 465413a8..ca069a60 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -27,7 +27,7 @@ from cloudinit import helpers from cloudinit import log as logging from cloudinit import util from cloudinit import net -from cloudinit.net.renderers import eni +from cloudinit.net import eni from cloudinit.distros.parsers.hostname import HostnameConf diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py new file mode 100644 index 00000000..b427012e --- /dev/null +++ b/cloudinit/net/eni.py @@ -0,0 +1,399 @@ +# vi: ts=4 expandtab +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import glob +import os +import re + +from cloudinit.net import LINKS_FNAME_PREFIX +from cloudinit.net import ParserError +from cloudinit.net.udev import generate_udev_rule +from cloudinit import util + + +NET_CONFIG_COMMANDS = [ + "pre-up", "up", "post-up", "down", "pre-down", "post-down", +] + +NET_CONFIG_BRIDGE_OPTIONS = [ + "bridge_ageing", "bridge_bridgeprio", "bridge_fd", "bridge_gcinit", + "bridge_hello", "bridge_maxage", "bridge_maxwait", "bridge_stp", +] + +NET_CONFIG_OPTIONS = [ + "address", "netmask", "broadcast", "network", "metric", "gateway", + "pointtopoint", "media", "mtu", "hostname", "leasehours", "leasetime", + "vendor", "client", "bootfile", "server", "hwaddr", "provider", "frame", + "netnum", "endpoint", "local", "ttl", +] + + +# TODO: switch valid_map based on mode inet/inet6 +def _iface_add_subnet(iface, subnet): + content = "" + valid_map = [ + 'address', + 'netmask', + 'broadcast', + 'metric', + 'gateway', + 'pointopoint', + 'mtu', + 'scope', + 'dns_search', + 'dns_nameservers', + ] + for key, value in subnet.items(): + if value and key in valid_map: + if type(value) == list: + value = " ".join(value) + if '_' in key: + key = key.replace('_', '-') + content += " {} {}\n".format(key, value) + + return content + + +# TODO: switch to valid_map for attrs +def _iface_add_attrs(iface): + content = "" + ignore_map = [ + 'control', + 'index', + 'inet', + 'mode', + 'name', + 'subnets', + 'type', + ] + if iface['type'] not in ['bond', 'bridge', 'vlan']: + ignore_map.append('mac_address') + + for key, value in iface.items(): + if value and key not in ignore_map: + if type(value) == list: + value = " ".join(value) + content += " {} {}\n".format(key, value) + + return content + + +def _iface_start_entry(iface, index): + fullname = iface['name'] + if index != 0: + fullname += ":%s" % index + + control = iface['control'] + if control == "auto": + cverb = "auto" + elif control in ("hotplug",): + cverb = "allow-" + control + else: + cverb = "# control-" + control + + subst = iface.copy() + subst.update({'fullname': fullname, 'cverb': cverb}) + + return ("{cverb} {fullname}\n" + "iface {fullname} {inet} {mode}\n").format(**subst) + + +def _parse_deb_config_data(ifaces, contents, src_dir, src_path): + """Parses the file contents, placing result into ifaces. + + '_source_path' is added to every dictionary entry to define which file + the configration information came from. + + :param ifaces: interface dictionary + :param contents: contents of interfaces file + :param src_dir: directory interfaces file was located + :param src_path: file path the `contents` was read + """ + currif = None + for line in contents.splitlines(): + line = line.strip() + if line.startswith('#'): + continue + split = line.split(' ') + option = split[0] + if option == "source-directory": + parsed_src_dir = split[1] + if not parsed_src_dir.startswith("/"): + parsed_src_dir = os.path.join(src_dir, parsed_src_dir) + for expanded_path in glob.glob(parsed_src_dir): + dir_contents = os.listdir(expanded_path) + dir_contents = [ + os.path.join(expanded_path, path) + for path in dir_contents + if (os.path.isfile(os.path.join(expanded_path, path)) and + re.match("^[a-zA-Z0-9_-]+$", path) is not None) + ] + for entry in dir_contents: + with open(entry, "r") as fp: + src_data = fp.read().strip() + abs_entry = os.path.abspath(entry) + _parse_deb_config_data( + ifaces, src_data, + os.path.dirname(abs_entry), abs_entry) + elif option == "source": + new_src_path = split[1] + if not new_src_path.startswith("/"): + new_src_path = os.path.join(src_dir, new_src_path) + for expanded_path in glob.glob(new_src_path): + with open(expanded_path, "r") as fp: + src_data = fp.read().strip() + abs_path = os.path.abspath(expanded_path) + _parse_deb_config_data( + ifaces, src_data, + os.path.dirname(abs_path), abs_path) + elif option == "auto": + for iface in split[1:]: + if iface not in ifaces: + ifaces[iface] = { + # Include the source path this interface was found in. + "_source_path": src_path + } + ifaces[iface]['auto'] = True + elif option == "iface": + iface, family, method = split[1:4] + if iface not in ifaces: + ifaces[iface] = { + # Include the source path this interface was found in. + "_source_path": src_path + } + elif 'family' in ifaces[iface]: + raise ParserError( + "Interface %s can only be defined once. " + "Re-defined in '%s'." % (iface, src_path)) + ifaces[iface]['family'] = family + ifaces[iface]['method'] = method + currif = iface + elif option == "hwaddress": + ifaces[currif]['hwaddress'] = split[1] + elif option in NET_CONFIG_OPTIONS: + ifaces[currif][option] = split[1] + elif option in NET_CONFIG_COMMANDS: + if option not in ifaces[currif]: + ifaces[currif][option] = [] + ifaces[currif][option].append(' '.join(split[1:])) + elif option.startswith('dns-'): + if 'dns' not in ifaces[currif]: + ifaces[currif]['dns'] = {} + if option == 'dns-search': + ifaces[currif]['dns']['search'] = [] + for domain in split[1:]: + ifaces[currif]['dns']['search'].append(domain) + elif option == 'dns-nameservers': + ifaces[currif]['dns']['nameservers'] = [] + for server in split[1:]: + ifaces[currif]['dns']['nameservers'].append(server) + elif option.startswith('bridge_'): + if 'bridge' not in ifaces[currif]: + ifaces[currif]['bridge'] = {} + if option in NET_CONFIG_BRIDGE_OPTIONS: + bridge_option = option.replace('bridge_', '', 1) + ifaces[currif]['bridge'][bridge_option] = split[1] + elif option == "bridge_ports": + ifaces[currif]['bridge']['ports'] = [] + for iface in split[1:]: + ifaces[currif]['bridge']['ports'].append(iface) + elif option == "bridge_hw" and split[1].lower() == "mac": + ifaces[currif]['bridge']['mac'] = split[2] + elif option == "bridge_pathcost": + if 'pathcost' not in ifaces[currif]['bridge']: + ifaces[currif]['bridge']['pathcost'] = {} + ifaces[currif]['bridge']['pathcost'][split[1]] = split[2] + elif option == "bridge_portprio": + if 'portprio' not in ifaces[currif]['bridge']: + ifaces[currif]['bridge']['portprio'] = {} + ifaces[currif]['bridge']['portprio'][split[1]] = split[2] + elif option.startswith('bond-'): + if 'bond' not in ifaces[currif]: + ifaces[currif]['bond'] = {} + bond_option = option.replace('bond-', '', 1) + ifaces[currif]['bond'][bond_option] = split[1] + for iface in ifaces.keys(): + if 'auto' not in ifaces[iface]: + ifaces[iface]['auto'] = False + + +def _parse_deb_config(path): + """Parses a debian network configuration file.""" + ifaces = {} + with open(path, "r") as fp: + contents = fp.read().strip() + abs_path = os.path.abspath(path) + _parse_deb_config_data( + ifaces, contents, + os.path.dirname(abs_path), abs_path) + return ifaces + + +class Renderer(object): + """Renders network information in a /etc/network/interfaces format.""" + + def render_persistent_net(self, network_state): + """Given state, emit udev rules to map mac to ifname.""" + content = "" + interfaces = network_state.get('interfaces') + for iface in interfaces.values(): + # for physical interfaces write out a persist net udev rule + if iface['type'] == 'physical' and \ + 'name' in iface and iface.get('mac_address'): + content += generate_udev_rule(iface['name'], + iface['mac_address']) + + return content + + def render_route(self, route, indent=""): + """ When rendering routes for an iface, in some cases applying a route + may result in the route command returning non-zero which produces + some confusing output for users manually using ifup/ifdown[1]. To + that end, we will optionally include an '|| true' postfix to each + route line allowing users to work with ifup/ifdown without using + --force option. + + We may at somepoint not want to emit this additional postfix, and + add a 'strict' flag to this function. When called with strict=True, + then we will not append the postfix. + + 1. http://askubuntu.com/questions/168033/ + how-to-set-static-routes-in-ubuntu-server + """ + content = "" + up = indent + "post-up route add" + down = indent + "pre-down route del" + eol = " || true\n" + mapping = { + 'network': '-net', + 'netmask': 'netmask', + 'gateway': 'gw', + 'metric': 'metric', + } + if route['network'] == '0.0.0.0' and route['netmask'] == '0.0.0.0': + default_gw = " default gw %s" % route['gateway'] + content += up + default_gw + eol + content += down + default_gw + eol + elif route['network'] == '::' and route['netmask'] == 0: + # ipv6! + default_gw = " -A inet6 default gw %s" % route['gateway'] + content += up + default_gw + eol + content += down + default_gw + eol + else: + route_line = "" + for k in ['network', 'netmask', 'gateway', 'metric']: + if k in route: + route_line += " %s %s" % (mapping[k], route[k]) + content += up + route_line + eol + content += down + route_line + eol + + return content + + def render_interfaces(self, network_state): + ''' Given state, emit etc/network/interfaces content ''' + + content = "" + interfaces = network_state.get('interfaces') + ''' Apply a sort order to ensure that we write out + the physical interfaces first; this is critical for + bonding + ''' + order = { + 'physical': 0, + 'bond': 1, + 'bridge': 2, + 'vlan': 3, + } + content += "auto lo\niface lo inet loopback\n" + for dnskey, value in network_state.get('dns', {}).items(): + if len(value): + content += " dns-{} {}\n".format(dnskey, " ".join(value)) + + for iface in sorted(interfaces.values(), + key=lambda k: (order[k['type']], k['name'])): + + if content[-2:] != "\n\n": + content += "\n" + subnets = iface.get('subnets', {}) + if subnets: + for index, subnet in zip(range(0, len(subnets)), subnets): + if content[-2:] != "\n\n": + content += "\n" + iface['index'] = index + iface['mode'] = subnet['type'] + iface['control'] = subnet.get('control', 'auto') + if iface['mode'].endswith('6'): + iface['inet'] += '6' + elif iface['mode'] == 'static' \ + and ":" in subnet['address']: + iface['inet'] += '6' + if iface['mode'].startswith('dhcp'): + iface['mode'] = 'dhcp' + + content += _iface_start_entry(iface, index) + content += _iface_add_subnet(iface, subnet) + content += _iface_add_attrs(iface) + else: + # ifenslave docs say to auto the slave devices + if 'bond-master' in iface: + content += "auto {name}\n".format(**iface) + content += "iface {name} {inet} {mode}\n".format(**iface) + content += _iface_add_attrs(iface) + + for route in network_state.get('routes'): + content += self.render_route(route) + + # global replacements until v2 format + content = content.replace('mac_address', 'hwaddress') + return content + + def render_network_state(self, + target, network_state, eni="etc/network/interfaces", + links_prefix=LINKS_FNAME_PREFIX, + netrules='etc/udev/rules.d/70-persistent-net.rules'): + + fpeni = os.path.sep.join((target, eni,)) + util.ensure_dir(os.path.dirname(fpeni)) + with open(fpeni, 'w+') as f: + f.write(self.render_interfaces(network_state)) + + if netrules: + netrules = os.path.sep.join((target, netrules,)) + util.ensure_dir(os.path.dirname(netrules)) + with open(netrules, 'w+') as f: + f.write(self.render_persistent_net(network_state)) + + if links_prefix: + self.render_systemd_links(target, network_state, links_prefix) + + def render_systemd_links(self, target, network_state, + links_prefix=LINKS_FNAME_PREFIX): + fp_prefix = os.path.sep.join((target, links_prefix)) + for f in glob.glob(fp_prefix + "*"): + os.unlink(f) + + interfaces = network_state.get('interfaces') + for iface in interfaces.values(): + if (iface['type'] == 'physical' and 'name' in iface and + iface.get('mac_address')): + fname = fp_prefix + iface['name'] + ".link" + with open(fname, "w") as fp: + fp.write("\n".join([ + "[Match]", + "MACAddress=" + iface['mac_address'], + "", + "[Link]", + "Name=" + iface['name'], + "" + ])) diff --git a/cloudinit/net/renderers/__init__.py b/cloudinit/net/renderers/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/cloudinit/net/renderers/eni.py b/cloudinit/net/renderers/eni.py deleted file mode 100644 index b427012e..00000000 --- a/cloudinit/net/renderers/eni.py +++ /dev/null @@ -1,399 +0,0 @@ -# vi: ts=4 expandtab -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import glob -import os -import re - -from cloudinit.net import LINKS_FNAME_PREFIX -from cloudinit.net import ParserError -from cloudinit.net.udev import generate_udev_rule -from cloudinit import util - - -NET_CONFIG_COMMANDS = [ - "pre-up", "up", "post-up", "down", "pre-down", "post-down", -] - -NET_CONFIG_BRIDGE_OPTIONS = [ - "bridge_ageing", "bridge_bridgeprio", "bridge_fd", "bridge_gcinit", - "bridge_hello", "bridge_maxage", "bridge_maxwait", "bridge_stp", -] - -NET_CONFIG_OPTIONS = [ - "address", "netmask", "broadcast", "network", "metric", "gateway", - "pointtopoint", "media", "mtu", "hostname", "leasehours", "leasetime", - "vendor", "client", "bootfile", "server", "hwaddr", "provider", "frame", - "netnum", "endpoint", "local", "ttl", -] - - -# TODO: switch valid_map based on mode inet/inet6 -def _iface_add_subnet(iface, subnet): - content = "" - valid_map = [ - 'address', - 'netmask', - 'broadcast', - 'metric', - 'gateway', - 'pointopoint', - 'mtu', - 'scope', - 'dns_search', - 'dns_nameservers', - ] - for key, value in subnet.items(): - if value and key in valid_map: - if type(value) == list: - value = " ".join(value) - if '_' in key: - key = key.replace('_', '-') - content += " {} {}\n".format(key, value) - - return content - - -# TODO: switch to valid_map for attrs -def _iface_add_attrs(iface): - content = "" - ignore_map = [ - 'control', - 'index', - 'inet', - 'mode', - 'name', - 'subnets', - 'type', - ] - if iface['type'] not in ['bond', 'bridge', 'vlan']: - ignore_map.append('mac_address') - - for key, value in iface.items(): - if value and key not in ignore_map: - if type(value) == list: - value = " ".join(value) - content += " {} {}\n".format(key, value) - - return content - - -def _iface_start_entry(iface, index): - fullname = iface['name'] - if index != 0: - fullname += ":%s" % index - - control = iface['control'] - if control == "auto": - cverb = "auto" - elif control in ("hotplug",): - cverb = "allow-" + control - else: - cverb = "# control-" + control - - subst = iface.copy() - subst.update({'fullname': fullname, 'cverb': cverb}) - - return ("{cverb} {fullname}\n" - "iface {fullname} {inet} {mode}\n").format(**subst) - - -def _parse_deb_config_data(ifaces, contents, src_dir, src_path): - """Parses the file contents, placing result into ifaces. - - '_source_path' is added to every dictionary entry to define which file - the configration information came from. - - :param ifaces: interface dictionary - :param contents: contents of interfaces file - :param src_dir: directory interfaces file was located - :param src_path: file path the `contents` was read - """ - currif = None - for line in contents.splitlines(): - line = line.strip() - if line.startswith('#'): - continue - split = line.split(' ') - option = split[0] - if option == "source-directory": - parsed_src_dir = split[1] - if not parsed_src_dir.startswith("/"): - parsed_src_dir = os.path.join(src_dir, parsed_src_dir) - for expanded_path in glob.glob(parsed_src_dir): - dir_contents = os.listdir(expanded_path) - dir_contents = [ - os.path.join(expanded_path, path) - for path in dir_contents - if (os.path.isfile(os.path.join(expanded_path, path)) and - re.match("^[a-zA-Z0-9_-]+$", path) is not None) - ] - for entry in dir_contents: - with open(entry, "r") as fp: - src_data = fp.read().strip() - abs_entry = os.path.abspath(entry) - _parse_deb_config_data( - ifaces, src_data, - os.path.dirname(abs_entry), abs_entry) - elif option == "source": - new_src_path = split[1] - if not new_src_path.startswith("/"): - new_src_path = os.path.join(src_dir, new_src_path) - for expanded_path in glob.glob(new_src_path): - with open(expanded_path, "r") as fp: - src_data = fp.read().strip() - abs_path = os.path.abspath(expanded_path) - _parse_deb_config_data( - ifaces, src_data, - os.path.dirname(abs_path), abs_path) - elif option == "auto": - for iface in split[1:]: - if iface not in ifaces: - ifaces[iface] = { - # Include the source path this interface was found in. - "_source_path": src_path - } - ifaces[iface]['auto'] = True - elif option == "iface": - iface, family, method = split[1:4] - if iface not in ifaces: - ifaces[iface] = { - # Include the source path this interface was found in. - "_source_path": src_path - } - elif 'family' in ifaces[iface]: - raise ParserError( - "Interface %s can only be defined once. " - "Re-defined in '%s'." % (iface, src_path)) - ifaces[iface]['family'] = family - ifaces[iface]['method'] = method - currif = iface - elif option == "hwaddress": - ifaces[currif]['hwaddress'] = split[1] - elif option in NET_CONFIG_OPTIONS: - ifaces[currif][option] = split[1] - elif option in NET_CONFIG_COMMANDS: - if option not in ifaces[currif]: - ifaces[currif][option] = [] - ifaces[currif][option].append(' '.join(split[1:])) - elif option.startswith('dns-'): - if 'dns' not in ifaces[currif]: - ifaces[currif]['dns'] = {} - if option == 'dns-search': - ifaces[currif]['dns']['search'] = [] - for domain in split[1:]: - ifaces[currif]['dns']['search'].append(domain) - elif option == 'dns-nameservers': - ifaces[currif]['dns']['nameservers'] = [] - for server in split[1:]: - ifaces[currif]['dns']['nameservers'].append(server) - elif option.startswith('bridge_'): - if 'bridge' not in ifaces[currif]: - ifaces[currif]['bridge'] = {} - if option in NET_CONFIG_BRIDGE_OPTIONS: - bridge_option = option.replace('bridge_', '', 1) - ifaces[currif]['bridge'][bridge_option] = split[1] - elif option == "bridge_ports": - ifaces[currif]['bridge']['ports'] = [] - for iface in split[1:]: - ifaces[currif]['bridge']['ports'].append(iface) - elif option == "bridge_hw" and split[1].lower() == "mac": - ifaces[currif]['bridge']['mac'] = split[2] - elif option == "bridge_pathcost": - if 'pathcost' not in ifaces[currif]['bridge']: - ifaces[currif]['bridge']['pathcost'] = {} - ifaces[currif]['bridge']['pathcost'][split[1]] = split[2] - elif option == "bridge_portprio": - if 'portprio' not in ifaces[currif]['bridge']: - ifaces[currif]['bridge']['portprio'] = {} - ifaces[currif]['bridge']['portprio'][split[1]] = split[2] - elif option.startswith('bond-'): - if 'bond' not in ifaces[currif]: - ifaces[currif]['bond'] = {} - bond_option = option.replace('bond-', '', 1) - ifaces[currif]['bond'][bond_option] = split[1] - for iface in ifaces.keys(): - if 'auto' not in ifaces[iface]: - ifaces[iface]['auto'] = False - - -def _parse_deb_config(path): - """Parses a debian network configuration file.""" - ifaces = {} - with open(path, "r") as fp: - contents = fp.read().strip() - abs_path = os.path.abspath(path) - _parse_deb_config_data( - ifaces, contents, - os.path.dirname(abs_path), abs_path) - return ifaces - - -class Renderer(object): - """Renders network information in a /etc/network/interfaces format.""" - - def render_persistent_net(self, network_state): - """Given state, emit udev rules to map mac to ifname.""" - content = "" - interfaces = network_state.get('interfaces') - for iface in interfaces.values(): - # for physical interfaces write out a persist net udev rule - if iface['type'] == 'physical' and \ - 'name' in iface and iface.get('mac_address'): - content += generate_udev_rule(iface['name'], - iface['mac_address']) - - return content - - def render_route(self, route, indent=""): - """ When rendering routes for an iface, in some cases applying a route - may result in the route command returning non-zero which produces - some confusing output for users manually using ifup/ifdown[1]. To - that end, we will optionally include an '|| true' postfix to each - route line allowing users to work with ifup/ifdown without using - --force option. - - We may at somepoint not want to emit this additional postfix, and - add a 'strict' flag to this function. When called with strict=True, - then we will not append the postfix. - - 1. http://askubuntu.com/questions/168033/ - how-to-set-static-routes-in-ubuntu-server - """ - content = "" - up = indent + "post-up route add" - down = indent + "pre-down route del" - eol = " || true\n" - mapping = { - 'network': '-net', - 'netmask': 'netmask', - 'gateway': 'gw', - 'metric': 'metric', - } - if route['network'] == '0.0.0.0' and route['netmask'] == '0.0.0.0': - default_gw = " default gw %s" % route['gateway'] - content += up + default_gw + eol - content += down + default_gw + eol - elif route['network'] == '::' and route['netmask'] == 0: - # ipv6! - default_gw = " -A inet6 default gw %s" % route['gateway'] - content += up + default_gw + eol - content += down + default_gw + eol - else: - route_line = "" - for k in ['network', 'netmask', 'gateway', 'metric']: - if k in route: - route_line += " %s %s" % (mapping[k], route[k]) - content += up + route_line + eol - content += down + route_line + eol - - return content - - def render_interfaces(self, network_state): - ''' Given state, emit etc/network/interfaces content ''' - - content = "" - interfaces = network_state.get('interfaces') - ''' Apply a sort order to ensure that we write out - the physical interfaces first; this is critical for - bonding - ''' - order = { - 'physical': 0, - 'bond': 1, - 'bridge': 2, - 'vlan': 3, - } - content += "auto lo\niface lo inet loopback\n" - for dnskey, value in network_state.get('dns', {}).items(): - if len(value): - content += " dns-{} {}\n".format(dnskey, " ".join(value)) - - for iface in sorted(interfaces.values(), - key=lambda k: (order[k['type']], k['name'])): - - if content[-2:] != "\n\n": - content += "\n" - subnets = iface.get('subnets', {}) - if subnets: - for index, subnet in zip(range(0, len(subnets)), subnets): - if content[-2:] != "\n\n": - content += "\n" - iface['index'] = index - iface['mode'] = subnet['type'] - iface['control'] = subnet.get('control', 'auto') - if iface['mode'].endswith('6'): - iface['inet'] += '6' - elif iface['mode'] == 'static' \ - and ":" in subnet['address']: - iface['inet'] += '6' - if iface['mode'].startswith('dhcp'): - iface['mode'] = 'dhcp' - - content += _iface_start_entry(iface, index) - content += _iface_add_subnet(iface, subnet) - content += _iface_add_attrs(iface) - else: - # ifenslave docs say to auto the slave devices - if 'bond-master' in iface: - content += "auto {name}\n".format(**iface) - content += "iface {name} {inet} {mode}\n".format(**iface) - content += _iface_add_attrs(iface) - - for route in network_state.get('routes'): - content += self.render_route(route) - - # global replacements until v2 format - content = content.replace('mac_address', 'hwaddress') - return content - - def render_network_state(self, - target, network_state, eni="etc/network/interfaces", - links_prefix=LINKS_FNAME_PREFIX, - netrules='etc/udev/rules.d/70-persistent-net.rules'): - - fpeni = os.path.sep.join((target, eni,)) - util.ensure_dir(os.path.dirname(fpeni)) - with open(fpeni, 'w+') as f: - f.write(self.render_interfaces(network_state)) - - if netrules: - netrules = os.path.sep.join((target, netrules,)) - util.ensure_dir(os.path.dirname(netrules)) - with open(netrules, 'w+') as f: - f.write(self.render_persistent_net(network_state)) - - if links_prefix: - self.render_systemd_links(target, network_state, links_prefix) - - def render_systemd_links(self, target, network_state, - links_prefix=LINKS_FNAME_PREFIX): - fp_prefix = os.path.sep.join((target, links_prefix)) - for f in glob.glob(fp_prefix + "*"): - os.unlink(f) - - interfaces = network_state.get('interfaces') - for iface in interfaces.values(): - if (iface['type'] == 'physical' and 'name' in iface and - iface.get('mac_address')): - fname = fp_prefix + iface['name'] + ".link" - with open(fname, "w") as fp: - fp.write("\n".join([ - "[Match]", - "MACAddress=" + iface['mac_address'], - "", - "[Link]", - "Name=" + iface['name'], - "" - ])) -- cgit v1.2.3 From 4b0d2430e7674d5abb8fb27ac9ddb129d2bc0715 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 10 May 2016 14:16:07 -0700 Subject: Fix removal of validate_command --- cloudinit/net/network_state.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 73be84e1..2530a601 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -39,26 +39,25 @@ def from_state_file(state_file): return network_state +def diff_keys(expected, actual): + missing = set(expected) + for key in actual: + missing.discard(key) + return missing + + class InvalidCommand(Exception): pass def ensure_command_keys(required_keys): - required_keys = frozenset(required_keys) - - def extract_missing(command): - missing_keys = set() - for key in required_keys: - if key not in command: - missing_keys.add(key) - return missing_keys def wrapper(func): @six.wraps(func) def decorator(self, command, *args, **kwargs): if required_keys: - missing_keys = extract_missing(command) + missing_keys = diff_keys(required_keys, command) if missing_keys: raise InvalidCommand("Command missing %s of required" " keys %s" % (missing_keys, @@ -120,10 +119,11 @@ class NetworkState(object): raise Exception('Invalid state, missing version field') required_keys = NETWORK_STATE_REQUIRED_KEYS[state['version']] - if not self.valid_command(state, required_keys): - msg = 'Invalid state, missing keys: {}'.format(required_keys) + missing_keys = diff_keys(required_keys, state) + if missing_keys: + msg = 'Invalid state, missing keys: %s'.format(missing_keys) LOG.error(msg) - raise Exception(msg) + raise ValueError(msg) # v1 - direct attr mapping, except version for key in [k for k in required_keys if k not in ['version']]: -- cgit v1.2.3 From c5eb65ed705475640fce1025c74a54052c6e9731 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 10 May 2016 15:12:44 -0700 Subject: Add some basic eni rendering tests --- cloudinit/net/__init__.py | 11 ++++--- cloudinit/net/network_state.py | 2 +- tests/unittests/test_net.py | 68 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 7 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index e911ed0c..0202cbd8 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -107,7 +107,7 @@ class ParserError(Exception): """Raised when parser has issue parsing the interfaces file.""" -def parse_net_config_data(net_config): +def parse_net_config_data(net_config, skip_broken=True): """Parses the config, returns NetworkState object :param net_config: curtin network config dict @@ -116,20 +116,19 @@ def parse_net_config_data(net_config): if 'version' in net_config and 'config' in net_config: ns = network_state.NetworkState(version=net_config.get('version'), config=net_config.get('config')) - ns.parse_config() + ns.parse_config(skip_broken=skip_broken) state = ns.network_state - return state -def parse_net_config(path): +def parse_net_config(path, skip_broken=True): """Parses a curtin network configuration file and return network state""" ns = None net_config = util.read_conf(path) if 'network' in net_config: - ns = parse_net_config_data(net_config.get('network')) - + ns = parse_net_config_data(net_config.get('network'), + skip_broken=skip_broken) return ns diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 2530a601..2feffa71 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -142,7 +142,7 @@ class NetworkState(object): raise RuntimeError("No handler found for" " command '%s'" % command_type) try: - handler(command) + handler(self, command) except InvalidCommand: if not skip_broken: raise diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 6daf9601..37b48efb 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -1,7 +1,9 @@ from cloudinit import util from cloudinit import net from cloudinit.net import cmdline +from cloudinit.net import eni from .helpers import TestCase +from .helpers import mock import base64 import copy @@ -9,6 +11,8 @@ import io import gzip import json import os +import shutil +import tempfile DHCP_CONTENT_1 = """ DEVICE='eth0' @@ -69,6 +73,70 @@ STATIC_EXPECTED_1 = { } +class TestEniNetRendering(TestCase): + + @mock.patch("cloudinit.net.sys_dev_path") + @mock.patch("cloudinit.net.sys_netdev_info") + @mock.patch("cloudinit.net.get_devicelist") + def test_generation(self, mock_get_devicelist, mock_sys_netdev_info, + mock_sys_dev_path): + mock_get_devicelist.return_value = ['eth1000', 'lo'] + + dev_characteristics = { + 'eth1000': { + "bridge": False, + "carrier": False, + "dormant": False, + "operstate": "down", + "address": "07-1C-C6-75-A4-BE", + } + } + + def netdev_info(name, field): + return dev_characteristics[name][field] + + mock_sys_netdev_info.side_effect = netdev_info + + tmp_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp_dir) + + def sys_dev_path(devname, path=""): + return tmp_dir + devname + "/" + path + + for dev in dev_characteristics: + os.makedirs(os.path.join(tmp_dir, dev)) + with open(os.path.join(tmp_dir, dev, 'operstate'), 'w') as fh: + fh.write("down") + + mock_sys_dev_path.side_effect = sys_dev_path + + network_cfg = net.generate_fallback_config() + network_state = net.parse_net_config_data(network_cfg, + skip_broken=False) + + render_dir = os.path.join(tmp_dir, "render") + os.makedirs(render_dir) + + renderer = eni.Renderer() + renderer.render_network_state(render_dir, network_state, + eni="interfaces", + links_prefix=None, + netrules=None) + + self.assertTrue(os.path.exists(os.path.join(render_dir, + 'interfaces'))) + with open(os.path.join(render_dir, 'interfaces')) as fh: + contents = fh.read() + + expected = """auto lo +iface lo inet loopback + +auto eth1000 +iface eth1000 inet dhcp +""" + self.assertEqual(expected, contents) + + class TestNetConfigParsing(TestCase): simple_cfg = { 'config': [{"type": "physical", "name": "eth0", -- cgit v1.2.3 From 26ea813d293467921ab6b1e32abd2ab8fcefa3bd Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 11 May 2016 14:18:02 -0700 Subject: Fix py26 for rhel (and older versions of python) --- cloudinit/net/cmdline.py | 26 ++++--- cloudinit/sources/DataSourceAzure.py | 2 +- cloudinit/sources/helpers/openstack.py | 8 +- cloudinit/util.py | 25 ++++++- requirements.txt | 8 +- setup.py | 1 - test-requirements.txt | 1 + tests/unittests/helpers.py | 86 +++------------------- tests/unittests/test__init__.py | 18 ++--- tests/unittests/test_cli.py | 7 +- tests/unittests/test_cs_util.py | 42 ++++------- tests/unittests/test_datasource/test_azure.py | 12 +-- .../unittests/test_datasource/test_azure_helper.py | 12 +-- tests/unittests/test_datasource/test_cloudsigma.py | 26 +++++-- tests/unittests/test_datasource/test_cloudstack.py | 11 +-- .../unittests/test_datasource/test_configdrive.py | 11 +-- tests/unittests/test_datasource/test_nocloud.py | 14 +--- tests/unittests/test_datasource/test_smartos.py | 21 ++++-- tests/unittests/test_net.py | 7 +- tests/unittests/test_reporting.py | 4 +- tests/unittests/test_rh_subscription.py | 23 ++++-- tox.ini | 36 ++++----- 22 files changed, 163 insertions(+), 238 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py index 958c264b..21bc35d9 100644 --- a/cloudinit/net/cmdline.py +++ b/cloudinit/net/cmdline.py @@ -20,7 +20,6 @@ import base64 import glob import gzip import io -import shlex from cloudinit.net import get_devicelist from cloudinit.net import sys_netdev_info @@ -34,13 +33,17 @@ def _load_shell_content(content, add_empty=False, empty_val=None): then add entries in to the returned dictionary for 'VAR=' variables. Set their value to empty_val.""" data = {} - for line in shlex.split(content): - key, value = line.split("=", 1) - if not value: - value = empty_val - if add_empty or value: - data[key] = value - + for line in util.shlex_split(content): + try: + key, value = line.split("=", 1) + except ValueError: + # Unsplittable line, skip it... + pass + else: + if not value: + value = empty_val + if add_empty or value: + data[key] = value return data @@ -60,6 +63,9 @@ def _klibc_to_config_entry(content, mac_addrs=None): if mac_addrs is None: mac_addrs = {} + print("Reading content") + print(content) + data = _load_shell_content(content) try: name = data['DEVICE'] @@ -185,7 +191,7 @@ def read_kernel_cmdline_config(files=None, mac_addrs=None, cmdline=None): return None if mac_addrs is None: - mac_addrs = {k: sys_netdev_info(k, 'address') - for k in get_devicelist()} + mac_addrs = dict((k, sys_netdev_info(k, 'address')) + for k in get_devicelist()) return config_from_klibc_net_cfg(files=files, mac_addrs=mac_addrs) diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 698f4cac..66c8ced8 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -421,7 +421,7 @@ def write_files(datadir, files, dirmode=None): elem.text = DEF_PASSWD_REDACTION return ET.tostring(root) except Exception: - LOG.critical("failed to redact userpassword in {}".format(fname)) + LOG.critical("failed to redact userpassword in %s", fname) return cnt if not datadir: diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 845ea971..b2acc648 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -534,13 +534,13 @@ def convert_net_json(network_json): config = [] for link in links: subnets = [] - cfg = {k: v for k, v in link.items() - if k in valid_keys['physical']} + cfg = dict((k, v) for k, v in link.items() + if k in valid_keys['physical']) cfg.update({'name': link['id']}) for network in [net for net in networks if net['link'] == link['id']]: - subnet = {k: v for k, v in network.items() - if k in valid_keys['subnet']} + subnet = dict((k, v) for k, v in network.items() + if k in valid_keys['subnet']) if 'dhcp' in network['type']: t = 'dhcp6' if network['type'].startswith('ipv6') else 'dhcp4' subnet.update({ diff --git a/cloudinit/util.py b/cloudinit/util.py index 0d21e11b..7562b97a 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -37,6 +37,7 @@ import pwd import random import re import shutil +import shlex import socket import stat import string @@ -81,6 +82,7 @@ CONTAINER_TESTS = (['systemd-detect-virt', '--quiet', '--container'], ['lxc-is-container']) PROC_CMDLINE = None +PY26 = sys.version_info[0:2] == (2, 6) def decode_binary(blob, encoding='utf-8'): @@ -171,7 +173,8 @@ class ProcessExecutionError(IOError): def __init__(self, stdout=None, stderr=None, exit_code=None, cmd=None, - description=None, reason=None): + description=None, reason=None, + errno=None): if not cmd: self.cmd = '-' else: @@ -202,6 +205,7 @@ class ProcessExecutionError(IOError): else: self.reason = '-' + self.errno = errno message = self.MESSAGE_TMPL % { 'description': self.description, 'cmd': self.cmd, @@ -1147,7 +1151,14 @@ def find_devs_with(criteria=None, oformat='device', options.append(path) cmd = blk_id_cmd + options # See man blkid for why 2 is added - (out, _err) = subp(cmd, rcs=[0, 2]) + try: + (out, _err) = subp(cmd, rcs=[0, 2]) + except ProcessExecutionError as e: + if e.errno == errno.ENOENT: + # blkid not found... + out = "" + else: + raise entries = [] for line in out.splitlines(): line = line.strip() @@ -1191,6 +1202,13 @@ def load_file(fname, read_cb=None, quiet=False, decode=True): return contents +def shlex_split(blob): + if PY26 and isinstance(blob, six.text_type): + # Older versions don't support unicode input + blob = blob.encode("utf8") + return shlex.split(blob) + + def get_cmdline(): if 'DEBUG_PROC_CMDLINE' in os.environ: return os.environ["DEBUG_PROC_CMDLINE"] @@ -1696,7 +1714,8 @@ def subp(args, data=None, rcs=None, env=None, capture=True, shell=False, sp = subprocess.Popen(args, **kws) (out, err) = sp.communicate(data) except OSError as e: - raise ProcessExecutionError(cmd=args, reason=e) + raise ProcessExecutionError(cmd=args, reason=e, + errno=e.errno) rc = sp.returncode if rc not in rcs: raise ProcessExecutionError(stdout=out, stderr=err, diff --git a/requirements.txt b/requirements.txt index 19c88857..cc1dc05f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,8 +11,12 @@ PrettyTable oauthlib # This one is currently used only by the CloudSigma and SmartOS datasources. -# If these datasources are removed, this is no longer needed -pyserial +# If these datasources are removed, this is no longer needed. +# +# This will not work in py2.6 so it is only optionally installed on +# python 2.7 and later. +# +# pyserial # This is only needed for places where we need to support configs in a manner # that the built-in config parser is not sufficent (ie diff --git a/setup.py b/setup.py index f86727b2..6b4fc031 100755 --- a/setup.py +++ b/setup.py @@ -197,7 +197,6 @@ requirements = read_requires() if sys.version_info < (3,): requirements.append('cheetah') - setuptools.setup( name='cloud-init', version=get_version(), diff --git a/test-requirements.txt b/test-requirements.txt index 9b3d07c5..d9c757e6 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -5,3 +5,4 @@ pep8==1.5.7 pyflakes contextlib2 setuptools +unittest2 diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index fb9c83a7..33f89254 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -7,12 +7,10 @@ import shutil import tempfile import unittest +import mock import six +import unittest2 -try: - from unittest import mock -except ImportError: - import mock try: from contextlib import ExitStack except ImportError: @@ -21,6 +19,9 @@ except ImportError: from cloudinit import helpers as ch from cloudinit import util +# Used for skipping tests +SkipTest = unittest2.SkipTest + # Used for detecting different python versions PY2 = False PY26 = False @@ -44,79 +45,6 @@ else: if _PY_MINOR == 4 and _PY_MICRO < 3: FIX_HTTPRETTY = True -if PY26: - # For now add these on, taken from python 2.7 + slightly adjusted. Drop - # all this once Python 2.6 is dropped as a minimum requirement. - class TestCase(unittest.TestCase): - def setUp(self): - super(TestCase, self).setUp() - self.__all_cleanups = ExitStack() - - def tearDown(self): - self.__all_cleanups.close() - unittest.TestCase.tearDown(self) - - def addCleanup(self, function, *args, **kws): - self.__all_cleanups.callback(function, *args, **kws) - - def assertIs(self, expr1, expr2, msg=None): - if expr1 is not expr2: - standardMsg = '%r is not %r' % (expr1, expr2) - self.fail(self._formatMessage(msg, standardMsg)) - - def assertIn(self, member, container, msg=None): - if member not in container: - standardMsg = '%r not found in %r' % (member, container) - self.fail(self._formatMessage(msg, standardMsg)) - - def assertNotIn(self, member, container, msg=None): - if member in container: - standardMsg = '%r unexpectedly found in %r' - standardMsg = standardMsg % (member, container) - self.fail(self._formatMessage(msg, standardMsg)) - - def assertIsNone(self, value, msg=None): - if value is not None: - standardMsg = '%r is not None' - standardMsg = standardMsg % (value) - self.fail(self._formatMessage(msg, standardMsg)) - - def assertIsInstance(self, obj, cls, msg=None): - """Same as self.assertTrue(isinstance(obj, cls)), with a nicer - default message.""" - if not isinstance(obj, cls): - standardMsg = '%s is not an instance of %r' % (repr(obj), cls) - self.fail(self._formatMessage(msg, standardMsg)) - - def assertDictContainsSubset(self, expected, actual, msg=None): - missing = [] - mismatched = [] - for k, v in expected.items(): - if k not in actual: - missing.append(k) - elif actual[k] != v: - mismatched.append('%r, expected: %r, actual: %r' - % (k, v, actual[k])) - - if len(missing) == 0 and len(mismatched) == 0: - return - - standardMsg = '' - if missing: - standardMsg = 'Missing: %r' % ','.join(m for m in missing) - if mismatched: - if standardMsg: - standardMsg += '; ' - standardMsg += 'Mismatched values: %s' % ','.join(mismatched) - - self.fail(self._formatMessage(msg, standardMsg)) - - -else: - class TestCase(unittest.TestCase): - pass - - # Makes the old path start # with new base instead of whatever # it previously had @@ -151,6 +79,10 @@ def retarget_many_wrapper(new_base, am, old_func): return wrapper +class TestCase(unittest2.TestCase): + pass + + class ResourceUsingTestCase(TestCase): def setUp(self): super(ResourceUsingTestCase, self).setUp() diff --git a/tests/unittests/test__init__.py b/tests/unittests/test__init__.py index 153f1658..a9b35afe 100644 --- a/tests/unittests/test__init__.py +++ b/tests/unittests/test__init__.py @@ -1,16 +1,7 @@ import os import shutil import tempfile -import unittest - -try: - from unittest import mock -except ImportError: - import mock -try: - from contextlib import ExitStack -except ImportError: - from contextlib2 import ExitStack +import unittest2 from cloudinit import handlers from cloudinit import helpers @@ -18,7 +9,7 @@ from cloudinit import settings from cloudinit import url_helper from cloudinit import util -from .helpers import TestCase +from .helpers import TestCase, ExitStack, mock class FakeModule(handlers.Handler): @@ -99,9 +90,10 @@ class TestWalkerHandleHandler(TestCase): self.assertEqual(self.data['handlercount'], 0) -class TestHandlerHandlePart(unittest.TestCase): +class TestHandlerHandlePart(TestCase): def setUp(self): + super(TestHandlerHandlePart, self).setUp() self.data = "fake data" self.ctype = "fake ctype" self.filename = "fake filename" @@ -177,7 +169,7 @@ class TestHandlerHandlePart(unittest.TestCase): self.data, self.ctype, self.filename, self.payload) -class TestCmdlineUrl(unittest.TestCase): +class TestCmdlineUrl(TestCase): def test_invalid_content(self): url = "http://example.com/foo" key = "mykey" diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py index ed863399..f8fe7c9b 100644 --- a/tests/unittests/test_cli.py +++ b/tests/unittests/test_cli.py @@ -4,12 +4,7 @@ import sys import six from . import helpers as test_helpers - -try: - from unittest import mock -except ImportError: - import mock - +mock = test_helpers.mock BIN_CLOUDINIT = "bin/cloud-init" diff --git a/tests/unittests/test_cs_util.py b/tests/unittests/test_cs_util.py index d7273035..8c9ac0cd 100644 --- a/tests/unittests/test_cs_util.py +++ b/tests/unittests/test_cs_util.py @@ -1,20 +1,14 @@ from __future__ import print_function -import sys -import unittest +from . import helpers as test_helpers -from cloudinit.cs_utils import Cepko +import unittest2 try: - skip = unittest.skip -except AttributeError: - # Python 2.6. Doesn't have to be high fidelity. - def skip(reason): - def decorator(func): - def wrapper(*args, **kws): - print(reason, file=sys.stderr) - return wrapper - return decorator + from cloudinit.cs_utils import Cepko + WILL_WORK = True +except ImportError: + WILL_WORK = False SERVER_CONTEXT = { @@ -32,29 +26,21 @@ SERVER_CONTEXT = { } -class CepkoMock(Cepko): - def all(self): - return SERVER_CONTEXT +if WILL_WORK: + class CepkoMock(Cepko): + def all(self): + return SERVER_CONTEXT - def get(self, key="", request_pattern=None): - return SERVER_CONTEXT['tags'] + def get(self, key="", request_pattern=None): + return SERVER_CONTEXT['tags'] # 2015-01-22 BAW: This test is completely useless because it only ever tests # the CepkoMock object. Even in its original form, I don't think it ever # touched the underlying Cepko class methods. -@skip('This test is completely useless') -class CepkoResultTests(unittest.TestCase): +class CepkoResultTests(test_helpers.TestCase): def setUp(self): - pass - # self.mocked = self.mocker.replace("cloudinit.cs_utils.Cepko", - # spec=CepkoMock, - # count=False, - # passthrough=False) - # self.mocked() - # self.mocker.result(CepkoMock()) - # self.mocker.replay() - # self.c = Cepko() + raise unittest2.SkipTest('This test is completely useless') def test_getitem(self): result = self.c.all() diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 444e2799..aafe1bc2 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -1,16 +1,8 @@ from cloudinit import helpers from cloudinit.util import b64e, decode_binary, load_file from cloudinit.sources import DataSourceAzure -from ..helpers import TestCase, populate_dir -try: - from unittest import mock -except ImportError: - import mock -try: - from contextlib import ExitStack -except ImportError: - from contextlib2 import ExitStack +from ..helpers import TestCase, populate_dir, mock, ExitStack, PY26, SkipTest import crypt import os @@ -83,6 +75,8 @@ class TestAzureDataSource(TestCase): def setUp(self): super(TestAzureDataSource, self).setUp() + if PY26: + raise SkipTest("Does not work on python 2.6") self.tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.tmp) diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 1134199b..736f4463 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -1,17 +1,11 @@ import os from cloudinit.sources.helpers import azure as azure_helper -from ..helpers import TestCase -try: - from unittest import mock -except ImportError: - import mock +from ..helpers import ExitStack +from ..helpers import TestCase -try: - from contextlib import ExitStack -except ImportError: - from contextlib2 import ExitStack +from ..helpers import mock GOAL_STATE_TEMPLATE = """\ diff --git a/tests/unittests/test_datasource/test_cloudsigma.py b/tests/unittests/test_datasource/test_cloudsigma.py index 772d189a..11968796 100644 --- a/tests/unittests/test_datasource/test_cloudsigma.py +++ b/tests/unittests/test_datasource/test_cloudsigma.py @@ -1,11 +1,18 @@ # coding: utf-8 + import copy -from cloudinit.cs_utils import Cepko -from cloudinit.sources import DataSourceCloudSigma +try: + # Serial does not work on py2.6 (anymore) + import pyserial + from cloudinit.cs_utils import Cepko + from cloudinit.sources import DataSourceCloudSigma + WILL_WORK = True +except ImportError: + WILL_WORK = False from .. import helpers as test_helpers - +from ..helpers import SkipTest SERVER_CONTEXT = { "cpu": 1000, @@ -29,17 +36,20 @@ SERVER_CONTEXT = { } -class CepkoMock(Cepko): - def __init__(self, mocked_context): - self.result = mocked_context +if WILL_WORK: + class CepkoMock(Cepko): + def __init__(self, mocked_context): + self.result = mocked_context - def all(self): - return self + def all(self): + return self class DataSourceCloudSigmaTest(test_helpers.TestCase): def setUp(self): super(DataSourceCloudSigmaTest, self).setUp() + if not WILL_WORK: + raise SkipTest("Datasource testing not supported") self.datasource = DataSourceCloudSigma.DataSourceCloudSigma("", "", "") self.datasource.is_running_in_cloudsigma = lambda: True self.datasource.cepko = CepkoMock(SERVER_CONTEXT) diff --git a/tests/unittests/test_datasource/test_cloudstack.py b/tests/unittests/test_datasource/test_cloudstack.py index 656d80d1..4d6b47b4 100644 --- a/tests/unittests/test_datasource/test_cloudstack.py +++ b/tests/unittests/test_datasource/test_cloudstack.py @@ -1,15 +1,6 @@ from cloudinit import helpers from cloudinit.sources.DataSourceCloudStack import DataSourceCloudStack -from ..helpers import TestCase - -try: - from unittest import mock -except ImportError: - import mock -try: - from contextlib import ExitStack -except ImportError: - from contextlib2 import ExitStack +from ..helpers import TestCase, mock, ExitStack class TestCloudStackPasswordFetching(TestCase): diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index 8beaf95e..14cc8266 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -5,22 +5,13 @@ import shutil import six import tempfile -try: - from unittest import mock -except ImportError: - import mock -try: - from contextlib import ExitStack -except ImportError: - from contextlib2 import ExitStack - from cloudinit import helpers from cloudinit import settings from cloudinit.sources import DataSourceConfigDrive as ds from cloudinit.sources.helpers import openstack from cloudinit import util -from ..helpers import TestCase +from ..helpers import TestCase, ExitStack, mock PUBKEY = u'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460\n' diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py index 2d5fc37c..a92dd3b3 100644 --- a/tests/unittests/test_datasource/test_nocloud.py +++ b/tests/unittests/test_datasource/test_nocloud.py @@ -1,22 +1,12 @@ from cloudinit import helpers from cloudinit.sources import DataSourceNoCloud from cloudinit import util -from ..helpers import TestCase, populate_dir +from ..helpers import TestCase, populate_dir, mock, ExitStack import os import yaml import shutil import tempfile -import unittest - -try: - from unittest import mock -except ImportError: - import mock -try: - from contextlib import ExitStack -except ImportError: - from contextlib2 import ExitStack class TestNoCloudDataSource(TestCase): @@ -139,7 +129,7 @@ class TestNoCloudDataSource(TestCase): self.assertTrue(ret) -class TestParseCommandLineData(unittest.TestCase): +class TestParseCommandLineData(TestCase): def test_parse_cmdline_data_valid(self): ds_id = "ds=nocloud" diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 5c49966a..6b628276 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -33,19 +33,21 @@ import tempfile import uuid from binascii import crc32 -import serial +try: + # Serial does not work on py2.6 (anymore) + import serial + from cloudinit.sources import DataSourceSmartOS + WILL_WORK = True +except ImportError: + WILL_WORK = False + import six from cloudinit import helpers as c_helpers -from cloudinit.sources import DataSourceSmartOS from cloudinit.util import b64e from .. import helpers - -try: - from unittest import mock -except ImportError: - import mock +from ..helpers import mock, SkipTest MOCK_RETURNS = { 'hostname': 'test-host', @@ -79,7 +81,8 @@ def get_mock_client(mockdata): class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): def setUp(self): super(TestSmartOSDataSource, self).setUp() - + if not WILL_WORK: + raise SkipTest("This test will not work") self.tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.tmp) self.legacy_user_d = tempfile.mkdtemp() @@ -445,6 +448,8 @@ class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase): def setUp(self): super(TestJoyentMetadataClient, self).setUp() + if not WILL_WORK: + raise SkipTest("This test will not work") self.serial = mock.MagicMock(spec=serial.Serial) self.request_id = 0xabcdef12 self.metadata_value = 'value' diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 005957de..ed2c6d0f 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -78,8 +78,9 @@ class TestEniNetRendering(TestCase): @mock.patch("cloudinit.net.sys_dev_path") @mock.patch("cloudinit.net.sys_netdev_info") @mock.patch("cloudinit.net.get_devicelist") - def test_generation(self, mock_get_devicelist, mock_sys_netdev_info, - mock_sys_dev_path): + def test_default_generation(self, mock_get_devicelist, + mock_sys_netdev_info, + mock_sys_dev_path): mock_get_devicelist.return_value = ['eth1000', 'lo'] dev_characteristics = { @@ -138,7 +139,7 @@ iface eth1000 inet dhcp self.assertEqual(expected.lstrip(), contents.lstrip()) -class TestNetConfigParsing(TestCase): +class TestCmdlineConfigParsing(TestCase): simple_cfg = { 'config': [{"type": "physical", "name": "eth0", "mac_address": "c0:d6:9f:2c:e8:80", diff --git a/tests/unittests/test_reporting.py b/tests/unittests/test_reporting.py index 32356ef9..493bb261 100644 --- a/tests/unittests/test_reporting.py +++ b/tests/unittests/test_reporting.py @@ -7,7 +7,9 @@ from cloudinit import reporting from cloudinit.reporting import handlers from cloudinit.reporting import events -from .helpers import (mock, TestCase) +import mock + +from .helpers import TestCase def _fake_registry(): diff --git a/tests/unittests/test_rh_subscription.py b/tests/unittests/test_rh_subscription.py index 8c586ad7..13045aaf 100644 --- a/tests/unittests/test_rh_subscription.py +++ b/tests/unittests/test_rh_subscription.py @@ -1,11 +1,24 @@ +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import logging + from cloudinit import util from cloudinit.config import cc_rh_subscription -import logging -import mock -import unittest + +from .helpers import TestCase, mock -class GoodTests(unittest.TestCase): +class GoodTests(TestCase): def setUp(self): super(GoodTests, self).setUp() self.name = "cc_rh_subscription" @@ -92,7 +105,7 @@ class GoodTests(unittest.TestCase): self.assertEqual(self.SM._sub_man_cli.call_count, 9) -class TestBadInput(unittest.TestCase): +class TestBadInput(TestCase): name = "cc_rh_subscription" cloud_init = None log = logging.getLogger("bad_tests") diff --git a/tox.ini b/tox.ini index bd7c27dd..3210b0ee 100644 --- a/tox.ini +++ b/tox.ini @@ -1,32 +1,32 @@ [tox] -envlist = py27,py3,pyflakes +envlist = py27,py26,py3,pyflakes recreate = True +usedevelop = True [testenv] commands = python -m nose {posargs:tests} deps = -r{toxinidir}/test-requirements.txt - -r{toxinidir}/requirements.txt - -[testenv:py3] -basepython = python3 + -r{toxinidir}/requirements.txt +setenv = + LC_ALL = en_US.utf-8 [testenv:pyflakes] basepython = python3 commands = {envpython} -m pyflakes {posargs:cloudinit/ tests/ tools/} - {envpython} -m pep8 {posargs:cloudinit/ tests/ tools/} + {envpython} -m pep8 {posargs:cloudinit/ tests/ tools/} -# https://github.com/gabrielfalcao/HTTPretty/issues/223 -setenv = - LC_ALL = en_US.utf-8 +[testenv:py3] +basepython = python3 +deps = -r{toxinidir}/test-requirements.txt + -r{toxinidir}/requirements.txt + pyserial + +[testenv:py27] +deps = -r{toxinidir}/test-requirements.txt + -r{toxinidir}/requirements.txt + pyserial [testenv:py26] commands = nosetests {posargs:tests} -deps = - contextlib2 - httpretty>=0.7.1 - mock - nose - pep8==1.5.7 - pyflakes -setenv = - LC_ALL = C +deps = -r{toxinidir}/test-requirements.txt + -r{toxinidir}/requirements.txt -- cgit v1.2.3 From e885f694c9951101b57ee182bebc000e398da563 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 11 May 2016 14:19:33 -0700 Subject: Remove stray prints leftover --- cloudinit/net/cmdline.py | 3 --- 1 file changed, 3 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py index 21bc35d9..f5712533 100644 --- a/cloudinit/net/cmdline.py +++ b/cloudinit/net/cmdline.py @@ -63,9 +63,6 @@ def _klibc_to_config_entry(content, mac_addrs=None): if mac_addrs is None: mac_addrs = {} - print("Reading content") - print(content) - data = _load_shell_content(content) try: name = data['DEVICE'] -- cgit v1.2.3 From 12d7ee2cb6589b866ab26b508b15c65326481d6c Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 11 May 2016 16:47:50 -0700 Subject: Use a fake serial module that will allow tests to contine Instead of aborting all serial using tests instead just create a serial module in cloudinit that will create a fake and broken serial class when pyserial is not actually installed. This allows for using the datasource and tests that exist in a more functional and tested manner (even when pyserial is not found). --- cloudinit/cs_utils.py | 3 +- cloudinit/serial.py | 50 ++++++++++++++++++++++ cloudinit/sources/DataSourceSmartOS.py | 4 +- tests/unittests/test_cs_util.py | 21 +++------ tests/unittests/test_datasource/test_cloudsigma.py | 23 +++------- tests/unittests/test_datasource/test_smartos.py | 15 ++----- 6 files changed, 71 insertions(+), 45 deletions(-) create mode 100644 cloudinit/serial.py (limited to 'cloudinit') diff --git a/cloudinit/cs_utils.py b/cloudinit/cs_utils.py index 83ac1a0e..412431f2 100644 --- a/cloudinit/cs_utils.py +++ b/cloudinit/cs_utils.py @@ -33,7 +33,8 @@ API Docs: http://cloudsigma-docs.readthedocs.org/en/latest/server_context.html import json import platform -import serial +from cloudinit import serial + # these high timeouts are necessary as read may read a lot of data. READ_TIMEOUT = 60 diff --git a/cloudinit/serial.py b/cloudinit/serial.py new file mode 100644 index 00000000..af45c13e --- /dev/null +++ b/cloudinit/serial.py @@ -0,0 +1,50 @@ +# vi: ts=4 expandtab +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +from __future__ import absolute_import + +try: + from serial import Serial +except ImportError: + # For older versions of python (ie 2.6) pyserial may not exist and/or + # work and/or be installed, so make a dummy/fake serial that blows up + # when used... + class Serial(object): + def __init__(self, *args, **kwargs): + pass + + @staticmethod + def isOpen(): + return False + + @staticmethod + def write(data): + raise IOError("Unable to perform serial `write` operation," + " pyserial not installed.") + + @staticmethod + def readline(): + raise IOError("Unable to perform serial `readline` operation," + " pyserial not installed.") + + @staticmethod + def flush(): + raise IOError("Unable to perform serial `flush` operation," + " pyserial not installed.") + + @staticmethod + def read(size=1): + raise IOError("Unable to perform serial `read` operation," + " pyserial not installed.") diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 6cbd8dfa..c7641eb3 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -40,13 +40,11 @@ import re import socket import stat -import serial - from cloudinit import log as logging +from cloudinit import serial from cloudinit import sources from cloudinit import util - LOG = logging.getLogger(__name__) SMARTOS_ATTRIB_MAP = { diff --git a/tests/unittests/test_cs_util.py b/tests/unittests/test_cs_util.py index 8c9ac0cd..56c9ce9e 100644 --- a/tests/unittests/test_cs_util.py +++ b/tests/unittests/test_cs_util.py @@ -2,13 +2,7 @@ from __future__ import print_function from . import helpers as test_helpers -import unittest2 - -try: - from cloudinit.cs_utils import Cepko - WILL_WORK = True -except ImportError: - WILL_WORK = False +from cloudinit.cs_utils import Cepko SERVER_CONTEXT = { @@ -26,13 +20,12 @@ SERVER_CONTEXT = { } -if WILL_WORK: - class CepkoMock(Cepko): - def all(self): - return SERVER_CONTEXT +class CepkoMock(Cepko): + def all(self): + return SERVER_CONTEXT - def get(self, key="", request_pattern=None): - return SERVER_CONTEXT['tags'] + def get(self, key="", request_pattern=None): + return SERVER_CONTEXT['tags'] # 2015-01-22 BAW: This test is completely useless because it only ever tests @@ -40,7 +33,7 @@ if WILL_WORK: # touched the underlying Cepko class methods. class CepkoResultTests(test_helpers.TestCase): def setUp(self): - raise unittest2.SkipTest('This test is completely useless') + raise test_helpers.SkipTest('This test is completely useless') def test_getitem(self): result = self.c.all() diff --git a/tests/unittests/test_datasource/test_cloudsigma.py b/tests/unittests/test_datasource/test_cloudsigma.py index 11968796..7950fc52 100644 --- a/tests/unittests/test_datasource/test_cloudsigma.py +++ b/tests/unittests/test_datasource/test_cloudsigma.py @@ -2,14 +2,8 @@ import copy -try: - # Serial does not work on py2.6 (anymore) - import pyserial - from cloudinit.cs_utils import Cepko - from cloudinit.sources import DataSourceCloudSigma - WILL_WORK = True -except ImportError: - WILL_WORK = False +from cloudinit.cs_utils import Cepko +from cloudinit.sources import DataSourceCloudSigma from .. import helpers as test_helpers from ..helpers import SkipTest @@ -36,20 +30,17 @@ SERVER_CONTEXT = { } -if WILL_WORK: - class CepkoMock(Cepko): - def __init__(self, mocked_context): - self.result = mocked_context +class CepkoMock(Cepko): + def __init__(self, mocked_context): + self.result = mocked_context - def all(self): - return self + def all(self): + return self class DataSourceCloudSigmaTest(test_helpers.TestCase): def setUp(self): super(DataSourceCloudSigmaTest, self).setUp() - if not WILL_WORK: - raise SkipTest("Datasource testing not supported") self.datasource = DataSourceCloudSigma.DataSourceCloudSigma("", "", "") self.datasource.is_running_in_cloudsigma = lambda: True self.datasource.cepko = CepkoMock(SERVER_CONTEXT) diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 6b628276..f536ef4f 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -33,13 +33,8 @@ import tempfile import uuid from binascii import crc32 -try: - # Serial does not work on py2.6 (anymore) - import serial - from cloudinit.sources import DataSourceSmartOS - WILL_WORK = True -except ImportError: - WILL_WORK = False +from cloudinit import serial +from cloudinit.sources import DataSourceSmartOS import six @@ -81,8 +76,7 @@ def get_mock_client(mockdata): class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): def setUp(self): super(TestSmartOSDataSource, self).setUp() - if not WILL_WORK: - raise SkipTest("This test will not work") + self.tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.tmp) self.legacy_user_d = tempfile.mkdtemp() @@ -448,8 +442,7 @@ class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase): def setUp(self): super(TestJoyentMetadataClient, self).setUp() - if not WILL_WORK: - raise SkipTest("This test will not work") + self.serial = mock.MagicMock(spec=serial.Serial) self.request_id = 0xabcdef12 self.metadata_value = 'value' -- cgit v1.2.3 From cca640d332eb8a6b068033a28b0b319873c7fbf6 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 09:21:50 +0200 Subject: allow to add keys without specifying a source --- cloudinit/config/cc_apt_configure.py | 42 ++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 16 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 702977cb..1d3eddff 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -164,6 +164,29 @@ def generate_sources_list(codename, mirrors, cloud, log): templater.render_to_file(template_fn, '/etc/apt/sources.list', params) +def add_key(ent, errorlist): + """ + add key to the system as defiend in entry (if any) + suppords raw keys or keyid's + The latter will as a first step fetched to get the raw key + """ + if ('keyid' in ent and 'key' not in ent): + keyserver = "keyserver.ubuntu.com" + if 'keyserver' in ent: + keyserver = ent['keyserver'] + try: + ent['key'] = getkeybyid(ent['keyid'], keyserver) + except: + errorlist.append([ent, "failed to get key from %s" % keyserver]) + return + + if 'key' in ent: + try: + util.subp(('apt-key', 'add', '-'), ent['key']) + except: + errorlist.append([ent, "failed add key"]) + + def add_sources(srclist, template_params=None, aa_repo_match=None): """ add entries in /etc/apt/sources.list.d for each abbreviated @@ -179,6 +202,9 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): errorlist = [] for ent in srclist: + # keys can be added without specifying a source + add_key(ent, errorlist) + if 'source' not in ent: errorlist.append(["", "missing source"]) continue @@ -201,22 +227,6 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): ent['filename'] = os.path.join("/etc/apt/sources.list.d/", ent['filename']) - if ('keyid' in ent and 'key' not in ent): - ks = "keyserver.ubuntu.com" - if 'keyserver' in ent: - ks = ent['keyserver'] - try: - ent['key'] = getkeybyid(ent['keyid'], ks) - except: - errorlist.append([source, "failed to get key from %s" % ks]) - continue - - if 'key' in ent: - try: - util.subp(('apt-key', 'add', '-'), ent['key']) - except: - errorlist.append([source, "failed add key"]) - try: contents = "%s\n" % (source) util.write_file(ent['filename'], contents, omode="ab") -- cgit v1.2.3 From 6fc583a4bcb49f7dbecdac095bc63e90dd6edaf3 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 13:43:59 +0200 Subject: test mirror list with failing mirror --- cloudinit/config/cc_apt_configure.py | 1 + .../test_handler_apt_configure_sources_list.py | 23 ++++++++++++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 1d3eddff..ccbdcbc1 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -57,6 +57,7 @@ def handle(name, cfg, cloud, log, _args): release = get_release() mirrors = find_apt_mirror_info(cloud, cfg) + print(mirrors) if not mirrors or "primary" not in mirrors: log.debug(("Skipping module named %s," " no package 'mirror' located"), name) diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py index aff272a3..bac2da24 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py @@ -58,14 +58,20 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): # TODO Later - custom template filename # TODO Later - custom template raw - def apt_source_list(self, distro, mirror): + def apt_source_list(self, distro, mirror, mirrorcheck=None): """ apt_source_list Test rendering of a source.list from template for a given distro """ self.patchOS(self.new_root) self.patchUtils(self.new_root) - cfg = {'apt_mirror': mirror} + if mirrorcheck is None: + mirrorcheck = mirror + + if isinstance(mirror, list): + cfg = {'apt_mirror_search': mirror} + else: + cfg = {'apt_mirror': mirror} mycloud = self._get_cloud(distro) with mock.patch.object(templater, 'render_to_file') as mocktmpl: @@ -81,9 +87,9 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): '/etc/apt/sources.list', {'codename': '', 'primary': - mirror, + mirrorcheck, 'mirror': - mirror}) + mirrorcheck}) def test_apt_source_list_ubuntu(self): @@ -100,4 +106,13 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): self.apt_source_list('debian', 'ftp.us.debian.org') + def test_apt_srcl_ubuntu_mirrorfail(self): + """ test_apt_source_list_ubuntu_mirrorfail + Test rendering of a source.list from template for ubuntu + """ + self.apt_source_list('ubuntu', ['http://does.not.exist', + 'http://archive.ubuntu.com/ubuntu/'], + 'http://archive.ubuntu.com/ubuntu/') + + # vi: ts=4 expandtab -- cgit v1.2.3 From 18864f8e7331da359399decb1b080e36fa343f5a Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 13:45:15 +0200 Subject: remove missed test print --- cloudinit/config/cc_apt_configure.py | 1 - .../test_handler/test_handler_apt_configure_sources_list.py | 9 +++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index ccbdcbc1..1d3eddff 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -57,7 +57,6 @@ def handle(name, cfg, cloud, log, _args): release = get_release() mirrors = find_apt_mirror_info(cloud, cfg) - print(mirrors) if not mirrors or "primary" not in mirrors: log.debug(("Skipping module named %s," " no package 'mirror' located"), name) diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py index d48167c9..b8fe03ae 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py @@ -106,6 +106,15 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): self.apt_source_list('ubuntu', 'http://archive.ubuntu.com/ubuntu/') + def test_apt_srcl_debian_mirrorfail(self): + """ test_apt_source_list_debian_mirrorfail + Test rendering of a source.list from template for debian + """ + self.apt_source_list('debian', ['http://does.not.exist', + 'ftp.us.debian.org'], + 'ftp.us.debian.org') + + def test_apt_srcl_ubuntu_mirrorfail(self): """ test_apt_source_list_ubuntu_mirrorfail Test rendering of a source.list from template for ubuntu -- cgit v1.2.3 From 0d8bf6f2c1464b5f4dab735841a50d02016d2caf Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 15:51:21 +0200 Subject: add feature to allow a custom template for source list --- cloudinit/config/cc_apt_configure.py | 18 +++++++++++++----- cloudinit/templater.py | 5 +++++ 2 files changed, 18 insertions(+), 5 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 1d3eddff..2ab5e86c 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -21,6 +21,7 @@ import glob import os import re +import tempfile from cloudinit import templater from cloudinit import util @@ -70,7 +71,7 @@ def handle(name, cfg, cloud, log, _args): if not util.get_cfg_option_bool(cfg, 'apt_preserve_sources_list', False): - generate_sources_list(release, mirrors, cloud, log) + generate_sources_list(cfg, release, mirrors, cloud, log) old_mirrors = cfg.get('apt_old_mirrors', {"primary": "archive.ubuntu.com/ubuntu", "security": "security.ubuntu.com/ubuntu"}) @@ -149,7 +150,17 @@ def get_release(): return stdout.strip() -def generate_sources_list(codename, mirrors, cloud, log): +def generate_sources_list(cfg, codename, mirrors, cloud, log): + params = {'codename': codename} + for k in mirrors: + params[k] = mirrors[k] + + custtmpl = cfg.get('apt_custom_sources_list', None) + if custtmpl is not None: + templater.render_string_to_file(custtmpl, + '/etc/apt/sources.list', params) + return + template_fn = cloud.get_template_filename('sources.list.%s' % (cloud.distro.name)) if not template_fn: @@ -158,9 +169,6 @@ def generate_sources_list(codename, mirrors, cloud, log): log.warn("No template found, not rendering /etc/apt/sources.list") return - params = {'codename': codename} - for k in mirrors: - params[k] = mirrors[k] templater.render_to_file(template_fn, '/etc/apt/sources.list', params) diff --git a/cloudinit/templater.py b/cloudinit/templater.py index a9231482..8a6ad417 100644 --- a/cloudinit/templater.py +++ b/cloudinit/templater.py @@ -142,6 +142,11 @@ def render_to_file(fn, outfn, params, mode=0o644): util.write_file(outfn, contents, mode=mode) +def render_string_to_file(content, outfn, params, mode=0o644): + contents = render_string(content, params) + util.write_file(outfn, contents, mode=mode) + + def render_string(content, params): if not params: params = {} -- cgit v1.2.3 From 9c6b5f54ad5b83131de6d997930bd9f4031e6a83 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 19:50:01 +0200 Subject: move errorlist.append out of add_key --- cloudinit/config/cc_apt_configure.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 2ab5e86c..91d02815 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -172,7 +172,7 @@ def generate_sources_list(cfg, codename, mirrors, cloud, log): templater.render_to_file(template_fn, '/etc/apt/sources.list', params) -def add_key(ent, errorlist): +def add_key(ent): """ add key to the system as defiend in entry (if any) suppords raw keys or keyid's @@ -185,14 +185,13 @@ def add_key(ent, errorlist): try: ent['key'] = getkeybyid(ent['keyid'], keyserver) except: - errorlist.append([ent, "failed to get key from %s" % keyserver]) - return + raise Exception('failed to get key from %s' % keyserver) if 'key' in ent: try: util.subp(('apt-key', 'add', '-'), ent['key']) except: - errorlist.append([ent, "failed add key"]) + raise Exception('failed add key') def add_sources(srclist, template_params=None, aa_repo_match=None): @@ -211,7 +210,10 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): errorlist = [] for ent in srclist: # keys can be added without specifying a source - add_key(ent, errorlist) + try: + add_key(ent) + except Exception as detail: + errorlist.append([ent, detail]) if 'source' not in ent: errorlist.append(["", "missing source"]) -- cgit v1.2.3 From e55ccfa5670e16aa7431a193d0838aa7d04db4d5 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 19:50:34 +0200 Subject: remove Unnecessary parens in add_key --- cloudinit/config/cc_apt_configure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 91d02815..492c3c00 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -178,7 +178,7 @@ def add_key(ent): suppords raw keys or keyid's The latter will as a first step fetched to get the raw key """ - if ('keyid' in ent and 'key' not in ent): + if 'keyid' in ent and 'key' not in ent: keyserver = "keyserver.ubuntu.com" if 'keyserver' in ent: keyserver = ent['keyserver'] -- cgit v1.2.3 From f11827aee59cf2678c3d2a157218a3fe2831f5c2 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 12 May 2016 10:56:26 -0700 Subject: Fix up a ton of flake8 issues --- cloudinit/config/cc_apt_configure.py | 6 +++--- cloudinit/config/cc_bootcmd.py | 4 ++-- cloudinit/config/cc_disk_setup.py | 2 +- cloudinit/config/cc_emit_upstart.py | 2 +- cloudinit/config/cc_fan.py | 2 +- cloudinit/config/cc_final_message.py | 2 +- cloudinit/config/cc_grub_dpkg.py | 2 +- cloudinit/config/cc_keys_to_console.py | 2 +- cloudinit/config/cc_lxd.py | 2 +- cloudinit/config/cc_mounts.py | 10 ++++----- cloudinit/config/cc_phone_home.py | 6 +++--- cloudinit/config/cc_rightscale_userdata.py | 4 ++-- cloudinit/config/cc_runcmd.py | 2 +- cloudinit/config/cc_scripts_per_boot.py | 2 +- cloudinit/config/cc_scripts_per_instance.py | 2 +- cloudinit/config/cc_scripts_per_once.py | 2 +- cloudinit/config/cc_scripts_user.py | 2 +- cloudinit/config/cc_scripts_vendor.py | 2 +- cloudinit/config/cc_seed_random.py | 2 +- cloudinit/config/cc_set_passwords.py | 2 +- cloudinit/config/cc_snappy.py | 4 ++-- cloudinit/config/cc_ssh.py | 6 +++--- cloudinit/config/cc_ssh_import_id.py | 4 ++-- cloudinit/config/cc_ubuntu_init_switch.py | 4 ++-- cloudinit/distros/__init__.py | 4 ++-- cloudinit/distros/debian.py | 4 ++-- cloudinit/handlers/__init__.py | 6 +++--- cloudinit/handlers/cloud_config.py | 2 +- cloudinit/handlers/upstart_job.py | 2 +- cloudinit/helpers.py | 10 ++++----- cloudinit/net/__init__.py | 18 +++++++--------- cloudinit/net/network_state.py | 7 +++--- cloudinit/net/udev.py | 2 +- cloudinit/netinfo.py | 7 +++--- cloudinit/reporting/handlers.py | 10 ++++----- cloudinit/sources/DataSourceAzure.py | 12 ++++++----- cloudinit/sources/DataSourceBigstep.py | 4 ++-- cloudinit/sources/DataSourceCloudSigma.py | 5 +++-- cloudinit/sources/DataSourceCloudStack.py | 5 +++-- cloudinit/sources/DataSourceDigitalOcean.py | 4 ++-- cloudinit/sources/DataSourceGCE.py | 4 ++-- cloudinit/sources/DataSourceNoCloud.py | 4 ++-- cloudinit/sources/DataSourceOVF.py | 25 ++++++++++++++-------- cloudinit/sources/__init__.py | 3 +-- cloudinit/sources/helpers/azure.py | 1 + cloudinit/sources/helpers/openstack.py | 4 ++-- cloudinit/sources/helpers/vmware/imc/boot_proto.py | 2 +- cloudinit/sources/helpers/vmware/imc/config.py | 2 +- cloudinit/sources/helpers/vmware/imc/config_nic.py | 2 +- .../sources/helpers/vmware/imc/config_source.py | 2 +- .../sources/helpers/vmware/imc/guestcust_error.py | 2 +- .../sources/helpers/vmware/imc/guestcust_event.py | 2 +- .../sources/helpers/vmware/imc/guestcust_state.py | 2 +- .../sources/helpers/vmware/imc/guestcust_util.py | 2 +- cloudinit/sources/helpers/vmware/imc/ipv4_mode.py | 2 +- cloudinit/sources/helpers/vmware/imc/nic_base.py | 6 +++--- cloudinit/stages.py | 4 ++-- cloudinit/url_helper.py | 5 +++-- cloudinit/user_data.py | 2 +- cloudinit/util.py | 18 ++++++++-------- test-requirements.txt | 3 ++- tox.ini | 5 +++++ 62 files changed, 150 insertions(+), 132 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 702977cb..e3fadc12 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -207,20 +207,20 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): ks = ent['keyserver'] try: ent['key'] = getkeybyid(ent['keyid'], ks) - except: + except Exception: errorlist.append([source, "failed to get key from %s" % ks]) continue if 'key' in ent: try: util.subp(('apt-key', 'add', '-'), ent['key']) - except: + except Exception: errorlist.append([source, "failed add key"]) try: contents = "%s\n" % (source) util.write_file(ent['filename'], contents, omode="ab") - except: + except Exception: errorlist.append([source, "failed write to file %s" % ent['filename']]) diff --git a/cloudinit/config/cc_bootcmd.py b/cloudinit/config/cc_bootcmd.py index a295cc4e..b763a3c3 100644 --- a/cloudinit/config/cc_bootcmd.py +++ b/cloudinit/config/cc_bootcmd.py @@ -38,7 +38,7 @@ def handle(name, cfg, cloud, log, _args): content = util.shellify(cfg["bootcmd"]) tmpf.write(util.encode_text(content)) tmpf.flush() - except: + except Exception: util.logexc(log, "Failed to shellify bootcmd") raise @@ -49,6 +49,6 @@ def handle(name, cfg, cloud, log, _args): env['INSTANCE_ID'] = str(iid) cmd = ['/bin/sh', tmpf.name] util.subp(cmd, env=env, capture=False) - except: + except Exception: util.logexc(log, "Failed to run bootcmd module %s", name) raise diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index bbaf9646..b642f1f8 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -198,7 +198,7 @@ def is_device_valid(name, partition=False): d_type = "" try: d_type = device_type(name) - except: + except Exception: LOG.warn("Query against device %s failed" % name) return False diff --git a/cloudinit/config/cc_emit_upstart.py b/cloudinit/config/cc_emit_upstart.py index 86ae97ab..06c53272 100644 --- a/cloudinit/config/cc_emit_upstart.py +++ b/cloudinit/config/cc_emit_upstart.py @@ -20,8 +20,8 @@ import os -from cloudinit.settings import PER_ALWAYS from cloudinit import log as logging +from cloudinit.settings import PER_ALWAYS from cloudinit import util frequency = PER_ALWAYS diff --git a/cloudinit/config/cc_fan.py b/cloudinit/config/cc_fan.py index 39e3850e..545fee22 100644 --- a/cloudinit/config/cc_fan.py +++ b/cloudinit/config/cc_fan.py @@ -37,8 +37,8 @@ If cloud-init sees a 'fan' entry in cloud-config it will """ from cloudinit import log as logging -from cloudinit import util from cloudinit.settings import PER_INSTANCE +from cloudinit import util LOG = logging.getLogger(__name__) diff --git a/cloudinit/config/cc_final_message.py b/cloudinit/config/cc_final_message.py index 4a51476f..c9021eb1 100644 --- a/cloudinit/config/cc_final_message.py +++ b/cloudinit/config/cc_final_message.py @@ -66,7 +66,7 @@ def handle(_name, cfg, cloud, log, args): try: contents = "%s - %s - v. %s\n" % (uptime, ts, cver) util.write_file(boot_fin_fn, contents) - except: + except Exception: util.logexc(log, "Failed to write boot finished file %s", boot_fin_fn) if cloud.datasource.is_disconnected: diff --git a/cloudinit/config/cc_grub_dpkg.py b/cloudinit/config/cc_grub_dpkg.py index 3c2d9985..156722d9 100644 --- a/cloudinit/config/cc_grub_dpkg.py +++ b/cloudinit/config/cc_grub_dpkg.py @@ -69,5 +69,5 @@ def handle(name, cfg, _cloud, log, _args): try: util.subp(['debconf-set-selections'], dconf_sel) - except: + except Exception: util.logexc(log, "Failed to run debconf-set-selections for grub-dpkg") diff --git a/cloudinit/config/cc_keys_to_console.py b/cloudinit/config/cc_keys_to_console.py index aa844ee9..9a02f056 100644 --- a/cloudinit/config/cc_keys_to_console.py +++ b/cloudinit/config/cc_keys_to_console.py @@ -57,6 +57,6 @@ def handle(name, cfg, cloud, log, _args): (stdout, _stderr) = util.subp(cmd) util.multi_log("%s\n" % (stdout.strip()), stderr=False, console=True) - except: + except Exception: log.warn("Writing keys to the system console failed!") raise diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py index bf735648..b1de8f84 100644 --- a/cloudinit/config/cc_lxd.py +++ b/cloudinit/config/cc_lxd.py @@ -111,7 +111,7 @@ def handle(name, cfg, cloud, log, args): data = "\n".join(["set %s %s" % (k, v) for k, v in debconf.items()]) + "\n" util.subp(['debconf-communicate'], data) - except: + except Exception: util.logexc(log, "Failed to run '%s' for lxd with" % dconf_comm) # Remove the existing configuration file (forces re-generation) diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 4fe3ee21..2b981935 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -244,7 +244,7 @@ def handle_swapcfg(swapcfg): LOG.debug("swap file %s already in use.", fname) return fname LOG.debug("swap file %s existed, but not in /proc/swaps", fname) - except: + except Exception: LOG.warn("swap file %s existed. Error reading /proc/swaps", fname) return fname @@ -379,7 +379,7 @@ def handle(_name, cfg, cloud, log, _args): toks = WS.split(line) if toks[3].find(comment) != -1: continue - except: + except Exception: pass fstab_lines.append(line) @@ -390,16 +390,16 @@ def handle(_name, cfg, cloud, log, _args): if needswap: try: util.subp(("swapon", "-a")) - except: + except Exception: util.logexc(log, "Activating swap via 'swapon -a' failed") for d in dirs: try: util.ensure_dir(d) - except: + except Exception: util.logexc(log, "Failed to make '%s' config-mount", d) try: util.subp(("mount", "-a")) - except: + except Exception: util.logexc(log, "Activating mounts via 'mount -a' failed") diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py index 3dcc9459..72176d42 100644 --- a/cloudinit/config/cc_phone_home.py +++ b/cloudinit/config/cc_phone_home.py @@ -65,7 +65,7 @@ def handle(name, cfg, cloud, log, args): tries = ph_cfg.get('tries') try: tries = int(tries) - except: + except Exception: tries = 10 util.logexc(log, "Configuration entry 'tries' is not an integer, " "using %s instead", tries) @@ -87,7 +87,7 @@ def handle(name, cfg, cloud, log, args): for (n, path) in pubkeys.items(): try: all_keys[n] = util.load_file(path) - except: + except Exception: util.logexc(log, "%s: failed to open, can not phone home that " "data!", path) @@ -117,6 +117,6 @@ def handle(name, cfg, cloud, log, args): util.read_file_or_url(url, data=real_submit_keys, retries=tries, sec_between=3, ssl_details=util.fetch_ssl_details(cloud.paths)) - except: + except Exception: util.logexc(log, "Failed to post phone home data to %s in %s tries", url, tries) diff --git a/cloudinit/config/cc_rightscale_userdata.py b/cloudinit/config/cc_rightscale_userdata.py index 0ecf3a4d..8118fac4 100644 --- a/cloudinit/config/cc_rightscale_userdata.py +++ b/cloudinit/config/cc_rightscale_userdata.py @@ -52,7 +52,7 @@ MY_HOOKNAME = 'CLOUD_INIT_REMOTE_HOOK' def handle(name, _cfg, cloud, log, _args): try: ud = cloud.get_userdata_raw() - except: + except Exception: log.debug("Failed to get raw userdata in module %s", name) return @@ -63,7 +63,7 @@ def handle(name, _cfg, cloud, log, _args): "did not find %s in parsed" " raw userdata"), name, MY_HOOKNAME) return - except: + except Exception: util.logexc(log, "Failed to parse query string %s into a dictionary", ud) raise diff --git a/cloudinit/config/cc_runcmd.py b/cloudinit/config/cc_runcmd.py index 66dc3363..bc09d38c 100644 --- a/cloudinit/config/cc_runcmd.py +++ b/cloudinit/config/cc_runcmd.py @@ -34,5 +34,5 @@ def handle(name, cfg, cloud, log, _args): try: content = util.shellify(cmd) util.write_file(out_fn, content, 0o700) - except: + except Exception: util.logexc(log, "Failed to shellify %s into file %s", cmd, out_fn) diff --git a/cloudinit/config/cc_scripts_per_boot.py b/cloudinit/config/cc_scripts_per_boot.py index 42b987eb..ee3b6c9f 100644 --- a/cloudinit/config/cc_scripts_per_boot.py +++ b/cloudinit/config/cc_scripts_per_boot.py @@ -35,7 +35,7 @@ def handle(name, _cfg, cloud, log, _args): runparts_path = os.path.join(cloud.get_cpath(), 'scripts', SCRIPT_SUBDIR) try: util.runparts(runparts_path) - except: + except Exception: log.warn("Failed to run module %s (%s in %s)", name, SCRIPT_SUBDIR, runparts_path) raise diff --git a/cloudinit/config/cc_scripts_per_instance.py b/cloudinit/config/cc_scripts_per_instance.py index b5d71c13..c0d62b12 100644 --- a/cloudinit/config/cc_scripts_per_instance.py +++ b/cloudinit/config/cc_scripts_per_instance.py @@ -35,7 +35,7 @@ def handle(name, _cfg, cloud, log, _args): runparts_path = os.path.join(cloud.get_cpath(), 'scripts', SCRIPT_SUBDIR) try: util.runparts(runparts_path) - except: + except Exception: log.warn("Failed to run module %s (%s in %s)", name, SCRIPT_SUBDIR, runparts_path) raise diff --git a/cloudinit/config/cc_scripts_per_once.py b/cloudinit/config/cc_scripts_per_once.py index d77d36d5..ecb527f6 100644 --- a/cloudinit/config/cc_scripts_per_once.py +++ b/cloudinit/config/cc_scripts_per_once.py @@ -35,7 +35,7 @@ def handle(name, _cfg, cloud, log, _args): runparts_path = os.path.join(cloud.get_cpath(), 'scripts', SCRIPT_SUBDIR) try: util.runparts(runparts_path) - except: + except Exception: log.warn("Failed to run module %s (%s in %s)", name, SCRIPT_SUBDIR, runparts_path) raise diff --git a/cloudinit/config/cc_scripts_user.py b/cloudinit/config/cc_scripts_user.py index 5c53014f..699857d1 100644 --- a/cloudinit/config/cc_scripts_user.py +++ b/cloudinit/config/cc_scripts_user.py @@ -36,7 +36,7 @@ def handle(name, _cfg, cloud, log, _args): runparts_path = os.path.join(cloud.get_ipath_cur(), SCRIPT_SUBDIR) try: util.runparts(runparts_path) - except: + except Exception: log.warn("Failed to run module %s (%s in %s)", name, SCRIPT_SUBDIR, runparts_path) raise diff --git a/cloudinit/config/cc_scripts_vendor.py b/cloudinit/config/cc_scripts_vendor.py index 0c9e504e..80bf10ff 100644 --- a/cloudinit/config/cc_scripts_vendor.py +++ b/cloudinit/config/cc_scripts_vendor.py @@ -37,7 +37,7 @@ def handle(name, cfg, cloud, log, _args): try: util.runparts(runparts_path, exe_prefix=prefix) - except: + except Exception: log.warn("Failed to run module %s (%s in %s)", name, SCRIPT_SUBDIR, runparts_path) raise diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py index 1b011216..5085c23a 100644 --- a/cloudinit/config/cc_seed_random.py +++ b/cloudinit/config/cc_seed_random.py @@ -24,8 +24,8 @@ import os from six import BytesIO -from cloudinit.settings import PER_INSTANCE from cloudinit import log as logging +from cloudinit.settings import PER_INSTANCE from cloudinit import util frequency = PER_INSTANCE diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index 58e1b713..5c8c23b8 100644 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -155,7 +155,7 @@ def handle(_name, cfg, cloud, log, args): cmd = filter(None, cmd) # Remove empty arguments util.subp(cmd) log.debug("Restarted the ssh daemon") - except: + except Exception: util.logexc(log, "Restarting of the ssh daemon failed") if len(errors): diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index fa9d54a0..1a485ee6 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -47,12 +47,12 @@ Example config: """ from cloudinit import log as logging -from cloudinit import util from cloudinit.settings import PER_INSTANCE +from cloudinit import util import glob -import tempfile import os +import tempfile LOG = logging.getLogger(__name__) diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py index d24e43c0..cb9b70aa 100644 --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -57,7 +57,7 @@ def handle(_name, cfg, cloud, log, _args): for f in glob.glob(key_pth): try: util.del_file(f) - except: + except Exception: util.logexc(log, "Failed deleting key file %s", f) if "ssh_keys" in cfg: @@ -78,7 +78,7 @@ def handle(_name, cfg, cloud, log, _args): with util.SeLinuxGuard("/etc/ssh", recursive=True): util.subp(cmd, capture=False) log.debug("Generated a key for %s from %s", pair[0], pair[1]) - except: + except Exception: util.logexc(log, "Failed generated a key for %s from %s", pair[0], pair[1]) else: @@ -122,7 +122,7 @@ def handle(_name, cfg, cloud, log, _args): keys.extend(cfgkeys) apply_credentials(keys, user, disable_root, disable_root_opts) - except: + except Exception: util.logexc(log, "Applying ssh credentials failed!") diff --git a/cloudinit/config/cc_ssh_import_id.py b/cloudinit/config/cc_ssh_import_id.py index 2d480d7e..28c4585b 100644 --- a/cloudinit/config/cc_ssh_import_id.py +++ b/cloudinit/config/cc_ssh_import_id.py @@ -52,14 +52,14 @@ def handle(_name, cfg, cloud, log, args): else: try: import_ids = user_cfg['ssh_import_id'] - except: + except Exception: log.debug("User %s is not configured for ssh_import_id", user) continue try: import_ids = util.uniq_merge(import_ids) import_ids = [str(i) for i in import_ids] - except: + except Exception: log.debug("User %s is not correctly configured for ssh_import_id", user) continue diff --git a/cloudinit/config/cc_ubuntu_init_switch.py b/cloudinit/config/cc_ubuntu_init_switch.py index 7e88ed85..884d79f1 100644 --- a/cloudinit/config/cc_ubuntu_init_switch.py +++ b/cloudinit/config/cc_ubuntu_init_switch.py @@ -40,10 +40,10 @@ It can be configured with the following option structure:: mechanism you've used to switch the init system. """ -from cloudinit.settings import PER_INSTANCE +from cloudinit.distros import ubuntu from cloudinit import log as logging +from cloudinit.settings import PER_INSTANCE from cloudinit import util -from cloudinit.distros import ubuntu import os import time diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 5879dabf..0f222c8c 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -50,8 +50,8 @@ OSFAMILIES = { LOG = logging.getLogger(__name__) +@six.add_metaclass(abc.ABCMeta) class Distro(object): - __metaclass__ = abc.ABCMeta usr_lib_exec = "/usr/lib" hosts_fn = "/etc/hosts" @@ -97,7 +97,7 @@ class Distro(object): try: res = os.lstat('/run/systemd/system') return stat.S_ISDIR(res.st_mode) - except: + except Exception: return False @abc.abstractmethod diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 75ab340f..32bef1cd 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -25,8 +25,8 @@ import os from cloudinit import distros from cloudinit import helpers from cloudinit import log as logging -from cloudinit import util from cloudinit import net +from cloudinit import util from cloudinit.distros.parsers.hostname import HostnameConf @@ -221,7 +221,7 @@ def _maybe_remove_legacy_eth0(path="/etc/network/interfaces.d/eth0.cfg"): msg = "removed %s with known contents" % path else: msg = (bmsg + " '%s' exists with user configured content." % path) - except: + except Exception: msg = bmsg + " %s exists, but could not be read." % path LOG.warn(msg) diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index 53d5604a..b6c43ce8 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -71,8 +71,8 @@ INCLUSION_SRCH = sorted(list(INCLUSION_TYPES_MAP.keys()), key=(lambda e: 0 - len(e))) +@six.add_metaclass(abc.ABCMeta) class Handler(object): - __metaclass__ = abc.ABCMeta def __init__(self, frequency, version=2): self.handler_version = version @@ -118,7 +118,7 @@ def run_part(mod, data, filename, payload, frequency, headers): mod.handle_part(data, content_type, filename, payload) else: raise ValueError("Unknown module version %s" % (mod_ver)) - except: + except Exception: util.logexc(LOG, "Failed calling handler %s (%s, %s, %s) with " "frequency %s", mod, content_type, filename, mod_ver, frequency) @@ -157,7 +157,7 @@ def walker_handle_handler(pdata, _ctype, _filename, payload): # register if it fails starting. handlers.register(mod, initialized=True) pdata['handlercount'] = curcount + 1 - except: + except Exception: util.logexc(LOG, "Failed at registering python file: %s (part " "handler %s)", modfname, curcount) diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py index 07b6d0e0..cad4dc0f 100644 --- a/cloudinit/handlers/cloud_config.py +++ b/cloudinit/handlers/cloud_config.py @@ -158,6 +158,6 @@ class CloudConfigPartHandler(handlers.Handler): for i in ("\n", "\r", "\t"): filename = filename.replace(i, " ") self.file_names.append(filename.strip()) - except: + except Exception: util.logexc(LOG, "Failed at merging in cloud config part from %s", filename) diff --git a/cloudinit/handlers/upstart_job.py b/cloudinit/handlers/upstart_job.py index c5bea711..ab381e00 100644 --- a/cloudinit/handlers/upstart_job.py +++ b/cloudinit/handlers/upstart_job.py @@ -80,7 +80,7 @@ def _has_suitable_upstart(): return False try: (version_out, _err) = util.subp(["initctl", "version"]) - except: + except Exception: util.logexc(LOG, "initctl version failed") return False diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index 7f00bf1f..09d75e65 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -86,7 +86,7 @@ class FileSemaphores(object): name = canon_sem_name(name) try: yield self._acquire(name, freq) - except: + except Exception: if clear_on_fail: self.clear(name, freq) raise @@ -219,7 +219,7 @@ class ConfigMerger(object): ds_cfg = self._ds.get_config_obj() if ds_cfg and isinstance(ds_cfg, (dict)): d_cfgs.append(ds_cfg) - except: + except Exception: util.logexc(LOG, "Failed loading of datasource config object " "from %s", self._ds) return d_cfgs @@ -230,7 +230,7 @@ class ConfigMerger(object): e_fn = os.environ[CFG_ENV_NAME] try: e_cfgs.append(util.read_conf(e_fn)) - except: + except Exception: util.logexc(LOG, 'Failed loading of env. config from %s', e_fn) return e_cfgs @@ -251,7 +251,7 @@ class ConfigMerger(object): if cc_fn and os.path.isfile(cc_fn): try: i_cfgs.append(util.read_conf(cc_fn)) - except: + except Exception: util.logexc(LOG, 'Failed loading of cloud-config from %s', cc_fn) return i_cfgs @@ -268,7 +268,7 @@ class ConfigMerger(object): for c_fn in self._fns: try: cfgs.append(util.read_conf(c_fn)) - except: + except Exception: util.logexc(LOG, "Failed loading of configuration from %s", c_fn) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 31544fd8..91e36aca 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -26,9 +26,9 @@ import re import shlex from cloudinit import log as logging +from cloudinit.net import network_state +from cloudinit.net.udev import generate_udev_rule from cloudinit import util -from .udev import generate_udev_rule -from . import network_state LOG = logging.getLogger(__name__) @@ -40,16 +40,16 @@ NET_CONFIG_OPTIONS = [ "pointtopoint", "media", "mtu", "hostname", "leasehours", "leasetime", "vendor", "client", "bootfile", "server", "hwaddr", "provider", "frame", "netnum", "endpoint", "local", "ttl", - ] +] NET_CONFIG_COMMANDS = [ "pre-up", "up", "post-up", "down", "pre-down", "post-down", - ] +] NET_CONFIG_BRIDGE_OPTIONS = [ "bridge_ageing", "bridge_bridgeprio", "bridge_fd", "bridge_gcinit", "bridge_hello", "bridge_maxage", "bridge_maxwait", "bridge_stp", - ] +] DEFAULT_PRIMARY_INTERFACE = 'eth0' @@ -399,9 +399,7 @@ def config_from_klibc_net_cfg(files=None, mac_addrs=None): def render_persistent_net(network_state): - ''' Given state, emit udev rules to map - mac to ifname - ''' + '''Given state, emit udev rules to map mac to ifname.''' content = "" interfaces = network_state.get('interfaces') for iface in interfaces.values(): @@ -465,7 +463,7 @@ def iface_add_attrs(iface): def render_route(route, indent=""): - """ When rendering routes for an iface, in some cases applying a route + """When rendering routes for an iface, in some cases applying a route may result in the route command returning non-zero which produces some confusing output for users manually using ifup/ifdown[1]. To that end, we will optionally include an '|| true' postfix to each @@ -530,7 +528,7 @@ def iface_start_entry(iface, index): def render_interfaces(network_state): - ''' Given state, emit etc/network/interfaces content ''' + '''Given state, emit etc/network/interfaces content.''' content = "" interfaces = network_state.get('interfaces') diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index e32d2cdf..4c726ab4 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -36,7 +36,7 @@ def from_state_file(state_file): return network_state -class NetworkState: +class NetworkState(object): def __init__(self, version=NETWORK_STATE_VERSION, config=None): self.version = version self.config = config @@ -53,7 +53,7 @@ class NetworkState: def get_command_handlers(self): METHOD_PREFIX = 'handle_' methods = filter(lambda x: callable(getattr(self, x)) and - x.startswith(METHOD_PREFIX), dir(self)) + x.startswith(METHOD_PREFIX), dir(self)) handlers = {} for m in methods: key = m.replace(METHOD_PREFIX, '') @@ -379,8 +379,9 @@ def mask2cidr(mask): if __name__ == '__main__': - import sys import random + import sys + from cloudinit import net def load_config(nc): diff --git a/cloudinit/net/udev.py b/cloudinit/net/udev.py index 6435ace0..09188295 100644 --- a/cloudinit/net/udev.py +++ b/cloudinit/net/udev.py @@ -48,7 +48,7 @@ def generate_udev_rule(interface, mac): compose_udev_equality('DRIVERS', '?*'), compose_udev_attr_equality('address', mac), compose_udev_setting('NAME', interface), - ]) + ]) return '%s\n' % rule # vi: ts=4 expandtab syntax=python diff --git a/cloudinit/netinfo.py b/cloudinit/netinfo.py index e30d6fb5..5d24062f 100644 --- a/cloudinit/netinfo.py +++ b/cloudinit/netinfo.py @@ -20,10 +20,11 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util -from cloudinit.log import logging import re +from cloudinit import logging +from cloudinit import util + from prettytable import PrettyTable LOG = logging.getLogger() @@ -163,7 +164,7 @@ def route_info(): def getgateway(): try: routes = route_info() - except: + except Exception: pass else: for r in routes.get('ipv4', []): diff --git a/cloudinit/reporting/handlers.py b/cloudinit/reporting/handlers.py index 3212d173..dff20ecb 100644 --- a/cloudinit/reporting/handlers.py +++ b/cloudinit/reporting/handlers.py @@ -4,9 +4,9 @@ import abc import json import six -from ..registry import DictRegistry -from .. import (url_helper, util) -from .. import log as logging +from cloudinit import log as logging +from cloudinit.registry import DictRegistry +from cloudinit import (url_helper, util) LOG = logging.getLogger(__name__) @@ -36,7 +36,7 @@ class LogHandler(ReportingHandler): input_level = level try: level = getattr(logging, level.upper()) - except: + except Exception: LOG.warn("invalid level '%s', using WARN", input_level) level = logging.WARN self.level = level @@ -81,7 +81,7 @@ class WebHookHandler(ReportingHandler): self.endpoint, data=json.dumps(event.as_dict()), timeout=self.timeout, retries=self.retries, ssl_details=self.ssl_details) - except: + except Exception: LOG.warn("failed posting event: %s" % event.as_string()) diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 698f4cac..2d046600 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -27,11 +27,12 @@ import xml.etree.ElementTree as ET from xml.dom import minidom +from cloudinit.sources.helpers.azure import get_metadata_from_fabric + from cloudinit import log as logging from cloudinit.settings import PER_ALWAYS from cloudinit import sources from cloudinit import util -from cloudinit.sources.helpers.azure import get_metadata_from_fabric LOG = logging.getLogger(__name__) @@ -40,7 +41,8 @@ DEFAULT_METADATA = {"instance-id": "iid-AZURE-NODE"} AGENT_START = ['service', 'walinuxagent', 'start'] BOUNCE_COMMAND = [ 'sh', '-xc', - "i=$interface; x=0; ifdown $i || x=$?; ifup $i || x=$?; exit $x"] + "i=$interface; x=0; ifdown $i || x=$?; ifup $i || x=$?; exit $x" +] BUILTIN_DS_CONFIG = { 'agent_command': AGENT_START, @@ -51,7 +53,7 @@ BUILTIN_DS_CONFIG = { 'policy': True, 'command': BOUNCE_COMMAND, 'hostname_command': 'hostname', - }, + }, 'disk_aliases': {'ephemeral0': '/dev/sdb'}, } @@ -60,7 +62,7 @@ BUILTIN_CLOUD_CONFIG = { 'ephemeral0': {'table_type': 'gpt', 'layout': [100], 'overwrite': True}, - }, + }, 'fs_setup': [{'filesystem': 'ext4', 'device': 'ephemeral0.1', 'replace_fs': 'ntfs'}], @@ -312,7 +314,7 @@ def support_new_ephemeral(cfg): file_count = 0 try: file_count = util.mount_cb(device, count_files) - except: + except Exception: return None LOG.debug("fabric prepared ephmeral0.1 has %s files on it", file_count) diff --git a/cloudinit/sources/DataSourceBigstep.py b/cloudinit/sources/DataSourceBigstep.py index b5ee4129..f80956a5 100644 --- a/cloudinit/sources/DataSourceBigstep.py +++ b/cloudinit/sources/DataSourceBigstep.py @@ -4,13 +4,13 @@ # Author: Alexandru Sirbu # -import json import errno +import json from cloudinit import log as logging from cloudinit import sources -from cloudinit import util from cloudinit import url_helper +from cloudinit import util LOG = logging.getLogger(__name__) diff --git a/cloudinit/sources/DataSourceCloudSigma.py b/cloudinit/sources/DataSourceCloudSigma.py index d7d4e844..33fe78b9 100644 --- a/cloudinit/sources/DataSourceCloudSigma.py +++ b/cloudinit/sources/DataSourceCloudSigma.py @@ -19,10 +19,11 @@ from base64 import b64decode import os import re +from cloudinit.cs_utils import Cepko + from cloudinit import log as logging from cloudinit import sources from cloudinit import util -from cloudinit.cs_utils import Cepko LOG = logging.getLogger(__name__) @@ -77,7 +78,7 @@ class DataSourceCloudSigma(sources.DataSource): try: server_context = self.cepko.all().result server_meta = server_context['meta'] - except: + except Exception: # TODO: check for explicit "config on", and then warn # but since no explicit config is available now, just debug. LOG.debug("CloudSigma: Unable to read from serial port") diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 4ba019df..4de1f563 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -25,14 +25,15 @@ # along with this program. If not, see . import os -import time from socket import inet_ntoa from struct import pack +import time from cloudinit import ec2_utils as ec2 from cloudinit import log as logging +from cloudinit import sources from cloudinit import url_helper as uhelp -from cloudinit import sources, util +from cloudinit import util LOG = logging.getLogger(__name__) diff --git a/cloudinit/sources/DataSourceDigitalOcean.py b/cloudinit/sources/DataSourceDigitalOcean.py index 12e863d2..44a17a00 100644 --- a/cloudinit/sources/DataSourceDigitalOcean.py +++ b/cloudinit/sources/DataSourceDigitalOcean.py @@ -14,10 +14,10 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from cloudinit import ec2_utils from cloudinit import log as logging -from cloudinit import util from cloudinit import sources -from cloudinit import ec2_utils +from cloudinit import util import functools diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 7e7fc033..9234d1f8 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -18,9 +18,9 @@ from base64 import b64decode from cloudinit import log as logging -from cloudinit import util from cloudinit import sources from cloudinit import url_helper +from cloudinit import util LOG = logging.getLogger(__name__) @@ -71,7 +71,7 @@ class DataSourceGCE(sources.DataSource): index = public_key.index(':') if index > 0: return public_key[(index + 1):] - except: + except Exception: return public_key def get_data(self): diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 74d0e5ec..48c61a90 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -61,7 +61,7 @@ class DataSourceNoCloud(sources.DataSource): if parse_cmdline_data(self.cmdline_id, md): found.append("cmdline") mydata = _merge_new_seed(mydata, {'meta-data': md}) - except: + except Exception: util.logexc(LOG, "Unable to parse command line data") return False @@ -288,7 +288,7 @@ def parse_cmdline_data(ds_id, fill, cmdline=None): continue try: (k, v) = item.split("=", 1) - except: + except Exception: k = item v = None if k in s2l: diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 2a6cd050..f2bb9366 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -30,13 +30,20 @@ import time from cloudinit import log as logging from cloudinit import sources from cloudinit import util -from .helpers.vmware.imc.config import Config -from .helpers.vmware.imc.config_file import ConfigFile -from .helpers.vmware.imc.config_nic import NicConfigurator -from .helpers.vmware.imc.guestcust_event import GuestCustEventEnum -from .helpers.vmware.imc.guestcust_state import GuestCustStateEnum -from .helpers.vmware.imc.guestcust_error import GuestCustErrorEnum -from .helpers.vmware.imc.guestcust_util import ( + +from cloudinit.sources.helpers.vmware.imc.config \ + import Config +from cloudinit.sources.helpers.vmware.imc.config_file \ + import ConfigFile +from cloudinit.sources.helpers.vmware.imc.config_nic \ + import NicConfigurator +from cloudinit.sources.helpers.vmware.imc.guestcust_event \ + import GuestCustEventEnum +from cloudinit.sources.helpers.vmware.imc.guestcust_state \ + import GuestCustStateEnum +from cloudinit.sourceshelpers.vmware.imc.guestcust_error \ + import GuestCustErrorEnum +from cloudinit.sourceshelpers.vmware.imc.guestcust_util import ( set_customization_status, get_nics_to_enable, enable_nics @@ -262,7 +269,7 @@ def read_ovf_environment(contents): elif prop == "user-data": try: ud = base64.decodestring(val) - except: + except Exception: ud = val return (md, ud, cfg) @@ -277,7 +284,7 @@ def get_ovf_env(dirname): try: contents = util.load_file(full_fn) return (fname, contents) - except: + except Exception: util.logexc(LOG, "Failed loading ovf file %s", full_fn) return (None, False) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 6bf2c33b..43e4fd57 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -45,10 +45,9 @@ class DataSourceNotFoundException(Exception): pass +@six.add_metaclass(abc.ABCMeta) class DataSource(object): - __metaclass__ = abc.ABCMeta - def __init__(self, sys_cfg, distro, paths, ud_proc=None): self.sys_cfg = sys_cfg self.distro = distro diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 018cac6d..bbfc283c 100644 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -5,6 +5,7 @@ import socket import struct import tempfile import time + from contextlib import contextmanager from xml.etree import ElementTree diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 1aa6bbae..156aba6c 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -145,8 +145,8 @@ class SourceMixin(object): return device +@six.add_metaclass(abc.ABCMeta) class BaseReader(object): - __metaclass__ = abc.ABCMeta def __init__(self, base_path): self.base_path = base_path @@ -475,7 +475,7 @@ class MetadataReader(BaseReader): def convert_vendordata_json(data, recurse=True): - """ data: a loaded json *object* (strings, arrays, dicts). + """data: a loaded json *object* (strings, arrays, dicts). return something suitable for cloudinit vendordata_raw. if data is: diff --git a/cloudinit/sources/helpers/vmware/imc/boot_proto.py b/cloudinit/sources/helpers/vmware/imc/boot_proto.py index faba5887..204fbcc4 100644 --- a/cloudinit/sources/helpers/vmware/imc/boot_proto.py +++ b/cloudinit/sources/helpers/vmware/imc/boot_proto.py @@ -18,7 +18,7 @@ # along with this program. If not, see . -class BootProtoEnum: +class BootProtoEnum(object): """Specifies the NIC Boot Settings.""" DHCP = 'dhcp' diff --git a/cloudinit/sources/helpers/vmware/imc/config.py b/cloudinit/sources/helpers/vmware/imc/config.py index aebc12a0..1dcd053a 100644 --- a/cloudinit/sources/helpers/vmware/imc/config.py +++ b/cloudinit/sources/helpers/vmware/imc/config.py @@ -20,7 +20,7 @@ from .nic import Nic -class Config: +class Config(object): """ Stores the Contents specified in the Customization Specification file. diff --git a/cloudinit/sources/helpers/vmware/imc/config_nic.py b/cloudinit/sources/helpers/vmware/imc/config_nic.py index 77098a05..511cc918 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_nic.py +++ b/cloudinit/sources/helpers/vmware/imc/config_nic.py @@ -26,7 +26,7 @@ from cloudinit import util logger = logging.getLogger(__name__) -class NicConfigurator: +class NicConfigurator(object): def __init__(self, nics): """ Initialize the Nic Configurator diff --git a/cloudinit/sources/helpers/vmware/imc/config_source.py b/cloudinit/sources/helpers/vmware/imc/config_source.py index a367e476..8a650871 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_source.py +++ b/cloudinit/sources/helpers/vmware/imc/config_source.py @@ -18,6 +18,6 @@ # along with this program. If not, see . -class ConfigSource: +class ConfigSource(object): """Specifies a source for the Config Content.""" pass diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_error.py b/cloudinit/sources/helpers/vmware/imc/guestcust_error.py index 1b04161f..750be1e3 100644 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_error.py +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_error.py @@ -18,7 +18,7 @@ # along with this program. If not, see . -class GuestCustErrorEnum: +class GuestCustErrorEnum(object): """Specifies different errors of Guest Customization engine""" GUESTCUST_ERROR_SUCCESS = 0 diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_event.py b/cloudinit/sources/helpers/vmware/imc/guestcust_event.py index fc22568f..e13b791d 100644 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_event.py +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_event.py @@ -18,7 +18,7 @@ # along with this program. If not, see . -class GuestCustEventEnum: +class GuestCustEventEnum(object): """Specifies different types of Guest Customization Events""" GUESTCUST_EVENT_CUSTOMIZE_FAILED = 100 diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_state.py b/cloudinit/sources/helpers/vmware/imc/guestcust_state.py index f255be5f..b9ddf513 100644 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_state.py +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_state.py @@ -18,7 +18,7 @@ # along with this program. If not, see . -class GuestCustStateEnum: +class GuestCustStateEnum(object): """Specifies different states of Guest Customization engine""" GUESTCUST_STATE_RUNNING = 4 diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py index d39f0a65..020ab613 100644 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py @@ -23,8 +23,8 @@ import time from cloudinit import util -from .guestcust_state import GuestCustStateEnum from .guestcust_event import GuestCustEventEnum +from .guestcust_state import GuestCustStateEnum logger = logging.getLogger(__name__) diff --git a/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py b/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py index 33f88726..873ddc3b 100644 --- a/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py +++ b/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py @@ -18,7 +18,7 @@ # along with this program. If not, see . -class Ipv4ModeEnum: +class Ipv4ModeEnum(object): """ The IPv4 configuration mode which directly represents the user's goal. diff --git a/cloudinit/sources/helpers/vmware/imc/nic_base.py b/cloudinit/sources/helpers/vmware/imc/nic_base.py index 030ba311..3c892db0 100644 --- a/cloudinit/sources/helpers/vmware/imc/nic_base.py +++ b/cloudinit/sources/helpers/vmware/imc/nic_base.py @@ -18,7 +18,7 @@ # along with this program. If not, see . -class NicBase: +class NicBase(object): """ Define what are expected of each nic. The following properties should be provided in an implementation class. @@ -93,7 +93,7 @@ class NicBase: raise NotImplementedError('Check constraints on properties') -class StaticIpv4Base: +class StaticIpv4Base(object): """ Define what are expected of a static IPv4 setting The following properties should be provided in an implementation class. @@ -124,7 +124,7 @@ class StaticIpv4Base: raise NotImplementedError('Ipv4 GATEWAY') -class StaticIpv6Base: +class StaticIpv6Base(object): """Define what are expected of a static IPv6 setting The following properties should be provided in an implementation class. """ diff --git a/cloudinit/stages.py b/cloudinit/stages.py index ffb15165..62d066de 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -44,10 +44,10 @@ from cloudinit import helpers from cloudinit import importer from cloudinit import log as logging from cloudinit import net +from cloudinit.reporting import events from cloudinit import sources from cloudinit import type_utils from cloudinit import util -from cloudinit.reporting import events LOG = logging.getLogger(__name__) @@ -483,7 +483,7 @@ class Init(object): c_handlers.initialized.remove(mod) try: handlers.call_end(mod, data, frequency) - except: + except Exception: util.logexc(LOG, "Failed to finalize handler: %s", mod) try: diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 936f7da5..c05e9d90 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -28,8 +28,9 @@ import time from email.utils import parsedate from functools import partial -from requests import exceptions + import oauthlib.oauth1 as oauth1 +from requests import exceptions from six.moves.urllib.parse import ( urlparse, urlunparse, @@ -61,7 +62,7 @@ try: SSL_ENABLED = True if _REQ_VER >= LooseVersion('0.7.0') and _REQ_VER < LooseVersion('1.0.0'): CONFIG_ENABLED = True -except: +except ImportError: pass diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index f7c5787c..f0631906 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -178,7 +178,7 @@ class UserDataProcessor(object): payload = util.load_yaml(msg.get_payload(decode=True)) if payload: payload_idx = payload.get('launch-index') - except: + except Exception: pass # Header overrides contents, for now (?) or the other way around? if header_idx is not None: diff --git a/cloudinit/util.py b/cloudinit/util.py index 0d21e11b..8d6cbb4b 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -288,7 +288,7 @@ def fork_cb(child_cb, *args, **kwargs): try: child_cb(*args, **kwargs) os._exit(0) - except: + except Exception: logexc(LOG, "Failed forking and calling callback %s", type_utils.obj_name(child_cb)) os._exit(1) @@ -472,7 +472,7 @@ def is_ipv4(instr): try: toks = [x for x in toks if int(x) < 256 and int(x) >= 0] - except: + except Exception: return False return len(toks) == 4 @@ -1210,7 +1210,7 @@ def get_cmdline(): else: try: cmdline = load_file("/proc/cmdline").strip() - except: + except Exception: cmdline = "" PROC_CMDLINE = cmdline @@ -1380,7 +1380,7 @@ def read_write_cmdline_url(target_fn): if not os.path.exists(target_fn): try: (key, url, content) = get_cmdline_url() - except: + except Exception: logexc(LOG, "Failed fetching command line url") return try: @@ -1391,7 +1391,7 @@ def read_write_cmdline_url(target_fn): elif key and not content: LOG.debug(("Command line key %s with url" " %s had no contents"), key, url) - except: + except Exception: logexc(LOG, "Failed writing url content to %s", target_fn) @@ -1449,7 +1449,7 @@ def mounts(): mp = m.group(2) fstype = m.group(3) opts = m.group(4) - except: + except Exception: continue # If the name of the mount point contains spaces these # can be escaped as '\040', so undo that.. @@ -1575,7 +1575,7 @@ def copy(src, dest): def time_rfc2822(): try: ts = time.strftime("%a, %d %b %Y %H:%M:%S %z", time.gmtime()) - except: + except Exception: ts = "??" return ts @@ -1601,7 +1601,7 @@ def uptime(): bootup = buf.value uptime_str = now - bootup - except: + except Exception: logexc(LOG, "Unable to read uptime using method: %s" % method) return uptime_str @@ -2055,7 +2055,7 @@ def log_time(logfunc, msg, func, args=None, kwargs=None, get_uptime=False): tmsg += " (N/A)" try: logfunc(msg + tmsg) - except: + except Exception: pass return ret diff --git a/test-requirements.txt b/test-requirements.txt index 9b3d07c5..170856ad 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,7 +1,8 @@ httpretty>=0.7.1 mock nose -pep8==1.5.7 +pep8>=1.5.7 pyflakes contextlib2 setuptools +flake8>=2.5.4 diff --git a/tox.ini b/tox.ini index bd7c27dd..7a144efe 100644 --- a/tox.ini +++ b/tox.ini @@ -30,3 +30,8 @@ deps = pyflakes setenv = LC_ALL = C + +[flake8] + +ignore=H404,H405,H105,H301,H104,H403,H101 +exclude = .venv,.tox,dist,doc,*egg,.git,build,tools,tests -- cgit v1.2.3 From 3f468f3f25319020e233e8c5db02d43dce0499e4 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 12 May 2016 11:00:19 -0700 Subject: Fix logging import --- cloudinit/netinfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/netinfo.py b/cloudinit/netinfo.py index 5d24062f..d8698a5d 100644 --- a/cloudinit/netinfo.py +++ b/cloudinit/netinfo.py @@ -22,7 +22,7 @@ import re -from cloudinit import logging +from cloudinit import log as logging from cloudinit import util from prettytable import PrettyTable -- cgit v1.2.3 From 53834934e4c520b2fb8b5acffca641213ddd688a Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 20:29:33 +0200 Subject: fix EXPORT_GPG_KEYID for long key fingerprints --- cloudinit/config/cc_apt_configure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 492c3c00..28f20939 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -43,7 +43,7 @@ EXPORT_GPG_KEYID = """ [ -n "$k" ] || exit 1; armour=$(gpg --list-keys --armour "${k}") if [ -z "${armour}" ]; then - gpg --keyserver ${ks} --recv $k >/dev/null && + gpg --keyserver ${ks} --recv "${k}" >/dev/null && armour=$(gpg --export --armour "${k}") && gpg --batch --yes --delete-keys "${k}" fi -- cgit v1.2.3 From 2202494b72cae19cbf9d34a8f3176d7021becb13 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 20:53:12 +0200 Subject: split add_key and add_key_raw fior better testability --- cloudinit/config/cc_apt_configure.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 28f20939..e7b8a9b3 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -171,10 +171,19 @@ def generate_sources_list(cfg, codename, mirrors, cloud, log): templater.render_to_file(template_fn, '/etc/apt/sources.list', params) +def add_key_raw(key): + """ + actual adding of a key as defined in key argument + to the system + """ + try: + util.subp(('apt-key', 'add', '-'), key) + except: + raise Exception('failed add key') def add_key(ent): """ - add key to the system as defiend in entry (if any) + add key to the system as defiend in ent (if any) suppords raw keys or keyid's The latter will as a first step fetched to get the raw key """ @@ -188,10 +197,7 @@ def add_key(ent): raise Exception('failed to get key from %s' % keyserver) if 'key' in ent: - try: - util.subp(('apt-key', 'add', '-'), ent['key']) - except: - raise Exception('failed add key') + add_key_raw(ent['key']) def add_sources(srclist, template_params=None, aa_repo_match=None): -- cgit v1.2.3 From 454de24c7d457b980c91849b128efe4faee62032 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 21:21:36 +0200 Subject: make pep8 happy with a few spaces --- cloudinit/config/cc_apt_configure.py | 2 ++ tests/unittests/test_handler/test_handler_apt_source.py | 1 + 2 files changed, 3 insertions(+) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index e7b8a9b3..e5a962ac 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -171,6 +171,7 @@ def generate_sources_list(cfg, codename, mirrors, cloud, log): templater.render_to_file(template_fn, '/etc/apt/sources.list', params) + def add_key_raw(key): """ actual adding of a key as defined in key argument @@ -181,6 +182,7 @@ def add_key_raw(key): except: raise Exception('failed add key') + def add_key(ent): """ add key to the system as defiend in ent (if any) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 439bd038..e130392c 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -30,6 +30,7 @@ S0ORP6HXET3+jC8BMG4tBWCTK/XEZw== =ACB2 -----END PGP PUBLIC KEY BLOCK-----""" + def load_tfile_or_url(*args, **kwargs): """ load_tfile_or_url load file and return content after decoding -- cgit v1.2.3 From 4ba4df2f00ab1763920280f76e2b4497898858af Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 12 May 2016 16:43:11 -0400 Subject: run flake8 instead of pyflakes in tox. expect tests/ to pass flake8. --- cloudinit/sources/helpers/vmware/imc/boot_proto.py | 50 ++-- cloudinit/sources/helpers/vmware/imc/config.py | 190 +++++++-------- .../sources/helpers/vmware/imc/config_namespace.py | 50 ++-- .../sources/helpers/vmware/imc/config_source.py | 46 ++-- .../sources/helpers/vmware/imc/guestcust_error.py | 48 ++-- .../sources/helpers/vmware/imc/guestcust_event.py | 54 ++--- .../sources/helpers/vmware/imc/guestcust_state.py | 50 ++-- .../sources/helpers/vmware/imc/guestcust_util.py | 256 ++++++++++----------- tests/unittests/helpers.py | 2 +- tests/unittests/test_builtin_handlers.py | 4 +- tests/unittests/test_cli.py | 4 +- tests/unittests/test_data.py | 49 ++-- tests/unittests/test_datasource/test_altcloud.py | 48 ++-- tests/unittests/test_datasource/test_azure.py | 18 +- .../unittests/test_datasource/test_azure_helper.py | 1 + tests/unittests/test_datasource/test_cloudstack.py | 1 + .../unittests/test_datasource/test_configdrive.py | 8 +- .../unittests/test_datasource/test_digitalocean.py | 2 +- tests/unittests/test_datasource/test_gce.py | 2 +- tests/unittests/test_datasource/test_maas.py | 4 +- tests/unittests/test_datasource/test_nocloud.py | 2 +- tests/unittests/test_datasource/test_openstack.py | 70 +++--- tests/unittests/test_datasource/test_smartos.py | 110 ++++----- tests/unittests/test_distros/test_generic.py | 10 +- tests/unittests/test_distros/test_hostname.py | 12 +- tests/unittests/test_distros/test_hosts.py | 18 +- tests/unittests/test_distros/test_netconfig.py | 37 ++- tests/unittests/test_distros/test_resolv.py | 21 +- tests/unittests/test_distros/test_sysconfig.py | 27 +-- .../test_distros/test_user_data_normalize.py | 74 +++--- tests/unittests/test_ec2_util.py | 36 +-- tests/unittests/test_filters/test_launch_index.py | 2 +- .../test_handler/test_handler_apt_configure.py | 2 +- .../test_handler/test_handler_ca_certs.py | 14 +- tests/unittests/test_handler/test_handler_chef.py | 14 +- .../test_handler/test_handler_growpart.py | 2 +- .../unittests/test_handler/test_handler_locale.py | 2 +- tests/unittests/test_handler/test_handler_lxd.py | 22 +- .../test_handler/test_handler_power_state.py | 2 +- .../unittests/test_handler/test_handler_rsyslog.py | 2 +- .../test_handler/test_handler_seed_random.py | 12 +- .../test_handler/test_handler_set_hostname.py | 16 +- .../unittests/test_handler/test_handler_snappy.py | 1 + .../test_handler/test_handler_timezone.py | 10 +- .../test_handler/test_handler_write_files.py | 4 +- .../test_handler/test_handler_yum_add_repo.py | 13 +- tests/unittests/test_merging.py | 18 +- tests/unittests/test_net.py | 5 +- tests/unittests/test_reporting.py | 2 +- tests/unittests/test_rh_subscription.py | 3 +- tests/unittests/test_runs/test_merge_run.py | 6 +- tests/unittests/test_runs/test_simple_run.py | 4 +- tests/unittests/test_templating.py | 8 +- tox.ini | 10 +- 54 files changed, 731 insertions(+), 747 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/helpers/vmware/imc/boot_proto.py b/cloudinit/sources/helpers/vmware/imc/boot_proto.py index 204fbcc4..fb53ec1d 100644 --- a/cloudinit/sources/helpers/vmware/imc/boot_proto.py +++ b/cloudinit/sources/helpers/vmware/imc/boot_proto.py @@ -1,25 +1,25 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2015 Canonical Ltd. -# Copyright (C) 2015 VMware Inc. -# -# Author: Sankar Tanguturi -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -class BootProtoEnum(object): - """Specifies the NIC Boot Settings.""" - - DHCP = 'dhcp' - STATIC = 'static' +# vi: ts=4 expandtab +# +# Copyright (C) 2015 Canonical Ltd. +# Copyright (C) 2015 VMware Inc. +# +# Author: Sankar Tanguturi +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +class BootProtoEnum(object): + """Specifies the NIC Boot Settings.""" + + DHCP = 'dhcp' + STATIC = 'static' diff --git a/cloudinit/sources/helpers/vmware/imc/config.py b/cloudinit/sources/helpers/vmware/imc/config.py index 1dcd053a..d645c497 100644 --- a/cloudinit/sources/helpers/vmware/imc/config.py +++ b/cloudinit/sources/helpers/vmware/imc/config.py @@ -1,95 +1,95 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2015 Canonical Ltd. -# Copyright (C) 2015 VMware Inc. -# -# Author: Sankar Tanguturi -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from .nic import Nic - - -class Config(object): - """ - Stores the Contents specified in the Customization - Specification file. - """ - - DNS = 'DNS|NAMESERVER|' - SUFFIX = 'DNS|SUFFIX|' - PASS = 'PASSWORD|-PASS' - TIMEZONE = 'DATETIME|TIMEZONE' - UTC = 'DATETIME|UTC' - HOSTNAME = 'NETWORK|HOSTNAME' - DOMAINNAME = 'NETWORK|DOMAINNAME' - - def __init__(self, configFile): - self._configFile = configFile - - @property - def host_name(self): - """Return the hostname.""" - return self._configFile.get(Config.HOSTNAME, None) - - @property - def domain_name(self): - """Return the domain name.""" - return self._configFile.get(Config.DOMAINNAME, None) - - @property - def timezone(self): - """Return the timezone.""" - return self._configFile.get(Config.TIMEZONE, None) - - @property - def utc(self): - """Retrieves whether to set time to UTC or Local.""" - return self._configFile.get(Config.UTC, None) - - @property - def admin_password(self): - """Return the root password to be set.""" - return self._configFile.get(Config.PASS, None) - - @property - def name_servers(self): - """Return the list of DNS servers.""" - res = [] - cnt = self._configFile.get_count_with_prefix(Config.DNS) - for i in range(1, cnt + 1): - key = Config.DNS + str(i) - res.append(self._configFile[key]) - - return res - - @property - def dns_suffixes(self): - """Return the list of DNS Suffixes.""" - res = [] - cnt = self._configFile.get_count_with_prefix(Config.SUFFIX) - for i in range(1, cnt + 1): - key = Config.SUFFIX + str(i) - res.append(self._configFile[key]) - - return res - - @property - def nics(self): - """Return the list of associated NICs.""" - res = [] - nics = self._configFile['NIC-CONFIG|NICS'] - for nic in nics.split(','): - res.append(Nic(nic, self._configFile)) - - return res +# vi: ts=4 expandtab +# +# Copyright (C) 2015 Canonical Ltd. +# Copyright (C) 2015 VMware Inc. +# +# Author: Sankar Tanguturi +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from .nic import Nic + + +class Config(object): + """ + Stores the Contents specified in the Customization + Specification file. + """ + + DNS = 'DNS|NAMESERVER|' + SUFFIX = 'DNS|SUFFIX|' + PASS = 'PASSWORD|-PASS' + TIMEZONE = 'DATETIME|TIMEZONE' + UTC = 'DATETIME|UTC' + HOSTNAME = 'NETWORK|HOSTNAME' + DOMAINNAME = 'NETWORK|DOMAINNAME' + + def __init__(self, configFile): + self._configFile = configFile + + @property + def host_name(self): + """Return the hostname.""" + return self._configFile.get(Config.HOSTNAME, None) + + @property + def domain_name(self): + """Return the domain name.""" + return self._configFile.get(Config.DOMAINNAME, None) + + @property + def timezone(self): + """Return the timezone.""" + return self._configFile.get(Config.TIMEZONE, None) + + @property + def utc(self): + """Retrieves whether to set time to UTC or Local.""" + return self._configFile.get(Config.UTC, None) + + @property + def admin_password(self): + """Return the root password to be set.""" + return self._configFile.get(Config.PASS, None) + + @property + def name_servers(self): + """Return the list of DNS servers.""" + res = [] + cnt = self._configFile.get_count_with_prefix(Config.DNS) + for i in range(1, cnt + 1): + key = Config.DNS + str(i) + res.append(self._configFile[key]) + + return res + + @property + def dns_suffixes(self): + """Return the list of DNS Suffixes.""" + res = [] + cnt = self._configFile.get_count_with_prefix(Config.SUFFIX) + for i in range(1, cnt + 1): + key = Config.SUFFIX + str(i) + res.append(self._configFile[key]) + + return res + + @property + def nics(self): + """Return the list of associated NICs.""" + res = [] + nics = self._configFile['NIC-CONFIG|NICS'] + for nic in nics.split(','): + res.append(Nic(nic, self._configFile)) + + return res diff --git a/cloudinit/sources/helpers/vmware/imc/config_namespace.py b/cloudinit/sources/helpers/vmware/imc/config_namespace.py index 7266b699..b28830f5 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_namespace.py +++ b/cloudinit/sources/helpers/vmware/imc/config_namespace.py @@ -1,25 +1,25 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2015 Canonical Ltd. -# Copyright (C) 2015 VMware Inc. -# -# Author: Sankar Tanguturi -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from .config_source import ConfigSource - - -class ConfigNamespace(ConfigSource): - """Specifies the Config Namespace.""" - pass +# vi: ts=4 expandtab +# +# Copyright (C) 2015 Canonical Ltd. +# Copyright (C) 2015 VMware Inc. +# +# Author: Sankar Tanguturi +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from .config_source import ConfigSource + + +class ConfigNamespace(ConfigSource): + """Specifies the Config Namespace.""" + pass diff --git a/cloudinit/sources/helpers/vmware/imc/config_source.py b/cloudinit/sources/helpers/vmware/imc/config_source.py index 8a650871..28ef306a 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_source.py +++ b/cloudinit/sources/helpers/vmware/imc/config_source.py @@ -1,23 +1,23 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2015 Canonical Ltd. -# Copyright (C) 2015 VMware Inc. -# -# Author: Sankar Tanguturi -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -class ConfigSource(object): - """Specifies a source for the Config Content.""" - pass +# vi: ts=4 expandtab +# +# Copyright (C) 2015 Canonical Ltd. +# Copyright (C) 2015 VMware Inc. +# +# Author: Sankar Tanguturi +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +class ConfigSource(object): + """Specifies a source for the Config Content.""" + pass diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_error.py b/cloudinit/sources/helpers/vmware/imc/guestcust_error.py index 750be1e3..d1546852 100644 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_error.py +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_error.py @@ -1,24 +1,24 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2016 Canonical Ltd. -# Copyright (C) 2016 VMware Inc. -# -# Author: Sankar Tanguturi -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -class GuestCustErrorEnum(object): - """Specifies different errors of Guest Customization engine""" - - GUESTCUST_ERROR_SUCCESS = 0 +# vi: ts=4 expandtab +# +# Copyright (C) 2016 Canonical Ltd. +# Copyright (C) 2016 VMware Inc. +# +# Author: Sankar Tanguturi +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +class GuestCustErrorEnum(object): + """Specifies different errors of Guest Customization engine""" + + GUESTCUST_ERROR_SUCCESS = 0 diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_event.py b/cloudinit/sources/helpers/vmware/imc/guestcust_event.py index e13b791d..ce90c898 100644 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_event.py +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_event.py @@ -1,27 +1,27 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2016 Canonical Ltd. -# Copyright (C) 2016 VMware Inc. -# -# Author: Sankar Tanguturi -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -class GuestCustEventEnum(object): - """Specifies different types of Guest Customization Events""" - - GUESTCUST_EVENT_CUSTOMIZE_FAILED = 100 - GUESTCUST_EVENT_NETWORK_SETUP_FAILED = 101 - GUESTCUST_EVENT_ENABLE_NICS = 103 - GUESTCUST_EVENT_QUERY_NICS = 104 +# vi: ts=4 expandtab +# +# Copyright (C) 2016 Canonical Ltd. +# Copyright (C) 2016 VMware Inc. +# +# Author: Sankar Tanguturi +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +class GuestCustEventEnum(object): + """Specifies different types of Guest Customization Events""" + + GUESTCUST_EVENT_CUSTOMIZE_FAILED = 100 + GUESTCUST_EVENT_NETWORK_SETUP_FAILED = 101 + GUESTCUST_EVENT_ENABLE_NICS = 103 + GUESTCUST_EVENT_QUERY_NICS = 104 diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_state.py b/cloudinit/sources/helpers/vmware/imc/guestcust_state.py index b9ddf513..422a096d 100644 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_state.py +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_state.py @@ -1,25 +1,25 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2016 Canonical Ltd. -# Copyright (C) 2016 VMware Inc. -# -# Author: Sankar Tanguturi -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -class GuestCustStateEnum(object): - """Specifies different states of Guest Customization engine""" - - GUESTCUST_STATE_RUNNING = 4 - GUESTCUST_STATE_DONE = 5 +# vi: ts=4 expandtab +# +# Copyright (C) 2016 Canonical Ltd. +# Copyright (C) 2016 VMware Inc. +# +# Author: Sankar Tanguturi +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +class GuestCustStateEnum(object): + """Specifies different states of Guest Customization engine""" + + GUESTCUST_STATE_RUNNING = 4 + GUESTCUST_STATE_DONE = 5 diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py index 020ab613..c07c5949 100644 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py @@ -1,128 +1,128 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2016 Canonical Ltd. -# Copyright (C) 2016 VMware Inc. -# -# Author: Sankar Tanguturi -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import logging -import os -import time - -from cloudinit import util - -from .guestcust_event import GuestCustEventEnum -from .guestcust_state import GuestCustStateEnum - -logger = logging.getLogger(__name__) - - -CLOUDINIT_LOG_FILE = "/var/log/cloud-init.log" -QUERY_NICS_SUPPORTED = "queryNicsSupported" -NICS_STATUS_CONNECTED = "connected" - - -# This will send a RPC command to the underlying -# VMware Virtualization Platform. -def send_rpc(rpc): - if not rpc: - return None - - out = "" - err = "Error sending the RPC command" - - try: - logger.debug("Sending RPC command: %s", rpc) - (out, err) = util.subp(["vmware-rpctool", rpc], rcs=[0]) - # Remove the trailing newline in the output. - if out: - out = out.rstrip() - except Exception as e: - logger.debug("Failed to send RPC command") - logger.exception(e) - - return (out, err) - - -# This will send the customization status to the -# underlying VMware Virtualization Platform. -def set_customization_status(custstate, custerror, errormessage=None): - message = "" - - if errormessage: - message = CLOUDINIT_LOG_FILE + "@" + errormessage - else: - message = CLOUDINIT_LOG_FILE - - rpc = "deployPkg.update.state %d %d %s" % (custstate, custerror, message) - (out, err) = send_rpc(rpc) - return (out, err) - - -# This will read the file nics.txt in the specified directory -# and return the content -def get_nics_to_enable(dirpath): - if not dirpath: - return None - - NICS_SIZE = 1024 - nicsfilepath = os.path.join(dirpath, "nics.txt") - if not os.path.exists(nicsfilepath): - return None - - with open(nicsfilepath, 'r') as fp: - nics = fp.read(NICS_SIZE) - - return nics - - -# This will send a RPC command to the underlying VMware Virtualization platform -# and enable nics. -def enable_nics(nics): - if not nics: - logger.warning("No Nics found") - return - - enableNicsWaitRetries = 5 - enableNicsWaitCount = 5 - enableNicsWaitSeconds = 1 - - for attempt in range(0, enableNicsWaitRetries): - logger.debug("Trying to connect interfaces, attempt %d", attempt) - (out, err) = set_customization_status( - GuestCustStateEnum.GUESTCUST_STATE_RUNNING, - GuestCustEventEnum.GUESTCUST_EVENT_ENABLE_NICS, - nics) - if not out: - time.sleep(enableNicsWaitCount * enableNicsWaitSeconds) - continue - - if out != QUERY_NICS_SUPPORTED: - logger.warning("NICS connection status query is not supported") - return - - for count in range(0, enableNicsWaitCount): - (out, err) = set_customization_status( - GuestCustStateEnum.GUESTCUST_STATE_RUNNING, - GuestCustEventEnum.GUESTCUST_EVENT_QUERY_NICS, - nics) - if out and out == NICS_STATUS_CONNECTED: - logger.info("NICS are connected on %d second", count) - return - - time.sleep(enableNicsWaitSeconds) - - logger.warning("Can't connect network interfaces after %d attempts", - enableNicsWaitRetries) +# vi: ts=4 expandtab +# +# Copyright (C) 2016 Canonical Ltd. +# Copyright (C) 2016 VMware Inc. +# +# Author: Sankar Tanguturi +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import logging +import os +import time + +from cloudinit import util + +from .guestcust_event import GuestCustEventEnum +from .guestcust_state import GuestCustStateEnum + +logger = logging.getLogger(__name__) + + +CLOUDINIT_LOG_FILE = "/var/log/cloud-init.log" +QUERY_NICS_SUPPORTED = "queryNicsSupported" +NICS_STATUS_CONNECTED = "connected" + + +# This will send a RPC command to the underlying +# VMware Virtualization Platform. +def send_rpc(rpc): + if not rpc: + return None + + out = "" + err = "Error sending the RPC command" + + try: + logger.debug("Sending RPC command: %s", rpc) + (out, err) = util.subp(["vmware-rpctool", rpc], rcs=[0]) + # Remove the trailing newline in the output. + if out: + out = out.rstrip() + except Exception as e: + logger.debug("Failed to send RPC command") + logger.exception(e) + + return (out, err) + + +# This will send the customization status to the +# underlying VMware Virtualization Platform. +def set_customization_status(custstate, custerror, errormessage=None): + message = "" + + if errormessage: + message = CLOUDINIT_LOG_FILE + "@" + errormessage + else: + message = CLOUDINIT_LOG_FILE + + rpc = "deployPkg.update.state %d %d %s" % (custstate, custerror, message) + (out, err) = send_rpc(rpc) + return (out, err) + + +# This will read the file nics.txt in the specified directory +# and return the content +def get_nics_to_enable(dirpath): + if not dirpath: + return None + + NICS_SIZE = 1024 + nicsfilepath = os.path.join(dirpath, "nics.txt") + if not os.path.exists(nicsfilepath): + return None + + with open(nicsfilepath, 'r') as fp: + nics = fp.read(NICS_SIZE) + + return nics + + +# This will send a RPC command to the underlying VMware Virtualization platform +# and enable nics. +def enable_nics(nics): + if not nics: + logger.warning("No Nics found") + return + + enableNicsWaitRetries = 5 + enableNicsWaitCount = 5 + enableNicsWaitSeconds = 1 + + for attempt in range(0, enableNicsWaitRetries): + logger.debug("Trying to connect interfaces, attempt %d", attempt) + (out, err) = set_customization_status( + GuestCustStateEnum.GUESTCUST_STATE_RUNNING, + GuestCustEventEnum.GUESTCUST_EVENT_ENABLE_NICS, + nics) + if not out: + time.sleep(enableNicsWaitCount * enableNicsWaitSeconds) + continue + + if out != QUERY_NICS_SUPPORTED: + logger.warning("NICS connection status query is not supported") + return + + for count in range(0, enableNicsWaitCount): + (out, err) = set_customization_status( + GuestCustStateEnum.GUESTCUST_STATE_RUNNING, + GuestCustEventEnum.GUESTCUST_EVENT_QUERY_NICS, + nics) + if out and out == NICS_STATUS_CONNECTED: + logger.info("NICS are connected on %d second", count) + return + + time.sleep(enableNicsWaitSeconds) + + logger.warning("Can't connect network interfaces after %d attempts", + enableNicsWaitRetries) diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index fb9c83a7..50b2bd72 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -2,8 +2,8 @@ from __future__ import print_function import functools import os -import sys import shutil +import sys import tempfile import unittest diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py index ad32d0b2..dea908d7 100644 --- a/tests/unittests/test_builtin_handlers.py +++ b/tests/unittests/test_builtin_handlers.py @@ -40,7 +40,7 @@ class TestBuiltins(test_helpers.FilesystemMockingTestCase): 'test.conf', 'blah', freq) h.handle_part('', handlers.CONTENT_END, None, None, None) - self.assertEquals(0, len(os.listdir(up_root))) + self.assertEqual(0, len(os.listdir(up_root))) def test_upstart_frequency_single(self): # files should be written out when frequency is ! per-instance @@ -67,7 +67,7 @@ class TestBuiltins(test_helpers.FilesystemMockingTestCase): h.handle_part('', handlers.CONTENT_END, None, None, None) - self.assertEquals(len(os.listdir('/etc/upstart')), 1) + self.assertEqual(len(os.listdir('/etc/upstart')), 1) mockobj.assert_called_once_with( ['initctl', 'reload-configuration'], capture=False) diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py index ed863399..f537bd83 100644 --- a/tests/unittests/test_cli.py +++ b/tests/unittests/test_cli.py @@ -1,7 +1,7 @@ import imp import os -import sys import six +import sys from . import helpers as test_helpers @@ -31,7 +31,7 @@ class TestCLI(test_helpers.FilesystemMockingTestCase): 'cli', open(BIN_CLOUDINIT), '', ('', 'r', imp.PY_SOURCE)) try: return cli.main() - except: + except Exception: pass @test_helpers.skipIf(not os.path.isfile(BIN_CLOUDINIT), "no bin/cloudinit") diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py index 9c1ec1d4..1923e2af 100644 --- a/tests/unittests/test_data.py +++ b/tests/unittests/test_data.py @@ -106,9 +106,9 @@ class TestConsumeUserData(helpers.FilesystemMockingTestCase): ci.consume_data() cc_contents = util.load_file(ci.paths.get_ipath("cloud_config")) cc = util.load_yaml(cc_contents) - self.assertEquals(2, len(cc)) - self.assertEquals('qux', cc['baz']) - self.assertEquals('qux2', cc['bar']) + self.assertEqual(2, len(cc)) + self.assertEqual('qux', cc['baz']) + self.assertEqual('qux2', cc['bar']) def test_simple_jsonp_vendor_and_user(self): # test that user-data wins over vendor @@ -145,9 +145,9 @@ class TestConsumeUserData(helpers.FilesystemMockingTestCase): (_which_ran, _failures) = mods.run_section('cloud_init_modules') cfg = mods.cfg self.assertIn('vendor_data', cfg) - self.assertEquals('qux', cfg['baz']) - self.assertEquals('qux2', cfg['bar']) - self.assertEquals('quxC', cfg['foo']) + self.assertEqual('qux', cfg['baz']) + self.assertEqual('qux2', cfg['bar']) + self.assertEqual('quxC', cfg['foo']) def test_simple_jsonp_no_vendor_consumed(self): # make sure that vendor data is not consumed @@ -184,8 +184,8 @@ class TestConsumeUserData(helpers.FilesystemMockingTestCase): mods = stages.Modules(initer) (_which_ran, _failures) = mods.run_section('cloud_init_modules') cfg = mods.cfg - self.assertEquals('qux', cfg['baz']) - self.assertEquals('qux2', cfg['bar']) + self.assertEqual('qux', cfg['baz']) + self.assertEqual('qux2', cfg['bar']) self.assertNotIn('foo', cfg) def test_mixed_cloud_config(self): @@ -222,8 +222,8 @@ c: d ci.consume_data() cc_contents = util.load_file(ci.paths.get_ipath("cloud_config")) cc = util.load_yaml(cc_contents) - self.assertEquals(1, len(cc)) - self.assertEquals('c', cc['a']) + self.assertEqual(1, len(cc)) + self.assertEqual('c', cc['a']) def test_vendor_user_yaml_cloud_config(self): vendor_blob = ''' @@ -263,8 +263,8 @@ run: (_which_ran, _failures) = mods.run_section('cloud_init_modules') cfg = mods.cfg self.assertIn('vendor_data', cfg) - self.assertEquals('c', cfg['a']) - self.assertEquals('user', cfg['name']) + self.assertEqual('c', cfg['a']) + self.assertEqual('user', cfg['name']) self.assertNotIn('x', cfg['run']) self.assertNotIn('y', cfg['run']) self.assertIn('z', cfg['run']) @@ -358,10 +358,10 @@ p: 1 None) contents = util.load_file(paths.get_ipath('cloud_config')) contents = util.load_yaml(contents) - self.assertEquals(contents['run'], ['b', 'c', 'stuff', 'morestuff']) - self.assertEquals(contents['a'], 'be') - self.assertEquals(contents['e'], [1, 2, 3]) - self.assertEquals(contents['p'], 1) + self.assertEqual(contents['run'], ['b', 'c', 'stuff', 'morestuff']) + self.assertEqual(contents['a'], 'be') + self.assertEqual(contents['e'], [1, 2, 3]) + self.assertEqual(contents['p'], 1) def test_unhandled_type_warning(self): """Raw text without magic is ignored but shows warning.""" @@ -411,10 +411,10 @@ c: 4 contents = util.load_file(ci.paths.get_ipath("cloud_config")) contents = util.load_yaml(contents) self.assertTrue(isinstance(contents, dict)) - self.assertEquals(3, len(contents)) - self.assertEquals(2, contents['a']) - self.assertEquals(3, contents['b']) - self.assertEquals(4, contents['c']) + self.assertEqual(3, len(contents)) + self.assertEqual(2, contents['a']) + self.assertEqual(3, contents['b']) + self.assertEqual(4, contents['c']) def test_mime_text_plain(self): """Mime message of type text/plain is ignored but shows warning.""" @@ -449,8 +449,7 @@ c: 4 mockobj.assert_has_calls([ mock.call(outpath, script, 0o700), - mock.call(ci.paths.get_ipath("cloud_config"), "", 0o600), - ]) + mock.call(ci.paths.get_ipath("cloud_config"), "", 0o600)]) def test_mime_text_x_shellscript(self): """Mime message of type text/x-shellscript is treated as script.""" @@ -470,8 +469,7 @@ c: 4 mockobj.assert_has_calls([ mock.call(outpath, script, 0o700), - mock.call(ci.paths.get_ipath("cloud_config"), "", 0o600), - ]) + mock.call(ci.paths.get_ipath("cloud_config"), "", 0o600)]) def test_mime_text_plain_shell(self): """Mime type text/plain starting #!/bin/sh is treated as script.""" @@ -491,8 +489,7 @@ c: 4 mockobj.assert_has_calls([ mock.call(outpath, script, 0o700), - mock.call(ci.paths.get_ipath("cloud_config"), "", 0o600), - ]) + mock.call(ci.paths.get_ipath("cloud_config"), "", 0o600)]) def test_mime_application_octet_stream(self): """Mime type application/octet-stream is ignored but shows warning.""" diff --git a/tests/unittests/test_datasource/test_altcloud.py b/tests/unittests/test_datasource/test_altcloud.py index 85759c68..12966563 100644 --- a/tests/unittests/test_datasource/test_altcloud.py +++ b/tests/unittests/test_datasource/test_altcloud.py @@ -134,7 +134,7 @@ class TestGetCloudType(TestCase): ''' util.read_dmi_data = _dmi_data('RHEV') dsrc = DataSourceAltCloud({}, None, self.paths) - self.assertEquals('RHEV', dsrc.get_cloud_type()) + self.assertEqual('RHEV', dsrc.get_cloud_type()) def test_vsphere(self): ''' @@ -143,7 +143,7 @@ class TestGetCloudType(TestCase): ''' util.read_dmi_data = _dmi_data('VMware Virtual Platform') dsrc = DataSourceAltCloud({}, None, self.paths) - self.assertEquals('VSPHERE', dsrc.get_cloud_type()) + self.assertEqual('VSPHERE', dsrc.get_cloud_type()) def test_unknown(self): ''' @@ -152,7 +152,7 @@ class TestGetCloudType(TestCase): ''' util.read_dmi_data = _dmi_data('Unrecognized Platform') dsrc = DataSourceAltCloud({}, None, self.paths) - self.assertEquals('UNKNOWN', dsrc.get_cloud_type()) + self.assertEqual('UNKNOWN', dsrc.get_cloud_type()) class TestGetDataCloudInfoFile(TestCase): @@ -187,7 +187,7 @@ class TestGetDataCloudInfoFile(TestCase): _write_cloud_info_file('RHEV') dsrc = DataSourceAltCloud({}, None, self.paths) dsrc.user_data_rhevm = lambda: True - self.assertEquals(True, dsrc.get_data()) + self.assertEqual(True, dsrc.get_data()) def test_vsphere(self): '''Success Test module get_data() forcing VSPHERE.''' @@ -195,7 +195,7 @@ class TestGetDataCloudInfoFile(TestCase): _write_cloud_info_file('VSPHERE') dsrc = DataSourceAltCloud({}, None, self.paths) dsrc.user_data_vsphere = lambda: True - self.assertEquals(True, dsrc.get_data()) + self.assertEqual(True, dsrc.get_data()) def test_fail_rhev(self): '''Failure Test module get_data() forcing RHEV.''' @@ -203,7 +203,7 @@ class TestGetDataCloudInfoFile(TestCase): _write_cloud_info_file('RHEV') dsrc = DataSourceAltCloud({}, None, self.paths) dsrc.user_data_rhevm = lambda: False - self.assertEquals(False, dsrc.get_data()) + self.assertEqual(False, dsrc.get_data()) def test_fail_vsphere(self): '''Failure Test module get_data() forcing VSPHERE.''' @@ -211,14 +211,14 @@ class TestGetDataCloudInfoFile(TestCase): _write_cloud_info_file('VSPHERE') dsrc = DataSourceAltCloud({}, None, self.paths) dsrc.user_data_vsphere = lambda: False - self.assertEquals(False, dsrc.get_data()) + self.assertEqual(False, dsrc.get_data()) def test_unrecognized(self): '''Failure Test module get_data() forcing unrecognized.''' _write_cloud_info_file('unrecognized') dsrc = DataSourceAltCloud({}, None, self.paths) - self.assertEquals(False, dsrc.get_data()) + self.assertEqual(False, dsrc.get_data()) class TestGetDataNoCloudInfoFile(TestCase): @@ -250,7 +250,7 @@ class TestGetDataNoCloudInfoFile(TestCase): util.read_dmi_data = _dmi_data('RHEV Hypervisor') dsrc = DataSourceAltCloud({}, None, self.paths) dsrc.user_data_rhevm = lambda: True - self.assertEquals(True, dsrc.get_data()) + self.assertEqual(True, dsrc.get_data()) def test_vsphere_no_cloud_file(self): '''Test No cloud info file module get_data() forcing VSPHERE.''' @@ -258,14 +258,14 @@ class TestGetDataNoCloudInfoFile(TestCase): util.read_dmi_data = _dmi_data('VMware Virtual Platform') dsrc = DataSourceAltCloud({}, None, self.paths) dsrc.user_data_vsphere = lambda: True - self.assertEquals(True, dsrc.get_data()) + self.assertEqual(True, dsrc.get_data()) def test_failure_no_cloud_file(self): '''Test No cloud info file module get_data() forcing unrecognized.''' util.read_dmi_data = _dmi_data('Unrecognized Platform') dsrc = DataSourceAltCloud({}, None, self.paths) - self.assertEquals(False, dsrc.get_data()) + self.assertEqual(False, dsrc.get_data()) class TestUserDataRhevm(TestCase): @@ -305,7 +305,7 @@ class TestUserDataRhevm(TestCase): dsrc = DataSourceAltCloud({}, None, self.paths) - self.assertEquals(False, dsrc.user_data_rhevm()) + self.assertEqual(False, dsrc.user_data_rhevm()) def test_modprobe_fails(self): '''Test user_data_rhevm() where modprobe fails.''' @@ -315,7 +315,7 @@ class TestUserDataRhevm(TestCase): dsrc = DataSourceAltCloud({}, None, self.paths) - self.assertEquals(False, dsrc.user_data_rhevm()) + self.assertEqual(False, dsrc.user_data_rhevm()) def test_no_modprobe_cmd(self): '''Test user_data_rhevm() with no modprobe command.''' @@ -325,7 +325,7 @@ class TestUserDataRhevm(TestCase): dsrc = DataSourceAltCloud({}, None, self.paths) - self.assertEquals(False, dsrc.user_data_rhevm()) + self.assertEqual(False, dsrc.user_data_rhevm()) def test_udevadm_fails(self): '''Test user_data_rhevm() where udevadm fails.''' @@ -335,7 +335,7 @@ class TestUserDataRhevm(TestCase): dsrc = DataSourceAltCloud({}, None, self.paths) - self.assertEquals(False, dsrc.user_data_rhevm()) + self.assertEqual(False, dsrc.user_data_rhevm()) def test_no_udevadm_cmd(self): '''Test user_data_rhevm() with no udevadm command.''' @@ -345,7 +345,7 @@ class TestUserDataRhevm(TestCase): dsrc = DataSourceAltCloud({}, None, self.paths) - self.assertEquals(False, dsrc.user_data_rhevm()) + self.assertEqual(False, dsrc.user_data_rhevm()) class TestUserDataVsphere(TestCase): @@ -380,7 +380,7 @@ class TestUserDataVsphere(TestCase): dsrc = DataSourceAltCloud({}, None, self.paths) - self.assertEquals(False, dsrc.user_data_vsphere()) + self.assertEqual(False, dsrc.user_data_vsphere()) class TestReadUserDataCallback(TestCase): @@ -408,8 +408,8 @@ class TestReadUserDataCallback(TestCase): def test_callback_both(self): '''Test read_user_data_callback() with both files.''' - self.assertEquals('test user data', - read_user_data_callback(self.mount_dir)) + self.assertEqual('test user data', + read_user_data_callback(self.mount_dir)) def test_callback_dc(self): '''Test read_user_data_callback() with only DC file.''' @@ -418,8 +418,8 @@ class TestReadUserDataCallback(TestCase): dc_file=False, non_dc_file=True) - self.assertEquals('test user data', - read_user_data_callback(self.mount_dir)) + self.assertEqual('test user data', + read_user_data_callback(self.mount_dir)) def test_callback_non_dc(self): '''Test read_user_data_callback() with only non-DC file.''' @@ -428,14 +428,14 @@ class TestReadUserDataCallback(TestCase): dc_file=True, non_dc_file=False) - self.assertEquals('test user data', - read_user_data_callback(self.mount_dir)) + self.assertEqual('test user data', + read_user_data_callback(self.mount_dir)) def test_callback_none(self): '''Test read_user_data_callback() no files are found.''' _remove_user_data_files(self.mount_dir) - self.assertEquals(None, read_user_data_callback(self.mount_dir)) + self.assertEqual(None, read_user_data_callback(self.mount_dir)) def force_arch(arch=None): diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 444e2799..5f3eb31f 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -14,11 +14,11 @@ except ImportError: import crypt import os -import stat -import yaml import shutil +import stat import tempfile import xml.etree.ElementTree as ET +import yaml def construct_valid_ovf_env(data=None, pubkeys=None, userdata=None): @@ -165,7 +165,7 @@ class TestAzureDataSource(TestCase): def tags_equal(x, y): for x_tag, x_val in x.items(): y_val = y.get(x_val.tag) - self.assertEquals(x_val.text, y_val.text) + self.assertEqual(x_val.text, y_val.text) old_cnt = create_tag_index(oxml) new_cnt = create_tag_index(nxml) @@ -354,8 +354,8 @@ class TestAzureDataSource(TestCase): self.assertTrue(ret) cfg = dsrc.get_config_obj() - self.assertEquals(dsrc.device_name_to_device("ephemeral0"), - "/dev/sdb") + self.assertEqual(dsrc.device_name_to_device("ephemeral0"), + "/dev/sdb") assert 'disk_setup' in cfg assert 'fs_setup' in cfg self.assertIsInstance(cfg['disk_setup'], dict) @@ -404,15 +404,15 @@ class TestAzureDataSource(TestCase): self.xml_notequals(data['ovfcontent'], on_disk_ovf) # Make sure that the redacted password on disk is not used by CI - self.assertNotEquals(dsrc.cfg.get('password'), - DataSourceAzure.DEF_PASSWD_REDACTION) + self.assertNotEqual(dsrc.cfg.get('password'), + DataSourceAzure.DEF_PASSWD_REDACTION) # Make sure that the password was really encrypted et = ET.fromstring(on_disk_ovf) for elem in et.iter(): if 'UserPassword' in elem.tag: - self.assertEquals(DataSourceAzure.DEF_PASSWD_REDACTION, - elem.text) + self.assertEqual(DataSourceAzure.DEF_PASSWD_REDACTION, + elem.text) def test_ovf_env_arrives_in_waagent_dir(self): xml = construct_valid_ovf_env(data={}, userdata="FOODATA") diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 1134199b..921bc978 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -1,6 +1,7 @@ import os from cloudinit.sources.helpers import azure as azure_helper + from ..helpers import TestCase try: diff --git a/tests/unittests/test_datasource/test_cloudstack.py b/tests/unittests/test_datasource/test_cloudstack.py index 656d80d1..974b3704 100644 --- a/tests/unittests/test_datasource/test_cloudstack.py +++ b/tests/unittests/test_datasource/test_cloudstack.py @@ -1,5 +1,6 @@ from cloudinit import helpers from cloudinit.sources.DataSourceCloudStack import DataSourceCloudStack + from ..helpers import TestCase try: diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index 89b15f54..195b8207 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -151,7 +151,7 @@ class TestConfigDriveDataSource(TestCase): mock.patch.object(os.path, 'exists', side_effect=exists_side_effect())) device = cfg_ds.device_name_to_device(name) - self.assertEquals(dev_name, device) + self.assertEqual(dev_name, device) find_mock.assert_called_once_with(mock.ANY) self.assertEqual(exists_mock.call_count, 2) @@ -179,7 +179,7 @@ class TestConfigDriveDataSource(TestCase): mock.patch.object(os.path, 'exists', return_value=True)) device = cfg_ds.device_name_to_device(name) - self.assertEquals(dev_name, device) + self.assertEqual(dev_name, device) find_mock.assert_called_once_with(mock.ANY) exists_mock.assert_called_once_with(mock.ANY) @@ -214,7 +214,7 @@ class TestConfigDriveDataSource(TestCase): with mock.patch.object(os.path, 'exists', side_effect=exists_side_effect()): device = cfg_ds.device_name_to_device(name) - self.assertEquals(dev_name, device) + self.assertEqual(dev_name, device) # We don't assert the call count for os.path.exists() because # not all of the entries in name_tests results in two calls to # that function. Specifically, 'root2k' doesn't seem to call @@ -242,7 +242,7 @@ class TestConfigDriveDataSource(TestCase): for name, dev_name in name_tests.items(): with mock.patch.object(os.path, 'exists', return_value=True): device = cfg_ds.device_name_to_device(name) - self.assertEquals(dev_name, device) + self.assertEqual(dev_name, device) def test_dir_valid(self): """Verify a dir is read as such.""" diff --git a/tests/unittests/test_datasource/test_digitalocean.py b/tests/unittests/test_datasource/test_digitalocean.py index 679d1b82..8936a1e3 100644 --- a/tests/unittests/test_datasource/test_digitalocean.py +++ b/tests/unittests/test_datasource/test_digitalocean.py @@ -19,8 +19,8 @@ import re from six.moves.urllib_parse import urlparse -from cloudinit import settings from cloudinit import helpers +from cloudinit import settings from cloudinit.sources import DataSourceDigitalOcean from .. import helpers as test_helpers diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py index fa714070..1f7eb99e 100644 --- a/tests/unittests/test_datasource/test_gce.py +++ b/tests/unittests/test_datasource/test_gce.py @@ -20,8 +20,8 @@ import re from base64 import b64encode, b64decode from six.moves.urllib_parse import urlparse -from cloudinit import settings from cloudinit import helpers +from cloudinit import settings from cloudinit.sources import DataSourceGCE from .. import helpers as test_helpers diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index 77d15cac..f66f1c6d 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -104,13 +104,13 @@ class TestMAASDataSource(TestCase): 'meta-data/local-hostname': 'test-hostname', 'meta-data/public-keys': 'test-hostname', 'user-data': b'foodata', - } + } valid_order = [ 'meta-data/local-hostname', 'meta-data/instance-id', 'meta-data/public-keys', 'user-data', - ] + ] my_seed = "http://example.com/xmeta" my_ver = "1999-99-99" my_headers = {'header1': 'value1', 'header2': 'value2'} diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py index 2d5fc37c..3c528c23 100644 --- a/tests/unittests/test_datasource/test_nocloud.py +++ b/tests/unittests/test_datasource/test_nocloud.py @@ -4,10 +4,10 @@ from cloudinit import util from ..helpers import TestCase, populate_dir import os -import yaml import shutil import tempfile import unittest +import yaml try: from unittest import mock diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index 0aa1ba84..4140d054 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -22,8 +22,8 @@ import re from .. import helpers as test_helpers -from six import StringIO from six.moves.urllib.parse import urlparse +from six import StringIO from cloudinit import helpers from cloudinit import settings @@ -142,34 +142,34 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): def test_successful(self): _register_uris(self.VERSION, EC2_FILES, EC2_META, OS_FILES) f = ds.read_metadata_service(BASE_URL) - self.assertEquals(VENDOR_DATA, f.get('vendordata')) - self.assertEquals(CONTENT_0, f['files']['/etc/foo.cfg']) - self.assertEquals(CONTENT_1, f['files']['/etc/bar/bar.cfg']) - self.assertEquals(2, len(f['files'])) - self.assertEquals(USER_DATA, f.get('userdata')) - self.assertEquals(EC2_META, f.get('ec2-metadata')) - self.assertEquals(2, f.get('version')) + self.assertEqual(VENDOR_DATA, f.get('vendordata')) + self.assertEqual(CONTENT_0, f['files']['/etc/foo.cfg']) + self.assertEqual(CONTENT_1, f['files']['/etc/bar/bar.cfg']) + self.assertEqual(2, len(f['files'])) + self.assertEqual(USER_DATA, f.get('userdata')) + self.assertEqual(EC2_META, f.get('ec2-metadata')) + self.assertEqual(2, f.get('version')) metadata = f['metadata'] - self.assertEquals('nova', metadata.get('availability_zone')) - self.assertEquals('sm-foo-test.novalocal', metadata.get('hostname')) - self.assertEquals('sm-foo-test.novalocal', - metadata.get('local-hostname')) - self.assertEquals('sm-foo-test', metadata.get('name')) - self.assertEquals('b0fa911b-69d4-4476-bbe2-1c92bff6535c', - metadata.get('uuid')) - self.assertEquals('b0fa911b-69d4-4476-bbe2-1c92bff6535c', - metadata.get('instance-id')) + self.assertEqual('nova', metadata.get('availability_zone')) + self.assertEqual('sm-foo-test.novalocal', metadata.get('hostname')) + self.assertEqual('sm-foo-test.novalocal', + metadata.get('local-hostname')) + self.assertEqual('sm-foo-test', metadata.get('name')) + self.assertEqual('b0fa911b-69d4-4476-bbe2-1c92bff6535c', + metadata.get('uuid')) + self.assertEqual('b0fa911b-69d4-4476-bbe2-1c92bff6535c', + metadata.get('instance-id')) @hp.activate def test_no_ec2(self): _register_uris(self.VERSION, {}, {}, OS_FILES) f = ds.read_metadata_service(BASE_URL) - self.assertEquals(VENDOR_DATA, f.get('vendordata')) - self.assertEquals(CONTENT_0, f['files']['/etc/foo.cfg']) - self.assertEquals(CONTENT_1, f['files']['/etc/bar/bar.cfg']) - self.assertEquals(USER_DATA, f.get('userdata')) - self.assertEquals({}, f.get('ec2-metadata')) - self.assertEquals(2, f.get('version')) + self.assertEqual(VENDOR_DATA, f.get('vendordata')) + self.assertEqual(CONTENT_0, f['files']['/etc/foo.cfg']) + self.assertEqual(CONTENT_1, f['files']['/etc/bar/bar.cfg']) + self.assertEqual(USER_DATA, f.get('userdata')) + self.assertEqual({}, f.get('ec2-metadata')) + self.assertEqual(2, f.get('version')) @hp.activate def test_bad_metadata(self): @@ -201,9 +201,9 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): os_files.pop(k, None) _register_uris(self.VERSION, {}, {}, os_files) f = ds.read_metadata_service(BASE_URL) - self.assertEquals(VENDOR_DATA, f.get('vendordata')) - self.assertEquals(CONTENT_0, f['files']['/etc/foo.cfg']) - self.assertEquals(CONTENT_1, f['files']['/etc/bar/bar.cfg']) + self.assertEqual(VENDOR_DATA, f.get('vendordata')) + self.assertEqual(CONTENT_0, f['files']['/etc/foo.cfg']) + self.assertEqual(CONTENT_1, f['files']['/etc/bar/bar.cfg']) self.assertFalse(f.get('userdata')) @hp.activate @@ -214,8 +214,8 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): os_files.pop(k, None) _register_uris(self.VERSION, {}, {}, os_files) f = ds.read_metadata_service(BASE_URL) - self.assertEquals(CONTENT_0, f['files']['/etc/foo.cfg']) - self.assertEquals(CONTENT_1, f['files']['/etc/bar/bar.cfg']) + self.assertEqual(CONTENT_0, f['files']['/etc/foo.cfg']) + self.assertEqual(CONTENT_1, f['files']['/etc/bar/bar.cfg']) self.assertFalse(f.get('vendordata')) @hp.activate @@ -247,16 +247,16 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): self.assertIsNone(ds_os.version) found = ds_os.get_data() self.assertTrue(found) - self.assertEquals(2, ds_os.version) + self.assertEqual(2, ds_os.version) md = dict(ds_os.metadata) md.pop('instance-id', None) md.pop('local-hostname', None) - self.assertEquals(OSTACK_META, md) - self.assertEquals(EC2_META, ds_os.ec2_metadata) - self.assertEquals(USER_DATA, ds_os.userdata_raw) - self.assertEquals(2, len(ds_os.files)) - self.assertEquals(VENDOR_DATA, ds_os.vendordata_pure) - self.assertEquals(ds_os.vendordata_raw, None) + self.assertEqual(OSTACK_META, md) + self.assertEqual(EC2_META, ds_os.ec2_metadata) + self.assertEqual(USER_DATA, ds_os.userdata_raw) + self.assertEqual(2, len(ds_os.files)) + self.assertEqual(VENDOR_DATA, ds_os.vendordata_pure) + self.assertEqual(ds_os.vendordata_raw, None) @hp.activate def test_bad_datasource_meta(self): diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 5c49966a..cdd04a72 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -24,6 +24,7 @@ from __future__ import print_function +from binascii import crc32 import os import os.path import re @@ -31,7 +32,6 @@ import shutil import stat import tempfile import uuid -from binascii import crc32 import serial import six @@ -157,16 +157,16 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): dsrc = self._get_ds() ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals('kvm', dsrc.smartos_type) - self.assertEquals('/dev/ttyS1', dsrc.seed) + self.assertEqual('kvm', dsrc.smartos_type) + self.assertEqual('/dev/ttyS1', dsrc.seed) def test_seed_lxbrand(self): # default seed should be /dev/ttyS1 dsrc = self._get_ds(is_lxbrand=True) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals('lx-brand', dsrc.smartos_type) - self.assertEquals('/native/.zonecontrol/metadata.sock', dsrc.seed) + self.assertEqual('lx-brand', dsrc.smartos_type) + self.assertEqual('/native/.zonecontrol/metadata.sock', dsrc.seed) def test_issmartdc(self): dsrc = self._get_ds() @@ -190,29 +190,29 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(MOCK_RETURNS['sdc:uuid'], - dsrc.metadata['instance-id']) + self.assertEqual(MOCK_RETURNS['sdc:uuid'], + dsrc.metadata['instance-id']) def test_root_keys(self): dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(MOCK_RETURNS['root_authorized_keys'], - dsrc.metadata['public-keys']) + self.assertEqual(MOCK_RETURNS['root_authorized_keys'], + dsrc.metadata['public-keys']) def test_hostname_b64(self): dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(MOCK_RETURNS['hostname'], - dsrc.metadata['local-hostname']) + self.assertEqual(MOCK_RETURNS['hostname'], + dsrc.metadata['local-hostname']) def test_hostname(self): dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(MOCK_RETURNS['hostname'], - dsrc.metadata['local-hostname']) + self.assertEqual(MOCK_RETURNS['hostname'], + dsrc.metadata['local-hostname']) def test_base64_all(self): # metadata provided base64_all of true @@ -224,16 +224,16 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): dsrc = self._get_ds(mockdata=my_returns) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(MOCK_RETURNS['hostname'], - dsrc.metadata['local-hostname']) - self.assertEquals(MOCK_RETURNS['cloud-init:user-data'], - dsrc.userdata_raw) - self.assertEquals(MOCK_RETURNS['root_authorized_keys'], - dsrc.metadata['public-keys']) - self.assertEquals(MOCK_RETURNS['disable_iptables_flag'], - dsrc.metadata['iptables_disable']) - self.assertEquals(MOCK_RETURNS['enable_motd_sys_info'], - dsrc.metadata['motd_sys_info']) + self.assertEqual(MOCK_RETURNS['hostname'], + dsrc.metadata['local-hostname']) + self.assertEqual(MOCK_RETURNS['cloud-init:user-data'], + dsrc.userdata_raw) + self.assertEqual(MOCK_RETURNS['root_authorized_keys'], + dsrc.metadata['public-keys']) + self.assertEqual(MOCK_RETURNS['disable_iptables_flag'], + dsrc.metadata['iptables_disable']) + self.assertEqual(MOCK_RETURNS['enable_motd_sys_info'], + dsrc.metadata['motd_sys_info']) def test_b64_userdata(self): my_returns = MOCK_RETURNS.copy() @@ -245,12 +245,12 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): dsrc = self._get_ds(mockdata=my_returns) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(MOCK_RETURNS['hostname'], - dsrc.metadata['local-hostname']) - self.assertEquals(MOCK_RETURNS['cloud-init:user-data'], - dsrc.userdata_raw) - self.assertEquals(MOCK_RETURNS['root_authorized_keys'], - dsrc.metadata['public-keys']) + self.assertEqual(MOCK_RETURNS['hostname'], + dsrc.metadata['local-hostname']) + self.assertEqual(MOCK_RETURNS['cloud-init:user-data'], + dsrc.userdata_raw) + self.assertEqual(MOCK_RETURNS['root_authorized_keys'], + dsrc.metadata['public-keys']) def test_b64_keys(self): my_returns = MOCK_RETURNS.copy() @@ -261,39 +261,39 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): dsrc = self._get_ds(mockdata=my_returns) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(MOCK_RETURNS['hostname'], - dsrc.metadata['local-hostname']) - self.assertEquals(MOCK_RETURNS['cloud-init:user-data'], - dsrc.userdata_raw) + self.assertEqual(MOCK_RETURNS['hostname'], + dsrc.metadata['local-hostname']) + self.assertEqual(MOCK_RETURNS['cloud-init:user-data'], + dsrc.userdata_raw) def test_userdata(self): dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(MOCK_RETURNS['user-data'], - dsrc.metadata['legacy-user-data']) - self.assertEquals(MOCK_RETURNS['cloud-init:user-data'], - dsrc.userdata_raw) + self.assertEqual(MOCK_RETURNS['user-data'], + dsrc.metadata['legacy-user-data']) + self.assertEqual(MOCK_RETURNS['cloud-init:user-data'], + dsrc.userdata_raw) def test_sdc_scripts(self): dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(MOCK_RETURNS['user-script'], - dsrc.metadata['user-script']) + self.assertEqual(MOCK_RETURNS['user-script'], + dsrc.metadata['user-script']) legacy_script_f = "%s/user-script" % self.legacy_user_d self.assertTrue(os.path.exists(legacy_script_f)) self.assertTrue(os.path.islink(legacy_script_f)) user_script_perm = oct(os.stat(legacy_script_f)[stat.ST_MODE])[-3:] - self.assertEquals(user_script_perm, '700') + self.assertEqual(user_script_perm, '700') def test_scripts_shebanged(self): dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(MOCK_RETURNS['user-script'], - dsrc.metadata['user-script']) + self.assertEqual(MOCK_RETURNS['user-script'], + dsrc.metadata['user-script']) legacy_script_f = "%s/user-script" % self.legacy_user_d self.assertTrue(os.path.exists(legacy_script_f)) @@ -301,9 +301,9 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): shebang = None with open(legacy_script_f, 'r') as f: shebang = f.readlines()[0].strip() - self.assertEquals(shebang, "#!/bin/bash") + self.assertEqual(shebang, "#!/bin/bash") user_script_perm = oct(os.stat(legacy_script_f)[stat.ST_MODE])[-3:] - self.assertEquals(user_script_perm, '700') + self.assertEqual(user_script_perm, '700') def test_scripts_shebang_not_added(self): """ @@ -319,8 +319,8 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): dsrc = self._get_ds(mockdata=my_returns) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(my_returns['user-script'], - dsrc.metadata['user-script']) + self.assertEqual(my_returns['user-script'], + dsrc.metadata['user-script']) legacy_script_f = "%s/user-script" % self.legacy_user_d self.assertTrue(os.path.exists(legacy_script_f)) @@ -328,7 +328,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): shebang = None with open(legacy_script_f, 'r') as f: shebang = f.readlines()[0].strip() - self.assertEquals(shebang, "#!/usr/bin/perl") + self.assertEqual(shebang, "#!/usr/bin/perl") def test_userdata_removed(self): """ @@ -358,7 +358,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): if re.match(r'.*\/mdata-user-data$', name_f): found_new = True print(name_f) - self.assertEquals(permissions, '400') + self.assertEqual(permissions, '400') self.assertFalse(found_new) @@ -366,8 +366,8 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(MOCK_RETURNS['sdc:vendor-data'], - dsrc.metadata['vendor-data']) + self.assertEqual(MOCK_RETURNS['sdc:vendor-data'], + dsrc.metadata['vendor-data']) def test_default_vendor_data(self): my_returns = MOCK_RETURNS.copy() @@ -376,7 +376,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): dsrc = self._get_ds(mockdata=my_returns) ret = dsrc.get_data() self.assertTrue(ret) - self.assertNotEquals(def_op_script, dsrc.metadata['vendor-data']) + self.assertNotEqual(def_op_script, dsrc.metadata['vendor-data']) # we expect default vendor-data is a boothook self.assertTrue(dsrc.vendordata_raw.startswith("#cloud-boothook")) @@ -385,15 +385,15 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(MOCK_RETURNS['disable_iptables_flag'], - dsrc.metadata['iptables_disable']) + self.assertEqual(MOCK_RETURNS['disable_iptables_flag'], + dsrc.metadata['iptables_disable']) def test_motd_sys_info(self): dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(MOCK_RETURNS['enable_motd_sys_info'], - dsrc.metadata['motd_sys_info']) + self.assertEqual(MOCK_RETURNS['enable_motd_sys_info'], + dsrc.metadata['motd_sys_info']) def test_default_ephemeral(self): # Test to make sure that the builtin config has the ephemeral diff --git a/tests/unittests/test_distros/test_generic.py b/tests/unittests/test_distros/test_generic.py index 6ed1704c..96fa0811 100644 --- a/tests/unittests/test_distros/test_generic.py +++ b/tests/unittests/test_distros/test_generic.py @@ -87,13 +87,13 @@ class TestGenericDistro(helpers.FilesystemMockingTestCase): rules = 'ALL=(ALL:ALL) ALL' contents = self._write_load_sudoers('harlowja', rules) expected = ['harlowja ALL=(ALL:ALL) ALL'] - self.assertEquals(len(expected), self._count_in(expected, contents)) + self.assertEqual(len(expected), self._count_in(expected, contents)) not_expected = [ 'harlowja A', 'harlowja L', 'harlowja L', ] - self.assertEquals(0, self._count_in(not_expected, contents)) + self.assertEqual(0, self._count_in(not_expected, contents)) def test_sudoers_ensure_rules_list(self): rules = [ @@ -107,13 +107,13 @@ class TestGenericDistro(helpers.FilesystemMockingTestCase): 'harlowja B-ALL=(ALL:ALL) ALL', 'harlowja C-ALL=(ALL:ALL) ALL', ] - self.assertEquals(len(expected), self._count_in(expected, contents)) + self.assertEqual(len(expected), self._count_in(expected, contents)) not_expected = [ 'harlowja A', 'harlowja L', 'harlowja L', ] - self.assertEquals(0, self._count_in(not_expected, contents)) + self.assertEqual(0, self._count_in(not_expected, contents)) def test_sudoers_ensure_new(self): cls = distros.fetch("ubuntu") @@ -136,7 +136,7 @@ class TestGenericDistro(helpers.FilesystemMockingTestCase): self.assertIn("includedir /b", contents) self.assertTrue(os.path.isdir("/b")) self.assertIn("josh", contents) - self.assertEquals(2, contents.count("josh")) + self.assertEqual(2, contents.count("josh")) def test_arch_package_mirror_info_unknown(self): """for an unknown arch, we should get back that with arch 'default'.""" diff --git a/tests/unittests/test_distros/test_hostname.py b/tests/unittests/test_distros/test_hostname.py index 143e29a9..5f28a868 100644 --- a/tests/unittests/test_distros/test_hostname.py +++ b/tests/unittests/test_distros/test_hostname.py @@ -15,24 +15,24 @@ BASE_HOSTNAME = BASE_HOSTNAME.strip() class TestHostnameHelper(unittest.TestCase): def test_parse_same(self): hn = hostname.HostnameConf(BASE_HOSTNAME) - self.assertEquals(str(hn).strip(), BASE_HOSTNAME) - self.assertEquals(hn.hostname, 'blahblah') + self.assertEqual(str(hn).strip(), BASE_HOSTNAME) + self.assertEqual(hn.hostname, 'blahblah') def test_no_adjust_hostname(self): hn = hostname.HostnameConf(BASE_HOSTNAME) prev_name = hn.hostname hn.set_hostname("") - self.assertEquals(hn.hostname, prev_name) + self.assertEqual(hn.hostname, prev_name) def test_adjust_hostname(self): hn = hostname.HostnameConf(BASE_HOSTNAME) prev_name = hn.hostname - self.assertEquals(prev_name, 'blahblah') + self.assertEqual(prev_name, 'blahblah') hn.set_hostname("bbbbd") - self.assertEquals(hn.hostname, 'bbbbd') + self.assertEqual(hn.hostname, 'bbbbd') expected_out = ''' # My super-duper-hostname bbbbd ''' - self.assertEquals(str(hn).strip(), expected_out.strip()) + self.assertEqual(str(hn).strip(), expected_out.strip()) diff --git a/tests/unittests/test_distros/test_hosts.py b/tests/unittests/test_distros/test_hosts.py index fc701eaa..ab867c6f 100644 --- a/tests/unittests/test_distros/test_hosts.py +++ b/tests/unittests/test_distros/test_hosts.py @@ -17,25 +17,25 @@ BASE_ETC = BASE_ETC.strip() class TestHostsHelper(unittest.TestCase): def test_parse(self): eh = hosts.HostsConf(BASE_ETC) - self.assertEquals(eh.get_entry('127.0.0.1'), [['localhost']]) - self.assertEquals(eh.get_entry('192.168.1.10'), - [['foo.mydomain.org', 'foo'], - ['bar.mydomain.org', 'bar']]) + self.assertEqual(eh.get_entry('127.0.0.1'), [['localhost']]) + self.assertEqual(eh.get_entry('192.168.1.10'), + [['foo.mydomain.org', 'foo'], + ['bar.mydomain.org', 'bar']]) eh = str(eh) self.assertTrue(eh.startswith('# Example')) def test_add(self): eh = hosts.HostsConf(BASE_ETC) eh.add_entry('127.0.0.0', 'blah') - self.assertEquals(eh.get_entry('127.0.0.0'), [['blah']]) + self.assertEqual(eh.get_entry('127.0.0.0'), [['blah']]) eh.add_entry('127.0.0.3', 'blah', 'blah2', 'blah3') - self.assertEquals(eh.get_entry('127.0.0.3'), - [['blah', 'blah2', 'blah3']]) + self.assertEqual(eh.get_entry('127.0.0.3'), + [['blah', 'blah2', 'blah3']]) def test_del(self): eh = hosts.HostsConf(BASE_ETC) eh.add_entry('127.0.0.0', 'blah') - self.assertEquals(eh.get_entry('127.0.0.0'), [['blah']]) + self.assertEqual(eh.get_entry('127.0.0.0'), [['blah']]) eh.del_entries('127.0.0.0') - self.assertEquals(eh.get_entry('127.0.0.0'), []) + self.assertEqual(eh.get_entry('127.0.0.0'), []) diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py index 2c2a424d..9172e3aa 100644 --- a/tests/unittests/test_distros/test_netconfig.py +++ b/tests/unittests/test_distros/test_netconfig.py @@ -1,4 +1,5 @@ import os +from six import StringIO try: from unittest import mock @@ -9,16 +10,14 @@ try: except ImportError: from contextlib2 import ExitStack -from six import StringIO from ..helpers import TestCase from cloudinit import distros +from cloudinit.distros.parsers.sys_conf import SysConf from cloudinit import helpers from cloudinit import settings from cloudinit import util -from cloudinit.distros.parsers.sys_conf import SysConf - BASE_NET_CFG = ''' auto lo @@ -108,23 +107,23 @@ class TestNetCfgDistro(TestCase): ub_distro.apply_network(BASE_NET_CFG, False) - self.assertEquals(len(write_bufs), 1) + self.assertEqual(len(write_bufs), 1) eni_name = '/etc/network/interfaces.d/50-cloud-init.cfg' self.assertIn(eni_name, write_bufs) write_buf = write_bufs[eni_name] - self.assertEquals(str(write_buf).strip(), BASE_NET_CFG.strip()) - self.assertEquals(write_buf.mode, 0o644) + self.assertEqual(str(write_buf).strip(), BASE_NET_CFG.strip()) + self.assertEqual(write_buf.mode, 0o644) def assertCfgEquals(self, blob1, blob2): b1 = dict(SysConf(blob1.strip().splitlines())) b2 = dict(SysConf(blob2.strip().splitlines())) - self.assertEquals(b1, b2) + self.assertEqual(b1, b2) for (k, v) in b1.items(): self.assertIn(k, b2) for (k, v) in b2.items(): self.assertIn(k, b1) for (k, v) in b1.items(): - self.assertEquals(v, b2[k]) + self.assertEqual(v, b2[k]) def test_simple_write_rh(self): rh_distro = self._get_distro('rhel') @@ -148,7 +147,7 @@ class TestNetCfgDistro(TestCase): rh_distro.apply_network(BASE_NET_CFG, False) - self.assertEquals(len(write_bufs), 4) + self.assertEqual(len(write_bufs), 4) self.assertIn('/etc/sysconfig/network-scripts/ifcfg-lo', write_bufs) write_buf = write_bufs['/etc/sysconfig/network-scripts/ifcfg-lo'] @@ -157,7 +156,7 @@ DEVICE="lo" ONBOOT=yes ''' self.assertCfgEquals(expected_buf, str(write_buf)) - self.assertEquals(write_buf.mode, 0o644) + self.assertEqual(write_buf.mode, 0o644) self.assertIn('/etc/sysconfig/network-scripts/ifcfg-eth0', write_bufs) @@ -172,7 +171,7 @@ GATEWAY="192.168.1.254" BROADCAST="192.168.1.0" ''' self.assertCfgEquals(expected_buf, str(write_buf)) - self.assertEquals(write_buf.mode, 0o644) + self.assertEqual(write_buf.mode, 0o644) self.assertIn('/etc/sysconfig/network-scripts/ifcfg-eth1', write_bufs) @@ -183,7 +182,7 @@ BOOTPROTO="dhcp" ONBOOT=yes ''' self.assertCfgEquals(expected_buf, str(write_buf)) - self.assertEquals(write_buf.mode, 0o644) + self.assertEqual(write_buf.mode, 0o644) self.assertIn('/etc/sysconfig/network', write_bufs) write_buf = write_bufs['/etc/sysconfig/network'] @@ -192,7 +191,7 @@ ONBOOT=yes NETWORKING=yes ''' self.assertCfgEquals(expected_buf, str(write_buf)) - self.assertEquals(write_buf.mode, 0o644) + self.assertEqual(write_buf.mode, 0o644) def test_write_ipv6_rhel(self): rh_distro = self._get_distro('rhel') @@ -216,7 +215,7 @@ NETWORKING=yes rh_distro.apply_network(BASE_NET_CFG_IPV6, False) - self.assertEquals(len(write_bufs), 4) + self.assertEqual(len(write_bufs), 4) self.assertIn('/etc/sysconfig/network-scripts/ifcfg-lo', write_bufs) write_buf = write_bufs['/etc/sysconfig/network-scripts/ifcfg-lo'] @@ -225,7 +224,7 @@ DEVICE="lo" ONBOOT=yes ''' self.assertCfgEquals(expected_buf, str(write_buf)) - self.assertEquals(write_buf.mode, 0o644) + self.assertEqual(write_buf.mode, 0o644) self.assertIn('/etc/sysconfig/network-scripts/ifcfg-eth0', write_bufs) @@ -243,7 +242,7 @@ IPV6ADDR="2607:f0d0:1002:0011::2" IPV6_DEFAULTGW="2607:f0d0:1002:0011::1" ''' self.assertCfgEquals(expected_buf, str(write_buf)) - self.assertEquals(write_buf.mode, 0o644) + self.assertEqual(write_buf.mode, 0o644) self.assertIn('/etc/sysconfig/network-scripts/ifcfg-eth1', write_bufs) write_buf = write_bufs['/etc/sysconfig/network-scripts/ifcfg-eth1'] @@ -260,7 +259,7 @@ IPV6ADDR="2607:f0d0:1002:0011::3" IPV6_DEFAULTGW="2607:f0d0:1002:0011::1" ''' self.assertCfgEquals(expected_buf, str(write_buf)) - self.assertEquals(write_buf.mode, 0o644) + self.assertEqual(write_buf.mode, 0o644) self.assertIn('/etc/sysconfig/network', write_bufs) write_buf = write_bufs['/etc/sysconfig/network'] @@ -271,7 +270,7 @@ NETWORKING_IPV6=yes IPV6_AUTOCONF=no ''' self.assertCfgEquals(expected_buf, str(write_buf)) - self.assertEquals(write_buf.mode, 0o644) + self.assertEqual(write_buf.mode, 0o644) def test_simple_write_freebsd(self): fbsd_distro = self._get_distro('freebsd') @@ -319,4 +318,4 @@ ifconfig_vtnet1="DHCP" defaultrouter="192.168.1.254" ''' self.assertCfgEquals(expected_buf, str(write_buf)) - self.assertEquals(write_buf.mode, 0o644) + self.assertEqual(write_buf.mode, 0o644) diff --git a/tests/unittests/test_distros/test_resolv.py b/tests/unittests/test_distros/test_resolv.py index 9edeb6e7..9402b5ea 100644 --- a/tests/unittests/test_distros/test_resolv.py +++ b/tests/unittests/test_distros/test_resolv.py @@ -1,9 +1,10 @@ from cloudinit.distros.parsers import resolv_conf from cloudinit.distros import rhel_util +from ..helpers import TestCase + import re import tempfile -from ..helpers import TestCase BASE_RESOLVE = ''' @@ -19,7 +20,7 @@ class TestResolvHelper(TestCase): def test_parse_same(self): rp = resolv_conf.ResolvConf(BASE_RESOLVE) rp_r = str(rp).strip() - self.assertEquals(BASE_RESOLVE, rp_r) + self.assertEqual(BASE_RESOLVE, rp_r) def test_write_works(self): with tempfile.NamedTemporaryFile() as fh: @@ -27,10 +28,10 @@ class TestResolvHelper(TestCase): def test_local_domain(self): rp = resolv_conf.ResolvConf(BASE_RESOLVE) - self.assertEquals(None, rp.local_domain) + self.assertEqual(None, rp.local_domain) rp.local_domain = "bob" - self.assertEquals('bob', rp.local_domain) + self.assertEqual('bob', rp.local_domain) self.assertIn('domain bob', str(rp)) def test_nameservers(self): @@ -41,7 +42,7 @@ class TestResolvHelper(TestCase): self.assertIn('10.2', rp.nameservers) self.assertIn('nameserver 10.2', str(rp)) self.assertNotIn('10.3', rp.nameservers) - self.assertEquals(len(rp.nameservers), 3) + self.assertEqual(len(rp.nameservers), 3) rp.add_nameserver('10.2') self.assertRaises(ValueError, rp.add_nameserver, '10.3') self.assertNotIn('10.3', rp.nameservers) @@ -55,12 +56,12 @@ class TestResolvHelper(TestCase): self.assertTrue(re.search(r'search(.*)bbb.y.com(.*)', str(rp))) self.assertIn('bbb.y.com', rp.search_domains) rp.add_search_domain('bbb.y.com') - self.assertEquals(len(rp.search_domains), 3) + self.assertEqual(len(rp.search_domains), 3) rp.add_search_domain('bbb2.y.com') - self.assertEquals(len(rp.search_domains), 4) + self.assertEqual(len(rp.search_domains), 4) rp.add_search_domain('bbb3.y.com') - self.assertEquals(len(rp.search_domains), 5) + self.assertEqual(len(rp.search_domains), 5) rp.add_search_domain('bbb4.y.com') - self.assertEquals(len(rp.search_domains), 6) + self.assertEqual(len(rp.search_domains), 6) self.assertRaises(ValueError, rp.add_search_domain, 'bbb5.y.com') - self.assertEquals(len(rp.search_domains), 6) + self.assertEqual(len(rp.search_domains), 6) diff --git a/tests/unittests/test_distros/test_sysconfig.py b/tests/unittests/test_distros/test_sysconfig.py index 03d89a10..8cb55522 100644 --- a/tests/unittests/test_distros/test_sysconfig.py +++ b/tests/unittests/test_distros/test_sysconfig.py @@ -1,6 +1,7 @@ import re from cloudinit.distros.parsers.sys_conf import SysConf + from ..helpers import TestCase @@ -27,34 +28,34 @@ IPV6TO4_ROUTING='eth0-:0004::1/64 eth1-:0005::1/64' ETHTOOL_OPTS="-K ${DEVICE} tso on; -G ${DEVICE} rx 256 tx 256" USEMD5=no''' conf = SysConf(contents.splitlines()) - self.assertEquals(conf['HOSTNAME'], 'blahblah') - self.assertEquals(conf['SHORTDATE'], '$(date +%y:%m:%d:%H:%M)') + self.assertEqual(conf['HOSTNAME'], 'blahblah') + self.assertEqual(conf['SHORTDATE'], '$(date +%y:%m:%d:%H:%M)') # Should be unquoted - self.assertEquals(conf['ETHTOOL_OPTS'], ('-K ${DEVICE} tso on; ' - '-G ${DEVICE} rx 256 tx 256')) - self.assertEquals(contents, str(conf)) + self.assertEqual(conf['ETHTOOL_OPTS'], ('-K ${DEVICE} tso on; ' + '-G ${DEVICE} rx 256 tx 256')) + self.assertEqual(contents, str(conf)) def test_parse_shell_vars(self): contents = 'USESMBAUTH=$XYZ' conf = SysConf(contents.splitlines()) - self.assertEquals(contents, str(conf)) + self.assertEqual(contents, str(conf)) conf = SysConf('') conf['B'] = '${ZZ}d apples' # Should be quoted - self.assertEquals('B="${ZZ}d apples"', str(conf)) + self.assertEqual('B="${ZZ}d apples"', str(conf)) conf = SysConf('') conf['B'] = '$? d apples' - self.assertEquals('B="$? d apples"', str(conf)) + self.assertEqual('B="$? d apples"', str(conf)) contents = 'IPMI_WATCHDOG_OPTIONS="timeout=60"' conf = SysConf(contents.splitlines()) - self.assertEquals('IPMI_WATCHDOG_OPTIONS=timeout=60', str(conf)) + self.assertEqual('IPMI_WATCHDOG_OPTIONS=timeout=60', str(conf)) def test_parse_adjust(self): contents = 'IPV6TO4_ROUTING="eth0-:0004::1/64 eth1-:0005::1/64"' conf = SysConf(contents.splitlines()) # Should be unquoted - self.assertEquals('eth0-:0004::1/64 eth1-:0005::1/64', - conf['IPV6TO4_ROUTING']) + self.assertEqual('eth0-:0004::1/64 eth1-:0005::1/64', + conf['IPV6TO4_ROUTING']) conf['IPV6TO4_ROUTING'] = "blah \tblah" contents2 = str(conf).strip() # Should be requoted due to whitespace @@ -65,12 +66,12 @@ USEMD5=no''' conf = SysConf(''.splitlines()) conf['B'] = ' $(time)' contents = str(conf) - self.assertEquals('B= $(time)', contents) + self.assertEqual('B= $(time)', contents) def test_parse_empty(self): contents = '' conf = SysConf(contents.splitlines()) - self.assertEquals('', str(conf).strip()) + self.assertEqual('', str(conf).strip()) def test_parse_add_new(self): contents = 'BLAH=b' diff --git a/tests/unittests/test_distros/test_user_data_normalize.py b/tests/unittests/test_distros/test_user_data_normalize.py index 4525f487..a887a930 100644 --- a/tests/unittests/test_distros/test_user_data_normalize.py +++ b/tests/unittests/test_distros/test_user_data_normalize.py @@ -33,20 +33,19 @@ class TestUGNormalize(TestCase): def test_group_dict(self): distro = self._make_distro('ubuntu') - g = {'groups': [ - {'ubuntu': ['foo', 'bar'], - 'bob': 'users'}, - 'cloud-users', - {'bob': 'users2'} - ]} + g = {'groups': + [{'ubuntu': ['foo', 'bar'], + 'bob': 'users'}, + 'cloud-users', + {'bob': 'users2'}]} (_users, groups) = self._norm(g, distro) self.assertIn('ubuntu', groups) ub_members = groups['ubuntu'] - self.assertEquals(sorted(['foo', 'bar']), sorted(ub_members)) + self.assertEqual(sorted(['foo', 'bar']), sorted(ub_members)) self.assertIn('bob', groups) b_members = groups['bob'] - self.assertEquals(sorted(['users', 'users2']), - sorted(b_members)) + self.assertEqual(sorted(['users', 'users2']), + sorted(b_members)) def test_basic_groups(self): distro = self._make_distro('ubuntu') @@ -55,7 +54,7 @@ class TestUGNormalize(TestCase): } (users, groups) = self._norm(ug_cfg, distro) self.assertIn('bob', groups) - self.assertEquals({}, users) + self.assertEqual({}, users) def test_csv_groups(self): distro = self._make_distro('ubuntu') @@ -66,7 +65,7 @@ class TestUGNormalize(TestCase): self.assertIn('bob', groups) self.assertIn('joe', groups) self.assertIn('steve', groups) - self.assertEquals({}, users) + self.assertEqual({}, users) def test_more_groups(self): distro = self._make_distro('ubuntu') @@ -77,7 +76,7 @@ class TestUGNormalize(TestCase): self.assertIn('bob', groups) self.assertIn('joe', groups) self.assertIn('steve', groups) - self.assertEquals({}, users) + self.assertEqual({}, users) def test_member_groups(self): distro = self._make_distro('ubuntu') @@ -90,11 +89,11 @@ class TestUGNormalize(TestCase): } (users, groups) = self._norm(ug_cfg, distro) self.assertIn('bob', groups) - self.assertEquals(['s'], groups['bob']) - self.assertEquals([], groups['joe']) + self.assertEqual(['s'], groups['bob']) + self.assertEqual([], groups['joe']) self.assertIn('joe', groups) self.assertIn('steve', groups) - self.assertEquals({}, users) + self.assertEqual({}, users) def test_users_simple_dict(self): distro = self._make_distro('ubuntu', bcfg) @@ -128,14 +127,14 @@ class TestUGNormalize(TestCase): } } (users, _groups) = self._norm(ug_cfg, distro) - self.assertEquals({}, users) + self.assertEqual({}, users) ug_cfg = { 'users': { 'default': 'no', } } (users, _groups) = self._norm(ug_cfg, distro) - self.assertEquals({}, users) + self.assertEqual({}, users) def test_users_simple_csv(self): distro = self._make_distro('ubuntu') @@ -145,8 +144,8 @@ class TestUGNormalize(TestCase): (users, _groups) = self._norm(ug_cfg, distro) self.assertIn('joe', users) self.assertIn('bob', users) - self.assertEquals({'default': False}, users['joe']) - self.assertEquals({'default': False}, users['bob']) + self.assertEqual({'default': False}, users['joe']) + self.assertEqual({'default': False}, users['bob']) def test_users_simple(self): distro = self._make_distro('ubuntu') @@ -159,8 +158,8 @@ class TestUGNormalize(TestCase): (users, _groups) = self._norm(ug_cfg, distro) self.assertIn('joe', users) self.assertIn('bob', users) - self.assertEquals({'default': False}, users['joe']) - self.assertEquals({'default': False}, users['bob']) + self.assertEqual({'default': False}, users['joe']) + self.assertEqual({'default': False}, users['bob']) def test_users_old_user(self): distro = self._make_distro('ubuntu', bcfg) @@ -211,8 +210,8 @@ class TestUGNormalize(TestCase): self.assertIn('zetta', users) ug_cfg = {} (users, groups) = self._norm(ug_cfg, distro) - self.assertEquals({}, users) - self.assertEquals({}, groups) + self.assertEqual({}, users) + self.assertEqual({}, groups) def test_users_dict_default_additional(self): distro = self._make_distro('ubuntu', bcfg) @@ -223,12 +222,10 @@ class TestUGNormalize(TestCase): } (users, _groups) = self._norm(ug_cfg, distro) self.assertIn('bob', users) - self.assertEquals(",".join(distro.get_default_user()['groups']), - users['bob']['groups']) - self.assertEquals(True, - users['bob']['blah']) - self.assertEquals(True, - users['bob']['default']) + self.assertEqual(",".join(distro.get_default_user()['groups']), + users['bob']['groups']) + self.assertEqual(True, users['bob']['blah']) + self.assertEqual(True, users['bob']['default']) def test_users_dict_extract(self): distro = self._make_distro('ubuntu', bcfg) @@ -240,7 +237,7 @@ class TestUGNormalize(TestCase): (users, _groups) = self._norm(ug_cfg, distro) self.assertIn('bob', users) (name, config) = distros.extract_default(users) - self.assertEquals(name, 'bob') + self.assertEqual(name, 'bob') expected_config = {} def_config = None try: @@ -255,7 +252,7 @@ class TestUGNormalize(TestCase): expected_config.pop('name', None) expected_config.pop('groups', None) config.pop('groups', None) - self.assertEquals(config, expected_config) + self.assertEqual(config, expected_config) def test_users_dict_default(self): distro = self._make_distro('ubuntu', bcfg) @@ -266,10 +263,9 @@ class TestUGNormalize(TestCase): } (users, _groups) = self._norm(ug_cfg, distro) self.assertIn('bob', users) - self.assertEquals(",".join(distro.get_default_user()['groups']), - users['bob']['groups']) - self.assertEquals(True, - users['bob']['default']) + self.assertEqual(",".join(distro.get_default_user()['groups']), + users['bob']['groups']) + self.assertEqual(True, users['bob']['default']) def test_users_dict_trans(self): distro = self._make_distro('ubuntu') @@ -283,8 +279,8 @@ class TestUGNormalize(TestCase): (users, _groups) = self._norm(ug_cfg, distro) self.assertIn('joe', users) self.assertIn('bob', users) - self.assertEquals({'tr_me': True, 'default': False}, users['joe']) - self.assertEquals({'default': False}, users['bob']) + self.assertEqual({'tr_me': True, 'default': False}, users['joe']) + self.assertEqual({'default': False}, users['bob']) def test_users_dict(self): distro = self._make_distro('ubuntu') @@ -297,5 +293,5 @@ class TestUGNormalize(TestCase): (users, _groups) = self._norm(ug_cfg, distro) self.assertIn('joe', users) self.assertIn('bob', users) - self.assertEquals({'default': False}, users['joe']) - self.assertEquals({'default': False}, users['bob']) + self.assertEqual({'default': False}, users['joe']) + self.assertEqual({'default': False}, users['bob']) diff --git a/tests/unittests/test_ec2_util.py b/tests/unittests/test_ec2_util.py index 99fc54be..d6cf17fa 100644 --- a/tests/unittests/test_ec2_util.py +++ b/tests/unittests/test_ec2_util.py @@ -16,7 +16,7 @@ class TestEc2Util(helpers.HttprettyTestCase): body='stuff', status=200) userdata = eu.get_instance_userdata(self.VERSION) - self.assertEquals('stuff', userdata.decode('utf-8')) + self.assertEqual('stuff', userdata.decode('utf-8')) @hp.activate def test_userdata_fetch_fail_not_found(self): @@ -24,7 +24,7 @@ class TestEc2Util(helpers.HttprettyTestCase): 'http://169.254.169.254/%s/user-data' % (self.VERSION), status=404) userdata = eu.get_instance_userdata(self.VERSION, retries=0) - self.assertEquals('', userdata) + self.assertEqual('', userdata) @hp.activate def test_userdata_fetch_fail_server_dead(self): @@ -32,7 +32,7 @@ class TestEc2Util(helpers.HttprettyTestCase): 'http://169.254.169.254/%s/user-data' % (self.VERSION), status=500) userdata = eu.get_instance_userdata(self.VERSION, retries=0) - self.assertEquals('', userdata) + self.assertEqual('', userdata) @hp.activate def test_userdata_fetch_fail_server_not_found(self): @@ -40,7 +40,7 @@ class TestEc2Util(helpers.HttprettyTestCase): 'http://169.254.169.254/%s/user-data' % (self.VERSION), status=404) userdata = eu.get_instance_userdata(self.VERSION) - self.assertEquals('', userdata) + self.assertEqual('', userdata) @hp.activate def test_metadata_fetch_no_keys(self): @@ -56,9 +56,9 @@ class TestEc2Util(helpers.HttprettyTestCase): hp.register_uri(hp.GET, uh.combine_url(base_url, 'ami-launch-index'), status=200, body='1') md = eu.get_instance_metadata(self.VERSION, retries=0) - self.assertEquals(md['hostname'], 'ec2.fake.host.name.com') - self.assertEquals(md['instance-id'], '123') - self.assertEquals(md['ami-launch-index'], '1') + self.assertEqual(md['hostname'], 'ec2.fake.host.name.com') + self.assertEqual(md['instance-id'], '123') + self.assertEqual(md['ami-launch-index'], '1') @hp.activate def test_metadata_fetch_key(self): @@ -77,9 +77,9 @@ class TestEc2Util(helpers.HttprettyTestCase): uh.combine_url(base_url, 'public-keys/0/openssh-key'), status=200, body='ssh-rsa AAAA.....wZEf my-public-key') md = eu.get_instance_metadata(self.VERSION, retries=0, timeout=0.1) - self.assertEquals(md['hostname'], 'ec2.fake.host.name.com') - self.assertEquals(md['instance-id'], '123') - self.assertEquals(1, len(md['public-keys'])) + self.assertEqual(md['hostname'], 'ec2.fake.host.name.com') + self.assertEqual(md['instance-id'], '123') + self.assertEqual(1, len(md['public-keys'])) @hp.activate def test_metadata_fetch_with_2_keys(self): @@ -102,9 +102,9 @@ class TestEc2Util(helpers.HttprettyTestCase): uh.combine_url(base_url, 'public-keys/1/openssh-key'), status=200, body='ssh-rsa AAAA.....wZEf my-other-key') md = eu.get_instance_metadata(self.VERSION, retries=0, timeout=0.1) - self.assertEquals(md['hostname'], 'ec2.fake.host.name.com') - self.assertEquals(md['instance-id'], '123') - self.assertEquals(2, len(md['public-keys'])) + self.assertEqual(md['hostname'], 'ec2.fake.host.name.com') + self.assertEqual(md['instance-id'], '123') + self.assertEqual(2, len(md['public-keys'])) @hp.activate def test_metadata_fetch_bdm(self): @@ -131,9 +131,9 @@ class TestEc2Util(helpers.HttprettyTestCase): status=200, body="sdc") md = eu.get_instance_metadata(self.VERSION, retries=0, timeout=0.1) - self.assertEquals(md['hostname'], 'ec2.fake.host.name.com') - self.assertEquals(md['instance-id'], '123') + self.assertEqual(md['hostname'], 'ec2.fake.host.name.com') + self.assertEqual(md['instance-id'], '123') bdm = md['block-device-mapping'] - self.assertEquals(2, len(bdm)) - self.assertEquals(bdm['ami'], 'sdb') - self.assertEquals(bdm['ephemeral0'], 'sdc') + self.assertEqual(2, len(bdm)) + self.assertEqual(bdm['ami'], 'sdb') + self.assertEqual(bdm['ephemeral0'], 'sdc') diff --git a/tests/unittests/test_filters/test_launch_index.py b/tests/unittests/test_filters/test_launch_index.py index 95d24b9b..395713e6 100644 --- a/tests/unittests/test_filters/test_launch_index.py +++ b/tests/unittests/test_filters/test_launch_index.py @@ -25,7 +25,7 @@ class TestLaunchFilter(helpers.ResourceUsingTestCase): for (index, count) in expected_counts.items(): index = util.safe_int(index) filtered_message = launch_index.Filter(index).apply(message) - self.assertEquals(count_messages(filtered_message), count) + self.assertEqual(count_messages(filtered_message), count) # Ensure original message still ok/not modified self.assertTrue(self.equivalentMessage(message, orig_message)) diff --git a/tests/unittests/test_handler/test_handler_apt_configure.py b/tests/unittests/test_handler/test_handler_apt_configure.py index 1ed185ca..d1dca2c4 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure.py +++ b/tests/unittests/test_handler/test_handler_apt_configure.py @@ -1,6 +1,6 @@ +from cloudinit.config import cc_apt_configure from cloudinit import util -from cloudinit.config import cc_apt_configure from ..helpers import TestCase import os diff --git a/tests/unittests/test_handler/test_handler_ca_certs.py b/tests/unittests/test_handler/test_handler_ca_certs.py index a6b9c0fd..5e771731 100644 --- a/tests/unittests/test_handler/test_handler_ca_certs.py +++ b/tests/unittests/test_handler/test_handler_ca_certs.py @@ -1,8 +1,8 @@ from cloudinit import cloud +from cloudinit.config import cc_ca_certs from cloudinit import helpers from cloudinit import util -from cloudinit.config import cc_ca_certs from ..helpers import TestCase import logging @@ -176,8 +176,7 @@ class TestAddCaCerts(TestCase): mock_write.assert_has_calls([ mock.call("/usr/share/ca-certificates/cloud-init-ca-certs.crt", cert, mode=0o644), - mock.call("/etc/ca-certificates.conf", expected, omode="wb"), - ]) + mock.call("/etc/ca-certificates.conf", expected, omode="wb")]) mock_load.assert_called_once_with("/etc/ca-certificates.conf") def test_single_cert_no_trailing_cr(self): @@ -202,8 +201,7 @@ class TestAddCaCerts(TestCase): mock.call("/etc/ca-certificates.conf", "%s\n%s\n" % (ca_certs_content, "cloud-init-ca-certs.crt"), - omode="wb"), - ]) + omode="wb")]) mock_load.assert_called_once_with("/etc/ca-certificates.conf") @@ -228,8 +226,7 @@ class TestAddCaCerts(TestCase): mock.call("/etc/ca-certificates.conf", "%s\n%s\n" % (ca_certs_content, "cloud-init-ca-certs.crt"), - omode='wb'), - ]) + omode='wb')]) mock_load.assert_called_once_with("/etc/ca-certificates.conf") @@ -264,8 +261,7 @@ class TestRemoveDefaultCaCerts(TestCase): mock_delete.assert_has_calls([ mock.call("/usr/share/ca-certificates/"), - mock.call("/etc/ssl/certs/"), - ]) + mock.call("/etc/ssl/certs/")]) mock_write.assert_called_once_with( "/etc/ca-certificates.conf", "", mode=0o644) diff --git a/tests/unittests/test_handler/test_handler_chef.py b/tests/unittests/test_handler/test_handler_chef.py index 7763f23b..7a1bc317 100644 --- a/tests/unittests/test_handler/test_handler_chef.py +++ b/tests/unittests/test_handler/test_handler_chef.py @@ -1,21 +1,19 @@ import json +import logging import os - -from cloudinit.config import cc_chef +import shutil +import six +import tempfile from cloudinit import cloud +from cloudinit.config import cc_chef from cloudinit import distros from cloudinit import helpers -from cloudinit import util from cloudinit.sources import DataSourceNone +from cloudinit import util from .. import helpers as t_help -import six -import logging -import shutil -import tempfile - LOG = logging.getLogger(__name__) CLIENT_TEMPL = os.path.sep.join(["templates", "chef_client.rb.tmpl"]) diff --git a/tests/unittests/test_handler/test_handler_growpart.py b/tests/unittests/test_handler/test_handler_growpart.py index bef0d80d..e653488a 100644 --- a/tests/unittests/test_handler/test_handler_growpart.py +++ b/tests/unittests/test_handler/test_handler_growpart.py @@ -1,7 +1,7 @@ from cloudinit import cloud +from cloudinit.config import cc_growpart from cloudinit import util -from cloudinit.config import cc_growpart from ..helpers import TestCase import errno diff --git a/tests/unittests/test_handler/test_handler_locale.py b/tests/unittests/test_handler/test_handler_locale.py index de85eff6..c91908f4 100644 --- a/tests/unittests/test_handler/test_handler_locale.py +++ b/tests/unittests/test_handler/test_handler_locale.py @@ -64,4 +64,4 @@ class TestLocale(t_help.FilesystemMockingTestCase): contents = util.load_file('/etc/sysconfig/language', decode=False) n_cfg = ConfigObj(BytesIO(contents)) - self.assertEquals({'RC_LANG': cfg['locale']}, dict(n_cfg)) + self.assertEqual({'RC_LANG': cfg['locale']}, dict(n_cfg)) diff --git a/tests/unittests/test_handler/test_handler_lxd.py b/tests/unittests/test_handler/test_handler_lxd.py index 5f61ba6a..6f90defb 100644 --- a/tests/unittests/test_handler/test_handler_lxd.py +++ b/tests/unittests/test_handler/test_handler_lxd.py @@ -1,6 +1,6 @@ from cloudinit.config import cc_lxd -from cloudinit import (distros, helpers, cloud) from cloudinit.sources import DataSourceNoCloud +from cloudinit import (distros, helpers, cloud) from .. import helpers as t_help import logging @@ -42,11 +42,11 @@ class TestLxd(t_help.TestCase): cc_lxd.handle('cc_lxd', self.lxd_cfg, cc, LOG, []) self.assertTrue(mock_util.which.called) init_call = mock_util.subp.call_args_list[0][0][0] - self.assertEquals(init_call, - ['lxd', 'init', '--auto', - '--network-address=0.0.0.0', - '--storage-backend=zfs', - '--storage-pool=poolname']) + self.assertEqual(init_call, + ['lxd', 'init', '--auto', + '--network-address=0.0.0.0', + '--storage-backend=zfs', + '--storage-pool=poolname']) @mock.patch("cloudinit.config.cc_lxd.util") def test_lxd_install(self, mock_util): @@ -56,7 +56,7 @@ class TestLxd(t_help.TestCase): cc_lxd.handle('cc_lxd', self.lxd_cfg, cc, LOG, []) self.assertTrue(cc.distro.install_packages.called) install_pkg = cc.distro.install_packages.call_args_list[0][0][0] - self.assertEquals(sorted(install_pkg), ['lxd', 'zfs']) + self.assertEqual(sorted(install_pkg), ['lxd', 'zfs']) @mock.patch("cloudinit.config.cc_lxd.util") def test_no_init_does_nothing(self, mock_util): @@ -87,7 +87,7 @@ class TestLxd(t_help.TestCase): "ipv6_netmask": "64", "ipv6_nat": "true", "domain": "lxd"} - self.assertEquals( + self.assertEqual( cc_lxd.bridge_to_debconf(data), {"lxd/setup-bridge": "true", "lxd/bridge-name": "testbr0", @@ -109,7 +109,7 @@ class TestLxd(t_help.TestCase): "ipv6_address": "fd98:9e0:3744::1", "ipv6_netmask": "64", "ipv6_nat": "true"} - self.assertEquals( + self.assertEqual( cc_lxd.bridge_to_debconf(data), {"lxd/setup-bridge": "true", "lxd/bridge-ipv6": "true", @@ -120,7 +120,7 @@ class TestLxd(t_help.TestCase): def test_lxd_debconf_existing(self): data = {"mode": "existing", "name": "testbr0"} - self.assertEquals( + self.assertEqual( cc_lxd.bridge_to_debconf(data), {"lxd/setup-bridge": "false", "lxd/use-existing-bridge": "true", @@ -128,7 +128,7 @@ class TestLxd(t_help.TestCase): def test_lxd_debconf_none(self): data = {"mode": "none"} - self.assertEquals( + self.assertEqual( cc_lxd.bridge_to_debconf(data), {"lxd/setup-bridge": "false", "lxd/bridge-name": ""}) diff --git a/tests/unittests/test_handler/test_handler_power_state.py b/tests/unittests/test_handler/test_handler_power_state.py index 04ce5687..feff319d 100644 --- a/tests/unittests/test_handler/test_handler_power_state.py +++ b/tests/unittests/test_handler/test_handler_power_state.py @@ -119,7 +119,7 @@ def check_lps_ret(psc_return, mode=None): try: float(timeout) - except: + except Exception: errs.append("timeout failed convert to float") if len(errs): diff --git a/tests/unittests/test_handler/test_handler_rsyslog.py b/tests/unittests/test_handler/test_handler_rsyslog.py index b932165c..38636063 100644 --- a/tests/unittests/test_handler/test_handler_rsyslog.py +++ b/tests/unittests/test_handler/test_handler_rsyslog.py @@ -31,7 +31,7 @@ class TestLoadConfig(t_help.TestCase): 'config_dir': "mydir", 'config_filename': 'myfilename', 'service_reload_command': 'auto'} - ) + ) self.assertEqual(found, self.basecfg) diff --git a/tests/unittests/test_handler/test_handler_seed_random.py b/tests/unittests/test_handler/test_handler_seed_random.py index 98bc9b81..a0390da9 100644 --- a/tests/unittests/test_handler/test_handler_seed_random.py +++ b/tests/unittests/test_handler/test_handler_seed_random.py @@ -92,7 +92,7 @@ class TestRandomSeed(t_help.TestCase): } cc_seed_random.handle('test', cfg, self._get_cloud('ubuntu'), LOG, []) contents = util.load_file(self._seed_file) - self.assertEquals("tiny-tim-was-here", contents) + self.assertEqual("tiny-tim-was-here", contents) def test_append_random_unknown_encoding(self): data = self._compress(b"tiny-toe") @@ -117,7 +117,7 @@ class TestRandomSeed(t_help.TestCase): } cc_seed_random.handle('test', cfg, self._get_cloud('ubuntu'), LOG, []) contents = util.load_file(self._seed_file) - self.assertEquals("tiny-toe", contents) + self.assertEqual("tiny-toe", contents) def test_append_random_gz(self): data = self._compress(b"big-toe") @@ -130,7 +130,7 @@ class TestRandomSeed(t_help.TestCase): } cc_seed_random.handle('test', cfg, self._get_cloud('ubuntu'), LOG, []) contents = util.load_file(self._seed_file) - self.assertEquals("big-toe", contents) + self.assertEqual("big-toe", contents) def test_append_random_base64(self): data = util.b64e('bubbles') @@ -143,7 +143,7 @@ class TestRandomSeed(t_help.TestCase): } cc_seed_random.handle('test', cfg, self._get_cloud('ubuntu'), LOG, []) contents = util.load_file(self._seed_file) - self.assertEquals("bubbles", contents) + self.assertEqual("bubbles", contents) def test_append_random_b64(self): data = util.b64e('kit-kat') @@ -156,7 +156,7 @@ class TestRandomSeed(t_help.TestCase): } cc_seed_random.handle('test', cfg, self._get_cloud('ubuntu'), LOG, []) contents = util.load_file(self._seed_file) - self.assertEquals("kit-kat", contents) + self.assertEqual("kit-kat", contents) def test_append_random_metadata(self): cfg = { @@ -168,7 +168,7 @@ class TestRandomSeed(t_help.TestCase): c = self._get_cloud('ubuntu', {'random_seed': '-so-was-josh'}) cc_seed_random.handle('test', cfg, c, LOG, []) contents = util.load_file(self._seed_file) - self.assertEquals('tiny-tim-was-here-so-was-josh', contents) + self.assertEqual('tiny-tim-was-here-so-was-josh', contents) def test_seed_command_provided_and_available(self): c = self._get_cloud('ubuntu', {}) diff --git a/tests/unittests/test_handler/test_handler_set_hostname.py b/tests/unittests/test_handler/test_handler_set_hostname.py index d358b069..7effa124 100644 --- a/tests/unittests/test_handler/test_handler_set_hostname.py +++ b/tests/unittests/test_handler/test_handler_set_hostname.py @@ -7,13 +7,11 @@ from cloudinit import util from .. import helpers as t_help -import shutil -import tempfile +from configobj import ConfigObj import logging - +import shutil from six import BytesIO - -from configobj import ConfigObj +import tempfile LOG = logging.getLogger(__name__) @@ -43,8 +41,8 @@ class TestHostname(t_help.FilesystemMockingTestCase): if not distro.uses_systemd(): contents = util.load_file("/etc/sysconfig/network", decode=False) n_cfg = ConfigObj(BytesIO(contents)) - self.assertEquals({'HOSTNAME': 'blah.blah.blah.yahoo.com'}, - dict(n_cfg)) + self.assertEqual({'HOSTNAME': 'blah.blah.blah.yahoo.com'}, + dict(n_cfg)) def test_write_hostname_debian(self): cfg = { @@ -58,7 +56,7 @@ class TestHostname(t_help.FilesystemMockingTestCase): cc_set_hostname.handle('cc_set_hostname', cfg, cc, LOG, []) contents = util.load_file("/etc/hostname") - self.assertEquals('blah', contents.strip()) + self.assertEqual('blah', contents.strip()) def test_write_hostname_sles(self): cfg = { @@ -71,4 +69,4 @@ class TestHostname(t_help.FilesystemMockingTestCase): self.patchUtils(self.tmp) cc_set_hostname.handle('cc_set_hostname', cfg, cc, LOG, []) contents = util.load_file("/etc/HOSTNAME") - self.assertEquals('blah', contents.strip()) + self.assertEqual('blah', contents.strip()) diff --git a/tests/unittests/test_handler/test_handler_snappy.py b/tests/unittests/test_handler/test_handler_snappy.py index 8aeff53c..57dce1bc 100644 --- a/tests/unittests/test_handler/test_handler_snappy.py +++ b/tests/unittests/test_handler/test_handler_snappy.py @@ -1,6 +1,7 @@ from cloudinit.config.cc_snappy import ( makeop, get_package_ops, render_snap_op) from cloudinit import util + from .. import helpers as t_help import os diff --git a/tests/unittests/test_handler/test_handler_timezone.py b/tests/unittests/test_handler/test_handler_timezone.py index e3df8759..b7e6b03d 100644 --- a/tests/unittests/test_handler/test_handler_timezone.py +++ b/tests/unittests/test_handler/test_handler_timezone.py @@ -28,12 +28,10 @@ from cloudinit.sources import DataSourceNoCloud from .. import helpers as t_help from configobj import ConfigObj - -from six import BytesIO - +import logging import shutil +from six import BytesIO import tempfile -import logging LOG = logging.getLogger(__name__) @@ -72,7 +70,7 @@ class TestTimezone(t_help.FilesystemMockingTestCase): contents = util.load_file('/etc/sysconfig/clock', decode=False) n_cfg = ConfigObj(BytesIO(contents)) - self.assertEquals({'TIMEZONE': cfg['timezone']}, dict(n_cfg)) + self.assertEqual({'TIMEZONE': cfg['timezone']}, dict(n_cfg)) contents = util.load_file('/etc/localtime') - self.assertEquals(dummy_contents, contents.strip()) + self.assertEqual(dummy_contents, contents.strip()) diff --git a/tests/unittests/test_handler/test_handler_write_files.py b/tests/unittests/test_handler/test_handler_write_files.py index f1c7f7b4..466e45f8 100644 --- a/tests/unittests/test_handler/test_handler_write_files.py +++ b/tests/unittests/test_handler/test_handler_write_files.py @@ -1,6 +1,6 @@ -from cloudinit import util -from cloudinit import log as logging from cloudinit.config.cc_write_files import write_files +from cloudinit import log as logging +from cloudinit import util from ..helpers import FilesystemMockingTestCase diff --git a/tests/unittests/test_handler/test_handler_yum_add_repo.py b/tests/unittests/test_handler/test_handler_yum_add_repo.py index 3a8aa7c1..28b060f8 100644 --- a/tests/unittests/test_handler/test_handler_yum_add_repo.py +++ b/tests/unittests/test_handler/test_handler_yum_add_repo.py @@ -1,16 +1,13 @@ -from cloudinit import util - from cloudinit.config import cc_yum_add_repo +from cloudinit import util from .. import helpers -import shutil -import tempfile +import configobj import logging - +import shutil from six import BytesIO - -import configobj +import tempfile LOG = logging.getLogger(__name__) @@ -68,4 +65,4 @@ class TestConfig(helpers.FilesystemMockingTestCase): 'gpgcheck': '1', } } - self.assertEquals(expected, dict(contents)) + self.assertEqual(expected, dict(contents)) diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py index 976d8283..681f3780 100644 --- a/tests/unittests/test_merging.py +++ b/tests/unittests/test_merging.py @@ -133,7 +133,7 @@ class TestSimpleRun(helpers.ResourceUsingTestCase): for test in test_dicts: c = _old_mergemanydict(*test) d = util.mergemanydict(test) - self.assertEquals(c, d) + self.assertEqual(c, d) def test_merge_cc_samples(self): tests = self._load_merge_files() @@ -155,7 +155,7 @@ class TestSimpleRun(helpers.ResourceUsingTestCase): fail_msg = fail_msg % (expected_fn, ",".join(merging_fns), merged_buf, expected_merge) - self.assertEquals(expected_merge, merged_buf, msg=fail_msg) + self.assertEqual(expected_merge, merged_buf, msg=fail_msg) def test_compat_merges_dict(self): a = { @@ -167,7 +167,7 @@ class TestSimpleRun(helpers.ResourceUsingTestCase): } c = _old_mergedict(a, b) d = util.mergemanydict([a, b]) - self.assertEquals(c, d) + self.assertEqual(c, d) def test_compat_merges_dict2(self): a = { @@ -182,7 +182,7 @@ class TestSimpleRun(helpers.ResourceUsingTestCase): } c = _old_mergedict(a, b) d = util.mergemanydict([a, b]) - self.assertEquals(c, d) + self.assertEqual(c, d) def test_compat_merges_list(self): a = {'b': [1, 2, 3]} @@ -190,7 +190,7 @@ class TestSimpleRun(helpers.ResourceUsingTestCase): c = {'b': [6, 7]} e = _old_mergemanydict(a, b, c) f = util.mergemanydict([a, b, c]) - self.assertEquals(e, f) + self.assertEqual(e, f) def test_compat_merges_str(self): a = {'b': "hi"} @@ -198,7 +198,7 @@ class TestSimpleRun(helpers.ResourceUsingTestCase): c = {'b': "hallo"} e = _old_mergemanydict(a, b, c) f = util.mergemanydict([a, b, c]) - self.assertEquals(e, f) + self.assertEqual(e, f) def test_compat_merge_sub_dict(self): a = { @@ -222,7 +222,7 @@ class TestSimpleRun(helpers.ResourceUsingTestCase): } c = _old_mergedict(a, b) d = util.mergemanydict([a, b]) - self.assertEquals(c, d) + self.assertEqual(c, d) def test_compat_merge_sub_dict2(self): a = { @@ -238,7 +238,7 @@ class TestSimpleRun(helpers.ResourceUsingTestCase): } c = _old_mergedict(a, b) d = util.mergemanydict([a, b]) - self.assertEquals(c, d) + self.assertEqual(c, d) def test_compat_merge_sub_list(self): a = { @@ -254,4 +254,4 @@ class TestSimpleRun(helpers.ResourceUsingTestCase): } c = _old_mergedict(a, b) d = util.mergemanydict([a, b]) - self.assertEquals(c, d) + self.assertEqual(c, d) diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 09235c4d..624a9aa8 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -1,11 +1,12 @@ -from cloudinit import util from cloudinit import net +from cloudinit import util + from .helpers import TestCase import base64 import copy -import io import gzip +import io import json import os diff --git a/tests/unittests/test_reporting.py b/tests/unittests/test_reporting.py index 32356ef9..5cad8406 100644 --- a/tests/unittests/test_reporting.py +++ b/tests/unittests/test_reporting.py @@ -4,8 +4,8 @@ # vi: ts=4 expandtab from cloudinit import reporting -from cloudinit.reporting import handlers from cloudinit.reporting import events +from cloudinit.reporting import handlers from .helpers import (mock, TestCase) diff --git a/tests/unittests/test_rh_subscription.py b/tests/unittests/test_rh_subscription.py index 8c586ad7..b84c807b 100644 --- a/tests/unittests/test_rh_subscription.py +++ b/tests/unittests/test_rh_subscription.py @@ -1,5 +1,6 @@ -from cloudinit import util from cloudinit.config import cc_rh_subscription +from cloudinit import util + import logging import mock import unittest diff --git a/tests/unittests/test_runs/test_merge_run.py b/tests/unittests/test_runs/test_merge_run.py index d0ec36a9..ce43798e 100644 --- a/tests/unittests/test_runs/test_merge_run.py +++ b/tests/unittests/test_runs/test_merge_run.py @@ -42,13 +42,13 @@ class TestMergeRun(helpers.FilesystemMockingTestCase): args=[PER_INSTANCE], freq=PER_INSTANCE) mirrors = initer.distro.get_option('package_mirrors') - self.assertEquals(1, len(mirrors)) + self.assertEqual(1, len(mirrors)) mirror = mirrors[0] - self.assertEquals(mirror['arches'], ['i386', 'amd64', 'blah']) + self.assertEqual(mirror['arches'], ['i386', 'amd64', 'blah']) mods = stages.Modules(initer) (which_ran, failures) = mods.run_section('cloud_init_modules') self.assertTrue(len(failures) == 0) self.assertTrue(os.path.exists('/etc/blah.ini')) self.assertIn('write-files', which_ran) contents = util.load_file('/etc/blah.ini') - self.assertEquals(contents, 'blah') + self.assertEqual(contents, 'blah') diff --git a/tests/unittests/test_runs/test_simple_run.py b/tests/unittests/test_runs/test_simple_run.py index e19e65cd..07e7b1a8 100644 --- a/tests/unittests/test_runs/test_simple_run.py +++ b/tests/unittests/test_runs/test_simple_run.py @@ -63,7 +63,7 @@ class TestSimpleRun(helpers.FilesystemMockingTestCase): initer.fetch() iid = initer.instancify() - self.assertEquals(iid, 'iid-datasource-none') + self.assertEqual(iid, 'iid-datasource-none') initer.update() self.assertTrue(os.path.islink("var/lib/cloud/instance")) @@ -78,4 +78,4 @@ class TestSimpleRun(helpers.FilesystemMockingTestCase): self.assertTrue(os.path.exists('/etc/blah.ini')) self.assertIn('write-files', which_ran) contents = util.load_file('/etc/blah.ini') - self.assertEquals(contents, 'blah') + self.assertEqual(contents, 'blah') diff --git a/tests/unittests/test_templating.py b/tests/unittests/test_templating.py index b9863650..94b6e061 100644 --- a/tests/unittests/test_templating.py +++ b/tests/unittests/test_templating.py @@ -58,7 +58,7 @@ class TestTemplates(test_helpers.TestCase): blob = "blahblah $blah" (template_type, renderer, contents) = templater.detect_template(blob) self.assertIn("cheetah", template_type) - self.assertEquals(blob, contents) + self.assertEqual(blob, contents) blob = '##template:something-new' self.assertRaises(ValueError, templater.detect_template, blob) @@ -67,18 +67,18 @@ class TestTemplates(test_helpers.TestCase): blob = '''## template:cheetah $a,$b''' c = templater.render_string(blob, {"a": 1, "b": 2}) - self.assertEquals("1,2", c) + self.assertEqual("1,2", c) def test_render_jinja(self): blob = '''## template:jinja {{a}},{{b}}''' c = templater.render_string(blob, {"a": 1, "b": 2}) - self.assertEquals("1,2", c) + self.assertEqual("1,2", c) def test_render_default(self): blob = '''$a,$b''' c = templater.render_string(blob, {"a": 1, "b": 2}) - self.assertEquals("1,2", c) + self.assertEqual("1,2", c) def test_render_basic_deeper(self): hn = 'myfoohost.yahoo.com' diff --git a/tox.ini b/tox.ini index 7a144efe..dafaaf6d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py3,pyflakes +envlist = py27,py3,flake8 recreate = True [testenv] @@ -10,10 +10,9 @@ deps = -r{toxinidir}/test-requirements.txt [testenv:py3] basepython = python3 -[testenv:pyflakes] +[testenv:flake8] basepython = python3 -commands = {envpython} -m pyflakes {posargs:cloudinit/ tests/ tools/} - {envpython} -m pep8 {posargs:cloudinit/ tests/ tools/} +commands = {envpython} -m flake8 {posargs:cloudinit/ tests/ tools/} # https://github.com/gabrielfalcao/HTTPretty/issues/223 setenv = @@ -32,6 +31,5 @@ setenv = LC_ALL = C [flake8] - ignore=H404,H405,H105,H301,H104,H403,H101 -exclude = .venv,.tox,dist,doc,*egg,.git,build,tools,tests +exclude = .venv,.tox,dist,doc,*egg,.git,build,tools -- cgit v1.2.3 From 811f7b5b3c4590ee4a16662db37b1f37541d9822 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 12 May 2016 16:52:56 -0400 Subject: fix last flake8 error --- cloudinit/sources/helpers/azure.py | 2 +- tests/unittests/test_datasource/test_azure_helper.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index bbfc283c..63ccf10e 100644 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -221,7 +221,7 @@ class WALinuxAgentShim(object): if 'unknown-245' in line: value = line.strip(' ').split(' ', 2)[-1].strip(';\n"') if value is None: - raise Exception('No endpoint found in DHCP config.') + raise ValueError('No endpoint found in DHCP config.') endpoint_ip_address = WALinuxAgentShim.get_ip_from_lease_value(value) LOG.debug('Azure endpoint found at %s', endpoint_ip_address) return endpoint_ip_address diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 921bc978..d07a1f07 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -71,7 +71,7 @@ class TestFindEndpoint(TestCase): def test_missing_special_azure_line(self): self.load_file.return_value = '' - self.assertRaises(Exception, + self.assertRaises(ValueError, azure_helper.WALinuxAgentShim.find_endpoint) @staticmethod -- cgit v1.2.3 From f2665c246a3e6dec55064eced09919d912ae0e52 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 16 May 2016 16:08:19 -0700 Subject: Fix slow tests Timeouts and retries were triggering so make it so that tests do not use the typical timesouts and retries so that the tests finish faster. --- cloudinit/sources/DataSourceOpenStack.py | 12 ++++++--- tests/unittests/test_datasource/test_openstack.py | 32 +++++++++++------------ 2 files changed, 24 insertions(+), 20 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index 3af17b10..dfd96035 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -103,7 +103,7 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): self.metadata_address = url2base.get(avail_url) return bool(avail_url) - def get_data(self): + def get_data(self, retries=5, timeout=5): try: if not self.wait_for_metadata_service(): return False @@ -115,7 +115,9 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): 'Crawl of openstack metadata service', read_metadata_service, args=[self.metadata_address], - kwargs={'ssl_details': self.ssl_details}) + kwargs={'ssl_details': self.ssl_details, + 'retries': retries, + 'timeout': timeout}) except openstack.NonReadable: return False except (openstack.BrokenMetadata, IOError): @@ -153,8 +155,10 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): return sources.instance_id_matches_system_uuid(self.get_instance_id()) -def read_metadata_service(base_url, ssl_details=None): - reader = openstack.MetadataReader(base_url, ssl_details=ssl_details) +def read_metadata_service(base_url, ssl_details=None, + timeout=5, retries=5): + reader = openstack.MetadataReader(base_url, ssl_details=ssl_details, + timeout=timeout, retries=retries) return reader.read_v2() diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index 4140d054..5c8592c5 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -135,13 +135,17 @@ def _register_uris(version, ec2_files, ec2_meta, os_files): body=get_request_callback) +def _read_metadata_service(): + return ds.read_metadata_service(BASE_URL, retries=0, timeout=0.1) + + class TestOpenStackDataSource(test_helpers.HttprettyTestCase): VERSION = 'latest' @hp.activate def test_successful(self): _register_uris(self.VERSION, EC2_FILES, EC2_META, OS_FILES) - f = ds.read_metadata_service(BASE_URL) + f = _read_metadata_service() self.assertEqual(VENDOR_DATA, f.get('vendordata')) self.assertEqual(CONTENT_0, f['files']['/etc/foo.cfg']) self.assertEqual(CONTENT_1, f['files']['/etc/bar/bar.cfg']) @@ -163,7 +167,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): @hp.activate def test_no_ec2(self): _register_uris(self.VERSION, {}, {}, OS_FILES) - f = ds.read_metadata_service(BASE_URL) + f = _read_metadata_service() self.assertEqual(VENDOR_DATA, f.get('vendordata')) self.assertEqual(CONTENT_0, f['files']['/etc/foo.cfg']) self.assertEqual(CONTENT_1, f['files']['/etc/bar/bar.cfg']) @@ -178,8 +182,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): if k.endswith('meta_data.json'): os_files.pop(k, None) _register_uris(self.VERSION, {}, {}, os_files) - self.assertRaises(openstack.NonReadable, ds.read_metadata_service, - BASE_URL) + self.assertRaises(openstack.NonReadable, _read_metadata_service) @hp.activate def test_bad_uuid(self): @@ -190,8 +193,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): if k.endswith('meta_data.json'): os_files[k] = json.dumps(os_meta) _register_uris(self.VERSION, {}, {}, os_files) - self.assertRaises(openstack.BrokenMetadata, ds.read_metadata_service, - BASE_URL) + self.assertRaises(openstack.BrokenMetadata, _read_metadata_service) @hp.activate def test_userdata_empty(self): @@ -200,7 +202,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): if k.endswith('user_data'): os_files.pop(k, None) _register_uris(self.VERSION, {}, {}, os_files) - f = ds.read_metadata_service(BASE_URL) + f = _read_metadata_service() self.assertEqual(VENDOR_DATA, f.get('vendordata')) self.assertEqual(CONTENT_0, f['files']['/etc/foo.cfg']) self.assertEqual(CONTENT_1, f['files']['/etc/bar/bar.cfg']) @@ -213,7 +215,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): if k.endswith('vendor_data.json'): os_files.pop(k, None) _register_uris(self.VERSION, {}, {}, os_files) - f = ds.read_metadata_service(BASE_URL) + f = _read_metadata_service() self.assertEqual(CONTENT_0, f['files']['/etc/foo.cfg']) self.assertEqual(CONTENT_1, f['files']['/etc/bar/bar.cfg']) self.assertFalse(f.get('vendordata')) @@ -225,8 +227,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): if k.endswith('vendor_data.json'): os_files[k] = '{' # some invalid json _register_uris(self.VERSION, {}, {}, os_files) - self.assertRaises(openstack.BrokenMetadata, ds.read_metadata_service, - BASE_URL) + self.assertRaises(openstack.BrokenMetadata, _read_metadata_service) @hp.activate def test_metadata_invalid(self): @@ -235,8 +236,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): if k.endswith('meta_data.json'): os_files[k] = '{' # some invalid json _register_uris(self.VERSION, {}, {}, os_files) - self.assertRaises(openstack.BrokenMetadata, ds.read_metadata_service, - BASE_URL) + self.assertRaises(openstack.BrokenMetadata, _read_metadata_service) @hp.activate def test_datasource(self): @@ -245,7 +245,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): None, helpers.Paths({})) self.assertIsNone(ds_os.version) - found = ds_os.get_data() + found = ds_os.get_data(timeout=0.1, retries=0) self.assertTrue(found) self.assertEqual(2, ds_os.version) md = dict(ds_os.metadata) @@ -269,7 +269,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): None, helpers.Paths({})) self.assertIsNone(ds_os.version) - found = ds_os.get_data() + found = ds_os.get_data(timeout=0.1, retries=0) self.assertFalse(found) self.assertIsNone(ds_os.version) @@ -288,7 +288,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): 'timeout': 0, } self.assertIsNone(ds_os.version) - found = ds_os.get_data() + found = ds_os.get_data(timeout=0.1, retries=0) self.assertFalse(found) self.assertIsNone(ds_os.version) @@ -311,7 +311,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): 'timeout': 0, } self.assertIsNone(ds_os.version) - found = ds_os.get_data() + found = ds_os.get_data(timeout=0.1, retries=0) self.assertFalse(found) self.assertIsNone(ds_os.version) -- cgit v1.2.3 From abb3c00fadefea8056c300faf141260e124a5064 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 17 May 2016 17:29:59 -0700 Subject: Don't expose anything but 'render_network_state' This should be the visible api of a network renderer as anything more granular varies between the different render types and will not apply to those renderers. --- cloudinit/net/eni.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index b427012e..b1bdac24 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -243,7 +243,7 @@ def _parse_deb_config(path): class Renderer(object): """Renders network information in a /etc/network/interfaces format.""" - def render_persistent_net(self, network_state): + def _render_persistent_net(self, network_state): """Given state, emit udev rules to map mac to ifname.""" content = "" interfaces = network_state.get('interfaces') @@ -256,7 +256,7 @@ class Renderer(object): return content - def render_route(self, route, indent=""): + def _render_route(self, route, indent=""): """ When rendering routes for an iface, in some cases applying a route may result in the route command returning non-zero which produces some confusing output for users manually using ifup/ifdown[1]. To @@ -300,7 +300,7 @@ class Renderer(object): return content - def render_interfaces(self, network_state): + def _render_interfaces(self, network_state): ''' Given state, emit etc/network/interfaces content ''' content = "" @@ -352,7 +352,7 @@ class Renderer(object): content += _iface_add_attrs(iface) for route in network_state.get('routes'): - content += self.render_route(route) + content += self._render_route(route) # global replacements until v2 format content = content.replace('mac_address', 'hwaddress') @@ -366,19 +366,19 @@ class Renderer(object): fpeni = os.path.sep.join((target, eni,)) util.ensure_dir(os.path.dirname(fpeni)) with open(fpeni, 'w+') as f: - f.write(self.render_interfaces(network_state)) + f.write(self._render_interfaces(network_state)) if netrules: netrules = os.path.sep.join((target, netrules,)) util.ensure_dir(os.path.dirname(netrules)) with open(netrules, 'w+') as f: - f.write(self.render_persistent_net(network_state)) + f.write(self._render_persistent_net(network_state)) if links_prefix: - self.render_systemd_links(target, network_state, links_prefix) + self._render_systemd_links(target, network_state, links_prefix) - def render_systemd_links(self, target, network_state, - links_prefix=LINKS_FNAME_PREFIX): + def _render_systemd_links(self, target, network_state, + links_prefix=LINKS_FNAME_PREFIX): fp_prefix = os.path.sep.join((target, links_prefix)) for f in glob.glob(fp_prefix + "*"): os.unlink(f) -- cgit v1.2.3 From 880d9fc2f9c62abf19b1506595aa81e5417dea45 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 19 May 2016 14:13:07 -0700 Subject: Adjust net module to be more isolated This allows it to be used outside of cloudinit more easily in the future. --- cloudinit/net/__init__.py | 108 ++++++++++++++++++--------------- cloudinit/net/cmdline.py | 15 ++++- cloudinit/net/eni.py | 35 +++++------ cloudinit/net/network_state.py | 38 +++++++++--- cloudinit/sources/helpers/openstack.py | 4 ++ cloudinit/util.py | 9 --- tests/unittests/test_net.py | 7 ++- 7 files changed, 129 insertions(+), 87 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 0202cbd8..07e7307e 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -17,44 +17,81 @@ # along with Curtin. If not, see . import errno +import logging import os - -from cloudinit import log as logging -from cloudinit.net import network_state -from cloudinit import util - +import six +import yaml LOG = logging.getLogger(__name__) SYS_CLASS_NET = "/sys/class/net/" LINKS_FNAME_PREFIX = "etc/systemd/network/50-cloud-init-" DEFAULT_PRIMARY_INTERFACE = 'eth0' +# NOTE(harlowja): some of these are similar to what is in cloudinit main +# source or utils tree/module but the reason that is done is so that this +# whole module can be easily extracted and placed into other +# code-bases (curtin for example). -def sys_dev_path(devname, path=""): - return SYS_CLASS_NET + devname + "/" + path +def write_file(path, content): + """Simple writing a file helper.""" + base_path = os.path.dirname(path) + if not os.path.isdir(base_path): + os.makedirs(base_path) + with open(path, "wb+") as fh: + if isinstance(content, six.text_type): + content = content.encode("utf8") + fh.write(content) -def read_sys_net(devname, path, translate=None, enoent=None, keyerror=None): +def read_file(path, decode='utf8', enoent=None): try: - contents = "" - with open(sys_dev_path(devname, path), "r") as fp: - contents = fp.read().strip() - if translate is None: - return contents - - try: - return translate.get(contents) - except KeyError: - LOG.debug("found unexpected value '%s' in '%s/%s'", contents, - devname, path) - if keyerror is not None: - return keyerror - raise + with open(path, "rb") as fh: + contents = fh.load() except OSError as e: if e.errno == errno.ENOENT and enoent is not None: return enoent raise + if decode: + return contents.decode(decode) + return contents + + +def dump_yaml(obj): + return yaml.safe_dump(obj, + line_break="\n", + indent=4, + explicit_start=True, + explicit_end=True, + default_flow_style=False) + + +def read_yaml_file(path): + val = yaml.safe_load(read_file(path)) + if not isinstance(val, dict): + gotten_type_name = type(val).__name__ + raise TypeError("Expected dict to be loaded from %s, got" + " '%s' instead" % (path, gotten_type_name)) + return val + + +def sys_dev_path(devname, path=""): + return SYS_CLASS_NET + devname + "/" + path + + +def read_sys_net(devname, path, translate=None, enoent=None, keyerror=None): + contents = read_file(sys_dev_path(devname, path), enoent=enoent) + contents = contents.strip() + if translate is None: + return contents + try: + return translate.get(contents) + except KeyError: + LOG.debug("found unexpected value '%s' in '%s/%s'", contents, + devname, path) + if keyerror is not None: + return keyerror + raise def is_up(devname): @@ -107,31 +144,6 @@ class ParserError(Exception): """Raised when parser has issue parsing the interfaces file.""" -def parse_net_config_data(net_config, skip_broken=True): - """Parses the config, returns NetworkState object - - :param net_config: curtin network config dict - """ - state = None - if 'version' in net_config and 'config' in net_config: - ns = network_state.NetworkState(version=net_config.get('version'), - config=net_config.get('config')) - ns.parse_config(skip_broken=skip_broken) - state = ns.network_state - return state - - -def parse_net_config(path, skip_broken=True): - """Parses a curtin network configuration file and - return network state""" - ns = None - net_config = util.read_conf(path) - if 'network' in net_config: - ns = parse_net_config_data(net_config.get('network'), - skip_broken=skip_broken) - return ns - - def is_disabled_cfg(cfg): if not cfg or not isinstance(cfg, dict): return False @@ -146,7 +158,7 @@ def sys_netdev_info(name, field): fname = os.path.join(SYS_CLASS_NET, name, field) if not os.path.exists(fname): raise OSError("%s: could not find sysfs entry: %s" % (name, fname)) - data = util.load_file(fname) + data = read_file(fname) if data[-1] == '\n': data = data[:-1] return data diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py index f5712533..b85d4b0a 100644 --- a/cloudinit/net/cmdline.py +++ b/cloudinit/net/cmdline.py @@ -20,12 +20,25 @@ import base64 import glob import gzip import io +import shlex +import sys + +import six from cloudinit.net import get_devicelist from cloudinit.net import sys_netdev_info from cloudinit import util +PY26 = sys.version_info[0:2] == (2, 6) + + +def _shlex_split(blob): + if PY26 and isinstance(blob, six.text_type): + # Older versions don't support unicode input + blob = blob.encode("utf8") + return shlex.split(blob) + def _load_shell_content(content, add_empty=False, empty_val=None): """Given shell like syntax (key=value\nkey2=value2\n) in content @@ -33,7 +46,7 @@ def _load_shell_content(content, add_empty=False, empty_val=None): then add entries in to the returned dictionary for 'VAR=' variables. Set their value to empty_val.""" data = {} - for line in util.shlex_split(content): + for line in _shlex_split(content): try: key, value = line.split("=", 1) except ValueError: diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index b1bdac24..adb31c22 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -16,10 +16,11 @@ import glob import os import re +from cloudinit import net + from cloudinit.net import LINKS_FNAME_PREFIX from cloudinit.net import ParserError from cloudinit.net.udev import generate_udev_rule -from cloudinit import util NET_CONFIG_COMMANDS = [ @@ -363,16 +364,13 @@ class Renderer(object): links_prefix=LINKS_FNAME_PREFIX, netrules='etc/udev/rules.d/70-persistent-net.rules'): - fpeni = os.path.sep.join((target, eni,)) - util.ensure_dir(os.path.dirname(fpeni)) - with open(fpeni, 'w+') as f: - f.write(self._render_interfaces(network_state)) + fpeni = os.path.join(target, eni) + net.write_file(fpeni, self._render_interfaces(network_state)) if netrules: - netrules = os.path.sep.join((target, netrules,)) - util.ensure_dir(os.path.dirname(netrules)) - with open(netrules, 'w+') as f: - f.write(self._render_persistent_net(network_state)) + netrules = os.path.join(target, netrules) + net.write_file(netrules, + self._render_persistent_net(network_state)) if links_prefix: self._render_systemd_links(target, network_state, links_prefix) @@ -382,18 +380,17 @@ class Renderer(object): fp_prefix = os.path.sep.join((target, links_prefix)) for f in glob.glob(fp_prefix + "*"): os.unlink(f) - interfaces = network_state.get('interfaces') for iface in interfaces.values(): if (iface['type'] == 'physical' and 'name' in iface and iface.get('mac_address')): fname = fp_prefix + iface['name'] + ".link" - with open(fname, "w") as fp: - fp.write("\n".join([ - "[Match]", - "MACAddress=" + iface['mac_address'], - "", - "[Link]", - "Name=" + iface['name'], - "" - ])) + content = "\n".join([ + "[Match]", + "MACAddress=" + iface['mac_address'], + "", + "[Link]", + "Name=" + iface['name'], + "" + ]) + net.write_file(fname, content) diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 2feffa71..c5aeadb5 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -16,12 +16,11 @@ # along with Curtin. If not, see . import copy +import logging import six -from cloudinit import log as logging -from cloudinit import util -from cloudinit.util import yaml_dumps as dump_config +from cloudinit import net LOG = logging.getLogger(__name__) @@ -31,9 +30,34 @@ NETWORK_STATE_REQUIRED_KEYS = { } +def parse_net_config_data(net_config, skip_broken=True): + """Parses the config, returns NetworkState object + + :param net_config: curtin network config dict + """ + state = None + if 'version' in net_config and 'config' in net_config: + ns = NetworkState(version=net_config.get('version'), + config=net_config.get('config')) + ns.parse_config(skip_broken=skip_broken) + state = ns.network_state + return state + + +def parse_net_config(path, skip_broken=True): + """Parses a curtin network configuration file and + return network state""" + ns = None + net_config = net.read_yaml_file(path) + if 'network' in net_config: + ns = parse_net_config_data(net_config.get('network'), + skip_broken=skip_broken) + return ns + + def from_state_file(state_file): network_state = None - state = util.read_conf(state_file) + state = net.read_yaml_file(state_file) network_state = NetworkState() network_state.load(state) return network_state @@ -111,7 +135,7 @@ class NetworkState(object): 'config': self.config, 'network_state': self.network_state, } - return dump_config(state) + return net.dump_yaml(state) def load(self, state): if 'version' not in state: @@ -121,7 +145,7 @@ class NetworkState(object): required_keys = NETWORK_STATE_REQUIRED_KEYS[state['version']] missing_keys = diff_keys(required_keys, state) if missing_keys: - msg = 'Invalid state, missing keys: %s'.format(missing_keys) + msg = 'Invalid state, missing keys: %s' % (missing_keys) LOG.error(msg) raise ValueError(msg) @@ -130,7 +154,7 @@ class NetworkState(object): setattr(self, key, state[key]) def dump_network_state(self): - return dump_config(self.network_state) + return net.dump_yaml(self.network_state) def parse_config(self, skip_broken=True): # rebuild network state diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index b2acc648..f85fd864 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -551,6 +551,10 @@ def convert_net_json(network_json): 'type': 'static', 'address': network.get('ip_address'), }) + if network['type'] == 'ipv6': + subnet['ipv6'] = True + else: + subnet['ipv4'] = True subnets.append(subnet) cfg.update({'subnets': subnets}) if link['type'] in ['ethernet', 'vif', 'ovs', 'phy']: diff --git a/cloudinit/util.py b/cloudinit/util.py index 7562b97a..2bec476e 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -37,7 +37,6 @@ import pwd import random import re import shutil -import shlex import socket import stat import string @@ -82,7 +81,6 @@ CONTAINER_TESTS = (['systemd-detect-virt', '--quiet', '--container'], ['lxc-is-container']) PROC_CMDLINE = None -PY26 = sys.version_info[0:2] == (2, 6) def decode_binary(blob, encoding='utf-8'): @@ -1202,13 +1200,6 @@ def load_file(fname, read_cb=None, quiet=False, decode=True): return contents -def shlex_split(blob): - if PY26 and isinstance(blob, six.text_type): - # Older versions don't support unicode input - blob = blob.encode("utf8") - return shlex.split(blob) - - def get_cmdline(): if 'DEBUG_PROC_CMDLINE' in os.environ: return os.environ["DEBUG_PROC_CMDLINE"] diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index ed2c6d0f..75c433f6 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -2,6 +2,7 @@ from cloudinit import util from cloudinit import net from cloudinit.net import cmdline from cloudinit.net import eni +from cloudinit.net import network_state from .helpers import TestCase from .helpers import mock @@ -112,14 +113,14 @@ class TestEniNetRendering(TestCase): mock_sys_dev_path.side_effect = sys_dev_path network_cfg = net.generate_fallback_config() - network_state = net.parse_net_config_data(network_cfg, - skip_broken=False) + ns = network_state.parse_net_config_data(network_cfg, + skip_broken=False) render_dir = os.path.join(tmp_dir, "render") os.makedirs(render_dir) renderer = eni.Renderer() - renderer.render_network_state(render_dir, network_state, + renderer.render_network_state(render_dir, ns, eni="interfaces", links_prefix=None, netrules=None) -- cgit v1.2.3 From 2fb5af62229b8975910bf0ef63731047bd8d7e63 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 19 May 2016 15:33:15 -0700 Subject: Fix up tests and flake8 warnings --- cloudinit/net/__init__.py | 2 +- cloudinit/net/eni.py | 16 ++++----- cloudinit/sources/DataSourceConfigDrive.py | 1 - cloudinit/stages.py | 2 +- tests/unittests/helpers.py | 1 + tests/unittests/test__init__.py | 1 - tests/unittests/test_datasource/test_cloudsigma.py | 1 - .../unittests/test_datasource/test_configdrive.py | 41 ++++++++++++---------- tests/unittests/test_datasource/test_nocloud.py | 2 ++ tests/unittests/test_datasource/test_smartos.py | 2 +- tests/unittests/test_net.py | 6 ++-- tests/unittests/test_rh_subscription.py | 5 +-- tox.ini | 4 +-- 13 files changed, 41 insertions(+), 43 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index ad44911b..ba0e39ae 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -49,8 +49,8 @@ DEFAULT_PRIMARY_INTERFACE = 'eth0' # whole module can be easily extracted and placed into other # code-bases (curtin for example). + def write_file(path, content): - """Simple writing a file helper.""" base_path = os.path.dirname(path) if not os.path.isdir(base_path): os.makedirs(base_path) diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index adb31c22..18bae97a 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -258,7 +258,7 @@ class Renderer(object): return content def _render_route(self, route, indent=""): - """ When rendering routes for an iface, in some cases applying a route + """When rendering routes for an iface, in some cases applying a route may result in the route command returning non-zero which produces some confusing output for users manually using ifup/ifdown[1]. To that end, we will optionally include an '|| true' postfix to each @@ -302,7 +302,7 @@ class Renderer(object): return content def _render_interfaces(self, network_state): - ''' Given state, emit etc/network/interfaces content ''' + '''Given state, emit etc/network/interfaces content''' content = "" interfaces = network_state.get('interfaces') @@ -336,8 +336,8 @@ class Renderer(object): iface['control'] = subnet.get('control', 'auto') if iface['mode'].endswith('6'): iface['inet'] += '6' - elif iface['mode'] == 'static' \ - and ":" in subnet['address']: + elif (iface['mode'] == 'static' + and ":" in subnet['address']): iface['inet'] += '6' if iface['mode'].startswith('dhcp'): iface['mode'] = 'dhcp' @@ -359,10 +359,10 @@ class Renderer(object): content = content.replace('mac_address', 'hwaddress') return content - def render_network_state(self, - target, network_state, eni="etc/network/interfaces", - links_prefix=LINKS_FNAME_PREFIX, - netrules='etc/udev/rules.d/70-persistent-net.rules'): + def render_network_state( + self, target, network_state, + eni="etc/network/interfaces", links_prefix=LINKS_FNAME_PREFIX, + netrules='etc/udev/rules.d/70-persistent-net.rules'): fpeni = os.path.join(target, eni) net.write_file(fpeni, self._render_interfaces(network_state)) diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 70373b43..4478c4e2 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -18,7 +18,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import copy import os from cloudinit import log as logging diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 5dd31539..b837009a 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -44,8 +44,8 @@ from cloudinit import helpers from cloudinit import importer from cloudinit import log as logging from cloudinit import net -from cloudinit.reporting import events from cloudinit.net import cmdline +from cloudinit.reporting import events from cloudinit import sources from cloudinit import type_utils from cloudinit import util diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 7b4d44e8..8d46a8bf 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -45,6 +45,7 @@ else: if _PY_MINOR == 4 and _PY_MICRO < 3: FIX_HTTPRETTY = True + # Makes the old path start # with new base instead of whatever # it previously had diff --git a/tests/unittests/test__init__.py b/tests/unittests/test__init__.py index a9b35afe..0154784a 100644 --- a/tests/unittests/test__init__.py +++ b/tests/unittests/test__init__.py @@ -1,7 +1,6 @@ import os import shutil import tempfile -import unittest2 from cloudinit import handlers from cloudinit import helpers diff --git a/tests/unittests/test_datasource/test_cloudsigma.py b/tests/unittests/test_datasource/test_cloudsigma.py index 7950fc52..2a42ce0c 100644 --- a/tests/unittests/test_datasource/test_cloudsigma.py +++ b/tests/unittests/test_datasource/test_cloudsigma.py @@ -6,7 +6,6 @@ from cloudinit.cs_utils import Cepko from cloudinit.sources import DataSourceCloudSigma from .. import helpers as test_helpers -from ..helpers import SkipTest SERVER_CONTEXT = { "cpu": 1000, diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index 1db50798..5395e544 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -368,30 +368,32 @@ class TestNetJson(TestCase): self.assertEqual(myds.network_config, network_config) def test_network_config_conversions(self): - """Tests a bunch of input network json and checks the expected conversions.""" + """Tests a bunch of input network json and checks the + expected conversions.""" in_datas = [ NETWORK_DATA, { 'services': [{'type': 'dns', 'address': '172.19.0.12'}], - 'networks': [ - {'network_id': 'dacd568d-5be6-4786-91fe-750c374b78b4', - 'type': 'ipv4', 'netmask': '255.255.252.0', - 'link': 'tap1a81968a-79', - 'routes': [ - { - 'netmask': '0.0.0.0', - 'network': '0.0.0.0', - 'gateway': '172.19.3.254' - }, - ], - 'ip_address': '172.19.1.34', - 'id': 'network0', + 'networks': [{ + 'network_id': 'dacd568d-5be6-4786-91fe-750c374b78b4', + 'type': 'ipv4', + 'netmask': '255.255.252.0', + 'link': 'tap1a81968a-79', + 'routes': [{ + 'netmask': '0.0.0.0', + 'network': '0.0.0.0', + 'gateway': '172.19.3.254', + }], + 'ip_address': '172.19.1.34', + 'id': 'network0', + }], + 'links': [{ + 'type': 'bridge', + 'vif_id': '1a81968a-797a-400f-8a80-567f997eb93f', + 'ethernet_mac_address': 'fa:16:3e:ed:9a:59', + 'id': 'tap1a81968a-79', + 'mtu': None, }], - 'links': [ - {'type': 'bridge', - 'vif_id': '1a81968a-797a-400f-8a80-567f997eb93f', - 'ethernet_mac_address': 'fa:16:3e:ed:9a:59', - 'id': 'tap1a81968a-79', 'mtu': None}] }, ] out_datas = [ @@ -440,6 +442,7 @@ class TestNetJson(TestCase): 'address': '172.19.1.34', 'netmask': '255.255.252.0', 'type': 'static', + 'ipv4': True, 'routes': [{ 'gateway': '172.19.3.254', 'netmask': '0.0.0.0', diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py index 077603b4..b0fa1130 100644 --- a/tests/unittests/test_datasource/test_nocloud.py +++ b/tests/unittests/test_datasource/test_nocloud.py @@ -7,6 +7,8 @@ import os import shutil import tempfile +import yaml + class TestNoCloudDataSource(TestCase): diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 2f159ac4..28f56039 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -42,7 +42,7 @@ from cloudinit import helpers as c_helpers from cloudinit.util import b64e from .. import helpers -from ..helpers import mock, SkipTest +from ..helpers import mock MOCK_RETURNS = { 'hostname': 'test-host', diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index faf0f0fb..7998111a 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -1,13 +1,11 @@ -from cloudinit import net -from cloudinit import util - from cloudinit import net from cloudinit.net import cmdline from cloudinit.net import eni from cloudinit.net import network_state +from cloudinit import util -from .helpers import TestCase from .helpers import mock +from .helpers import TestCase import base64 import copy diff --git a/tests/unittests/test_rh_subscription.py b/tests/unittests/test_rh_subscription.py index e4dcc58b..891dbe77 100644 --- a/tests/unittests/test_rh_subscription.py +++ b/tests/unittests/test_rh_subscription.py @@ -10,13 +10,10 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . - import logging -import mock -import unittest - from cloudinit.config import cc_rh_subscription +from cloudinit import util from .helpers import TestCase, mock diff --git a/tox.ini b/tox.ini index b92ebac1..7802a291 100644 --- a/tox.ini +++ b/tox.ini @@ -1,11 +1,11 @@ [tox] -envlist = py27,py3,flake8 +envlist = py26,py27,py3,flake8 recreate = True usedevelop = True [testenv] -commands = python -m nose {posargs:tests} +commands = nosetests {posargs:tests} deps = -r{toxinidir}/test-requirements.txt -r{toxinidir}/requirements.txt setenv = -- cgit v1.2.3 From ee239517c5342cbd62c9fdeaf735d78d6fd1fbb8 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 11:59:14 +0200 Subject: support apt_sources to be a dictionary key is the filename, and "old" input shall be handled as it was all the time. For compatibility this will (continue to) overwrite the file of multiple options that did not specify an output file (they all get the same default). Yet it will process them all - as it always did - e.g. to add the keys of all of them. Any users of the new format won't have these issues, as they will always have a key. --- cloudinit/config/cc_apt_configure.py | 34 +++++++++++++++++++++++++--------- cloudinit/util.py | 8 ++++++++ 2 files changed, 33 insertions(+), 9 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index e5a962ac..a46ebb3e 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -215,8 +215,28 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): def aa_repo_match(x): return False + # convert old list format to new dict based format + if isinstance(srclist, list): + srcdict = {} + for srcent in srclist: + if 'filename' not in srcent: + # file collides for multiple !filename cases for compatibility + # yet we need them all processed, so not same dictionary key + srcent['filename'] = "cloud_config_sources.list" + key = util.rand_dict_key(srcdict, "cloud_config_sources.list") + else: + # all with filename use that as key (matching new format) + key = srcent['filename'] + srcdict[key] = srcent + else: + srcdict = srclist + errorlist = [] - for ent in srclist: + for filename in srcdict: + ent = srcdict[filename] + if 'filename' not in ent: + ent[filename] = filename + # keys can be added without specifying a source try: add_key(ent) @@ -226,10 +246,13 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): if 'source' not in ent: errorlist.append(["", "missing source"]) continue - source = ent['source'] source = templater.render_string(source, template_params) + if not ent['filename'].startswith("/"): + ent['filename'] = os.path.join("/etc/apt/sources.list.d/", + ent['filename']) + if aa_repo_match(source): try: util.subp(["add-apt-repository", source]) @@ -238,13 +261,6 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): ("add-apt-repository failed. " + str(e))]) continue - if 'filename' not in ent: - ent['filename'] = 'cloud_config_sources.list' - - if not ent['filename'].startswith("/"): - ent['filename'] = os.path.join("/etc/apt/sources.list.d/", - ent['filename']) - try: contents = "%s\n" % (source) util.write_file(ent['filename'], contents, omode="ab") diff --git a/cloudinit/util.py b/cloudinit/util.py index 0d21e11b..2931efbd 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -335,6 +335,14 @@ def rand_str(strlen=32, select_from=None): select_from = string.ascii_letters + string.digits return "".join([random.choice(select_from) for _x in range(0, strlen)]) +def rand_dict_key(dictionary, postfix=None): + if not postfix: + postfix = "" + while True: + newkey = rand_str(strlen=8) + "_" + postfix + if newkey not in dictionary: + break + return newkey def read_conf(fname): try: -- cgit v1.2.3 From a33f8c09863381006f708a1e9d49997ed9f7befa Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 12:12:26 +0200 Subject: warn about multiple colliding apt_source without filenames --- cloudinit/config/cc_apt_configure.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index a46ebb3e..327b543e 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -215,15 +215,22 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): def aa_repo_match(x): return False + errorlist = [] # convert old list format to new dict based format if isinstance(srclist, list): srcdict = {} + fnfallbackused = None for srcent in srclist: if 'filename' not in srcent: # file collides for multiple !filename cases for compatibility # yet we need them all processed, so not same dictionary key srcent['filename'] = "cloud_config_sources.list" key = util.rand_dict_key(srcdict, "cloud_config_sources.list") + if fnfallbackused is not None: + errorlist.append(["multiple apt_source entries without", + "filename will conflict: %s vs %s" % + (srcent, fnfallbackused)]) + fnfallbackused = srcent else: # all with filename use that as key (matching new format) key = srcent['filename'] @@ -231,7 +238,6 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): else: srcdict = srclist - errorlist = [] for filename in srcdict: ent = srcdict[filename] if 'filename' not in ent: -- cgit v1.2.3 From 4ed5251d17ee7a44ce12d38d9b3d4fa554279419 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 15:06:48 +0200 Subject: fix issue with dictionary style apt_sources handling filenames --- cloudinit/config/cc_apt_configure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 327b543e..a25d6af1 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -241,7 +241,7 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): for filename in srcdict: ent = srcdict[filename] if 'filename' not in ent: - ent[filename] = filename + ent['filename'] = filename # keys can be added without specifying a source try: -- cgit v1.2.3 From 2945e028477ddb031d9a51ada16d5b992380242a Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 16:12:18 +0200 Subject: make sure we only handle list or dict apt_sources and bail out for others --- cloudinit/config/cc_apt_configure.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index a25d6af1..dd199471 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -217,8 +217,8 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): errorlist = [] # convert old list format to new dict based format + srcdict = {} if isinstance(srclist, list): - srcdict = {} fnfallbackused = None for srcent in srclist: if 'filename' not in srcent: @@ -235,8 +235,10 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): # all with filename use that as key (matching new format) key = srcent['filename'] srcdict[key] = srcent - else: + elif isinstance(srclist, dict): srcdict = srclist + else: + errorlist.append(["srclist", "unknown apt_sources format"]) for filename in srcdict: ent = srcdict[filename] -- cgit v1.2.3 From a63a64a70def97730d2ab544b0df9f87f3484333 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 16:53:03 +0200 Subject: final pep8 check fixups --- cloudinit/util.py | 2 ++ tests/unittests/test_handler/test_handler_apt_source.py | 9 +++------ 2 files changed, 5 insertions(+), 6 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/util.py b/cloudinit/util.py index 2931efbd..0773af69 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -335,6 +335,7 @@ def rand_str(strlen=32, select_from=None): select_from = string.ascii_letters + string.digits return "".join([random.choice(select_from) for _x in range(0, strlen)]) + def rand_dict_key(dictionary, postfix=None): if not postfix: postfix = "" @@ -344,6 +345,7 @@ def rand_dict_key(dictionary, postfix=None): break return newkey + def read_conf(fname): try: return load_yaml(load_file(fname), default={}) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 237cf14d..c19904fb 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -55,7 +55,6 @@ class TestAptSourceConfig(TestCase): self.fallbackfn = os.path.join(self.tmp, "etc/apt/sources.list.d/", "cloud_config_sources.list") - @staticmethod def _get_default_params(): """ get_default_params @@ -68,9 +67,9 @@ class TestAptSourceConfig(TestCase): def myjoin(self, *args, **kwargs): """ myjoin - redir into writable tmpdir""" - if (args[0] == "/etc/apt/sources.list.d/" - and args[1] == "cloud_config_sources.list" - and len(args) == 2): + if (args[0] == "/etc/apt/sources.list.d/" and + args[1] == "cloud_config_sources.list" and + len(args) == 2): return self.join(self.tmp, args[0].lstrip("/"), args[1]) else: return self.join(*args, **kwargs) @@ -137,7 +136,6 @@ class TestAptSourceConfig(TestCase): "main universe multiverse restricted"), contents, flags=re.IGNORECASE)) - def test_apt_src_basic_tri(self): """ test_apt_src_basic_tri Test Fix three deb source string, has to overwrite mirror conf in @@ -232,7 +230,6 @@ class TestAptSourceConfig(TestCase): "universe"), contents, flags=re.IGNORECASE)) - def test_apt_src_replace_tri(self): """ test_apt_src_replace_tri Test three autoreplacements of MIRROR and RELEASE in source specs with -- cgit v1.2.3 From 2a20382249c35f368f931dc2bcd8225ea1fc4e84 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Tue, 24 May 2016 18:55:35 +0200 Subject: integrate further smaller review feedback --- cloudinit/config/cc_apt_configure.py | 52 +++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 25 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 8f22c446..02b16336 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -21,7 +21,6 @@ import glob import os import re -import tempfile from cloudinit import templater from cloudinit import util @@ -179,44 +178,29 @@ def add_key_raw(key): """ try: util.subp(('apt-key', 'add', '-'), key) - except: + except util.ProcessExecutionError: raise Exception('failed add key') def add_key(ent): """ - add key to the system as defiend in ent (if any) - suppords raw keys or keyid's - The latter will as a first step fetched to get the raw key + add key to the system as defined in ent (if any) + supports raw keys or keyid's + The latter will as a first step fetch the raw key from a keyserver """ if 'keyid' in ent and 'key' not in ent: keyserver = "keyserver.ubuntu.com" if 'keyserver' in ent: keyserver = ent['keyserver'] - try: - ent['key'] = getkeybyid(ent['keyid'], keyserver) - except: - raise Exception('failed to get key from %s' % keyserver) + ent['key'] = getkeybyid(ent['keyid'], keyserver) if 'key' in ent: add_key_raw(ent['key']) - -def add_sources(srclist, template_params=None, aa_repo_match=None): - """ - add entries in /etc/apt/sources.list.d for each abbreviated - sources.list entry in 'srclist'. When rendering template, also - include the values in dictionary searchList +def convert_to_new_format(srclist, errorlist): + """ convert_to_new_format + convert the old list based format to the new dict based one """ - if template_params is None: - template_params = {} - - if aa_repo_match is None: - def aa_repo_match(x): - return False - - errorlist = [] - # convert old list format to new dict based format srcdict = {} if isinstance(srclist, list): fnfallbackused = None @@ -240,6 +224,24 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): else: errorlist.append(["srclist", "unknown apt_sources format"]) + return srcdict + +def add_sources(srclist, template_params=None, aa_repo_match=None): + """ + add entries in /etc/apt/sources.list.d for each abbreviated + sources.list entry in 'srclist'. When rendering template, also + include the values in dictionary searchList + """ + if template_params is None: + template_params = {} + + if aa_repo_match is None: + def aa_repo_match(x): + return False + + errorlist = [] + srcdict = convert_to_new_format(srclist, errorlist) + for filename in srcdict: ent = srcdict[filename] if 'filename' not in ent: @@ -257,7 +259,7 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): source = ent['source'] source = templater.render_string(source, template_params) - if not ent['filename'].startswith("/"): + if not ent['filename'].startswith(os.path.sep): ent['filename'] = os.path.join("/etc/apt/sources.list.d/", ent['filename']) -- cgit v1.2.3 From c2cfbd6831912e6dfd6b16ed21afb929592ce51a Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Tue, 24 May 2016 19:06:55 +0200 Subject: pacify pep8 regarding the new changes --- cloudinit/config/cc_apt_configure.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 02b16336..ffbf7513 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -197,6 +197,7 @@ def add_key(ent): if 'key' in ent: add_key_raw(ent['key']) + def convert_to_new_format(srclist, errorlist): """ convert_to_new_format convert the old list based format to the new dict based one @@ -226,6 +227,7 @@ def convert_to_new_format(srclist, errorlist): return srcdict + def add_sources(srclist, template_params=None, aa_repo_match=None): """ add entries in /etc/apt/sources.list.d for each abbreviated -- cgit v1.2.3 From ef47b4f4c14ffe1337508f85f365b338a048a5a1 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 24 May 2016 12:50:31 -0700 Subject: Fix load -> read --- cloudinit/net/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index ba0e39ae..776af0d3 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -63,7 +63,7 @@ def write_file(path, content): def read_file(path, decode='utf8', enoent=None): try: with open(path, "rb") as fh: - contents = fh.load() + contents = fh.read() except OSError as e: if e.errno == errno.ENOENT and enoent is not None: return enoent -- cgit v1.2.3 From 85a53d66ad0241b2d6453d902487bb2edc1512b8 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 24 May 2016 13:48:50 -0700 Subject: Fix up some of the net usage and restore imports and add a mini compat module --- cloudinit/net/__init__.py | 5 +++-- cloudinit/net/cmdline.py | 15 +++++-------- cloudinit/net/compat.py | 51 ++++++++++++++++++++++++++++++++++++++++++ cloudinit/net/eni.py | 15 ++++++------- cloudinit/net/network_state.py | 23 ++++++++++--------- 5 files changed, 79 insertions(+), 30 deletions(-) create mode 100644 cloudinit/net/compat.py (limited to 'cloudinit') diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 776af0d3..f8df58f0 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -20,7 +20,8 @@ import errno import logging import os -import six +from .import compat + import yaml LOG = logging.getLogger(__name__) @@ -55,7 +56,7 @@ def write_file(path, content): if not os.path.isdir(base_path): os.makedirs(base_path) with open(path, "wb+") as fh: - if isinstance(content, six.text_type): + if isinstance(content, compat.text_type): content = content.encode("utf8") fh.write(content) diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py index b85d4b0a..41cba893 100644 --- a/cloudinit/net/cmdline.py +++ b/cloudinit/net/cmdline.py @@ -21,20 +21,17 @@ import glob import gzip import io import shlex -import sys -import six - -from cloudinit.net import get_devicelist -from cloudinit.net import sys_netdev_info +from . import compat +from . import get_devicelist +from . import read_file +from . import sys_netdev_info from cloudinit import util -PY26 = sys.version_info[0:2] == (2, 6) - def _shlex_split(blob): - if PY26 and isinstance(blob, six.text_type): + if compat.PY26 and isinstance(blob, compat.text_type): # Older versions don't support unicode input blob = blob.encode("utf8") return shlex.split(blob) @@ -143,7 +140,7 @@ def config_from_klibc_net_cfg(files=None, mac_addrs=None): entries = [] names = {} for cfg_file in files: - name, entry = _klibc_to_config_entry(util.load_file(cfg_file), + name, entry = _klibc_to_config_entry(read_file(cfg_file), mac_addrs=mac_addrs) if name in names: raise ValueError( diff --git a/cloudinit/net/compat.py b/cloudinit/net/compat.py new file mode 100644 index 00000000..8bf92ef5 --- /dev/null +++ b/cloudinit/net/compat.py @@ -0,0 +1,51 @@ +# Curtin is free software: you can redistribute it and/or modify it under +# the terms of the GNU Affero General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# Curtin is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for +# more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Curtin. If not, see . + +# Mini six like module (so that this code can be more easily extracted). + +import sys + +PY26 = sys.version_info[0:2] == (2, 6) +PY27 = sys.version_info[0:2] == (2, 7) +PY2 = PY26 or PY27 +PY3 = sys.version_info[0:2] >= (3, 0) + +if PY3: + text_type = str + binary_type = bytes + string_types = (text_type, text_type) + import io + StringIO = io.StringIO +else: + text_type = unicode + binary_type = bytes + string_types = (binary_type, text_type) + from StringIO import StringIO # noqa + + +# Taken from six (remove when we can actually directly use six) + +def add_metaclass(metaclass): + """Class decorator for creating a class with a metaclass.""" + def wrapper(cls): + orig_vars = cls.__dict__.copy() + slots = orig_vars.get('__slots__') + if slots is not None: + if isinstance(slots, str): + slots = [slots] + for slots_var in slots: + orig_vars.pop(slots_var) + orig_vars.pop('__dict__', None) + orig_vars.pop('__weakref__', None) + return metaclass(cls.__name__, cls.__bases__, orig_vars) + return wrapper diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index 18bae97a..f82c7f54 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -16,11 +16,11 @@ import glob import os import re -from cloudinit import net +from . import LINKS_FNAME_PREFIX +from . import ParserError +from . import write_file -from cloudinit.net import LINKS_FNAME_PREFIX -from cloudinit.net import ParserError -from cloudinit.net.udev import generate_udev_rule +from .udev import generate_udev_rule NET_CONFIG_COMMANDS = [ @@ -365,12 +365,11 @@ class Renderer(object): netrules='etc/udev/rules.d/70-persistent-net.rules'): fpeni = os.path.join(target, eni) - net.write_file(fpeni, self._render_interfaces(network_state)) + write_file(fpeni, self._render_interfaces(network_state)) if netrules: netrules = os.path.join(target, netrules) - net.write_file(netrules, - self._render_persistent_net(network_state)) + write_file(netrules, self._render_persistent_net(network_state)) if links_prefix: self._render_systemd_links(target, network_state, links_prefix) @@ -393,4 +392,4 @@ class Renderer(object): "Name=" + iface['name'], "" ]) - net.write_file(fname, content) + write_file(fname, content) diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index c5aeadb5..32c48229 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -16,11 +16,12 @@ # along with Curtin. If not, see . import copy +import functools import logging -import six - -from cloudinit import net +from . import compat +from . import dump_yaml +from . import read_yaml_file LOG = logging.getLogger(__name__) @@ -48,7 +49,7 @@ def parse_net_config(path, skip_broken=True): """Parses a curtin network configuration file and return network state""" ns = None - net_config = net.read_yaml_file(path) + net_config = read_yaml_file(path) if 'network' in net_config: ns = parse_net_config_data(net_config.get('network'), skip_broken=skip_broken) @@ -57,7 +58,7 @@ def parse_net_config(path, skip_broken=True): def from_state_file(state_file): network_state = None - state = net.read_yaml_file(state_file) + state = read_yaml_file(state_file) network_state = NetworkState() network_state.load(state) return network_state @@ -78,7 +79,7 @@ def ensure_command_keys(required_keys): def wrapper(func): - @six.wraps(func) + @functools.wraps(func) def decorator(self, command, *args, **kwargs): if required_keys: missing_keys = diff_keys(required_keys, command) @@ -102,8 +103,8 @@ class CommandHandlerMeta(type): """ def __new__(cls, name, parents, dct): command_handlers = {} - for attr_name, attr in six.iteritems(dct): - if six.callable(attr) and attr_name.startswith('handle_'): + for attr_name, attr in dct.items(): + if callable(attr) and attr_name.startswith('handle_'): handles_what = attr_name[len('handle_'):] if handles_what: command_handlers[handles_what] = attr @@ -112,7 +113,7 @@ class CommandHandlerMeta(type): parents, dct) -@six.add_metaclass(CommandHandlerMeta) +@compat.add_metaclass(CommandHandlerMeta) class NetworkState(object): initial_network_state = { @@ -135,7 +136,7 @@ class NetworkState(object): 'config': self.config, 'network_state': self.network_state, } - return net.dump_yaml(state) + return dump_yaml(state) def load(self, state): if 'version' not in state: @@ -154,7 +155,7 @@ class NetworkState(object): setattr(self, key, state[key]) def dump_network_state(self): - return net.dump_yaml(self.network_state) + return dump_yaml(self.network_state) def parse_config(self, skip_broken=True): # rebuild network state -- cgit v1.2.3 From 2ec72c91b7ba60b260ba0a50097df16f82960dd8 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 24 May 2016 19:27:08 -0400 Subject: fix logic error in ec2 get_instance_userdata and slow tests The change to get_instance_userdata is to fix an issue that was causing retry in the test when it was not desired. if user_data returned 404 it means "there was no user-data", so dont bother retrying. However, _skip_retry_on_codes was returning False indicating that readurl should retry. test_merging was creating 2500 random tests, shrink that down to 100. test_seed_runs is still on my system the slowest test, but taking < .5 seconds where it was taking > 3. --- cloudinit/ec2_utils.py | 7 ++++--- tests/unittests/test_merging.py | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index 37b92a83..90b34eff 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -144,9 +144,10 @@ def _skip_retry_on_codes(status_codes, _request_args, cause): """Returns if a request should retry based on a given set of codes that case retrying to be stopped/skipped. """ - if cause.code in status_codes: - return False - return True + print("status_codes=%s" % status_codes) + print("_request_args=%s" % _request_args) + print("cause=%s" % cause) + return cause.code in status_codes def get_instance_userdata(api_version='latest', diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py index 681f3780..a33ec184 100644 --- a/tests/unittests/test_merging.py +++ b/tests/unittests/test_merging.py @@ -125,9 +125,9 @@ class TestSimpleRun(helpers.ResourceUsingTestCase): def test_seed_runs(self): test_dicts = [] - for i in range(1, 50): + for i in range(1, 10): base_dicts = [] - for j in range(1, 50): + for j in range(1, 10): base_dicts.append(make_dict(5, i * j)) test_dicts.append(base_dicts) for test in test_dicts: -- cgit v1.2.3 From db54fca319954d72f3fd48eabd27aad8be31e7b3 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 25 May 2016 09:08:29 -0400 Subject: remove debug print statements --- cloudinit/ec2_utils.py | 3 --- 1 file changed, 3 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index 90b34eff..76dda042 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -144,9 +144,6 @@ def _skip_retry_on_codes(status_codes, _request_args, cause): """Returns if a request should retry based on a given set of codes that case retrying to be stopped/skipped. """ - print("status_codes=%s" % status_codes) - print("_request_args=%s" % _request_args) - print("cause=%s" % cause) return cause.code in status_codes -- cgit v1.2.3 From 7f2e99f5345c227d07849da68acdf8562b44c3e1 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 25 May 2016 17:05:09 -0400 Subject: commit to push for fear of loss. == background == DataSource Mode (dsmode) is present in many datasources in cloud-init. dsmode was originally added to cloud-init to specify when this datasource should be 'realized'. cloud-init has 4 stages of boot. a.) cloud-init --local . network is guaranteed not present. b.) cloud-init (--network). network is guaranteed present. c.) cloud-config d.) cloud-init final 'init_modules' [1] are run "as early as possible". And as such, are executed in either 'a' or 'b' based on the datasource. However, executing them means that user-data has been fully consumed. User-data and vendor-data may have '#include http://...' which then rely on the network being present. boothooks are an example of the things run in init_modules. The 'dsmode' was a way for a user to indicate that init_modules should run at 'a' (dsmode=local) or 'b' (dsmode=net) directly. Things were further confused when a datasource could provide networking configuration. Then, we needed to apply the networking config at 'a' but if the user had provided boothooks that expected networking, then the init_modules would need to be executed at 'b'. The config drive datasource hacked its way through this and applies networking if *it* detects it is a new instance. == Suggested Change == The plan is to 1. incorporate 'dsmode' into DataSource superclass 2. make all existing datasources default to network 3. apply any networking configuration from a datasource on first boot only apply_networking will always rename network devices when it runs. for bug 1579130. 4. run init_modules at cloud-init (network) time frame unless datasource is 'local'. 5. Datasources can provide a 'first_boot' method that will be called when a new instance_id is found. This will allow the config drive's write_files to be applied once. Over all, this will very much simplify things. We'll no longer have 2 sources like DataSourceNoCloud and DataSourceNoCloudNet, but would just have one source with a dsmode. == Concerns == Some things have odd reliance on dsmode. For example, OpenNebula's get_hostname uses it to determine if it should do a lookup of an ip address. == Bugs to fix here == http://pad.lv/1577982 ConfigDrive: cloud-init fails to configure network from network_data.json http://pad.lv/1579130 need to support systemd.link renaming of devices in container http://pad.lv/1577844 Drop unnecessary blocking of all net udev rules --- bin/cloud-init | 20 ++++--- cloudinit/distros/__init__.py | 2 + cloudinit/helpers.py | 23 ++++---- cloudinit/net/__init__.py | 45 +++++++++++++++ cloudinit/sources/DataSourceCloudSigma.py | 18 +----- cloudinit/sources/DataSourceConfigDrive.py | 90 ++++++++---------------------- cloudinit/sources/DataSourceNoCloud.py | 78 +++++++++++--------------- cloudinit/sources/DataSourceOpenNebula.py | 39 ++----------- cloudinit/sources/DataSourceOpenStack.py | 9 +-- cloudinit/sources/__init__.py | 33 +++++++++++ cloudinit/stages.py | 66 ++++++++++++++++------ tox.ini | 3 +- 12 files changed, 223 insertions(+), 203 deletions(-) (limited to 'cloudinit') diff --git a/bin/cloud-init b/bin/cloud-init index 5857af32..482b8402 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -236,6 +236,7 @@ def main_init(name, args): else: LOG.debug("Execution continuing, no previous run detected that" " would allow us to stop early.") + else: existing = "check" if util.get_cfg_option_bool(init.cfg, 'manual_cache_clean', False): @@ -265,17 +266,20 @@ def main_init(name, args): else: return (None, ["No instance datasource found."]) - if args.local: - if not init.ds_restored: - # if local mode and the datasource was not restored from cache - # (this is not first boot) then apply networking. - init.apply_network_config() - else: - LOG.debug("skipping networking config from restored datasource.") - # Stage 6 iid = init.instancify() LOG.debug("%s will now be targeting instance id: %s", name, iid) + + if init.is_new_instance(): + # on new instance, apply network config. if not in local mode, + # then we just bring up new networking as the OS has already + # brought up the configured networking. + init.apply_network_config(bringup=not args.local) + + if args.local and init.datasource.dsmode != sources.DSMODE_LOCAL: + return (init.datasource, []) + + # update fully realizes user-data (pulling in #include if necessary) init.update() # Stage 7 try: diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 0f222c8c..3bfbc484 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -128,6 +128,8 @@ class Distro(object): mirror_info=arch_info) def apply_network(self, settings, bring_up=True): + # this applies network where 'settings' is interfaces(5) style + # it is obsolete compared to apply_network_config # Write it out dev_names = self._write_network(settings) # Now try to bring them up diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index 09d75e65..abfb0cbb 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -328,6 +328,7 @@ class Paths(object): self.cfgs = path_cfgs # Populate all the initial paths self.cloud_dir = path_cfgs.get('cloud_dir', '/var/lib/cloud') + self.run_dir = path_cfgs.get('run_dir', '/run/cloud-init') self.instance_link = os.path.join(self.cloud_dir, 'instance') self.boot_finished = os.path.join(self.instance_link, "boot-finished") self.upstart_conf_d = path_cfgs.get('upstart_dir') @@ -349,26 +350,19 @@ class Paths(object): "data": "data", "vendordata_raw": "vendor-data.txt", "vendordata": "vendor-data.txt.i", + "instance_id": "instance-id", } # Set when a datasource becomes active self.datasource = ds # get_ipath_cur: get the current instance path for an item def get_ipath_cur(self, name=None): - ipath = self.instance_link - add_on = self.lookups.get(name) - if add_on: - ipath = os.path.join(ipath, add_on) - return ipath + return self._get_path(self.instance_link, name) # get_cpath : get the "clouddir" (/var/lib/cloud/) # for a name in dirmap def get_cpath(self, name=None): - cpath = self.cloud_dir - add_on = self.lookups.get(name) - if add_on: - cpath = os.path.join(cpath, add_on) - return cpath + return self._get_path(self.cloud_dir, name) # _get_ipath : get the instance path for a name in pathmap # (/var/lib/cloud/instances//) @@ -397,6 +391,15 @@ class Paths(object): else: return ipath + def _get_path(self, base, name=None): + add_on = self.lookups.get(name) + if not add_on: + return base + return os.path.join(base, add_on) + + def get_runpath(self, name=None): + return self._get_path(self.run_dir, name) + # This config parser will not throw when sections don't exist # and you are setting values on those sections which is useful diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 91e36aca..40d330b5 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -768,4 +768,49 @@ def read_kernel_cmdline_config(files=None, mac_addrs=None, cmdline=None): return config_from_klibc_net_cfg(files=files, mac_addrs=mac_addrs) +def convert_eni_data(eni_data): + # return a network config representation of what is in eni_data + ifaces = {} + parse_deb_config_data(ifaces, eni_data, src_dir=None, src_path=None) + return _ifaces_to_net_config_data(ifaces) + + +def _ifaces_to_net_config_data(ifaces): + """Return network config that represents the ifaces data provided. + ifaces = parse_deb_config("/etc/network/interfaces") + config = ifaces_to_net_config_data(ifaces) + state = parse_net_config_data(config).""" + devs = {} + for name, data in ifaces.items(): + # devname is 'eth0' for name='eth0:1' + devname = name.partition(":")[0] + if devname not in devs: + devs[devname] = {'type': 'physical', 'name': devname, + 'subnets': []} + # this isnt strictly correct, but some might specify + # hwaddress on a nic for matching / declaring name. + if 'hwaddress' in data: + devs[devname]['mac_address'] = data['hwaddress'] + subnet = {'_orig_eni_name': name, 'type': data['method']} + if data.get('auto'): + subnet['control'] = 'auto' + else: + subnet['control'] = 'manual' + + if data.get('method') == 'static': + subnet['address'] = data['address'] + + if 'gateway' in data: + subnet['gateway'] = data['gateway'] + + if 'dns' in data: + for n in ('nameservers', 'search'): + if n in data['dns'] and data['dns'][n]: + subnet['dns_' + n] = data['dns'][n] + devs[devname]['subnets'].append(subnet) + + return {'version': 1, + 'config': [devs[d] for d in sorted(devs)]} + + # vi: ts=4 expandtab syntax=python diff --git a/cloudinit/sources/DataSourceCloudSigma.py b/cloudinit/sources/DataSourceCloudSigma.py index 33fe78b9..07e8ae11 100644 --- a/cloudinit/sources/DataSourceCloudSigma.py +++ b/cloudinit/sources/DataSourceCloudSigma.py @@ -27,8 +27,6 @@ from cloudinit import util LOG = logging.getLogger(__name__) -VALID_DSMODES = ("local", "net", "disabled") - class DataSourceCloudSigma(sources.DataSource): """ @@ -38,7 +36,6 @@ class DataSourceCloudSigma(sources.DataSource): http://cloudsigma-docs.readthedocs.org/en/latest/server_context.html """ def __init__(self, sys_cfg, distro, paths): - self.dsmode = 'local' self.cepko = Cepko() self.ssh_public_key = '' sources.DataSource.__init__(self, sys_cfg, distro, paths) @@ -84,11 +81,9 @@ class DataSourceCloudSigma(sources.DataSource): LOG.debug("CloudSigma: Unable to read from serial port") return False - dsmode = server_meta.get('cloudinit-dsmode', self.dsmode) - if dsmode not in VALID_DSMODES: - LOG.warn("Invalid dsmode %s, assuming default of 'net'", dsmode) - dsmode = 'net' - if dsmode == "disabled" or dsmode != self.dsmode: + self.dsmode = self._determine_dsmode( + [server_meta.get('cloudinit-dsmode')]) + if dsmode == sources.DSMODE_DISABLED: return False base64_fields = server_meta.get('base64_fields', '').split(',') @@ -120,17 +115,10 @@ class DataSourceCloudSigma(sources.DataSource): return self.metadata['uuid'] -class DataSourceCloudSigmaNet(DataSourceCloudSigma): - def __init__(self, sys_cfg, distro, paths): - DataSourceCloudSigma.__init__(self, sys_cfg, distro, paths) - self.dsmode = 'net' - - # Used to match classes to dependencies. Since this datasource uses the serial # port network is not really required, so it's okay to load without it, too. datasources = [ (DataSourceCloudSigma, (sources.DEP_FILESYSTEM)), - (DataSourceCloudSigmaNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), ] diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 52a9f543..20df5fcd 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -22,6 +22,7 @@ import copy import os from cloudinit import log as logging +from cloudinit import net from cloudinit import sources from cloudinit import util @@ -35,7 +36,6 @@ DEFAULT_MODE = 'pass' DEFAULT_METADATA = { "instance-id": DEFAULT_IID, } -VALID_DSMODES = ("local", "net", "pass", "disabled") FS_TYPES = ('vfat', 'iso9660') LABEL_TYPES = ('config-2',) POSSIBLE_MOUNTS = ('sr', 'cd') @@ -47,12 +47,12 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): def __init__(self, sys_cfg, distro, paths): super(DataSourceConfigDrive, self).__init__(sys_cfg, distro, paths) self.source = None - self.dsmode = 'local' self.seed_dir = os.path.join(paths.seed_dir, 'config_drive') self.version = None self.ec2_metadata = None self._network_config = None self.network_json = None + self.network_eni = None self.files = {} def __str__(self): @@ -98,38 +98,22 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): md = results.get('metadata', {}) md = util.mergemanydict([md, DEFAULT_METADATA]) - user_dsmode = results.get('dsmode', None) - if user_dsmode not in VALID_DSMODES + (None,): - LOG.warn("User specified invalid mode: %s", user_dsmode) - user_dsmode = None - dsmode = get_ds_mode(cfgdrv_ver=results['version'], - ds_cfg=self.ds_cfg.get('dsmode'), - user=user_dsmode) + self.dsmode = self._determine_dsmode( + [results.get('dsmode'), self.ds_cfg.get('dsmode'), + sources.DSMODE_PASS if results['version'] == 1 else None]) - if dsmode == "disabled": - # most likely user specified + if self.dsmode == sources.DSMODE_DISABLED: return False - # TODO(smoser): fix this, its dirty. - # we want to do some things (writing files and network config) - # only on first boot, and even then, we want to do so in the - # local datasource (so they happen earlier) even if the configured - # dsmode is 'net' or 'pass'. To do this, we check the previous - # instance-id + # This is legacy and sneaky. If dsmode is 'pass' then write + # 'injected files' and apply legacy ENI network format. prev_iid = get_previous_iid(self.paths) cur_iid = md['instance-id'] - if prev_iid != cur_iid and self.dsmode == "local": + if prev_iid != cur_iid and self.dsmode == sources.DSMODE_PASS: on_first_boot(results, distro=self.distro) - - # dsmode != self.dsmode here if: - # * dsmode = "pass", pass means it should only copy files and then - # pass to another datasource - # * dsmode = "net" and self.dsmode = "local" - # so that user boothooks would be applied with network, the - # local datasource just gets out of the way, and lets the net claim - if dsmode != self.dsmode: - LOG.debug("%s: not claiming datasource, dsmode=%s", self, dsmode) + LOG.debug("%s: not claiming datasource, dsmode=%s", self, + self.dsmode) return False self.source = found @@ -147,12 +131,11 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): LOG.warn("Invalid content in vendor-data: %s", e) self.vendordata_raw = None - try: - self.network_json = results.get('networkdata') - except ValueError as e: - LOG.warn("Invalid content in network-data: %s", e) - self.network_json = None - + # network_config is an /etc/network/interfaces formated file and is + # obsolete compared to networkdata (from network_data.json) but both + # might be present. + self.network_eni = results.get("network_config") + self.network_json = results.get('networkdata') return True def check_instance_id(self, sys_cfg): @@ -164,40 +147,11 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): if self._network_config is None: if self.network_json is not None: self._network_config = convert_network_data(self.network_json) + elif self.network_eni is not None: + self._network_config = net.convert_eni_data(self.network_eni) return self._network_config -class DataSourceConfigDriveNet(DataSourceConfigDrive): - def __init__(self, sys_cfg, distro, paths): - DataSourceConfigDrive.__init__(self, sys_cfg, distro, paths) - self.dsmode = 'net' - - -def get_ds_mode(cfgdrv_ver, ds_cfg=None, user=None): - """Determine what mode should be used. - valid values are 'pass', 'disabled', 'local', 'net' - """ - # user passed data trumps everything - if user is not None: - return user - - if ds_cfg is not None: - return ds_cfg - - # at config-drive version 1, the default behavior was pass. That - # meant to not use use it as primary data source, but expect a ec2 metadata - # source. for version 2, we default to 'net', which means - # the DataSourceConfigDriveNet, would be used. - # - # this could change in the future. If there was definitive metadata - # that indicated presense of an openstack metadata service, then - # we could change to 'pass' by default also. The motivation for that - # would be 'cloud-init query' as the web service could be more dynamic - if cfgdrv_ver == 1: - return "pass" - return "net" - - def read_config_drive(source_dir): reader = openstack.ConfigDriveReader(source_dir) finders = [ @@ -231,9 +185,12 @@ def on_first_boot(data, distro=None): % (type(data))) net_conf = data.get("network_config", '') if net_conf and distro: - LOG.debug("Updating network interfaces from config drive") + LOG.warn("Updating network interfaces from config drive") distro.apply_network(net_conf) - files = data.get('files', {}) + write_injected_files(data.get('files')) + + +def write_injected_files(files): if files: LOG.debug("Writing %s injected files", len(files)) for (filename, content) in files.items(): @@ -296,7 +253,6 @@ def find_candidate_devs(probe_optical=True): # Used to match classes to dependencies datasources = [ (DataSourceConfigDrive, (sources.DEP_FILESYSTEM, )), - (DataSourceConfigDriveNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), ] diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 48c61a90..7e30118c 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -24,6 +24,7 @@ import errno import os from cloudinit import log as logging +from cloudinit import net from cloudinit import sources from cloudinit import util @@ -35,7 +36,6 @@ class DataSourceNoCloud(sources.DataSource): sources.DataSource.__init__(self, sys_cfg, distro, paths) self.dsmode = 'local' self.seed = None - self.cmdline_id = "ds=nocloud" self.seed_dirs = [os.path.join(paths.seed_dir, 'nocloud'), os.path.join(paths.seed_dir, 'nocloud-net')] self.seed_dir = None @@ -58,7 +58,7 @@ class DataSourceNoCloud(sources.DataSource): try: # Parse the kernel command line, getting data passed in md = {} - if parse_cmdline_data(self.cmdline_id, md): + if load_cmdline_data(md): found.append("cmdline") mydata = _merge_new_seed(mydata, {'meta-data': md}) except Exception: @@ -123,12 +123,6 @@ class DataSourceNoCloud(sources.DataSource): mydata = _merge_new_seed(mydata, seeded) - # For seed from a device, the default mode is 'net'. - # that is more likely to be what is desired. If they want - # dsmode of local, then they must specify that. - if 'dsmode' not in mydata['meta-data']: - mydata['meta-data']['dsmode'] = "net" - LOG.debug("Using data from %s", dev) found.append(dev) break @@ -144,7 +138,6 @@ class DataSourceNoCloud(sources.DataSource): if len(found) == 0: return False - seeded_network = None # The special argument "seedfrom" indicates we should # attempt to seed the userdata / metadata from its value # its primarily value is in allowing the user to type less @@ -160,10 +153,6 @@ class DataSourceNoCloud(sources.DataSource): LOG.debug("Seed from %s not supported by %s", seedfrom, self) return False - if (mydata['meta-data'].get('network-interfaces') or - mydata.get('network-config')): - seeded_network = self.dsmode - # This could throw errors, but the user told us to do it # so if errors are raised, let them raise (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) @@ -179,35 +168,21 @@ class DataSourceNoCloud(sources.DataSource): mydata['meta-data'] = util.mergemanydict([mydata['meta-data'], defaults]) - netdata = {'format': None, 'data': None} - if mydata['meta-data'].get('network-interfaces'): - netdata['format'] = 'interfaces' - netdata['data'] = mydata['meta-data']['network-interfaces'] - elif mydata.get('network-config'): - netdata['format'] = 'network-config' - netdata['data'] = mydata['network-config'] - - # if this is the local datasource or 'seedfrom' was used - # and the source of the seed was self.dsmode. - # Then see if there is network config to apply. - # note this is obsolete network-interfaces style seeding. - if self.dsmode in ("local", seeded_network): - if mydata['meta-data'].get('network-interfaces'): - LOG.debug("Updating network interfaces from %s", self) - self.distro.apply_network( - mydata['meta-data']['network-interfaces']) - - if mydata['meta-data']['dsmode'] == self.dsmode: - self.seed = ",".join(found) - self.metadata = mydata['meta-data'] - self.userdata_raw = mydata['user-data'] - self.vendordata_raw = mydata['vendor-data'] - self._network_config = mydata['network-config'] - return True + self.dsmode = self._determine_dsmode( + [mydata['meta-data'].get('dsmode')]) - LOG.debug("%s: not claiming datasource, dsmode=%s", self, - mydata['meta-data']['dsmode']) - return False + if self.dsmode == sources.DSMODE_DISABLED: + LOG.debug("%s: not claiming datasource, dsmode=%s", self, + self.dsmode) + return False + + self.seed = ",".join(found) + self.metadata = mydata['meta-data'] + self.userdata_raw = mydata['user-data'] + self.vendordata_raw = mydata['vendor-data'] + self._network_config = mydata['network-config'] + self._network_eni = mydata['meta-data'].get('network-interfaces') + return True def check_instance_id(self, sys_cfg): # quickly (local check only) if self.instance_id is still valid @@ -227,6 +202,9 @@ class DataSourceNoCloud(sources.DataSource): @property def network_config(self): + if self._network_config is None: + if self.network_eni is not None: + self._network_config = net.convert_eni_data(self.network_eni) return self._network_config @@ -254,8 +232,22 @@ def _quick_read_instance_id(cmdline_id, dirs=None): return None +def load_cmdline_data(fill, cmdline=None): + pairs = [("ds=nocloud", sources.DSMODE_LOCAL), + ("ds=nocloud-net", sources.DSMODE_NETWORK)] + for idstr, dsmode in pairs: + if parse_cmdline_data(idstr, fill, cmdline): + # if dsmode was explicitly in the commanad line, then + # prefer it to the dsmode based on the command line id + if 'dsmode' not in fill: + fill['dsmode'] = dsmode + return True + return False + + # Returns true or false indicating if cmdline indicated -# that this module should be used +# that this module should be used. Updates dictionary 'fill' +# with data that was found. # Example cmdline: # root=LABEL=uec-rootfs ro ds=nocloud def parse_cmdline_data(ds_id, fill, cmdline=None): @@ -319,9 +311,7 @@ def _merge_new_seed(cur, seeded): class DataSourceNoCloudNet(DataSourceNoCloud): def __init__(self, sys_cfg, distro, paths): DataSourceNoCloud.__init__(self, sys_cfg, distro, paths) - self.cmdline_id = "ds=nocloud-net" self.supported_seed_starts = ("http://", "https://", "ftp://") - self.dsmode = "net" # Used to match classes to dependencies diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 681f3a96..15819a4f 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -37,16 +37,13 @@ from cloudinit import util LOG = logging.getLogger(__name__) DEFAULT_IID = "iid-dsopennebula" -DEFAULT_MODE = 'net' DEFAULT_PARSEUSER = 'nobody' CONTEXT_DISK_FILES = ["context.sh"] -VALID_DSMODES = ("local", "net", "disabled") class DataSourceOpenNebula(sources.DataSource): def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) - self.dsmode = 'local' self.seed = None self.seed_dir = os.path.join(paths.seed_dir, 'opennebula') @@ -93,52 +90,27 @@ class DataSourceOpenNebula(sources.DataSource): md = util.mergemanydict([md, defaults]) # check for valid user specified dsmode - user_dsmode = results['metadata'].get('DSMODE', None) - if user_dsmode not in VALID_DSMODES + (None,): - LOG.warn("user specified invalid mode: %s", user_dsmode) - user_dsmode = None - - # decide dsmode - if user_dsmode: - dsmode = user_dsmode - elif self.ds_cfg.get('dsmode'): - dsmode = self.ds_cfg.get('dsmode') - else: - dsmode = DEFAULT_MODE - - if dsmode == "disabled": - # most likely user specified - return False - - # apply static network configuration only in 'local' dsmode - if ('network-interfaces' in results and self.dsmode == "local"): - LOG.debug("Updating network interfaces from %s", self) - self.distro.apply_network(results['network-interfaces']) + self.dsmode = self._determine_dsmode( + [results.get('DSMODE'), self.ds_cfg.get('dsmode')]) - if dsmode != self.dsmode: - LOG.debug("%s: not claiming datasource, dsmode=%s", self, dsmode) + if self.dsmode == sources.DSMODE_DISABLED: return False self.seed = seed + self.network_eni = results.get("network_config") self.metadata = md self.userdata_raw = results.get('userdata') return True def get_hostname(self, fqdn=False, resolve_ip=None): if resolve_ip is None: - if self.dsmode == 'net': + if self.dsmode == sources.DSMODE_NET: resolve_ip = True else: resolve_ip = False return sources.DataSource.get_hostname(self, fqdn, resolve_ip) -class DataSourceOpenNebulaNet(DataSourceOpenNebula): - def __init__(self, sys_cfg, distro, paths): - DataSourceOpenNebula.__init__(self, sys_cfg, distro, paths) - self.dsmode = 'net' - - class NonContextDiskDir(Exception): pass @@ -446,7 +418,6 @@ def read_context_disk_dir(source_dir, asuser=None): # Used to match classes to dependencies datasources = [ (DataSourceOpenNebula, (sources.DEP_FILESYSTEM, )), - (DataSourceOpenNebulaNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), ] diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index dfd96035..c06d17f3 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -33,13 +33,11 @@ DEFAULT_IID = "iid-dsopenstack" DEFAULT_METADATA = { "instance-id": DEFAULT_IID, } -VALID_DSMODES = ("net", "disabled") class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): def __init__(self, sys_cfg, distro, paths): super(DataSourceOpenStack, self).__init__(sys_cfg, distro, paths) - self.dsmode = 'net' self.metadata_address = None self.ssl_details = util.fetch_ssl_details(self.paths) self.version = None @@ -125,11 +123,8 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): self.metadata_address) return False - user_dsmode = results.get('dsmode', None) - if user_dsmode not in VALID_DSMODES + (None,): - LOG.warn("User specified invalid mode: %s", user_dsmode) - user_dsmode = None - if user_dsmode == 'disabled': + self.dsmode = self._determine_dsmode([results.get('dsmode')]) + if self.dsmode == sources.DSMODE_DISABLED: return False md = results.get('metadata', {}) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 43e4fd57..e0171e8c 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -34,6 +34,13 @@ from cloudinit import util from cloudinit.filters import launch_index from cloudinit.reporting import events +DSMODE_DISABLED = "disabled" +DSMODE_LOCAL = "net" +DSMODE_NETWORK = "local" +DSMODE_PASS = "pass" + +VALID_DSMODES = [DSMODE_DISABLED, DSMODE_LOCAL, DSMODE_NETWORK] + DEP_FILESYSTEM = "FILESYSTEM" DEP_NETWORK = "NETWORK" DS_PREFIX = 'DataSource' @@ -57,6 +64,7 @@ class DataSource(object): self.userdata_raw = None self.vendordata = None self.vendordata_raw = None + self.dsmode = DSMODE_NETWORK # find the datasource config name. # remove 'DataSource' from classname on front, and remove 'Net' on end. @@ -223,10 +231,35 @@ class DataSource(object): # quickly (local check only) if self.instance_id is still return False + @staticmethod + def _determine_dsmode(candidates, default=None, valid=None): + # return the first candidate that is non None, warn if not valid + if default is None: + default = DSMODE_NETWORK + + if valid is None: + valid = VALID_DSMODES + + for candidate in candidates: + if candidate is None: + continue + if candidate in valid: + return candidate + else: + LOG.warn("invalid dsmode '%s', using default=%s", + candidate, default) + return default + + return default + @property def network_config(self): return None + @property + def first_instance_boot(self): + return + def normalize_pubkey_data(pubkey_data): keys = [] diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 62d066de..53ebcb45 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -67,6 +67,7 @@ class Init(object): # Changed only when a fetch occurs self.datasource = NULL_DATA_SOURCE self.ds_restored = False + self._previous_iid = None if reporter is None: reporter = events.ReportEventStack( @@ -213,6 +214,31 @@ class Init(object): cfg_list = self.cfg.get('datasource_list') or [] return (cfg_list, pkg_list) + def _restore_from_checked_cache(self, existing): + if existing not in ("check", "trust"): + raise ValueError("Unexpected value for existing: %s" % existing) + + ds = self._restore_from_cache() + if not ds: + return (None, "no cache found") + + run_iid_fn = self.paths.get_runpath('instance-id') + if os.path.exists(run_iid_fn): + run_iid = util.load_file(run_iid_fn).strip() + else: + run_iid = None + + if run_iid == ds.get_instance_id: + return (ds, "restored from cache with run check: %s" % ds) + elif existing == "trust": + return (ds, "restored from cache: %s" % ds) + else: + if (hasattr(ds, 'check_instance_id') and + ds.check_instance_id(self.cfg)): + return (ds, "restored from checked cache: %s" % ds) + else: + return (None, "cache invalid in datasource: %s" % ds) + def _get_data_source(self, existing): if self.datasource is not NULL_DATA_SOURCE: return self.datasource @@ -221,19 +247,9 @@ class Init(object): name="check-cache", description="attempting to read from cache [%s]" % existing, parent=self.reporter) as myrep: - ds = self._restore_from_cache() - if ds and existing == "trust": - myrep.description = "restored from cache: %s" % ds - elif ds and existing == "check": - if (hasattr(ds, 'check_instance_id') and - ds.check_instance_id(self.cfg)): - myrep.description = "restored from checked cache: %s" % ds - else: - myrep.description = "cache invalid in datasource: %s" % ds - ds = None - else: - myrep.description = "no cache found" + ds, desc = self._restore_from_checked_cache(existing) + myrep.description = desc self.ds_restored = bool(ds) LOG.debug(myrep.description) @@ -301,15 +317,15 @@ class Init(object): # What the instance id was and is... iid = self.datasource.get_instance_id() - previous_iid = None iid_fn = os.path.join(dp, 'instance-id') try: previous_iid = util.load_file(iid_fn).strip() except Exception: - pass + previous_iid = None if not previous_iid: previous_iid = iid util.write_file(iid_fn, "%s\n" % iid) + util.write_file(self.paths.get_runpath('instance-id'), "%s\n" % iid) util.write_file(os.path.join(dp, 'previous-instance-id'), "%s\n" % (previous_iid)) # Ensure needed components are regenerated @@ -318,6 +334,21 @@ class Init(object): self._reset() return iid + def previous_iid(self): + if self._previous_iid is not None: + return self._previous_iid + + dp = self.paths.get_cpath('data') + iid_fn = os.path.join(dp, 'instance-id') + try: + self._previous_iid = util.load_file(iid_fn).strip() + except Exception: + pass + return self._previous_iid + + def is_new_instance(self): + return self.datasource.get_instance_id() == self.previous_iid() + def fetch(self, existing="check"): return self._get_data_source(existing=existing) @@ -593,15 +624,16 @@ class Init(object): return (ncfg, loc) return (net.generate_fallback_config(), "fallback") - def apply_network_config(self): + def apply_network_config(self, bringup): netcfg, src = self._find_networking_config() if netcfg is None: LOG.info("network config is disabled by %s", src) return - LOG.info("Applying network configuration from %s: %s", src, netcfg) + LOG.info("Applying network configuration from %s bringup=%s: %s", + src, bringup, netcfg) try: - return self.distro.apply_network_config(netcfg) + return self.distro.apply_network_config(netcfg, bringup=bringup) except NotImplementedError: LOG.warn("distro '%s' does not implement apply_network_config. " "networking may not be configured properly." % diff --git a/tox.ini b/tox.ini index dafaaf6d..18d059df 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,7 @@ [tox] envlist = py27,py3,flake8 -recreate = True +recreate = False +skip_install = True [testenv] commands = python -m nose {posargs:tests} -- cgit v1.2.3 From 443df65a45e42c354b8eb7638528970adf036290 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 25 May 2016 20:36:40 -0400 Subject: fix bring_up --- bin/cloud-init | 4 ++-- cloudinit/stages.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'cloudinit') diff --git a/bin/cloud-init b/bin/cloud-init index 482b8402..1d421acf 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -260,7 +260,7 @@ def main_init(name, args): util.logexc(LOG, ("No instance datasource found!" " Likely bad things to come!")) if not args.force: - init.apply_network_config() + init.apply_network_config(bring_up=not args.local) if args.local: return (None, []) else: @@ -274,7 +274,7 @@ def main_init(name, args): # on new instance, apply network config. if not in local mode, # then we just bring up new networking as the OS has already # brought up the configured networking. - init.apply_network_config(bringup=not args.local) + init.apply_network_config(bring_up=not args.local) if args.local and init.datasource.dsmode != sources.DSMODE_LOCAL: return (init.datasource, []) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 53ebcb45..35eddc3b 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -624,16 +624,16 @@ class Init(object): return (ncfg, loc) return (net.generate_fallback_config(), "fallback") - def apply_network_config(self, bringup): + def apply_network_config(self, bring_up): netcfg, src = self._find_networking_config() if netcfg is None: LOG.info("network config is disabled by %s", src) return LOG.info("Applying network configuration from %s bringup=%s: %s", - src, bringup, netcfg) + src, bring_up, netcfg) try: - return self.distro.apply_network_config(netcfg, bringup=bringup) + return self.distro.apply_network_config(netcfg, bring_up=bring_up) except NotImplementedError: LOG.warn("distro '%s' does not implement apply_network_config. " "networking may not be configured properly." % -- cgit v1.2.3 From 63501f44eff7ef2d6083900c47180faf444662fc Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 May 2016 09:02:17 -0400 Subject: kernel command line: override all local settings settings on the kernel command line (cc:) were documented to override all local settings, but a bug in implementation meant they would only override those that are in /etc/cloud/cloud.cfg, not any found in /etc/cloud/cloud.cfg.d. LP: #1582323 --- ChangeLog | 2 ++ cloudinit/stages.py | 10 +++++----- 2 files changed, 7 insertions(+), 5 deletions(-) (limited to 'cloudinit') diff --git a/ChangeLog b/ChangeLog index 6748e8fa..8db29e2e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -110,6 +110,8 @@ - Paths: fix instance path if datasource's id has a '/'. (LP: #1575938) [Robert Jennings] - Ec2: do not retry requests for user-data path on 404. + - settings on the kernel command line (cc:) override all local settings + rather than only those in /etc/cloud/cloud.cfg (LP: #1582323) 0.7.6: - open 0.7.6 diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 62d066de..002e5832 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -794,16 +794,16 @@ class Modules(object): def fetch_base_config(): base_cfgs = [] default_cfg = util.get_builtin_cfg() - kern_contents = util.read_cc_from_cmdline() - - # Kernel/cmdline parameters override system config - if kern_contents: - base_cfgs.append(util.load_yaml(kern_contents, default={})) # Anything in your conf.d location?? # or the 'default' cloud.cfg location??? base_cfgs.append(util.read_conf_with_confd(CLOUD_CONFIG)) + # Kernel/cmdline parameters override system config + kern_contents = util.read_cc_from_cmdline() + if kern_contents: + base_cfgs.append(util.load_yaml(kern_contents, default={})) + # And finally the default gets to play if default_cfg: base_cfgs.append(default_cfg) -- cgit v1.2.3 From 399eb29662ee67f7744d3a482f63e8af377e75db Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 May 2016 11:22:39 -0400 Subject: config drive: log where network config came from --- cloudinit/sources/DataSourceConfigDrive.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 20df5fcd..2d13a32f 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -146,9 +146,13 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): def network_config(self): if self._network_config is None: if self.network_json is not None: + LOG.debug("network config provided via network_json") self._network_config = convert_network_data(self.network_json) elif self.network_eni is not None: self._network_config = net.convert_eni_data(self.network_eni) + LOG.debug("network config provided via converted eni data") + else: + LOG.debug("no network configuration available") return self._network_config -- cgit v1.2.3 From b4a298b10c26ee79ee6f21a164cf32ab767ca14f Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 May 2016 11:22:52 -0400 Subject: cloudinit/helpers.py: _get_path raise KeyError if input is bad. previously, if you did: paths.get_ipath("bogus") it would silenetly hand you back just the directory. now it will fail, which seems much more sane. --- cloudinit/helpers.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index abfb0cbb..d4acbe7e 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -392,10 +392,9 @@ class Paths(object): return ipath def _get_path(self, base, name=None): - add_on = self.lookups.get(name) - if not add_on: + if name is None: return base - return os.path.join(base, add_on) + return os.path.join(base, self.lookups[name]) def get_runpath(self, name=None): return self._get_path(self.run_dir, name) -- cgit v1.2.3 From ac851b8ff106abb4314d2b71f2a7dc194b259899 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 May 2016 11:24:28 -0400 Subject: fix usage of instance-id to instance_id in runpath, update cache on instancify the fix for instance_id is clear and necessary. making instancify write the cache is required for how we are having the local datasource be relevant. --- cloudinit/stages.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 35eddc3b..5141612f 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -222,7 +222,7 @@ class Init(object): if not ds: return (None, "no cache found") - run_iid_fn = self.paths.get_runpath('instance-id') + run_iid_fn = self.paths.get_runpath('instance_id') if os.path.exists(run_iid_fn): run_iid = util.load_file(run_iid_fn).strip() else: @@ -325,9 +325,11 @@ class Init(object): if not previous_iid: previous_iid = iid util.write_file(iid_fn, "%s\n" % iid) - util.write_file(self.paths.get_runpath('instance-id'), "%s\n" % iid) + util.write_file(self.paths.get_runpath('instance_id'), "%s\n" % iid) util.write_file(os.path.join(dp, 'previous-instance-id'), "%s\n" % (previous_iid)) + + self._write_to_cache() # Ensure needed components are regenerated # after change of instance which may cause # change of configuration @@ -363,8 +365,6 @@ class Init(object): reporter=self.reporter) def update(self): - if not self._write_to_cache(): - return self._store_userdata() self._store_vendordata() -- cgit v1.2.3 From e86794ba646863ac54dcb348c016d2ae0be2c180 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 May 2016 15:50:40 -0400 Subject: compare instance id to get_instance_id() not the function itself. --- cloudinit/stages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 90c9016c..cbf4df8b 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -228,7 +228,7 @@ class Init(object): else: run_iid = None - if run_iid == ds.get_instance_id: + if run_iid == ds.get_instance_id(): return (ds, "restored from cache with run check: %s" % ds) elif existing == "trust": return (ds, "restored from cache: %s" % ds) -- cgit v1.2.3 From 48b7526425055a3c636f11135305f1e77469562a Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 May 2016 15:50:59 -0400 Subject: fix typos in names --- cloudinit/sources/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index e0171e8c..2a6b8d90 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -35,8 +35,8 @@ from cloudinit.filters import launch_index from cloudinit.reporting import events DSMODE_DISABLED = "disabled" -DSMODE_LOCAL = "net" -DSMODE_NETWORK = "local" +DSMODE_LOCAL = "local" +DSMODE_NETWORK = "net" DSMODE_PASS = "pass" VALID_DSMODES = [DSMODE_DISABLED, DSMODE_LOCAL, DSMODE_NETWORK] -- cgit v1.2.3 From 1174e19116f2934f927d89fe00ed8d6e22983d75 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 May 2016 16:02:53 -0400 Subject: fix logic in is_new_instance --- cloudinit/stages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index cbf4df8b..e34662b1 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -349,7 +349,7 @@ class Init(object): return self._previous_iid def is_new_instance(self): - return self.datasource.get_instance_id() == self.previous_iid() + return self.datasource.get_instance_id() != self.previous_iid() def fetch(self, existing="check"): return self._get_data_source(existing=existing) -- cgit v1.2.3 From 22bd075cd77ccc1021e049333f498afd56de2c6a Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Thu, 26 May 2016 16:10:22 -0500 Subject: Add smartos sdc:nics converter and network_config property for smartos configdrive --- cloudinit/sources/DataSourceSmartOS.py | 95 ++++++++++++++++++++++++++++++++-- 1 file changed, 91 insertions(+), 4 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 46cf117a..6355fc2a 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -64,6 +64,7 @@ SMARTOS_ATTRIB_MAP = { 'availability_zone': ('sdc:datacenter_name', True), 'vendor-data': ('sdc:vendor-data', False), 'operator-script': ('sdc:operator-script', False), + 'network-data': ('sdc:nics', False), } DS_NAME = 'SmartOS' @@ -176,6 +177,8 @@ class DataSourceSmartOS(sources.DataSource): BUILTIN_DS_CONFIG]) self.metadata = {} + self.network_data = None + self._network_config = None self.script_base_d = os.path.join(self.paths.get_cpath("scripts")) @@ -195,7 +198,6 @@ class DataSourceSmartOS(sources.DataSource): serial_device=self.ds_cfg['serial_device'], serial_timeout=self.ds_cfg['serial_timeout']) - def _set_provisioned(self): '''Mark the instance provisioning state as successful. @@ -221,7 +223,7 @@ class DataSourceSmartOS(sources.DataSource): if not self.smartos_env: LOG.debug("Not running on smartos") return False - + if not self.md_client.exists(): LOG.debug("No metadata device '%r' found for SmartOS datasource", self.md_client) @@ -279,6 +281,11 @@ class DataSourceSmartOS(sources.DataSource): self.metadata = util.mergemanydict([md, self.metadata]) self.userdata_raw = ud self.vendordata_raw = md['vendor-data'] + if not md['network-data']: + smartos_noun, strip = SMARTOS_ATTRIB_MAP.get('network-data') + self.network_data = self.md_client.get_json(smartos_noun, + strip=strip) + md['network-data'] = self.network_data self._set_provisioned() return True @@ -292,6 +299,14 @@ class DataSourceSmartOS(sources.DataSource): def get_instance_id(self): return self.metadata['instance-id'] + @property + def network_config(self): + if self._network_config is None: + if self.network_data is not None: + self._network_config = ( + convert_smartos_network_data(self.network_data)) + return self._network_config + class JoyentMetadataFetchException(Exception): pass @@ -386,8 +401,8 @@ class JoyentMetadataClient(object): result = result.strip() return result - def get_json(self, key, default=None): - result = self.get(key) + def get_json(self, key, default=None, strip=False): + result = self.get(key, default=default, strip=strip) if result is None: return default return json.loads(result) @@ -654,5 +669,77 @@ def get_datasource_list(depends): return sources.list_from_depends(depends, datasources) +# Covert SMARTOS 'sdc:nics' data to network_config yaml +def convert_smartos_network_data(network_data=None): + """Return a dictionary of network_config by parsing provided + SMARTOS sdc:nics configuration data + + sdc:nics data is a dictionary of properties of a nic and the ip + configuration desired. Additional nic dictionaries are appended + to the list. + + Converting the format is straightforward though it does include + duplicate information as well as data which appears to be relevant + to the hostOS rather than the guest. + + For each entry in the nics list returned from query sdc:nics, we + create a type: physical entry, and extract the interface properties: + 'mac' -> 'mac_address', 'mtu', 'interface' -> 'name'. The remaining + keys are related to ip configuration. For each ip in the 'ips' list + we create a subnet entry under 'subnets' pairing the ip to a one in + the 'gateways' list. + """ + + valid_keys = { + 'physical': [ + 'mac_address', + 'mtu', + 'name', + 'params', + 'subnets', + 'type', + ], + 'subnet': [ + 'address', + 'broadcast', + 'dns_nameservers', + 'dns_search', + 'gateway', + 'metric', + 'netmask', + 'pointopoint', + 'routes', + 'scope', + 'type', + ], + } + + config = [] + for nic in network_data: + cfg = {k: v for k, v in nic.items() + if k in valid_keys['physical']} + cfg.update({ + 'type': 'physical', + 'name': nic['interface'] + }) + if 'mac' in nic: + cfg.update({'mac_address': nic['mac']}) + + subnets = [] + for ip, gw in zip(nic['ips'], nic['gateways']): + subnet = {k: v for k, v in nic.items() + if k in valid_keys['subnet']} + subnet.update({ + 'type': 'static', + 'address': ip, + 'gateway': gw, + }) + subnets.append(subnet) + cfg.update({'subnets': subnets}) + config.append(cfg) + + return {'version': 1, 'config': config} + + if __name__ == "__main__": jmc = JoyentMetadataClient(seed_file).get_metadata(noun) -- cgit v1.2.3 From f63f16c31be4f3b993a671e95b74550150f5715f Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 May 2016 21:53:38 -0400 Subject: hide the instance_id file in /run/cloud-init by using .instance_id i dont want to expose this as i'd rather have some json there or write to /run/cloud-init/status.json . would also like to indicate 'first_boot' somewhere. --- cloudinit/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index d4acbe7e..fb95babc 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -350,7 +350,7 @@ class Paths(object): "data": "data", "vendordata_raw": "vendor-data.txt", "vendordata": "vendor-data.txt.i", - "instance_id": "instance-id", + "instance_id": ".instance-id", } # Set when a datasource becomes active self.datasource = ds -- cgit v1.2.3 From 3ce4cebd417043e1d0a3bb387998f9b71bb76e03 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 May 2016 21:54:33 -0400 Subject: fix is_new_instance() to work better --- cloudinit/stages.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index e34662b1..20c334b7 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -52,6 +52,7 @@ from cloudinit import util LOG = logging.getLogger(__name__) NULL_DATA_SOURCE = None +NO_PREVIOUS_INSTANCE_ID = "NO_PREVIOUS_INSTANCE_ID" class Init(object): @@ -68,6 +69,8 @@ class Init(object): self.datasource = NULL_DATA_SOURCE self.ds_restored = False self._previous_iid = None + # simply ensure this gets set + self.previous_iid() if reporter is None: reporter = events.ReportEventStack( @@ -345,11 +348,15 @@ class Init(object): try: self._previous_iid = util.load_file(iid_fn).strip() except Exception: + self._previous_iid = NO_PREVIOUS_INSTANCE_ID pass return self._previous_iid def is_new_instance(self): - return self.datasource.get_instance_id() != self.previous_iid() + previous = self.previous_iid() + ret = (previous == NO_PREVIOUS_INSTANCE_ID or + previous != self.datasource.get_instance_id()) + return ret def fetch(self, existing="check"): return self._get_data_source(existing=existing) -- cgit v1.2.3 From c1c1550d74d067d78cf4ba1cf64f38c2dae8c9e1 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Thu, 26 May 2016 21:17:29 -0500 Subject: Move sdc:nics to a JSON map. Add unittest for sdc:nics --- cloudinit/sources/DataSourceSmartOS.py | 19 +++++---- tests/unittests/test_datasource/test_smartos.py | 51 +++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 8 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 6355fc2a..35fa8c33 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -64,7 +64,11 @@ SMARTOS_ATTRIB_MAP = { 'availability_zone': ('sdc:datacenter_name', True), 'vendor-data': ('sdc:vendor-data', False), 'operator-script': ('sdc:operator-script', False), - 'network-data': ('sdc:nics', False), +} + +SMARTOS_ATTRIB_JSON = { + # Cloud-init Key : (SmartOS Key known JSON) + 'network-data': 'sdc:nics', } DS_NAME = 'SmartOS' @@ -233,6 +237,9 @@ class DataSourceSmartOS(sources.DataSource): smartos_noun, strip = attribute md[ci_noun] = self.md_client.get(smartos_noun, strip=strip) + for ci_noun, smartos_noun in SMARTOS_ATTRIB_JSON.items(): + md[ci_noun] = self.md_client.get_json(smartos_noun) + # @datadictionary: This key may contain a program that is written # to a file in the filesystem of the guest on each boot and then # executed. It may be of any format that would be considered @@ -281,11 +288,7 @@ class DataSourceSmartOS(sources.DataSource): self.metadata = util.mergemanydict([md, self.metadata]) self.userdata_raw = ud self.vendordata_raw = md['vendor-data'] - if not md['network-data']: - smartos_noun, strip = SMARTOS_ATTRIB_MAP.get('network-data') - self.network_data = self.md_client.get_json(smartos_noun, - strip=strip) - md['network-data'] = self.network_data + self.network_data = md['network-data'] self._set_provisioned() return True @@ -401,8 +404,8 @@ class JoyentMetadataClient(object): result = result.strip() return result - def get_json(self, key, default=None, strip=False): - result = self.get(key, default=default, strip=strip) + def get_json(self, key, default=None): + result = self.get(key, default=default) if result is None: return default return json.loads(result) diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 5c49966a..1ee64d60 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -24,6 +24,7 @@ from __future__ import print_function +import json import os import os.path import re @@ -47,6 +48,48 @@ try: except ImportError: import mock +SDC_NICS = json.loads(""" +[ + { + "nic_tag": "external", + "primary": true, + "mtu": 1500, + "model": "virtio", + "gateway": "8.12.42.1", + "netmask": "255.255.255.0", + "ip": "8.12.42.102", + "network_uuid": "992fc7ce-6aac-4b74-aed6-7b9d2c6c0bfe", + "gateways": [ + "8.12.42.1" + ], + "vlan_id": 324, + "mac": "90:b8:d0:f5:e4:f5", + "interface": "net0", + "ips": [ + "8.12.42.102/24" + ] + }, + { + "nic_tag": "sdc_overlay/16187209", + "gateway": "192.168.128.1", + "model": "virtio", + "mac": "90:b8:d0:a5:ff:cd", + "netmask": "255.255.252.0", + "ip": "192.168.128.93", + "network_uuid": "4cad71da-09bc-452b-986d-03562a03a0a9", + "gateways": [ + "192.168.128.1" + ], + "vlan_id": 2, + "mtu": 8500, + "interface": "net1", + "ips": [ + "192.168.128.93/22" + ] + } +] +""") + MOCK_RETURNS = { 'hostname': 'test-host', 'root_authorized_keys': 'ssh-rsa AAAAB3Nz...aC1yc2E= keyname', @@ -60,6 +103,7 @@ MOCK_RETURNS = { 'sdc:vendor-data': '\n'.join(['VENDOR_DATA', '']), 'user-data': '\n'.join(['something', '']), 'user-script': '\n'.join(['/bin/true', '']), + 'sdc:nics': SDC_NICS, } DMI_DATA_RETURN = 'smartdc' @@ -275,6 +319,13 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): self.assertEquals(MOCK_RETURNS['cloud-init:user-data'], dsrc.userdata_raw) + def test_sdc_nics(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEquals(MOCK_RETURNS['sdc:nics'], + dsrc.metadata['network-data']) + def test_sdc_scripts(self): dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() -- cgit v1.2.3 From 5cd30a36eaa6ca1a239019a5409faa603f063f6c Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 May 2016 09:24:23 -0400 Subject: fix pyflakes, move datasources= to bottom --- cloudinit/sources/DataSourceSmartOS.py | 37 ++++++++++++++++------------------ 1 file changed, 17 insertions(+), 20 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 35fa8c33..4224f2ba 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -34,13 +34,11 @@ import base64 import binascii -import contextlib import json import os import random import re import socket -import stat import serial @@ -391,9 +389,6 @@ class JoyentMetadataClient(object): return None value = self._get_value_from_frame(request_id, response) - if value is None: - return default - return value def get(self, key, default=None, strip=False): @@ -442,11 +437,11 @@ class JoyentMetadataClient(object): class JoyentMetadataSocketClient(JoyentMetadataClient): def __init__(self, socketpath): - self.socketpath = metadata_socketfile + self.socketpath = socketpath def open_transport(self): sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - sock.connect(path) + sock.connect(self.socketpath) self.fp = sock.makefile('rwb') def exists(self): @@ -577,7 +572,7 @@ def jmc_client_factory( device=serial_device, timeout=serial_timeout, smartos_type=smartos_type) elif smartos_type == 'lx-brand': - return JoyentMetadataSerialClient(socketpath=metadata_socketfile) + return JoyentMetadataSerialClient(socketpath=metadata_sockfile) raise ValueError("Unknown value for smartos_type: %s" % smartos_type) @@ -661,17 +656,6 @@ def get_smartos_environ(uname_version=None, product_name=None, return None -# Used to match classes to dependencies -datasources = [ - (DataSourceSmartOS, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), -] - - -# Return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return sources.list_from_depends(depends, datasources) - - # Covert SMARTOS 'sdc:nics' data to network_config yaml def convert_smartos_network_data(network_data=None): """Return a dictionary of network_config by parsing provided @@ -744,5 +728,18 @@ def convert_smartos_network_data(network_data=None): return {'version': 1, 'config': config} +# Used to match classes to dependencies +datasources = [ + (DataSourceSmartOS, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), +] + + +# Return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return sources.list_from_depends(depends, datasources) + + if __name__ == "__main__": - jmc = JoyentMetadataClient(seed_file).get_metadata(noun) + import sys + jmc = jmc_client_factory() + jmc.get_metadata(sys.argv[1]) -- cgit v1.2.3 From 949cebd48c9100d4fd00b74232bcf048980e6e0d Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 May 2016 09:49:29 -0400 Subject: fix pyflakes and some pylint errors/warnings --- cloudinit/sources/DataSourceSmartOS.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 4224f2ba..cf4e00e5 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -69,6 +69,9 @@ SMARTOS_ATTRIB_JSON = { 'network-data': 'sdc:nics', } +SMARTOS_ENV_LX_BRAND = "lx-brand" +SMARTOS_ENV_KVM = "kvm" + DS_NAME = 'SmartOS' DS_CFG_PATH = ['datasource', DS_NAME] NO_BASE64_DECODE = [ @@ -183,6 +186,7 @@ class DataSourceSmartOS(sources.DataSource): self._network_config = None self.script_base_d = os.path.join(self.paths.get_cpath("scripts")) + self.smartos_env = None self._init() @@ -295,7 +299,9 @@ class DataSourceSmartOS(sources.DataSource): return self.ds_cfg['disk_aliases'].get(name) def get_config_obj(self): - return self.cfg + if self.smartos_env == SMARTOS_ENV_KVM: + return BUILTIN_CLOUD_CONFIG + return None def get_instance_id(self): return self.metadata['instance-id'] @@ -434,6 +440,9 @@ class JoyentMetadataClient(object): self.close_transport() return + def open_transport(self): + raise NotImplementedError + class JoyentMetadataSocketClient(JoyentMetadataClient): def __init__(self, socketpath): @@ -519,7 +528,7 @@ class JoyentMetadataLegacySerialClient(JoyentMetadataSerialClient): # now add any b64- that has a true value for key in [k[3:] for k in keys if k.startswith("b64-")]: if util.is_true(self._get(key)): - b64_keys.append(key) + b64_keys.add(key) else: if key in b64_keys: b64_keys.remove(key) @@ -572,7 +581,7 @@ def jmc_client_factory( device=serial_device, timeout=serial_timeout, smartos_type=smartos_type) elif smartos_type == 'lx-brand': - return JoyentMetadataSerialClient(socketpath=metadata_sockfile) + return JoyentMetadataSocketClient(socketpath=metadata_sockfile) raise ValueError("Unknown value for smartos_type: %s" % smartos_type) @@ -647,11 +656,15 @@ def get_smartos_environ(uname_version=None, product_name=None, if uname_version is None: uname_version = uname[3] if uname_version.lower() == 'brandz virtual linux': - return 'lx-brand' + return SMARTOS_ENV_LX_BRAND + + if product_name is None: + system_type = util.read_dmi_data("system-product-name") + else: + system_type = product_name - system_type = util.read_dmi_data("system-product-name") if system_type and 'smartdc' in system_type.lower(): - return 'kvm' + return SMARTOS_ENV_KVM return None @@ -742,4 +755,4 @@ def get_datasource_list(depends): if __name__ == "__main__": import sys jmc = jmc_client_factory() - jmc.get_metadata(sys.argv[1]) + jmc.get(sys.argv[1]) -- cgit v1.2.3 From e6e768769d67960cde12dee0b0c2b58deb2eb376 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 May 2016 14:06:59 -0400 Subject: fix a bunch of the tests --- cloudinit/sources/DataSourceSmartOS.py | 12 +- tests/unittests/test_datasource/test_smartos.py | 312 ++++++++++++++++++++++-- 2 files changed, 298 insertions(+), 26 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index cf4e00e5..e64dea68 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -186,7 +186,7 @@ class DataSourceSmartOS(sources.DataSource): self._network_config = None self.script_base_d = os.path.join(self.paths.get_cpath("scripts")) - self.smartos_env = None + self.smartos_type = None self._init() @@ -196,10 +196,11 @@ class DataSourceSmartOS(sources.DataSource): def _init(self): if self.smartos_environ == self._unset: - self.smartos_env = get_smartos_environ() + self.smartos_type = get_smartos_environ() if self.md_client == self._unset: self.md_client = jmc_client_factory( + smartos_type=self.smartos_type, metadata_sockfile=self.ds_cfg['metadata_sockfile'], serial_device=self.ds_cfg['serial_device'], serial_timeout=self.ds_cfg['serial_timeout']) @@ -226,7 +227,7 @@ class DataSourceSmartOS(sources.DataSource): md = {} ud = "" - if not self.smartos_env: + if not self.smartos_type: LOG.debug("Not running on smartos") return False @@ -299,7 +300,7 @@ class DataSourceSmartOS(sources.DataSource): return self.ds_cfg['disk_aliases'].get(name) def get_config_obj(self): - if self.smartos_env == SMARTOS_ENV_KVM: + if self.smartos_type == SMARTOS_ENV_KVM: return BUILTIN_CLOUD_CONFIG return None @@ -608,6 +609,7 @@ def write_boot_content(content, content_f, link=None, shebang=False, bit and to the SmartOS default of assuming that bash. """ + print("content_f=%s" % content_f) if not content and os.path.exists(content_f): os.unlink(content_f) if link and os.path.islink(link): @@ -639,7 +641,7 @@ def write_boot_content(content, content_f, link=None, shebang=False, util.ensure_dir(os.path.dirname(link)) os.symlink(content_f, link) except IOError as e: - util.logexc(LOG, "failed establishing content link", e) + util.logexc(LOG, "failed establishing content link: %s", e) def get_smartos_environ(uname_version=None, product_name=None, diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index ea20777a..946286bd 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -40,13 +40,9 @@ import six from cloudinit import helpers as c_helpers from cloudinit.sources import DataSourceSmartOS from cloudinit.util import b64e +from cloudinit import util -from .. import helpers - -try: - from unittest import mock -except ImportError: - import mock +from ..helpers import mock, TestCase, FilesystemMockingTestCase SDC_NICS = json.loads(""" [ @@ -103,7 +99,7 @@ MOCK_RETURNS = { 'sdc:vendor-data': '\n'.join(['VENDOR_DATA', '']), 'user-data': '\n'.join(['something', '']), 'user-script': '\n'.join(['/bin/true', '']), - 'sdc:nics': SDC_NICS, + 'sdc:nics': json.dumps(SDC_NICS), } DMI_DATA_RETURN = 'smartdc' @@ -115,12 +111,286 @@ def get_mock_client(mockdata): def __init__(self, serial): pass - def get_metadata(self, metadata_key): + def get(self, metadata_key): return mockdata.get(metadata_key) return MockMetadataClient -class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): +class PsuedoJoyentClient(object): + def __init__(self, data=None): + if data is None: + data = MOCK_RETURNS.copy() + self.data = data + return + + def get(self, key, default=None, strip=False): + if key in self.data: + r = self.data[key] + if strip: + r = r.strip() + else: + r = default + return r + + def get_json(self, key, default=None): + result = self.get(key, default=default) + if result is None: + return default + return json.loads(result) + + def exists(self): + return True + + +class TestSmartOSDataSource(FilesystemMockingTestCase): + def setUp(self): + super(TestSmartOSDataSource, self).setUp() + + dsmos = 'cloudinit.sources.DataSourceSmartOS' + patcher = mock.patch(dsmos + ".jmc_client_factory") + self.jmc_cfact = patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch(dsmos + ".get_smartos_environ") + self.get_smartos_environ = patcher.start() + self.addCleanup(patcher.stop) + + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp) + self.paths = c_helpers.Paths({'cloud_dir': self.tmp}) + + self.legacy_user_d = tempfile.mkdtemp() + self.orig_lud = DataSourceSmartOS.LEGACY_USER_D + DataSourceSmartOS.LEGACY_USER_D = self.legacy_user_d + + def tearDown(self): + DataSourceSmartOS.LEGACY_USER_D = self.orig_lud + super(TestSmartOSDataSource, self).tearDown() + + def _get_ds(self, mockdata=None, mode=DataSourceSmartOS.SMARTOS_ENV_KVM, + sys_cfg=None, ds_cfg=None): + self.jmc_cfact.return_value = PsuedoJoyentClient(mockdata) + self.get_smartos_environ.return_value = mode + + if sys_cfg is None: + sys_cfg = {} + + if ds_cfg is not None: + sys_cfg['datasource'] = sys_cfg.get('datasource', {}) + sys_cfg['datasource']['SmartOS'] = ds_cfg + + return DataSourceSmartOS.DataSourceSmartOS( + sys_cfg, distro=None, paths=self.paths) + + def test_it_got_here(self): + dsrc = self._get_ds() + ret = dsrc.get_data() + + def test_no_base64(self): + ds_cfg = {'no_base64_decode': ['test_var1'], 'all_base': True} + dsrc = self._get_ds(ds_cfg=ds_cfg) + ret = dsrc.get_data() + self.assertTrue(ret) + + def test_uuid(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(MOCK_RETURNS['sdc:uuid'], + dsrc.metadata['instance-id']) + + def test_root_keys(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(MOCK_RETURNS['root_authorized_keys'], + dsrc.metadata['public-keys']) + + def test_hostname_b64(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(MOCK_RETURNS['hostname'], + dsrc.metadata['local-hostname']) + + def test_hostname(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(MOCK_RETURNS['hostname'], + dsrc.metadata['local-hostname']) + + def test_userdata(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(MOCK_RETURNS['user-data'], + dsrc.metadata['legacy-user-data']) + self.assertEqual(MOCK_RETURNS['cloud-init:user-data'], + dsrc.userdata_raw) + + def test_sdc_nics(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEquals(json.loads(MOCK_RETURNS['sdc:nics']), + dsrc.metadata['network-data']) + + def test_sdc_scripts(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(MOCK_RETURNS['user-script'], + dsrc.metadata['user-script']) + + legacy_script_f = "%s/user-script" % self.legacy_user_d + self.assertTrue(os.path.exists(legacy_script_f)) + self.assertTrue(os.path.islink(legacy_script_f)) + user_script_perm = oct(os.stat(legacy_script_f)[stat.ST_MODE])[-3:] + self.assertEqual(user_script_perm, '700') + + def test_scripts_shebanged(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(MOCK_RETURNS['user-script'], + dsrc.metadata['user-script']) + + legacy_script_f = "%s/user-script" % self.legacy_user_d + self.assertTrue(os.path.exists(legacy_script_f)) + self.assertTrue(os.path.islink(legacy_script_f)) + shebang = None + with open(legacy_script_f, 'r') as f: + shebang = f.readlines()[0].strip() + self.assertEqual(shebang, "#!/bin/bash") + user_script_perm = oct(os.stat(legacy_script_f)[stat.ST_MODE])[-3:] + self.assertEqual(user_script_perm, '700') + + def test_scripts_shebang_not_added(self): + """ + Test that the SmartOS requirement that plain text scripts + are executable. This test makes sure that plain texts scripts + with out file magic have it added appropriately by cloud-init. + """ + + my_returns = MOCK_RETURNS.copy() + my_returns['user-script'] = '\n'.join(['#!/usr/bin/perl', + 'print("hi")', '']) + + dsrc = self._get_ds(mockdata=my_returns) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(my_returns['user-script'], + dsrc.metadata['user-script']) + + legacy_script_f = "%s/user-script" % self.legacy_user_d + self.assertTrue(os.path.exists(legacy_script_f)) + self.assertTrue(os.path.islink(legacy_script_f)) + shebang = None + with open(legacy_script_f, 'r') as f: + shebang = f.readlines()[0].strip() + self.assertEqual(shebang, "#!/usr/bin/perl") + + def test_userdata_removed(self): + """ + User-data in the SmartOS world is supposed to be written to a file + each and every boot. This tests to make sure that in the event the + legacy user-data is removed, the existing user-data is backed-up + and there is no /var/db/user-data left. + """ + + user_data_f = "%s/mdata-user-data" % self.legacy_user_d + with open(user_data_f, 'w') as f: + f.write("PREVIOUS") + + my_returns = MOCK_RETURNS.copy() + del my_returns['user-data'] + + dsrc = self._get_ds(mockdata=my_returns) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertFalse(dsrc.metadata.get('legacy-user-data')) + + found_new = False + for root, _dirs, files in os.walk(self.legacy_user_d): + for name in files: + name_f = os.path.join(root, name) + permissions = oct(os.stat(name_f)[stat.ST_MODE])[-3:] + if re.match(r'.*\/mdata-user-data$', name_f): + found_new = True + print(name_f) + self.assertEqual(permissions, '400') + + self.assertFalse(found_new) + + def test_vendor_data_not_default(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(MOCK_RETURNS['sdc:vendor-data'], + dsrc.metadata['vendor-data']) + + def test_default_vendor_data(self): + my_returns = MOCK_RETURNS.copy() + def_op_script = my_returns['sdc:vendor-data'] + del my_returns['sdc:vendor-data'] + dsrc = self._get_ds(mockdata=my_returns) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertNotEqual(def_op_script, dsrc.metadata['vendor-data']) + + # we expect default vendor-data is a boothook + self.assertTrue(dsrc.vendordata_raw.startswith("#cloud-boothook")) + + def test_disable_iptables_flag(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(MOCK_RETURNS['disable_iptables_flag'], + dsrc.metadata['iptables_disable']) + + def test_motd_sys_info(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(MOCK_RETURNS['enable_motd_sys_info'], + dsrc.metadata['motd_sys_info']) + + def test_default_ephemeral(self): + # Test to make sure that the builtin config has the ephemeral + # configuration. + dsrc = self._get_ds() + cfg = dsrc.get_config_obj() + + ret = dsrc.get_data() + self.assertTrue(ret) + + assert 'disk_setup' in cfg + assert 'fs_setup' in cfg + self.assertIsInstance(cfg['disk_setup'], dict) + self.assertIsInstance(cfg['fs_setup'], list) + + def test_override_disk_aliases(self): + # Test to make sure that the built-in DS is overriden + builtin = DataSourceSmartOS.BUILTIN_DS_CONFIG + + mydscfg = {'disk_aliases': {'FOO': '/dev/bar'}} + + # expect that these values are in builtin, or this is pointless + for k in mydscfg: + self.assertIn(k, builtin) + + dsrc = self._get_ds(ds_cfg=mydscfg) + ret = dsrc.get_data() + self.assertTrue(ret) + + self.assertEqual(mydscfg['disk_aliases']['FOO'], + dsrc.ds_cfg['disk_aliases']['FOO']) + + self.assertEqual(dsrc.device_name_to_device('FOO'), + mydscfg['disk_aliases']['FOO']) + + +class OldTestSmartOSDataSource(FilesystemMockingTestCase): def setUp(self): super(TestSmartOSDataSource, self).setUp() @@ -141,7 +411,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): super(TestSmartOSDataSource, self).setUp() def tearDown(self): - helpers.FilesystemMockingTestCase.tearDown(self) + FilesystemMockingTestCase.tearDown(self) if self._log_handler and self._log: self._log.removeHandler(self._log_handler) apply_patches([i for i in reversed(self.unapply)]) @@ -166,7 +436,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): if dmi_data is None: dmi_data = DMI_DATA_RETURN - def _dmi_data(): + def _dmi_data(item): return dmi_data def _os_uname(): @@ -188,12 +458,12 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): self.apply_patches([(mod, 'LEGACY_USER_D', self.legacy_user_d)]) self.apply_patches([ (mod, 'JoyentMetadataClient', get_mock_client(mockdata))]) - self.apply_patches([(mod, 'dmi_data', _dmi_data)]) + self.apply_patches([(util, 'read_dmi_data', _dmi_data)]) self.apply_patches([(os, 'uname', _os_uname)]) - self.apply_patches([(mod, 'device_exists', lambda d: True)]) + self.apply_patches([(os.path, 'exists', lambda d: True)]) dsrc = mod.DataSourceSmartOS(sys_cfg, distro=None, paths=self.paths) - self.apply_patches([(dsrc, '_get_seed_file_object', mock.MagicMock())]) + #self.apply_patches([(dsrc, '_get_seed_file_object', mock.MagicMock())]) return dsrc def test_seed(self): @@ -492,7 +762,7 @@ def apply_patches(patches): return ret -class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase): +class TestJoyentMetadataClient(FilesystemMockingTestCase): def setUp(self): super(TestJoyentMetadataClient, self).setUp() @@ -532,7 +802,7 @@ class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase): mock.Mock(return_value=self.request_id))) def _get_client(self): - return DataSourceSmartOS.JoyentMetadataClient(self.serial) + return DataSourceSmartOS.JoyentMetadataSerialClient(self.serial) def assertEndsWith(self, haystack, prefix): self.assertTrue(haystack.endswith(prefix), @@ -546,7 +816,7 @@ class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase): def test_get_metadata_writes_a_single_line(self): client = self._get_client() - client.get_metadata('some_key') + client.get('some_key') self.assertEqual(1, self.serial.write.call_count) written_line = self.serial.write.call_args[0][0] print(type(written_line)) @@ -556,7 +826,7 @@ class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase): def _get_written_line(self, key='some_key'): client = self._get_client() - client.get_metadata(key) + client.get(key) return self.serial.write.call_args[0][0] def test_get_metadata_writes_bytes(self): @@ -600,12 +870,12 @@ class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase): def test_get_metadata_reads_a_line(self): client = self._get_client() - client.get_metadata('some_key') + client.get('some_key') self.assertEqual(self.metasource_data_len, self.serial.read.call_count) def test_get_metadata_returns_valid_value(self): client = self._get_client() - value = client.get_metadata('some_key') + value = client.get('some_key') self.assertEqual(self.metadata_value, value) def test_get_metadata_throws_exception_for_incorrect_length(self): @@ -633,4 +903,4 @@ class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase): self.response_parts['length'] = 17 client = self._get_client() client._checksum = lambda _: self.response_parts['crc'] - self.assertIsNone(client.get_metadata('some_key')) + self.assertIsNone(client.get('some_key')) -- cgit v1.2.3 From 8a148ed934156c63a76a28c9d3a33278e52d71d1 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 May 2016 14:27:50 -0400 Subject: fix the remaining tests --- cloudinit/sources/DataSourceSmartOS.py | 4 +- tests/unittests/test_datasource/test_smartos.py | 391 +----------------------- 2 files changed, 7 insertions(+), 388 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index e64dea68..e9ff1235 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -332,11 +332,11 @@ class JoyentMetadataClient(object): r' (?P(?P[0-9a-f]+) (?PSUCCESS|NOTFOUND)' r'( (?P.+))?)') - def __init__(self, smartos_type=None): + def __init__(self, smartos_type=None, fp=None): if smartos_type is None: smartos_type = get_smartos_environ() self.smartos_type = smartos_type - self.fp = None + self.fp = fp def _checksum(self, body): return '{0:08x}'.format( diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 946286bd..56d85f99 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -105,17 +105,6 @@ MOCK_RETURNS = { DMI_DATA_RETURN = 'smartdc' -def get_mock_client(mockdata): - class MockMetadataClient(object): - - def __init__(self, serial): - pass - - def get(self, metadata_key): - return mockdata.get(metadata_key) - return MockMetadataClient - - class PsuedoJoyentClient(object): def __init__(self, data=None): if data is None: @@ -390,377 +379,6 @@ class TestSmartOSDataSource(FilesystemMockingTestCase): mydscfg['disk_aliases']['FOO']) -class OldTestSmartOSDataSource(FilesystemMockingTestCase): - def setUp(self): - super(TestSmartOSDataSource, self).setUp() - - self.tmp = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, self.tmp) - self.legacy_user_d = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, self.legacy_user_d) - - # If you should want to watch the logs... - self._log = None - self._log_file = None - self._log_handler = None - - # patch cloud_dir, so our 'seed_dir' is guaranteed empty - self.paths = c_helpers.Paths({'cloud_dir': self.tmp}) - - self.unapply = [] - super(TestSmartOSDataSource, self).setUp() - - def tearDown(self): - FilesystemMockingTestCase.tearDown(self) - if self._log_handler and self._log: - self._log.removeHandler(self._log_handler) - apply_patches([i for i in reversed(self.unapply)]) - super(TestSmartOSDataSource, self).tearDown() - - def _patchIn(self, root): - self.restore() - self.patchOS(root) - self.patchUtils(root) - - def apply_patches(self, patches): - ret = apply_patches(patches) - self.unapply += ret - - def _get_ds(self, sys_cfg=None, ds_cfg=None, mockdata=None, dmi_data=None, - is_lxbrand=False): - mod = DataSourceSmartOS - - if mockdata is None: - mockdata = MOCK_RETURNS - - if dmi_data is None: - dmi_data = DMI_DATA_RETURN - - def _dmi_data(item): - return dmi_data - - def _os_uname(): - if not is_lxbrand: - # LP: #1243287. tests assume this runs, but running test on - # arm would cause them all to fail. - return ('LINUX', 'NODENAME', 'RELEASE', 'VERSION', 'x86_64') - else: - return ('LINUX', 'NODENAME', 'RELEASE', 'BRANDZ VIRTUAL LINUX', - 'X86_64') - - if sys_cfg is None: - sys_cfg = {} - - if ds_cfg is not None: - sys_cfg['datasource'] = sys_cfg.get('datasource', {}) - sys_cfg['datasource']['SmartOS'] = ds_cfg - - self.apply_patches([(mod, 'LEGACY_USER_D', self.legacy_user_d)]) - self.apply_patches([ - (mod, 'JoyentMetadataClient', get_mock_client(mockdata))]) - self.apply_patches([(util, 'read_dmi_data', _dmi_data)]) - self.apply_patches([(os, 'uname', _os_uname)]) - self.apply_patches([(os.path, 'exists', lambda d: True)]) - dsrc = mod.DataSourceSmartOS(sys_cfg, distro=None, - paths=self.paths) - #self.apply_patches([(dsrc, '_get_seed_file_object', mock.MagicMock())]) - return dsrc - - def test_seed(self): - # default seed should be /dev/ttyS1 - dsrc = self._get_ds() - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual('kvm', dsrc.smartos_type) - self.assertEqual('/dev/ttyS1', dsrc.seed) - - def test_seed_lxbrand(self): - # default seed should be /dev/ttyS1 - dsrc = self._get_ds(is_lxbrand=True) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual('lx-brand', dsrc.smartos_type) - self.assertEqual('/native/.zonecontrol/metadata.sock', dsrc.seed) - - def test_issmartdc(self): - dsrc = self._get_ds() - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertTrue(dsrc.is_smartdc) - - def test_issmartdc_lxbrand(self): - dsrc = self._get_ds(is_lxbrand=True) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertTrue(dsrc.is_smartdc) - - def test_no_base64(self): - ds_cfg = {'no_base64_decode': ['test_var1'], 'all_base': True} - dsrc = self._get_ds(ds_cfg=ds_cfg) - ret = dsrc.get_data() - self.assertTrue(ret) - - def test_uuid(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(MOCK_RETURNS['sdc:uuid'], - dsrc.metadata['instance-id']) - - def test_root_keys(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(MOCK_RETURNS['root_authorized_keys'], - dsrc.metadata['public-keys']) - - def test_hostname_b64(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(MOCK_RETURNS['hostname'], - dsrc.metadata['local-hostname']) - - def test_hostname(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(MOCK_RETURNS['hostname'], - dsrc.metadata['local-hostname']) - - def test_base64_all(self): - # metadata provided base64_all of true - my_returns = MOCK_RETURNS.copy() - my_returns['base64_all'] = "true" - for k in ('hostname', 'cloud-init:user-data'): - my_returns[k] = b64e(my_returns[k]) - - dsrc = self._get_ds(mockdata=my_returns) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(MOCK_RETURNS['hostname'], - dsrc.metadata['local-hostname']) - self.assertEqual(MOCK_RETURNS['cloud-init:user-data'], - dsrc.userdata_raw) - self.assertEqual(MOCK_RETURNS['root_authorized_keys'], - dsrc.metadata['public-keys']) - self.assertEqual(MOCK_RETURNS['disable_iptables_flag'], - dsrc.metadata['iptables_disable']) - self.assertEqual(MOCK_RETURNS['enable_motd_sys_info'], - dsrc.metadata['motd_sys_info']) - - def test_b64_userdata(self): - my_returns = MOCK_RETURNS.copy() - my_returns['b64-cloud-init:user-data'] = "true" - my_returns['b64-hostname'] = "true" - for k in ('hostname', 'cloud-init:user-data'): - my_returns[k] = b64e(my_returns[k]) - - dsrc = self._get_ds(mockdata=my_returns) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(MOCK_RETURNS['hostname'], - dsrc.metadata['local-hostname']) - self.assertEqual(MOCK_RETURNS['cloud-init:user-data'], - dsrc.userdata_raw) - self.assertEqual(MOCK_RETURNS['root_authorized_keys'], - dsrc.metadata['public-keys']) - - def test_b64_keys(self): - my_returns = MOCK_RETURNS.copy() - my_returns['base64_keys'] = 'hostname,ignored' - for k in ('hostname',): - my_returns[k] = b64e(my_returns[k]) - - dsrc = self._get_ds(mockdata=my_returns) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(MOCK_RETURNS['hostname'], - dsrc.metadata['local-hostname']) - self.assertEqual(MOCK_RETURNS['cloud-init:user-data'], - dsrc.userdata_raw) - - def test_userdata(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(MOCK_RETURNS['user-data'], - dsrc.metadata['legacy-user-data']) - self.assertEqual(MOCK_RETURNS['cloud-init:user-data'], - dsrc.userdata_raw) - - def test_sdc_nics(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEquals(MOCK_RETURNS['sdc:nics'], - dsrc.metadata['network-data']) - - def test_sdc_scripts(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(MOCK_RETURNS['user-script'], - dsrc.metadata['user-script']) - - legacy_script_f = "%s/user-script" % self.legacy_user_d - self.assertTrue(os.path.exists(legacy_script_f)) - self.assertTrue(os.path.islink(legacy_script_f)) - user_script_perm = oct(os.stat(legacy_script_f)[stat.ST_MODE])[-3:] - self.assertEqual(user_script_perm, '700') - - def test_scripts_shebanged(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(MOCK_RETURNS['user-script'], - dsrc.metadata['user-script']) - - legacy_script_f = "%s/user-script" % self.legacy_user_d - self.assertTrue(os.path.exists(legacy_script_f)) - self.assertTrue(os.path.islink(legacy_script_f)) - shebang = None - with open(legacy_script_f, 'r') as f: - shebang = f.readlines()[0].strip() - self.assertEqual(shebang, "#!/bin/bash") - user_script_perm = oct(os.stat(legacy_script_f)[stat.ST_MODE])[-3:] - self.assertEqual(user_script_perm, '700') - - def test_scripts_shebang_not_added(self): - """ - Test that the SmartOS requirement that plain text scripts - are executable. This test makes sure that plain texts scripts - with out file magic have it added appropriately by cloud-init. - """ - - my_returns = MOCK_RETURNS.copy() - my_returns['user-script'] = '\n'.join(['#!/usr/bin/perl', - 'print("hi")', '']) - - dsrc = self._get_ds(mockdata=my_returns) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(my_returns['user-script'], - dsrc.metadata['user-script']) - - legacy_script_f = "%s/user-script" % self.legacy_user_d - self.assertTrue(os.path.exists(legacy_script_f)) - self.assertTrue(os.path.islink(legacy_script_f)) - shebang = None - with open(legacy_script_f, 'r') as f: - shebang = f.readlines()[0].strip() - self.assertEqual(shebang, "#!/usr/bin/perl") - - def test_userdata_removed(self): - """ - User-data in the SmartOS world is supposed to be written to a file - each and every boot. This tests to make sure that in the event the - legacy user-data is removed, the existing user-data is backed-up - and there is no /var/db/user-data left. - """ - - user_data_f = "%s/mdata-user-data" % self.legacy_user_d - with open(user_data_f, 'w') as f: - f.write("PREVIOUS") - - my_returns = MOCK_RETURNS.copy() - del my_returns['user-data'] - - dsrc = self._get_ds(mockdata=my_returns) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertFalse(dsrc.metadata.get('legacy-user-data')) - - found_new = False - for root, _dirs, files in os.walk(self.legacy_user_d): - for name in files: - name_f = os.path.join(root, name) - permissions = oct(os.stat(name_f)[stat.ST_MODE])[-3:] - if re.match(r'.*\/mdata-user-data$', name_f): - found_new = True - print(name_f) - self.assertEqual(permissions, '400') - - self.assertFalse(found_new) - - def test_vendor_data_not_default(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(MOCK_RETURNS['sdc:vendor-data'], - dsrc.metadata['vendor-data']) - - def test_default_vendor_data(self): - my_returns = MOCK_RETURNS.copy() - def_op_script = my_returns['sdc:vendor-data'] - del my_returns['sdc:vendor-data'] - dsrc = self._get_ds(mockdata=my_returns) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertNotEqual(def_op_script, dsrc.metadata['vendor-data']) - - # we expect default vendor-data is a boothook - self.assertTrue(dsrc.vendordata_raw.startswith("#cloud-boothook")) - - def test_disable_iptables_flag(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(MOCK_RETURNS['disable_iptables_flag'], - dsrc.metadata['iptables_disable']) - - def test_motd_sys_info(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(MOCK_RETURNS['enable_motd_sys_info'], - dsrc.metadata['motd_sys_info']) - - def test_default_ephemeral(self): - # Test to make sure that the builtin config has the ephemeral - # configuration. - dsrc = self._get_ds() - cfg = dsrc.get_config_obj() - - ret = dsrc.get_data() - self.assertTrue(ret) - - assert 'disk_setup' in cfg - assert 'fs_setup' in cfg - self.assertIsInstance(cfg['disk_setup'], dict) - self.assertIsInstance(cfg['fs_setup'], list) - - def test_override_disk_aliases(self): - # Test to make sure that the built-in DS is overriden - builtin = DataSourceSmartOS.BUILTIN_DS_CONFIG - - mydscfg = {'disk_aliases': {'FOO': '/dev/bar'}} - - # expect that these values are in builtin, or this is pointless - for k in mydscfg: - self.assertIn(k, builtin) - - dsrc = self._get_ds(ds_cfg=mydscfg) - ret = dsrc.get_data() - self.assertTrue(ret) - - self.assertEqual(mydscfg['disk_aliases']['FOO'], - dsrc.ds_cfg['disk_aliases']['FOO']) - - self.assertEqual(dsrc.device_name_to_device('FOO'), - mydscfg['disk_aliases']['FOO']) - - -def apply_patches(patches): - ret = [] - for (ref, name, replace) in patches: - if replace is None: - continue - orig = getattr(ref, name) - setattr(ref, name, replace) - ret.append((ref, name, orig)) - return ret - class TestJoyentMetadataClient(FilesystemMockingTestCase): @@ -802,7 +420,8 @@ class TestJoyentMetadataClient(FilesystemMockingTestCase): mock.Mock(return_value=self.request_id))) def _get_client(self): - return DataSourceSmartOS.JoyentMetadataSerialClient(self.serial) + return DataSourceSmartOS.JoyentMetadataClient( + fp=self.serial, smartos_type=DataSourceSmartOS.SMARTOS_ENV_KVM) def assertEndsWith(self, haystack, prefix): self.assertTrue(haystack.endswith(prefix), @@ -882,20 +501,20 @@ class TestJoyentMetadataClient(FilesystemMockingTestCase): self.response_parts['length'] = 0 client = self._get_client() self.assertRaises(DataSourceSmartOS.JoyentMetadataFetchException, - client.get_metadata, 'some_key') + client.get, 'some_key') def test_get_metadata_throws_exception_for_incorrect_crc(self): self.response_parts['crc'] = 'deadbeef' client = self._get_client() self.assertRaises(DataSourceSmartOS.JoyentMetadataFetchException, - client.get_metadata, 'some_key') + client.get, 'some_key') def test_get_metadata_throws_exception_for_request_id_mismatch(self): self.response_parts['request_id'] = 'deadbeef' client = self._get_client() client._checksum = lambda _: self.response_parts['crc'] self.assertRaises(DataSourceSmartOS.JoyentMetadataFetchException, - client.get_metadata, 'some_key') + client.get, 'some_key') def test_get_metadata_returns_None_if_value_not_found(self): self.response_parts['payload'] = '' -- cgit v1.2.3 From 91734552a2d338938ed0e3aa4885f77b99409ead Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 May 2016 14:32:43 -0400 Subject: fix pyflakes and flake8 --- cloudinit/sources/DataSourceSmartOS.py | 3 +-- tests/unittests/test_datasource/test_smartos.py | 10 ++-------- 2 files changed, 3 insertions(+), 10 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index e9ff1235..f5820696 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -722,8 +722,7 @@ def convert_smartos_network_data(network_data=None): if k in valid_keys['physical']} cfg.update({ 'type': 'physical', - 'name': nic['interface'] - }) + 'name': nic['interface']}) if 'mac' in nic: cfg.update({'mac_address': nic['mac']}) diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 56d85f99..ae45513d 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -40,9 +40,8 @@ import six from cloudinit import helpers as c_helpers from cloudinit.sources import DataSourceSmartOS from cloudinit.util import b64e -from cloudinit import util -from ..helpers import mock, TestCase, FilesystemMockingTestCase +from ..helpers import mock, FilesystemMockingTestCase SDC_NICS = json.loads(""" [ @@ -111,7 +110,7 @@ class PsuedoJoyentClient(object): data = MOCK_RETURNS.copy() self.data = data return - + def get(self, key, default=None, strip=False): if key in self.data: r = self.data[key] @@ -169,10 +168,6 @@ class TestSmartOSDataSource(FilesystemMockingTestCase): return DataSourceSmartOS.DataSourceSmartOS( sys_cfg, distro=None, paths=self.paths) - - def test_it_got_here(self): - dsrc = self._get_ds() - ret = dsrc.get_data() def test_no_base64(self): ds_cfg = {'no_base64_decode': ['test_var1'], 'all_base': True} @@ -379,7 +374,6 @@ class TestSmartOSDataSource(FilesystemMockingTestCase): mydscfg['disk_aliases']['FOO']) - class TestJoyentMetadataClient(FilesystemMockingTestCase): def setUp(self): -- cgit v1.2.3 From 71e4da45263e6cb3eb5d5938908656ed04c3db9f Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 May 2016 14:33:20 -0400 Subject: remove debug print --- cloudinit/sources/DataSourceSmartOS.py | 1 - 1 file changed, 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index f5820696..9c249ddf 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -609,7 +609,6 @@ def write_boot_content(content, content_f, link=None, shebang=False, bit and to the SmartOS default of assuming that bash. """ - print("content_f=%s" % content_f) if not content and os.path.exists(content_f): os.unlink(content_f) if link and os.path.islink(link): -- cgit v1.2.3 From a9533cd924e8eae89234a19d8359a87c23a30e12 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 May 2016 14:52:13 -0400 Subject: add nicer main --- cloudinit/sources/DataSourceSmartOS.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 9c249ddf..3d7297c9 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -755,4 +755,23 @@ def get_datasource_list(depends): if __name__ == "__main__": import sys jmc = jmc_client_factory() - jmc.get(sys.argv[1]) + if len(sys.argv) == 1: + keys = (list(SMARTOS_ATTRIB_JSON.keys()) + + list(SMARTOS_ATTRIB_MAP.keys())) + else: + keys = sys.argv[1:] + + data = {} + for key in keys: + if key in SMARTOS_ATTRIB_JSON: + keyname = SMARTOS_ATTRIB_JSON[key] + data[key] = jmc.get_json(keyname) + else: + if key in SMARTOS_ATTRIB_MAP: + keyname, strip = SMARTOS_ATTRIB_MAP[key] + else: + keyname, strip = (key, False) + val = jmc.get(keyname, strip=strip) + data[key] = jmc.get(keyname, strip=strip) + + print(json.dumps(data, indent=1)) -- cgit v1.2.3 From 9a43053c564210aa088f96ab7877a66a1c3b48fa Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 May 2016 14:55:52 -0400 Subject: Smartos datasource is local. --- cloudinit/sources/DataSourceSmartOS.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 3d7297c9..24331d37 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -743,7 +743,7 @@ def convert_smartos_network_data(network_data=None): # Used to match classes to dependencies datasources = [ - (DataSourceSmartOS, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), + (DataSourceConfigDrive, (sources.DEP_FILESYSTEM, )), ] -- cgit v1.2.3 From 7ab03cab899c5bd355c24a4b894c74d63c6dd8b6 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 May 2016 15:03:08 -0400 Subject: smartos is local, but it is named DataSourceSmartOS not DataSourceConfigDrive --- cloudinit/sources/DataSourceSmartOS.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 24331d37..2821402a 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -743,7 +743,7 @@ def convert_smartos_network_data(network_data=None): # Used to match classes to dependencies datasources = [ - (DataSourceConfigDrive, (sources.DEP_FILESYSTEM, )), + (DataSourceSmartOS, (sources.DEP_FILESYSTEM, )), ] -- cgit v1.2.3 From 75109ffd12ad4dcbbccf1b0603506efcae413433 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 May 2016 15:31:52 -0400 Subject: return dict not None on get_config_obj --- cloudinit/sources/DataSourceSmartOS.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 2821402a..7c8928b3 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -302,7 +302,7 @@ class DataSourceSmartOS(sources.DataSource): def get_config_obj(self): if self.smartos_type == SMARTOS_ENV_KVM: return BUILTIN_CLOUD_CONFIG - return None + return {} def get_instance_id(self): return self.metadata['instance-id'] -- cgit v1.2.3 From 653676831e70192d4a0322ee453d6a4c3e6541da Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 May 2016 16:57:18 -0400 Subject: fix test cases by avoiding rendering paths to early --- cloudinit/stages.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 20c334b7..f164d6f6 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -69,8 +69,6 @@ class Init(object): self.datasource = NULL_DATA_SOURCE self.ds_restored = False self._previous_iid = None - # simply ensure this gets set - self.previous_iid() if reporter is None: reporter = events.ReportEventStack( @@ -321,12 +319,8 @@ class Init(object): # What the instance id was and is... iid = self.datasource.get_instance_id() iid_fn = os.path.join(dp, 'instance-id') - try: - previous_iid = util.load_file(iid_fn).strip() - except Exception: - previous_iid = None - if not previous_iid: - previous_iid = iid + + previous_iid = self.previous_iid() util.write_file(iid_fn, "%s\n" % iid) util.write_file(self.paths.get_runpath('instance_id'), "%s\n" % iid) util.write_file(os.path.join(dp, 'previous-instance-id'), @@ -349,7 +343,8 @@ class Init(object): self._previous_iid = util.load_file(iid_fn).strip() except Exception: self._previous_iid = NO_PREVIOUS_INSTANCE_ID - pass + + LOG.debug("previous iid found to be %s", self._previous_iid) return self._previous_iid def is_new_instance(self): -- cgit v1.2.3 From aa236033b159b691f5ec31885750a8167c63b2a1 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 30 May 2016 12:27:40 +0200 Subject: drop errorlist from convert_to_new_format --- cloudinit/config/cc_apt_configure.py | 12 +++--------- tests/unittests/test_handler/test_handler_apt_source.py | 6 ++---- 2 files changed, 5 insertions(+), 13 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index ffbf7513..2dd48844 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -198,24 +198,18 @@ def add_key(ent): add_key_raw(ent['key']) -def convert_to_new_format(srclist, errorlist): +def convert_to_new_format(srclist): """ convert_to_new_format convert the old list based format to the new dict based one """ srcdict = {} if isinstance(srclist, list): - fnfallbackused = None for srcent in srclist: if 'filename' not in srcent: # file collides for multiple !filename cases for compatibility # yet we need them all processed, so not same dictionary key srcent['filename'] = "cloud_config_sources.list" key = util.rand_dict_key(srcdict, "cloud_config_sources.list") - if fnfallbackused is not None: - errorlist.append(["multiple apt_source entries without", - "filename will conflict: %s vs %s" % - (srcent, fnfallbackused)]) - fnfallbackused = srcent else: # all with filename use that as key (matching new format) key = srcent['filename'] @@ -223,7 +217,7 @@ def convert_to_new_format(srclist, errorlist): elif isinstance(srclist, dict): srcdict = srclist else: - errorlist.append(["srclist", "unknown apt_sources format"]) + raise ValueError("unknown apt_sources format") return srcdict @@ -242,7 +236,7 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): return False errorlist = [] - srcdict = convert_to_new_format(srclist, errorlist) + srcdict = convert_to_new_format(srclist) for filename in srcdict: ent = srcdict[filename] diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 753e86a8..4536c5b2 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -529,7 +529,6 @@ class TestAptSourceConfig(TestCase): 'filename': self.aptlistfile2} cfg3 = {'source': 'deb $MIRROR $RELEASE universe', 'filename': self.aptlistfile3} - errorlist = [] checkcfg = {self.aptlistfile: {'filename': self.aptlistfile, 'source': 'deb $MIRROR $RELEASE ' 'multiverse'}, @@ -539,11 +538,10 @@ class TestAptSourceConfig(TestCase): 'source': 'deb $MIRROR $RELEASE ' 'universe'}} - newcfg = cc_apt_configure.convert_to_new_format([cfg1, cfg2, cfg3], - errorlist) + newcfg = cc_apt_configure.convert_to_new_format([cfg1, cfg2, cfg3]) self.assertEqual(newcfg, checkcfg) - newcfg2 = cc_apt_configure.convert_to_new_format(newcfg, errorlist) + newcfg2 = cc_apt_configure.convert_to_new_format(newcfg) self.assertEqual(newcfg2, checkcfg) -- cgit v1.2.3 From 3c85315373306729443ef79fd8e54af46a7bc849 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 30 May 2016 13:07:22 +0200 Subject: fix EXPORT_GPG_KEYID for existing keys This was broken for keys already existing in the local keyring. There instead of the keycontent it reported the header like: pub 1024R/03683F77 2009-10-27 uid Launchpad PPA for Scott Moser --- cloudinit/config/cc_apt_configure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 2dd48844..d603f417 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -40,7 +40,7 @@ EXPORT_GPG_KEYID = """ k=${1} ks=${2}; exec 2>/dev/null [ -n "$k" ] || exit 1; - armour=$(gpg --list-keys --armour "${k}") + armour=$(gpg --export --armour "${k}") if [ -z "${armour}" ]; then gpg --keyserver ${ks} --recv "${k}" >/dev/null && armour=$(gpg --export --armour "${k}") && -- cgit v1.2.3 From 0ce31b75247458b735b1b52dd5985519190d48fb Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 31 May 2016 13:14:17 -0400 Subject: use constants for kvm and lx-brand --- cloudinit/sources/DataSourceSmartOS.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 7c8928b3..d6a9bf28 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -577,11 +577,11 @@ def jmc_client_factory( if smartos_type is None: smartos_type = get_smartos_environ(uname_version) - if smartos_type == 'kvm': + if smartos_type == SMARTOS_ENV_KVM: return JoyentMetadataLegacySerialClient( device=serial_device, timeout=serial_timeout, smartos_type=smartos_type) - elif smartos_type == 'lx-brand': + elif smartos_type == SMARTOS_ENV_LX_BRAND: return JoyentMetadataSocketClient(socketpath=metadata_sockfile) raise ValueError("Unknown value for smartos_type: %s" % smartos_type) -- cgit v1.2.3 From 1b8a09389654a29af7e618b803bffaed0185e9e8 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 31 May 2016 17:17:39 -0400 Subject: add renaming code for renaming interfaces currently does not work in lxc https://github.com/lxc/lxd/issues/2063 --- bin/cloud-init | 6 +-- cloudinit/distros/__init__.py | 4 ++ cloudinit/net/__init__.py | 86 +++++++++++++++++++++++++++++++++++++++++++ cloudinit/stages.py | 10 +++++ 4 files changed, 101 insertions(+), 5 deletions(-) (limited to 'cloudinit') diff --git a/bin/cloud-init b/bin/cloud-init index 29e9b521..21c3a684 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -280,11 +280,7 @@ def main_init(name, args): LOG.debug("[%s] %s will now be targeting instance id: %s. new=%s", mode, name, iid, init.is_new_instance()) - if init.is_new_instance(): - # on new instance, apply network config. - # in network mode 'bring_up' must be passed in as the OS - # has already brought up networking. - init.apply_network_config(bring_up=bool(mode != sources.DSMODE_LOCAL)) + init.apply_network_config(bring_up=bool(mode != sources.DSMODE_LOCAL)) if mode == sources.DSMODE_LOCAL: if init.datasource.dsmode != mode: diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 3bfbc484..5c29c804 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -31,6 +31,7 @@ import stat from cloudinit import importer from cloudinit import log as logging +from cloudinit import net from cloudinit import ssh_util from cloudinit import type_utils from cloudinit import util @@ -145,6 +146,9 @@ class Distro(object): return self._bring_up_interfaces(dev_names) return False + def apply_network_config_names(self, netconfig): + net.apply_network_config_names(netconfig) + @abc.abstractmethod def apply_locale(self, locale, out_fn=None): raise NotImplementedError() diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 40d330b5..ec1b3835 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -813,4 +813,90 @@ def _ifaces_to_net_config_data(ifaces): 'config': [devs[d] for d in sorted(devs)]} +def apply_network_config_names(netcfg, strict_present=True, strict_busy=True): + """read the network config and rename devices accordingly. + if strict_present is false, then do not raise exception if no devices + match. if strict_busy is false, then do not raise exception if the + device cannot be renamed because it is currently configured.""" + renames = [] + for ent in netcfg.get('config', {}): + if ent.get('type') != 'physical': + continue + mac = ent.get('mac_address') + name = ent.get('name') + if not mac: + continue + renames.append([mac, name]) + + return rename_interfaces(renames) + + +def rename_interfaces(renames, strict_present=True, strict_busy=True): + cur_bymac = {get_interface_mac(n): n for n in get_devicelist()} + expected = {mac: name for mac, name in renames} + cur_byname = {v: k for k, v in cur_bymac.items()} + + tmpname_fmt = "cirename%d" + tmpi = -1 + + moves = [] + changes = [] + errors = [] + for mac, new_name in expected.items(): + cur_name = cur_bymac.get(mac) + if cur_name == new_name: + # nothing to do + continue + + if not cur_name: + if strict_present: + errors.append( + "[nic not present] Cannot rename mac=%s to %s" + ", not available." % (mac, new_name)) + elif is_up(cur_name): + if strict_busy: + errors.append("[busy] Error renaming mac=%s from %s to %s." % + (mac, cur_name, new_name)) + elif new_name in cur_byname: + if is_up(new_name): + if strict_busy: + errors.append( + "[busy-target] Error renaming mac=%s from %s to %s." % + (mac, cur_name, new_name)) + else: + tmp_name = None + while tmp_name is None or tmp_name in cur_byname: + tmpi += 1 + tmp_name = tmpname_fmt % tmpi + moves.append((mac, cur_name, tmp_name)) + changes.append((mac, tmp_name, new_name)) + else: + changes.append((mac, cur_name, new_name)) + + def rename_dev(cur, new): + cmd = ["ip", "link", "set", cur, "name", new] + util.subp(cmd) + + for mac, cur, new in moves + changes: + try: + rename_dev(cur, new) + except util.ProcessExecutionError as e: + errors.append( + "[unknown] Error renaming mac=%s from %s to %s. (%s)" % + (mac, cur, new, e)) + + if len(errors): + raise Exception('\n'.join(errors)) + + +def get_interface_mac(ifname): + """Returns the string value of an interface's MAC Address""" + return read_sys_net(ifname, "address", enoent=False) + + +def get_ifname_mac_pairs(): + """Build a list of tuples (ifname, mac)""" + return [(ifname, get_interface_mac(ifname)) for ifname in get_devicelist()] + + # vi: ts=4 expandtab syntax=python diff --git a/cloudinit/stages.py b/cloudinit/stages.py index f164d6f6..211d2286 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -632,6 +632,16 @@ class Init(object): LOG.info("network config is disabled by %s", src) return + try: + LOG.debug("applying net config names for %s" % netcfg) + self.distro.apply_network_config_names(netcfg) + except Exception as e: + LOG.warn("Failed to rename devices: %s", e) + + if not self.is_new_instance(): + LOG.debug("not a new instance. network config is not applied.") + return + LOG.info("Applying network configuration from %s bringup=%s: %s", src, bring_up, netcfg) try: -- cgit v1.2.3 From f67e8f104b199c9402cf047637b939516526e0ac Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 1 Jun 2016 17:14:50 -0400 Subject: support renaming and fix logic in rename_interfaces The one issue i'm aware of currently is that tap devices (ip tuntap add mode tap user root mytap1) do not work correctly with 'is_up' which means the check does not bring them down and the rename fails. The LOG.debug message should be cleaned up too, as it currently references the function rather function.__name__ for nicer message. --- cloudinit/net/__init__.py | 133 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 98 insertions(+), 35 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index ec1b3835..f5ae7705 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -831,19 +831,65 @@ def apply_network_config_names(netcfg, strict_present=True, strict_busy=True): return rename_interfaces(renames) -def rename_interfaces(renames, strict_present=True, strict_busy=True): - cur_bymac = {get_interface_mac(n): n for n in get_devicelist()} - expected = {mac: name for mac, name in renames} - cur_byname = {v: k for k, v in cur_bymac.items()} +def _get_current_rename_info(check_downable=True): + """Collect information necessary for rename_interfaces.""" + names = get_devicelist() + bymac = {} + for n in names: + bymac[get_interface_mac(n)] = { + 'name': n, 'up': is_up(n), 'downable': None} + if check_downable: + nmatch = re.compile(r"[0-9]+:\s+(\w+)[@:]") + ipv6, _err = util.subp(['ip', '-6', 'addr', 'show', 'permanent', + 'scope', 'global'], capture=True) + ipv4, _err = util.subp(['ip', '-4', 'addr', 'show'], capture=True) + + nics_with_addresses = set() + for bytes_out in (ipv6, ipv4): + nics_with_addresses.update(nmatch.findall(bytes_out)) + + for d in bymac.values(): + d['downable'] = (d['up'] is False or + d['name'] not in nics_with_addresses) + + return bymac + + +def rename_interfaces(renames, strict_present=True, strict_busy=True, + current_info=None): + if current_info is None: + current_info = _get_current_rename_info() + + cur_bymac = {} + for mac, data in current_info.items(): + cur = data.copy() + cur['mac'] = mac + cur_bymac[mac] = cur + + def update_byname(bymac): + return {data['name']: data for data in bymac.values()} + + def rename(cur, new): + util.subp(["ip", "link", "set", cur, "name", new], capture=True) + + def down(name): + util.subp(["ip", "link", "set", name, "down"], capture=True) + + def up(name): + util.subp(["ip", "link", "set", name, "up"], capture=True) + + ops = [] + errors = [] + ups = [] + cur_byname = update_byname(cur_bymac) tmpname_fmt = "cirename%d" tmpi = -1 - moves = [] - changes = [] - errors = [] - for mac, new_name in expected.items(): - cur_name = cur_bymac.get(mac) + for mac, new_name in renames: + cur = cur_bymac.get(mac, {}) + cur_name = cur.get('name') + cur_ops = [] if cur_name == new_name: # nothing to do continue @@ -853,37 +899,54 @@ def rename_interfaces(renames, strict_present=True, strict_busy=True): errors.append( "[nic not present] Cannot rename mac=%s to %s" ", not available." % (mac, new_name)) - elif is_up(cur_name): - if strict_busy: - errors.append("[busy] Error renaming mac=%s from %s to %s." % - (mac, cur_name, new_name)) - elif new_name in cur_byname: - if is_up(new_name): + continue + + if cur['up']: + msg = "[busy] Error renaming mac=%s from %s to %s" + if not cur['downable']: if strict_busy: - errors.append( - "[busy-target] Error renaming mac=%s from %s to %s." % - (mac, cur_name, new_name)) - else: - tmp_name = None - while tmp_name is None or tmp_name in cur_byname: - tmpi += 1 - tmp_name = tmpname_fmt % tmpi - moves.append((mac, cur_name, tmp_name)) - changes.append((mac, tmp_name, new_name)) - else: - changes.append((mac, cur_name, new_name)) + errors.append(msg % (mac, cur_name, new_name)) + continue + cur['up'] = False + cur_ops.append((down, mac, new_name, (cur_name,))) + ups.append((up, mac, new_name, (new_name,))) + + if new_name in cur_byname: + target = cur_byname[new_name] + if target['up']: + msg = "[busy-target] Error renaming mac=%s from %s to %s." + if not target['downable']: + if strict_busy: + errors.append(msg % (mac, cur_name, new_name)) + continue + else: + cur_ops.append((down, mac, new_name, (new_name,))) + + tmp_name = None + while tmp_name is None or tmp_name in cur_byname: + tmpi += 1 + tmp_name = tmpname_fmt % tmpi + + cur_ops.append((rename, mac, new_name, (new_name, tmp_name))) + target['name'] = tmp_name + cur_byname = update_byname(cur_bymac) + if target['up']: + ups.append((up, mac, new_name, (tmp_name,))) + + cur_ops.append((rename, mac, new_name, (cur['name'], new_name))) + cur['name'] = new_name + cur_byname = update_byname(cur_bymac) + ops += cur_ops - def rename_dev(cur, new): - cmd = ["ip", "link", "set", cur, "name", new] - util.subp(cmd) + LOG.debug("achieving renaming of %s with ops %s", renames, ops + ups) - for mac, cur, new in moves + changes: + for op, mac, new_name, params in ops + ups: try: - rename_dev(cur, new) - except util.ProcessExecutionError as e: + op(*params) + except Exception as e: errors.append( - "[unknown] Error renaming mac=%s from %s to %s. (%s)" % - (mac, cur, new, e)) + "[unknown] Error performing %s%s for %s, %s: %s" % + (op.__name__, params, mac, new_name, e)) if len(errors): raise Exception('\n'.join(errors)) -- cgit v1.2.3 From 0fc5c222d2ff335b9a1de3e910bb8822b6d73031 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 1 Jun 2016 17:17:11 -0400 Subject: cloudinit/stages.py: if no datasource found, do not attempt is_new_instance if local does not find a datasource, then we try to apply networking. but that would then hit the NULL_DATA_SOURCE which does not work with is_new_instance. avoid that. --- cloudinit/stages.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 211d2286..5756e74d 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -638,7 +638,8 @@ class Init(object): except Exception as e: LOG.warn("Failed to rename devices: %s", e) - if not self.is_new_instance(): + if (self.datasource is not NULL_DATA_SOURCE and + not self.is_new_instance()): LOG.debug("not a new instance. network config is not applied.") return -- cgit v1.2.3 From 7f46de87ee543a82c9a95137478676edaba2acc1 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 1 Jun 2016 20:23:00 -0400 Subject: clean up log message a bit. --- cloudinit/net/__init__.py | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index f5ae7705..6d9ea575 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -908,8 +908,8 @@ def rename_interfaces(renames, strict_present=True, strict_busy=True, errors.append(msg % (mac, cur_name, new_name)) continue cur['up'] = False - cur_ops.append((down, mac, new_name, (cur_name,))) - ups.append((up, mac, new_name, (new_name,))) + cur_ops.append(("down", mac, new_name, (cur_name,))) + ups.append(("up", mac, new_name, (new_name,))) if new_name in cur_byname: target = cur_byname[new_name] @@ -920,33 +920,41 @@ def rename_interfaces(renames, strict_present=True, strict_busy=True, errors.append(msg % (mac, cur_name, new_name)) continue else: - cur_ops.append((down, mac, new_name, (new_name,))) + cur_ops.append(("down", mac, new_name, (new_name,))) tmp_name = None while tmp_name is None or tmp_name in cur_byname: tmpi += 1 tmp_name = tmpname_fmt % tmpi - cur_ops.append((rename, mac, new_name, (new_name, tmp_name))) + cur_ops.append(("rename", mac, new_name, (new_name, tmp_name))) target['name'] = tmp_name cur_byname = update_byname(cur_bymac) if target['up']: - ups.append((up, mac, new_name, (tmp_name,))) + ups.append(("up", mac, new_name, (tmp_name,))) - cur_ops.append((rename, mac, new_name, (cur['name'], new_name))) + cur_ops.append(("rename", mac, new_name, (cur['name'], new_name))) cur['name'] = new_name cur_byname = update_byname(cur_bymac) ops += cur_ops - LOG.debug("achieving renaming of %s with ops %s", renames, ops + ups) + opmap = {'rename': rename, 'down': down, 'up': up} - for op, mac, new_name, params in ops + ups: - try: - op(*params) - except Exception as e: - errors.append( - "[unknown] Error performing %s%s for %s, %s: %s" % - (op.__name__, params, mac, new_name, e)) + if len(ops) + len(ups) == 0: + if len(errors): + LOG.debug("unable to do any work for renaming of %s", renames) + else: + LOG.debug("no work necessary for renaming of %s", renames) + else: + LOG.debug("achieving renaming of %s with ops %s", renames, ops + ups) + + for op, mac, new_name, params in ops + ups: + try: + opmap.get(op)(*params) + except Exception as e: + errors.append( + "[unknown] Error performing %s%s for %s, %s: %s" % + (op, params, mac, new_name, e)) if len(errors): raise Exception('\n'.join(errors)) -- cgit v1.2.3 From 7c27cb1fcae01d34b32895630d57b2c9bc624caf Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 2 Jun 2016 09:47:29 -0400 Subject: fix log message in emit_upstart --- cloudinit/config/cc_emit_upstart.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_emit_upstart.py b/cloudinit/config/cc_emit_upstart.py index 06c53272..98828b9e 100644 --- a/cloudinit/config/cc_emit_upstart.py +++ b/cloudinit/config/cc_emit_upstart.py @@ -56,7 +56,7 @@ def handle(name, _cfg, cloud, log, args): event_names = ['cloud-config'] if not is_upstart_system(): - log.debug("not upstart system, '%s' disabled") + log.debug("not upstart system, '%s' disabled", name) return cfgpath = cloud.paths.get_ipath_cur("cloud_config") -- cgit v1.2.3 From b071e4bbbbe1b5a6ced02796696b05d2e1b16778 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 2 Jun 2016 13:18:23 -0400 Subject: openstack: support decoding when reading files, use that for network_config The network config file is /etc/network/interfaces formated. We will decode that here so that the user can expect that it is a string. The issue was that it was bytes but convert_eni_data was expecting a string. --- cloudinit/sources/helpers/openstack.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 156aba6c..3ccb11d3 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -190,14 +190,14 @@ class BaseReader(object): versions_available) return selected_version - def _read_content_path(self, item): + def _read_content_path(self, item, decode=False): path = item.get('content_path', '').lstrip("/") path_pieces = path.split("/") valid_pieces = [p for p in path_pieces if len(p)] if not valid_pieces: raise BrokenMetadata("Item %s has no valid content path" % (item)) path = self._path_join(self.base_path, "openstack", *path_pieces) - return self._path_read(path) + return self._path_read(path, decode=decode) def read_v2(self): """Reads a version 2 formatted location. @@ -298,7 +298,8 @@ class BaseReader(object): net_item = metadata.get("network_config", None) if net_item: try: - results['network_config'] = self._read_content_path(net_item) + content = self._read_content_path(net_item, decode=True) + results['network_config'] = content except IOError as e: raise BrokenMetadata("Failed to read network" " configuration: %s" % (e)) @@ -333,8 +334,8 @@ class ConfigDriveReader(BaseReader): components = [base] + list(add_ons) return os.path.join(*components) - def _path_read(self, path): - return util.load_file(path, decode=False) + def _path_read(self, path, decode=False): + return util.load_file(path, decode=decode) def _fetch_available_versions(self): if self._versions is None: @@ -446,7 +447,7 @@ class MetadataReader(BaseReader): self._versions = found return self._versions - def _path_read(self, path): + def _path_read(self, path, decode=False): def should_retry_cb(_request_args, cause): try: @@ -463,7 +464,10 @@ class MetadataReader(BaseReader): ssl_details=self.ssl_details, timeout=self.timeout, exception_cb=should_retry_cb) - return response.contents + if decode: + return response.contents.decode() + else: + return response.contents def _path_join(self, base, *add_ons): return url_helper.combine_url(base, *add_ons) -- cgit v1.2.3 From 80648a623fe6c7ae397629da30c04e52d79759f2 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 2 Jun 2016 13:19:50 -0400 Subject: eni parsing: support 'ether' in hwaddress, netmask and broadcast this adds ability to support ENI that has: hwadress ether 36:4c:e1:3b:14:31 or hwaddress 36:4c:e1:3b:14:31 the former is written by openstack (at least on dreamhost). Also, in the conversion of eni to network config support broadcast and netmask. --- cloudinit/net/__init__.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 6d9ea575..05152ead 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -201,7 +201,11 @@ def parse_deb_config_data(ifaces, contents, src_dir, src_path): ifaces[iface]['method'] = method currif = iface elif option == "hwaddress": - ifaces[currif]['hwaddress'] = split[1] + if split[1] == "ether": + val = split[2] + else: + val = split[1] + ifaces[currif]['hwaddress'] = val elif option in NET_CONFIG_OPTIONS: ifaces[currif][option] = split[1] elif option in NET_CONFIG_COMMANDS: @@ -800,8 +804,9 @@ def _ifaces_to_net_config_data(ifaces): if data.get('method') == 'static': subnet['address'] = data['address'] - if 'gateway' in data: - subnet['gateway'] = data['gateway'] + for copy_key in ('netmask', 'gateway', 'broadcast'): + if copy_key in data: + subnet[copy_key] = data[copy_key] if 'dns' in data: for n in ('nameservers', 'search'): -- cgit v1.2.3 From 1abff1453a24ed14375cb6294364295a0c2c7ee3 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 2 Jun 2016 14:08:44 -0400 Subject: smartos: do not raise error when not on smartos if get_smartos_environ() returned a None, then the datasoure would raise a ValueError when get_data was called. Fix that. --- cloudinit/sources/DataSourceSmartOS.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index d6a9bf28..55dc94bb 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -171,7 +171,7 @@ LEGACY_USER_D = "/var/db" class DataSourceSmartOS(sources.DataSource): _unset = "_unset" - smartos_environ = _unset + smartos_type = _unset md_client = _unset def __init__(self, sys_cfg, distro, paths): @@ -195,8 +195,10 @@ class DataSourceSmartOS(sources.DataSource): return "%s [client=%s]" % (root, self.md_client) def _init(self): - if self.smartos_environ == self._unset: + if self.smartos_type == self._unset: self.smartos_type = get_smartos_environ() + if self.smartos_type is None: + self.md_client = None if self.md_client == self._unset: self.md_client = jmc_client_factory( @@ -577,7 +579,9 @@ def jmc_client_factory( if smartos_type is None: smartos_type = get_smartos_environ(uname_version) - if smartos_type == SMARTOS_ENV_KVM: + if smartos_type is None: + return None + elif smartos_type == SMARTOS_ENV_KVM: return JoyentMetadataLegacySerialClient( device=serial_device, timeout=serial_timeout, smartos_type=smartos_type) @@ -755,6 +759,9 @@ def get_datasource_list(depends): if __name__ == "__main__": import sys jmc = jmc_client_factory() + if jmc is None: + print("Do not appear to be on smartos.") + sys.exit(1) if len(sys.argv) == 1: keys = (list(SMARTOS_ATTRIB_JSON.keys()) + list(SMARTOS_ATTRIB_MAP.keys())) -- cgit v1.2.3 From bd31ab1e78f59c88b4aba031ffdaca506b3b04ae Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 2 Jun 2016 14:36:51 -0400 Subject: fix untested previous change to smartos --- cloudinit/sources/DataSourceSmartOS.py | 1 - 1 file changed, 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 55dc94bb..0e03b04f 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -186,7 +186,6 @@ class DataSourceSmartOS(sources.DataSource): self._network_config = None self.script_base_d = os.path.join(self.paths.get_cpath("scripts")) - self.smartos_type = None self._init() -- cgit v1.2.3 From d709524941ba2b4e06940a9eb0861f0819d5560f Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 2 Jun 2016 15:18:27 -0400 Subject: re-add the 'Net' classes for datasources When the .pkl file is loaded, the module that it is loaded from must have the same symbol. Ie, if booted once and got DataSourceConfigDriveNet then upgraded and rebooted, then next boot would show Can't get attribute 'DataSourceConfigDriveNet' --- cloudinit/sources/DataSourceCloudSigma.py | 3 +++ cloudinit/sources/DataSourceConfigDrive.py | 25 ++++++++++++++----------- cloudinit/sources/DataSourceOpenNebula.py | 3 +++ 3 files changed, 20 insertions(+), 11 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceCloudSigma.py b/cloudinit/sources/DataSourceCloudSigma.py index 07e8ae11..d1f806d6 100644 --- a/cloudinit/sources/DataSourceCloudSigma.py +++ b/cloudinit/sources/DataSourceCloudSigma.py @@ -115,6 +115,9 @@ class DataSourceCloudSigma(sources.DataSource): return self.metadata['uuid'] +# Legacy: Must be present in case we load an old pkl object +DataSourceCloudSigmaNet = DataSourceCloudSigma + # Used to match classes to dependencies. Since this datasource uses the serial # port network is not really required, so it's okay to load without it, too. datasources = [ diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 2d13a32f..61d967d9 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -254,17 +254,6 @@ def find_candidate_devs(probe_optical=True): return devices -# Used to match classes to dependencies -datasources = [ - (DataSourceConfigDrive, (sources.DEP_FILESYSTEM, )), -] - - -# Return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return sources.list_from_depends(depends, datasources) - - # Convert OpenStack ConfigDrive NetworkData json to network_config yaml def convert_network_data(network_json=None): """Return a dictionary of network_config by parsing provided @@ -382,3 +371,17 @@ def convert_network_data(network_json=None): config.append(cfg) return {'version': 1, 'config': config} + + +# Legacy: Must be present in case we load an old pkl object +DataSourceConfigDriveNet = DataSourceConfigDrive + +# Used to match classes to dependencies +datasources = [ + (DataSourceConfigDrive, (sources.DEP_FILESYSTEM, )), +] + + +# Return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return sources.list_from_depends(depends, datasources) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 15819a4f..8f85b115 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -415,6 +415,9 @@ def read_context_disk_dir(source_dir, asuser=None): return results +# Legacy: Must be present in case we load an old pkl object +DataSourceOpenNebulaNet = DataSourceOpenNebula + # Used to match classes to dependencies datasources = [ (DataSourceOpenNebula, (sources.DEP_FILESYSTEM, )), -- cgit v1.2.3 From 6bd7fbc35ac8726a8a0f422cd802d290c236bf3b Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 2 Jun 2016 23:03:38 -0400 Subject: ConfigDrive: do not use 'id' on a link for the device name 'id' on a link in the openstack spec should be "Generic, generated ID". current implementation was to use the host's name for the host side nic. Which provided names like 'tap-adfasdffd'. We do not want to name devices like that as its quite unexpected and non user friendly. So here we use the system name for any nic that is present, but then require that the nics found also be present at the time of rendering. The end result is that if the system boots with net.ifnames=0 then it will get 'eth0' like names. and if it boots without net.ifnames then it will get enp0s1 like names. --- cloudinit/net/__init__.py | 15 +++++--- cloudinit/sources/DataSourceConfigDrive.py | 29 ++++++++++++--- .../unittests/test_datasource/test_configdrive.py | 41 ++++++++++++++++++++-- 3 files changed, 74 insertions(+), 11 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 05152ead..f47053b2 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -970,9 +970,16 @@ def get_interface_mac(ifname): return read_sys_net(ifname, "address", enoent=False) -def get_ifname_mac_pairs(): - """Build a list of tuples (ifname, mac)""" - return [(ifname, get_interface_mac(ifname)) for ifname in get_devicelist()] - +def get_interfaces_by_mac(devs=None): + """Build a dictionary of tuples {mac: name}""" + if devs is None: + devs = get_devicelist() + ret = {} + for name in devs: + mac = get_interface_mac(name) + # some devices may not have a mac (tun0) + if mac: + ret[mac] = name + return ret # vi: ts=4 expandtab syntax=python diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 61d967d9..3cc9155d 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -255,7 +255,7 @@ def find_candidate_devs(probe_optical=True): # Convert OpenStack ConfigDrive NetworkData json to network_config yaml -def convert_network_data(network_json=None): +def convert_network_data(network_json=None, known_macs=None): """Return a dictionary of network_config by parsing provided OpenStack ConfigDrive NetworkData json format @@ -319,9 +319,15 @@ def convert_network_data(network_json=None): subnets = [] cfg = {k: v for k, v in link.items() if k in valid_keys['physical']} - cfg.update({'name': link['id']}) - for network in [net for net in networks - if net['link'] == link['id']]: + # 'name' is not in openstack spec yet, but we will support it if it is + # present. The 'id' in the spec is currently implemented as the host + # nic's name, meaning something like 'tap-adfasdffd'. We do not want + # to name guest devices with such ugly names. + if 'name' in link: + cfg['name'] = link['name'] + + for network in [n for n in networks + if n['link'] == link['id']]: subnet = {k: v for k, v in network.items() if k in valid_keys['subnet']} if 'dhcp' in network['type']: @@ -365,6 +371,21 @@ def convert_network_data(network_json=None): config.append(cfg) + need_names = [d for d in config + if d.get('type') == 'physical' and 'name' not in d] + + if need_names: + if known_macs is None: + known_macs = net.get_interfaces_by_mac() + + for d in need_names: + mac = d.get('mac_address') + if not mac: + raise ValueError("No mac_address or name entry for %s" % d) + if mac not in known_macs: + raise ValueError("Unable to find a system nic for %s" % d) + d['name'] = known_macs[mac] + for service in services: cfg = service cfg.update({'type': 'nameserver'}) diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index 195b8207..1364b39d 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -73,7 +73,7 @@ NETWORK_DATA = { 'type': 'ovs', 'mtu': None, 'id': 'tap2f88d109-5b'}, {'vif_id': '1a5382f8-04c5-4d75-ab98-d666c1ef52cc', 'ethernet_mac_address': 'fa:16:3e:05:30:fe', - 'type': 'ovs', 'mtu': None, 'id': 'tap1a5382f8-04'} + 'type': 'ovs', 'mtu': None, 'id': 'tap1a5382f8-04', 'name': 'nic0'} ], 'networks': [ {'link': 'tap2ecc7709-b3', 'type': 'ipv4_dhcp', @@ -88,6 +88,10 @@ NETWORK_DATA = { ] } +KNOWN_MACS = { + 'fa:16:3e:69:b0:58': 'enp0s1', + 'fa:16:3e:d4:57:ad': 'enp0s2'} + CFG_DRIVE_FILES_V2 = { 'ec2/2009-04-04/meta-data.json': json.dumps(EC2_META), 'ec2/2009-04-04/user-data': USER_DATA, @@ -365,10 +369,40 @@ class TestConfigDriveDataSource(TestCase): """Verify that network_data is converted and present on ds object.""" populate_dir(self.tmp, CFG_DRIVE_FILES_V2) myds = cfg_ds_from_dir(self.tmp) - network_config = ds.convert_network_data(NETWORK_DATA) + network_config = ds.convert_network_data(NETWORK_DATA, + known_macs=KNOWN_MACS) self.assertEqual(myds.network_config, network_config) +class TestConvertNetworkData(TestCase): + def _getnames_in_config(self, ncfg): + return set([n['name'] for n in ncfg['config'] + if n['type'] == 'physical']) + + def test_conversion_fills_names(self): + ncfg = ds.convert_network_data(NETWORK_DATA, known_macs=KNOWN_MACS) + expected = set(['nic0', 'enp0s1', 'enp0s2']) + found = self._getnames_in_config(ncfg) + self.assertEqual(found, expected) + + @mock.patch('cloudinit.net.get_interfaces_by_mac') + def test_convert_reads_system_prefers_name(self, get_interfaces_by_mac): + macs = KNOWN_MACS.copy() + macs.update({'fa:16:3e:05:30:fe': 'foonic1', + 'fa:16:3e:69:b0:58': 'ens1'}) + get_interfaces_by_mac.return_value = macs + + ncfg = ds.convert_network_data(NETWORK_DATA) + expected = set(['nic0', 'ens1', 'enp0s2']) + found = self._getnames_in_config(ncfg) + self.assertEqual(found, expected) + + def test_convert_raises_value_error_on_missing_name(self): + macs = {'aa:aa:aa:aa:aa:00': 'ens1'} + self.assertRaises(ValueError, ds.convert_network_data, + NETWORK_DATA, known_macs=macs) + + def cfg_ds_from_dir(seed_d): found = ds.read_config_drive(seed_d) cfg_ds = ds.DataSourceConfigDrive(settings.CFG_BUILTIN, None, @@ -387,7 +421,8 @@ def populate_ds_from_read_config(cfg_ds, source, results): cfg_ds.userdata_raw = results.get('userdata') cfg_ds.version = results.get('version') cfg_ds.network_json = results.get('networkdata') - cfg_ds._network_config = ds.convert_network_data(cfg_ds.network_json) + cfg_ds._network_config = ds.convert_network_data( + cfg_ds.network_json, known_macs=KNOWN_MACS) def populate_dir(seed_dir, files): -- cgit v1.2.3 From e2c2d70cde19211b18e5ec333e1cb0382d93f14d Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 3 Jun 2016 00:42:22 -0400 Subject: lxd: fix log messsage --- cloudinit/config/cc_lxd.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py index b1de8f84..70d4e7c3 100644 --- a/cloudinit/config/cc_lxd.py +++ b/cloudinit/config/cc_lxd.py @@ -52,7 +52,8 @@ def handle(name, cfg, cloud, log, args): # Get config lxd_cfg = cfg.get('lxd') if not lxd_cfg: - log.debug("Skipping module named %s, not present or disabled by cfg") + log.debug("Skipping module named %s, not present or disabled by cfg", + name) return if not isinstance(lxd_cfg, dict): log.warn("lxd config must be a dictionary. found a '%s'", -- cgit v1.2.3 From 8a8d7eed2ed5696d58a825ec7301d8424c23ce5e Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 3 Jun 2016 14:23:34 -0400 Subject: avoid rendering 'lo' twice by not writing it in network config. --- cloudinit/net/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'cloudinit') diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index f47053b2..c72b6ff8 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -788,6 +788,10 @@ def _ifaces_to_net_config_data(ifaces): for name, data in ifaces.items(): # devname is 'eth0' for name='eth0:1' devname = name.partition(":")[0] + if devname == "lo": + # currently provding 'lo' in network config results in duplicate + # entries. in rendered interfaces file. so skip it. + continue if devname not in devs: devs[devname] = {'type': 'physical', 'name': devname, 'subnets': []} -- cgit v1.2.3 From f495947a701d5629b6dbfd2ff9e01dad7bd5166b Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 3 Jun 2016 14:58:51 -0400 Subject: fix issue with routes on subnets not getting rendered --- cloudinit/net/__init__.py | 2 ++ .../unittests/test_datasource/test_configdrive.py | 41 +++++++++++++++++++++- 2 files changed, 42 insertions(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index c72b6ff8..49e9d5c2 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -574,6 +574,8 @@ def render_interfaces(network_state): content += iface_start_entry(iface, index) content += iface_add_subnet(iface, subnet) content += iface_add_attrs(iface) + for route in subnet.get('routes', []): + content += render_route(route, indent=" ") else: # ifenslave docs say to auto the slave devices if 'bond-master' in iface: diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index 1364b39d..83d75f9c 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -15,6 +15,7 @@ except ImportError: from contextlib2 import ExitStack from cloudinit import helpers +from cloudinit import net from cloudinit import settings from cloudinit.sources import DataSourceConfigDrive as ds from cloudinit.sources.helpers import openstack @@ -88,9 +89,33 @@ NETWORK_DATA = { ] } +NETWORK_DATA_2 = { + "services": [ + {"type": "dns", "address": "1.1.1.191"}, + {"type": "dns", "address": "1.1.1.4"}], + "networks": [ + {"network_id": "d94bbe94-7abc-48d4-9c82-4628ea26164a", "type": "ipv4", + "netmask": "255.255.255.248", "link": "eth0", + "routes": [{"netmask": "0.0.0.0", "network": "0.0.0.0", + "gateway": "2.2.2.9"}], + "ip_address": "2.2.2.10", "id": "network0-ipv4"}, + {"network_id": "ca447c83-6409-499b-aaef-6ad1ae995348", "type": "ipv4", + "netmask": "255.255.255.224", "link": "eth1", + "routes": [], "ip_address": "3.3.3.24", "id": "network1-ipv4"}], + "links": [ + {"ethernet_mac_address": "fa:16:3e:dd:50:9a", "mtu": 1500, + "type": "vif", "id": "eth0", "vif_id": "vif-foo1"}, + {"ethernet_mac_address": "fa:16:3e:a8:14:69", "mtu": 1500, + "type": "vif", "id": "eth1", "vif_id": "vif-foo2"}] +} + + KNOWN_MACS = { 'fa:16:3e:69:b0:58': 'enp0s1', - 'fa:16:3e:d4:57:ad': 'enp0s2'} + 'fa:16:3e:d4:57:ad': 'enp0s2', + 'fa:16:3e:dd:50:9a': 'foo1', + 'fa:16:3e:a8:14:69': 'foo2', +} CFG_DRIVE_FILES_V2 = { 'ec2/2009-04-04/meta-data.json': json.dumps(EC2_META), @@ -402,6 +427,20 @@ class TestConvertNetworkData(TestCase): self.assertRaises(ValueError, ds.convert_network_data, NETWORK_DATA, known_macs=macs) + def test_conversion_with_route(self): + ncfg = ds.convert_network_data(NETWORK_DATA_2, known_macs=KNOWN_MACS) + # not the best test, but see that we get a route in the + # network config and that it gets rendered to an ENI file + routes = [] + for n in ncfg['config']: + for s in n.get('subnets', []): + routes.extend(s.get('routes', [])) + self.assertIn( + {'network': '0.0.0.0', 'netmask': '0.0.0.0', 'gateway': '2.2.2.9'}, + routes) + eni = net.render_interfaces(net.parse_net_config_data(ncfg)) + self.assertIn("route add default gw 2.2.2.9", eni) + def cfg_ds_from_dir(seed_d): found = ds.read_config_drive(seed_d) -- cgit v1.2.3 From 42a7d2b6d44be5fd6e41734902e08897b709015d Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 3 Jun 2016 15:06:55 -0400 Subject: config drive conversion: recognize 'bridge' as a physical type, fix mtu the network json in openstack provides a type of 'bridge' when the underlying (host) type is a bridge. Silly, but we need to consider that a physical device as it will be for us. also, the 'mtu' will appear on the link, not on the route --- cloudinit/sources/DataSourceConfigDrive.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 3cc9155d..c87f57fd 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -293,6 +293,7 @@ def convert_network_data(network_json=None, known_macs=None): 'mac_address', 'subnets', 'params', + 'mtu', ], 'subnet': [ 'type', @@ -302,7 +303,6 @@ def convert_network_data(network_json=None, known_macs=None): 'metric', 'gateway', 'pointopoint', - 'mtu', 'scope', 'dns_nameservers', 'dns_search', @@ -342,7 +342,7 @@ def convert_network_data(network_json=None, known_macs=None): }) subnets.append(subnet) cfg.update({'subnets': subnets}) - if link['type'] in ['ethernet', 'vif', 'ovs', 'phy']: + if link['type'] in ['ethernet', 'vif', 'ovs', 'phy', 'bridge']: cfg.update({ 'type': 'physical', 'mac_address': link['ethernet_mac_address']}) -- cgit v1.2.3 From 80931f7008971c9a7705c054fabc29fec7a133e2 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 3 Jun 2016 15:27:32 -0400 Subject: fix tox -e flake8 --- cloudinit/config/cc_apt_configure.py | 4 +- .../test_handler_apt_configure_sources_list.py | 17 +++--- .../test_handler/test_handler_apt_source.py | 66 +++++++++++----------- 3 files changed, 43 insertions(+), 44 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index d603f417..7a9777c0 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -199,8 +199,8 @@ def add_key(ent): def convert_to_new_format(srclist): - """ convert_to_new_format - convert the old list based format to the new dict based one + """convert_to_new_format + convert the old list based format to the new dict based one """ srcdict = {} if isinstance(srclist, list): diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py index 5cf386f8..5d0417a2 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py @@ -3,7 +3,6 @@ Test templating of sources list """ import logging import os -import re import shutil import tempfile @@ -62,14 +61,14 @@ deb-src http://archive.ubuntu.com/ubuntu/ fakerelease main restricted def load_tfile_or_url(*args, **kwargs): - """ load_tfile_or_url + """load_tfile_or_url load file and return content after decoding """ return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents) class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): - """ TestAptSourceConfigSourceList + """TestAptSourceConfigSourceList Main Class to test sources list rendering """ def setUp(self): @@ -89,7 +88,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): return cloud.Cloud(myds, paths, {}, mydist, None) def apt_source_list(self, distro, mirror, mirrorcheck=None): - """ apt_source_list + """apt_source_list Test rendering of a source.list from template for a given distro """ if mirrorcheck is None: @@ -115,19 +114,19 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): {'codename': '', 'primary': mirrorcheck, 'mirror': mirrorcheck}) def test_apt_source_list_debian(self): - """ test_apt_source_list_debian + """test_apt_source_list_debian Test rendering of a source.list from template for debian """ self.apt_source_list('debian', 'http://httpredir.debian.org/debian') def test_apt_source_list_ubuntu(self): - """ test_apt_source_list_ubuntu + """test_apt_source_list_ubuntu Test rendering of a source.list from template for ubuntu """ self.apt_source_list('ubuntu', 'http://archive.ubuntu.com/ubuntu/') def test_apt_srcl_debian_mirrorfail(self): - """ test_apt_source_list_debian_mirrorfail + """test_apt_source_list_debian_mirrorfail Test rendering of a source.list from template for debian """ self.apt_source_list('debian', ['http://does.not.exist', @@ -135,7 +134,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): 'http://httpredir.debian.org/debian') def test_apt_srcl_ubuntu_mirrorfail(self): - """ test_apt_source_list_ubuntu_mirrorfail + """test_apt_source_list_ubuntu_mirrorfail Test rendering of a source.list from template for ubuntu """ self.apt_source_list('ubuntu', ['http://does.not.exist', @@ -143,7 +142,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): 'http://archive.ubuntu.com/ubuntu/') def test_apt_srcl_custom(self): - """ test_apt_srcl_custom + """test_apt_srcl_custom Test rendering from a custom source.list template """ cfg = util.load_yaml(YAML_TEXT_CUSTOM_SL) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index fe2ffae5..4dbe69f0 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -33,14 +33,14 @@ S0ORP6HXET3+jC8BMG4tBWCTK/XEZw== def load_tfile_or_url(*args, **kwargs): - """ load_tfile_or_url + """load_tfile_or_url load file and return content after decoding """ return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents) class TestAptSourceConfig(TestCase): - """ TestAptSourceConfig + """TestAptSourceConfig Main Class to test apt_source configs """ def setUp(self): @@ -57,7 +57,7 @@ class TestAptSourceConfig(TestCase): @staticmethod def _get_default_params(): - """ get_default_params + """get_default_params Get the most basic default mrror and release info to be used in tests """ params = {} @@ -66,7 +66,7 @@ class TestAptSourceConfig(TestCase): return params def myjoin(self, *args, **kwargs): - """ myjoin - redir into writable tmpdir""" + """myjoin - redir into writable tmpdir""" if (args[0] == "/etc/apt/sources.list.d/" and args[1] == "cloud_config_sources.list" and len(args) == 2): @@ -75,7 +75,7 @@ class TestAptSourceConfig(TestCase): return self.join(*args, **kwargs) def apt_src_basic(self, filename, cfg): - """ apt_src_basic + """apt_src_basic Test Fix deb source string, has to overwrite mirror conf in params """ params = self._get_default_params() @@ -92,7 +92,7 @@ class TestAptSourceConfig(TestCase): contents, flags=re.IGNORECASE)) def test_apt_src_basic(self): - """ test_apt_src_basic + """test_apt_src_basic Test Fix deb source string, has to overwrite mirror conf in params. Test with a filename provided in config. """ @@ -103,7 +103,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_basic(self.aptlistfile, [cfg]) def test_apt_src_basic_dict(self): - """ test_apt_src_basic_dict + """test_apt_src_basic_dict Test Fix deb source string, has to overwrite mirror conf in params. Test with a filename provided in config. Provided in a dictionary with filename being the key (new format) @@ -115,7 +115,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_basic(self.aptlistfile, cfg) def apt_src_basic_tri(self, cfg): - """ apt_src_basic_tri + """apt_src_basic_tri Test Fix three deb source string, has to overwrite mirror conf in params. Test with filenames provided in config. generic part to check three files with different content @@ -137,7 +137,7 @@ class TestAptSourceConfig(TestCase): contents, flags=re.IGNORECASE)) def test_apt_src_basic_tri(self): - """ test_apt_src_basic_tri + """test_apt_src_basic_tri Test Fix three deb source string, has to overwrite mirror conf in params. Test with filenames provided in config. """ @@ -156,7 +156,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_basic_tri([cfg1, cfg2, cfg3]) def test_apt_src_basic_dict_tri(self): - """ test_apt_src_basic_dict_tri + """test_apt_src_basic_dict_tri Test Fix three deb source string, has to overwrite mirror conf in params. Test with filenames provided in config. Provided in a dictionary with filename being the key (new format) @@ -176,7 +176,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_basic_tri(cfg) def test_apt_src_basic_nofn(self): - """ test_apt_src_basic_nofn + """test_apt_src_basic_nofn Test Fix deb source string, has to overwrite mirror conf in params. Test without a filename provided in config and test for known fallback. """ @@ -187,7 +187,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_basic(self.fallbackfn, [cfg]) def apt_src_replacement(self, filename, cfg): - """ apt_src_replace + """apt_src_replace Test Autoreplacement of MIRROR and RELEASE in source specs """ params = self._get_default_params() @@ -202,7 +202,7 @@ class TestAptSourceConfig(TestCase): contents, flags=re.IGNORECASE)) def test_apt_src_replace(self): - """ test_apt_src_replace + """test_apt_src_replace Test Autoreplacement of MIRROR and RELEASE in source specs with Filename being set """ @@ -211,7 +211,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_replacement(self.aptlistfile, [cfg]) def apt_src_replace_tri(self, cfg): - """ apt_src_replace_tri + """apt_src_replace_tri Test three autoreplacements of MIRROR and RELEASE in source specs with generic part """ @@ -231,7 +231,7 @@ class TestAptSourceConfig(TestCase): contents, flags=re.IGNORECASE)) def test_apt_src_replace_tri(self): - """ test_apt_src_replace_tri + """test_apt_src_replace_tri Test three autoreplacements of MIRROR and RELEASE in source specs with Filename being set """ @@ -244,7 +244,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_replace_tri([cfg1, cfg2, cfg3]) def test_apt_src_replace_dict_tri(self): - """ test_apt_src_replace_dict_tri + """test_apt_src_replace_dict_tri Test three autoreplacements of MIRROR and RELEASE in source specs with Filename being set Provided in a dictionary with filename being the key (new format) @@ -252,13 +252,13 @@ class TestAptSourceConfig(TestCase): filenames to be overwritten inside the directory entry. """ cfg = {self.aptlistfile: {'source': 'deb $MIRROR $RELEASE multiverse'}, - 'notused': {'source': 'deb $MIRROR $RELEASE main', - 'filename': self.aptlistfile2}, + 'notused': {'source': 'deb $MIRROR $RELEASE main', + 'filename': self.aptlistfile2}, self.aptlistfile3: {'source': 'deb $MIRROR $RELEASE universe'}} self.apt_src_replace_tri(cfg) def test_apt_src_replace_nofn(self): - """ test_apt_src_replace_nofn + """test_apt_src_replace_nofn Test Autoreplacement of MIRROR and RELEASE in source specs with No filename being set """ @@ -267,7 +267,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_replacement(self.fallbackfn, [cfg]) def apt_src_keyid(self, filename, cfg, keynum): - """ apt_src_keyid + """apt_src_keyid Test specification of a source + keyid """ params = self._get_default_params() @@ -293,7 +293,7 @@ class TestAptSourceConfig(TestCase): contents, flags=re.IGNORECASE)) def test_apt_src_keyid(self): - """ test_apt_src_keyid + """test_apt_src_keyid Test specification of a source + keyid with filename being set """ cfg = {'source': ('deb ' @@ -305,7 +305,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_keyid(self.aptlistfile, [cfg], 1) def test_apt_src_keyid_tri(self): - """ test_apt_src_keyid_tri + """test_apt_src_keyid_tri Test specification of a source + keyid with filename being set Setting three of such, check for content and keys """ @@ -345,7 +345,7 @@ class TestAptSourceConfig(TestCase): contents, flags=re.IGNORECASE)) def test_apt_src_keyid_nofn(self): - """ test_apt_src_keyid_nofn + """test_apt_src_keyid_nofn Test specification of a source + keyid without filename being set """ cfg = {'source': ('deb ' @@ -357,7 +357,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_keyid(self.fallbackfn, [cfg], 1) def apt_src_key(self, filename, cfg): - """ apt_src_key + """apt_src_key Test specification of a source + key """ params = self._get_default_params() @@ -378,7 +378,7 @@ class TestAptSourceConfig(TestCase): contents, flags=re.IGNORECASE)) def test_apt_src_key(self): - """ test_apt_src_key + """test_apt_src_key Test specification of a source + key with filename being set """ cfg = {'source': ('deb ' @@ -390,7 +390,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_key(self.aptlistfile, cfg) def test_apt_src_key_nofn(self): - """ test_apt_src_key_nofn + """test_apt_src_key_nofn Test specification of a source + key without filename being set """ cfg = {'source': ('deb ' @@ -402,7 +402,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_key(self.fallbackfn, cfg) def test_apt_src_keyonly(self): - """ test_apt_src_keyonly + """test_apt_src_keyonly Test specification key without source """ params = self._get_default_params() @@ -419,7 +419,7 @@ class TestAptSourceConfig(TestCase): self.assertFalse(os.path.isfile(self.aptlistfile)) def test_apt_src_keyidonly(self): - """ test_apt_src_keyidonly + """test_apt_src_keyidonly Test specification of a keyid without source """ params = self._get_default_params() @@ -436,7 +436,7 @@ class TestAptSourceConfig(TestCase): self.assertFalse(os.path.isfile(self.aptlistfile)) def test_apt_src_keyid_real(self): - """ test_apt_src_keyid_real + """test_apt_src_keyid_real Test specification of a keyid without source incl up to addition of the key (nothing but add_key_raw mocked) """ @@ -454,7 +454,7 @@ class TestAptSourceConfig(TestCase): self.assertFalse(os.path.isfile(self.aptlistfile)) def test_apt_src_longkeyid_real(self): - """ test_apt_src_longkeyid_real + """test_apt_src_longkeyid_real Test specification of a long key fingerprint without source incl up to addition of the key (nothing but add_key_raw mocked) """ @@ -472,7 +472,7 @@ class TestAptSourceConfig(TestCase): self.assertFalse(os.path.isfile(self.aptlistfile)) def test_apt_src_ppa(self): - """ test_apt_src_ppa + """test_apt_src_ppa Test specification of a ppa """ params = self._get_default_params() @@ -491,7 +491,7 @@ class TestAptSourceConfig(TestCase): self.assertFalse(os.path.isfile(self.aptlistfile)) def test_apt_src_ppa_tri(self): - """ test_apt_src_ppa_tri + """test_apt_src_ppa_tri Test specification of a ppa """ params = self._get_default_params() @@ -519,7 +519,7 @@ class TestAptSourceConfig(TestCase): self.assertFalse(os.path.isfile(self.aptlistfile3)) def test_convert_to_new_format(self): - """ test_convert_to_new_format + """test_convert_to_new_format Test the conversion of old to new format And the noop conversion of new to new format as well """ -- cgit v1.2.3 From dca09871e61186deb216ce5cb1fc5db3b69c9fc2 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 6 Jun 2016 09:18:17 +0200 Subject: pythonify getkeybyid and move it to uitl.py --- cloudinit/config/cc_apt_configure.py | 27 +------------------------ cloudinit/util.py | 38 ++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 26 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 7a9777c0..2f270662 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -34,21 +34,6 @@ APT_PROXY_FN = "/etc/apt/apt.conf.d/95cloud-init-proxy" # this will match 'XXX:YYY' (ie, 'cloud-archive:foo' or 'ppa:bar') ADD_APT_REPO_MATCH = r"^[\w-]+:\w" -# A temporary shell program to get a given gpg key -# from a given keyserver -EXPORT_GPG_KEYID = """ - k=${1} ks=${2}; - exec 2>/dev/null - [ -n "$k" ] || exit 1; - armour=$(gpg --export --armour "${k}") - if [ -z "${armour}" ]; then - gpg --keyserver ${ks} --recv "${k}" >/dev/null && - armour=$(gpg --export --armour "${k}") && - gpg --batch --yes --delete-keys "${k}" - fi - [ -n "${armour}" ] && echo "${armour}" -""" - def handle(name, cfg, cloud, log, _args): if util.is_false(cfg.get('apt_configure_enabled', True)): @@ -108,16 +93,6 @@ def handle(name, cfg, cloud, log, _args): util.logexc(log, "Failed to run debconf-set-selections") -# get gpg keyid from keyserver -def getkeybyid(keyid, keyserver): - with util.ExtendedTemporaryFile(suffix='.sh', mode="w+", ) as fh: - fh.write(EXPORT_GPG_KEYID) - fh.flush() - cmd = ['/bin/sh', fh.name, keyid, keyserver] - (stdout, _stderr) = util.subp(cmd) - return stdout.strip() - - def mirror2lists_fileprefix(mirror): string = mirror # take off http:// or ftp:// @@ -192,7 +167,7 @@ def add_key(ent): keyserver = "keyserver.ubuntu.com" if 'keyserver' in ent: keyserver = ent['keyserver'] - ent['key'] = getkeybyid(ent['keyid'], keyserver) + ent['key'] = util.getkeybyid(ent['keyid'], keyserver) if 'key' in ent: add_key_raw(ent['key']) diff --git a/cloudinit/util.py b/cloudinit/util.py index d6b80dbe..d3b14f72 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2234,3 +2234,41 @@ def message_from_string(string): if sys.version_info[:2] < (2, 7): return email.message_from_file(six.StringIO(string)) return email.message_from_string(string) + + +def gpg_export_armour(key): + """Export gpg key, armoured key gets returned""" + (armour, _) = subp(["gpg", "--export", "--armour", key], capture=True) + return armour + + +def gpg_recv_key(key, keyserver): + """Receive gpg key from the specified keyserver""" + try: + subp(["gpg", "--keyserver", keyserver, "--recv", key], + capture=True) + except ProcessExecutionError as error: + raise ValueError('Failed to import key %s from server %s - error %s' % + (key, keyserver, error)) + + +def gpg_delete_key(key): + """Delete the specified key from the local gpg ring""" + subp(["gpg", "--batch", "--yes", "--delete-keys", key], capture=False) + + +def getkeybyid(keyid, keyserver): + """get gpg keyid from keyserver""" + armour = gpg_export_armour(keyid) + if not armour: + try: + gpg_recv_key(keyid, keyserver=keyserver) + except ValueError: + LOG.exception('Failed to obtain gpg key %s', keyid) + raise + + armour = gpg_export_armour(keyid) + # delete just imported key to leave environment as it was before + gpg_delete_key(keyid) + + return armour.rstrip('\n') -- cgit v1.2.3 From 89688593e9358433cd9383d1bdfae12dbcd58f72 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 6 Jun 2016 13:36:23 +0200 Subject: add_key_raw - make exceptions more specific --- cloudinit/config/cc_apt_configure.py | 4 +-- cloudinit/util.py | 1 + .../test_handler/test_handler_apt_source.py | 38 +++++++++++++++++++--- 3 files changed, 36 insertions(+), 7 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 2f270662..a9ac6ea8 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -154,7 +154,7 @@ def add_key_raw(key): try: util.subp(('apt-key', 'add', '-'), key) except util.ProcessExecutionError: - raise Exception('failed add key') + raise ValueError('failed to add apt GPG Key to apt keyring') def add_key(ent): @@ -221,7 +221,7 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): # keys can be added without specifying a source try: add_key(ent) - except Exception as detail: + except ValueError as detail: errorlist.append([ent, detail]) if 'source' not in ent: diff --git a/cloudinit/util.py b/cloudinit/util.py index d3b14f72..a1622946 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2244,6 +2244,7 @@ def gpg_export_armour(key): def gpg_recv_key(key, keyserver): """Receive gpg key from the specified keyserver""" + print("ORIGINAL gpg_recv_key") try: subp(["gpg", "--keyserver", keyserver, "--recv", key], capture=True) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index a4878124..65e375a0 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -4,6 +4,7 @@ Testing various config variations of the apt_source config import os import re import shutil +import socket import tempfile try: @@ -56,6 +57,7 @@ class TestAptSourceConfig(TestCase): # mock fallback filename into writable tmp dir self.fallbackfn = os.path.join(self.tmp, "etc/apt/sources.list.d/", "cloud_config_sources.list") + self.orig_gpg_recv_key = util.gpg_recv_key patcher = mock.patch("cloudinit.config.cc_apt_configure.get_release") get_rel = patcher.start() @@ -445,14 +447,40 @@ class TestAptSourceConfig(TestCase): def apt_src_keyid_real(self, cfg, expectedkey): """apt_src_keyid_real Test specification of a keyid without source including - up to addition of the key (nothing but add_key_raw mocked to keep the + up to addition of the key (add_key_raw mocked to keep the environment as is) """ params = self._get_default_params() - with mock.patch.object(cc_apt_configure, 'add_key_raw') as mockobj: - cc_apt_configure.add_sources([cfg], params) - mockobj.assert_called_with(expectedkey) + def fake_gpg_recv_key(self, key, keyserver): + """try original gpg_recv_key, but allow fall back""" + try: + print("Try orig orig_gpg_recv_key") + self.orig_gpg_recv_key(key, keyserver) + except ValueError: + print("Fail, test net") + # if this is a networking issue mock it's effect + testsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + testsock.connect((keyserver, 80)) + testsock.close() + except socket.error: + # as fallback add the known key as a working recv would + print("Fallback import expectedkey") + util.subp(("gpg", "--import", "-"), EXPECTEDKEY) + + print("FOO") + with mock.patch.object(cc_apt_configure, 'add_key_raw') as mockkey: + with mock.patch.object(util, 'gpg_recv_key', + side_effect=fake_gpg_recv_key) as mockrecv: + cc_apt_configure.add_sources([cfg], params) + + # since we might mock the recv path ensure it is called right + mockrecv.assert_called_with(cfg['keyid'], + keyserver=cfg.get('keyserver', + 'keyserver.ubuntu.com')) + # no matter if really imported or faked, ensure we add the right key + mockkey.assert_called_with(expectedkey) # filename should be ignored on key only self.assertFalse(os.path.isfile(self.aptlistfile)) @@ -477,7 +505,7 @@ class TestAptSourceConfig(TestCase): """test_apt_src_longkeyid_ks_real - Test long keyid from other ks""" keyid = "B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77" cfg = {'keyid': keyid, - 'keyserver': 'keys.gnupg.net', + 'keyserver': 'knorz.gnupg.net', 'filename': self.aptlistfile} self.apt_src_keyid_real(cfg, EXPECTEDKEY) -- cgit v1.2.3 From e9b333b49954d5863f8b53581454179b0363d978 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 6 Jun 2016 13:52:31 +0200 Subject: fixup key tests for cases where network isn't available --- cloudinit/util.py | 1 - tests/unittests/test_handler/test_handler_apt_source.py | 8 ++------ 2 files changed, 2 insertions(+), 7 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/util.py b/cloudinit/util.py index a1622946..d3b14f72 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2244,7 +2244,6 @@ def gpg_export_armour(key): def gpg_recv_key(key, keyserver): """Receive gpg key from the specified keyserver""" - print("ORIGINAL gpg_recv_key") try: subp(["gpg", "--keyserver", keyserver, "--recv", key], capture=True) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 65e375a0..c7eeb64c 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -452,13 +452,11 @@ class TestAptSourceConfig(TestCase): """ params = self._get_default_params() - def fake_gpg_recv_key(self, key, keyserver): + def fake_gpg_recv_key(key, keyserver): """try original gpg_recv_key, but allow fall back""" try: - print("Try orig orig_gpg_recv_key") self.orig_gpg_recv_key(key, keyserver) - except ValueError: - print("Fail, test net") + except ValueError as error: # if this is a networking issue mock it's effect testsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: @@ -466,10 +464,8 @@ class TestAptSourceConfig(TestCase): testsock.close() except socket.error: # as fallback add the known key as a working recv would - print("Fallback import expectedkey") util.subp(("gpg", "--import", "-"), EXPECTEDKEY) - print("FOO") with mock.patch.object(cc_apt_configure, 'add_key_raw') as mockkey: with mock.patch.object(util, 'gpg_recv_key', side_effect=fake_gpg_recv_key) as mockrecv: -- cgit v1.2.3 From c1ad09609805c6c4f5262eb21533799af814379a Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 6 Jun 2016 14:23:55 +0200 Subject: rename add_key / add_source to add_apt_key / add_apt_source The functions clearly are apt specific so the name should reflect that. --- cloudinit/config/cc_apt_configure.py | 14 ++++++------ .../test_handler/test_handler_apt_source.py | 25 +++++++++++----------- 2 files changed, 20 insertions(+), 19 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index a9ac6ea8..91a117f5 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -79,8 +79,8 @@ def handle(name, cfg, cloud, log, _args): def matcher(x): return False - errors = add_sources(cfg['apt_sources'], params, - aa_repo_match=matcher) + errors = add_apt_sources(cfg['apt_sources'], params, + aa_repo_match=matcher) for e in errors: log.warn("Add source error: %s", ':'.join(e)) @@ -146,7 +146,7 @@ def generate_sources_list(cfg, codename, mirrors, cloud, log): templater.render_to_file(template_fn, '/etc/apt/sources.list', params) -def add_key_raw(key): +def add_apt_key_raw(key): """ actual adding of a key as defined in key argument to the system @@ -157,7 +157,7 @@ def add_key_raw(key): raise ValueError('failed to add apt GPG Key to apt keyring') -def add_key(ent): +def add_apt_key(ent): """ add key to the system as defined in ent (if any) supports raw keys or keyid's @@ -170,7 +170,7 @@ def add_key(ent): ent['key'] = util.getkeybyid(ent['keyid'], keyserver) if 'key' in ent: - add_key_raw(ent['key']) + add_apt_key_raw(ent['key']) def convert_to_new_format(srclist): @@ -197,7 +197,7 @@ def convert_to_new_format(srclist): return srcdict -def add_sources(srclist, template_params=None, aa_repo_match=None): +def add_apt_sources(srclist, template_params=None, aa_repo_match=None): """ add entries in /etc/apt/sources.list.d for each abbreviated sources.list entry in 'srclist'. When rendering template, also @@ -220,7 +220,7 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): # keys can be added without specifying a source try: - add_key(ent) + add_apt_key(ent) except ValueError as detail: errorlist.append([ent, detail]) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index c7eeb64c..4a720213 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -89,7 +89,7 @@ class TestAptSourceConfig(TestCase): """ params = self._get_default_params() - cc_apt_configure.add_sources(cfg, params) + cc_apt_configure.add_apt_sources(cfg, params) self.assertTrue(os.path.isfile(filename)) @@ -200,7 +200,7 @@ class TestAptSourceConfig(TestCase): Test Autoreplacement of MIRROR and RELEASE in source specs """ params = self._get_default_params() - cc_apt_configure.add_sources(cfg, params) + cc_apt_configure.add_apt_sources(cfg, params) self.assertTrue(os.path.isfile(filename)) @@ -283,7 +283,7 @@ class TestAptSourceConfig(TestCase): with mock.patch.object(util, 'subp', return_value=('fakekey 1234', '')) as mockobj: - cc_apt_configure.add_sources(cfg, params) + cc_apt_configure.add_apt_sources(cfg, params) # check if it added the right ammount of keys calls = [] @@ -372,7 +372,7 @@ class TestAptSourceConfig(TestCase): params = self._get_default_params() with mock.patch.object(util, 'subp') as mockobj: - cc_apt_configure.add_sources([cfg], params) + cc_apt_configure.add_apt_sources([cfg], params) mockobj.assert_called_with(('apt-key', 'add', '-'), 'fakekey 4321') @@ -419,7 +419,7 @@ class TestAptSourceConfig(TestCase): 'filename': self.aptlistfile} with mock.patch.object(util, 'subp') as mockobj: - cc_apt_configure.add_sources([cfg], params) + cc_apt_configure.add_apt_sources([cfg], params) mockobj.assert_called_once_with(('apt-key', 'add', '-'), 'fakekey 4242') @@ -437,7 +437,7 @@ class TestAptSourceConfig(TestCase): with mock.patch.object(util, 'subp', return_value=('fakekey 1212', '')) as mockobj: - cc_apt_configure.add_sources([cfg], params) + cc_apt_configure.add_apt_sources([cfg], params) mockobj.assert_called_with(('apt-key', 'add', '-'), 'fakekey 1212') @@ -447,7 +447,7 @@ class TestAptSourceConfig(TestCase): def apt_src_keyid_real(self, cfg, expectedkey): """apt_src_keyid_real Test specification of a keyid without source including - up to addition of the key (add_key_raw mocked to keep the + up to addition of the key (add_apt_key_raw mocked to keep the environment as is) """ params = self._get_default_params() @@ -466,10 +466,10 @@ class TestAptSourceConfig(TestCase): # as fallback add the known key as a working recv would util.subp(("gpg", "--import", "-"), EXPECTEDKEY) - with mock.patch.object(cc_apt_configure, 'add_key_raw') as mockkey: + with mock.patch.object(cc_apt_configure, 'add_apt_key_raw') as mockkey: with mock.patch.object(util, 'gpg_recv_key', side_effect=fake_gpg_recv_key) as mockrecv: - cc_apt_configure.add_sources([cfg], params) + cc_apt_configure.add_apt_sources([cfg], params) # since we might mock the recv path ensure it is called right mockrecv.assert_called_with(cfg['keyid'], @@ -518,7 +518,8 @@ class TestAptSourceConfig(TestCase): matcher = re.compile(r'^[\w-]+:\w').search with mock.patch.object(util, 'subp') as mockobj: - cc_apt_configure.add_sources([cfg], params, aa_repo_match=matcher) + cc_apt_configure.add_apt_sources([cfg], params, + aa_repo_match=matcher) mockobj.assert_called_once_with(['add-apt-repository', 'ppa:smoser/cloud-init-test']) @@ -541,8 +542,8 @@ class TestAptSourceConfig(TestCase): matcher = re.compile(r'^[\w-]+:\w').search with mock.patch.object(util, 'subp') as mockobj: - cc_apt_configure.add_sources([cfg1, cfg2, cfg3], params, - aa_repo_match=matcher) + cc_apt_configure.add_apt_sources([cfg1, cfg2, cfg3], params, + aa_repo_match=matcher) calls = [call(['add-apt-repository', 'ppa:smoser/cloud-init-test']), call(['add-apt-repository', 'ppa:smoser/cloud-init-test2']), call(['add-apt-repository', 'ppa:smoser/cloud-init-test3'])] -- cgit v1.2.3 From 71ea8d200da164df6f5e28d28aea424b57fd8f1f Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 6 Jun 2016 15:18:59 +0200 Subject: rename fucntion to mirrorurl_to_apt_fileprefix to reflect what it actually does --- cloudinit/config/cc_apt_configure.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 91a117f5..be6324a2 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -93,7 +93,7 @@ def handle(name, cfg, cloud, log, _args): util.logexc(log, "Failed to run debconf-set-selections") -def mirror2lists_fileprefix(mirror): +def mirrorurl_to_apt_fileprefix(mirror): string = mirror # take off http:// or ftp:// if string.endswith("/"): @@ -110,8 +110,8 @@ def rename_apt_lists(old_mirrors, new_mirrors, lists_d="/var/lib/apt/lists"): nmirror = new_mirrors.get(name) if not nmirror: continue - oprefix = os.path.join(lists_d, mirror2lists_fileprefix(omirror)) - nprefix = os.path.join(lists_d, mirror2lists_fileprefix(nmirror)) + oprefix = os.path.join(lists_d, mirrorurl_to_apt_fileprefix(omirror)) + nprefix = os.path.join(lists_d, mirrorurl_to_apt_fileprefix(nmirror)) if oprefix == nprefix: continue olen = len(oprefix) -- cgit v1.2.3 From bb0bf81cb373fde0c59464127633ebc7527f2be5 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 6 Jun 2016 15:48:25 +0200 Subject: capture output of gpg calls to avoid messing up stdout/stderr --- cloudinit/util.py | 2 +- tests/unittests/test_handler/test_handler_apt_source.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/util.py b/cloudinit/util.py index d3b14f72..6d16532d 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2254,7 +2254,7 @@ def gpg_recv_key(key, keyserver): def gpg_delete_key(key): """Delete the specified key from the local gpg ring""" - subp(["gpg", "--batch", "--yes", "--delete-keys", key], capture=False) + subp(["gpg", "--batch", "--yes", "--delete-keys", key], capture=True) def getkeybyid(keyid, keyserver): diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 9aa6ff71..ea8aa17a 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -469,7 +469,8 @@ class TestAptSourceConfig(TestCase): testsock.close() except socket.error: # as fallback add the known key as a working recv would - util.subp(("gpg", "--import", "-"), EXPECTEDKEY) + util.subp(("gpg", "--import", "-"), EXPECTEDKEY, + capture=True) with mock.patch.object(cc_apt_configure, 'add_apt_key_raw') as mockkey: with mock.patch.object(util, 'gpg_recv_key', -- cgit v1.2.3 From c176089d9870f373e5876d246820fba6ff44d6c8 Mon Sep 17 00:00:00 2001 From: Andrew Jorgensen Date: Mon, 6 Jun 2016 09:19:10 -0700 Subject: Change the Cheetah warning to a debug message In the absence of cheetah, which is a fairly heavy templating engine, and not strictly needed by anything in cloud-init, the only warning we saw in the logs was this one from the templater. Degrading this to a debug message makes any other warnings more relevant. --- cloudinit/templater.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/templater.py b/cloudinit/templater.py index 8a6ad417..41ef27e3 100644 --- a/cloudinit/templater.py +++ b/cloudinit/templater.py @@ -3,10 +3,12 @@ # Copyright (C) 2012 Canonical Ltd. # Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # Copyright (C) 2012 Yahoo! Inc. +# Copyright (C) 2016 Amazon.com, Inc. or its affiliates. # # Author: Scott Moser # Author: Juerg Haefliger # Author: Joshua Harlow +# Author: Andrew Jorgensen # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -102,12 +104,11 @@ def detect_template(text): rest = '' type_match = TYPE_MATCHER.match(ident) if not type_match: - if not CHEETAH_AVAILABLE: - LOG.warn("Cheetah not available as the default renderer for" - " unknown template, reverting to the basic renderer.") - return ('basic', basic_render, text) - else: + if CHEETAH_AVAILABLE: + LOG.debug("Using Cheetah as the renderer for unknown template.") return ('cheetah', cheetah_render, text) + else: + return ('basic', basic_render, text) else: template_type = type_match.group(1).lower().strip() if template_type not in ('jinja', 'cheetah', 'basic'): -- cgit v1.2.3 From 600ce494084368c511848c00fff7220255b14e6b Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 6 Jun 2016 18:45:55 -0700 Subject: Get cmdline working again --- cloudinit/net/cmdline.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py index 39523be2..187f7026 100644 --- a/cloudinit/net/cmdline.py +++ b/cloudinit/net/cmdline.py @@ -24,7 +24,6 @@ import shlex from . import compat from . import get_devicelist -from . import read_file from . import sys_netdev_info from cloudinit import util @@ -136,7 +135,7 @@ def config_from_klibc_net_cfg(files=None, mac_addrs=None): entries = [] names = {} for cfg_file in files: - name, entry = _klibc_to_config_entry(read_file(cfg_file), + name, entry = _klibc_to_config_entry(util.load_file(cfg_file), mac_addrs=mac_addrs) if name in names: raise ValueError( -- cgit v1.2.3 From 6d0e6068f2692fd34c3c5cc93af7ec8ef5be4711 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 7 Jun 2016 10:59:27 -0700 Subject: For now just remove compat.py module Let's reduce the size of this change for now. --- cloudinit/net/cmdline.py | 10 ++++++--- cloudinit/net/compat.py | 51 ------------------------------------------ cloudinit/net/network_state.py | 4 ++-- 3 files changed, 9 insertions(+), 56 deletions(-) delete mode 100644 cloudinit/net/compat.py (limited to 'cloudinit') diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py index 187f7026..822a020b 100644 --- a/cloudinit/net/cmdline.py +++ b/cloudinit/net/cmdline.py @@ -21,16 +21,20 @@ import glob import gzip import io import shlex +import sys + +import six -from . import compat from . import get_devicelist from . import sys_netdev_info from cloudinit import util +PY26 = sys.version_info[0:2] == (2, 6) + def _shlex_split(blob): - if compat.PY26 and isinstance(blob, compat.text_type): + if PY26 and isinstance(blob, six.text_type): # Older versions don't support unicode input blob = blob.encode("utf8") return shlex.split(blob) @@ -42,7 +46,7 @@ def _load_shell_content(content, add_empty=False, empty_val=None): then add entries in to the returned dictionary for 'VAR=' variables. Set their value to empty_val.""" data = {} - for line in shlex.split(content): + for line in _shlex_split(content): key, value = line.split("=", 1) if not value: value = empty_val diff --git a/cloudinit/net/compat.py b/cloudinit/net/compat.py deleted file mode 100644 index 8bf92ef5..00000000 --- a/cloudinit/net/compat.py +++ /dev/null @@ -1,51 +0,0 @@ -# Curtin is free software: you can redistribute it and/or modify it under -# the terms of the GNU Affero General Public License as published by the -# Free Software Foundation, either version 3 of the License, or (at your -# option) any later version. -# -# Curtin is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for -# more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Curtin. If not, see . - -# Mini six like module (so that this code can be more easily extracted). - -import sys - -PY26 = sys.version_info[0:2] == (2, 6) -PY27 = sys.version_info[0:2] == (2, 7) -PY2 = PY26 or PY27 -PY3 = sys.version_info[0:2] >= (3, 0) - -if PY3: - text_type = str - binary_type = bytes - string_types = (text_type, text_type) - import io - StringIO = io.StringIO -else: - text_type = unicode - binary_type = bytes - string_types = (binary_type, text_type) - from StringIO import StringIO # noqa - - -# Taken from six (remove when we can actually directly use six) - -def add_metaclass(metaclass): - """Class decorator for creating a class with a metaclass.""" - def wrapper(cls): - orig_vars = cls.__dict__.copy() - slots = orig_vars.get('__slots__') - if slots is not None: - if isinstance(slots, str): - slots = [slots] - for slots_var in slots: - orig_vars.pop(slots_var) - orig_vars.pop('__dict__', None) - orig_vars.pop('__weakref__', None) - return metaclass(cls.__name__, cls.__bases__, orig_vars) - return wrapper diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 1e82e75d..a8be5e26 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -19,7 +19,7 @@ import copy import functools import logging -from . import compat +import six from cloudinit import util @@ -113,7 +113,7 @@ class CommandHandlerMeta(type): parents, dct) -@compat.add_metaclass(CommandHandlerMeta) +@six.add_metaclass(CommandHandlerMeta) class NetworkState(object): initial_network_state = { -- cgit v1.2.3 From dbeff5c80281ee1c236583d523feb323c7518c19 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 7 Jun 2016 14:34:57 -0400 Subject: system config of networking should override datasource provided while datasource provided networking is more dynamic in most cases, preference should still be given to networking configuration provided in the system. This is because the user of the image should be ultimately in control of the networking configuration if they so choose. LP: #1590104 --- cloudinit/stages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 5756e74d..4ba95318 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -618,7 +618,7 @@ class Init(object): dscfg = ('ds', self.datasource.network_config) sys_cfg = ('system_cfg', self.cfg.get('network')) - for loc, ncfg in (cmdline_cfg, dscfg, sys_cfg): + for loc, ncfg in (cmdline_cfg, sys_cfg, dscfg): if net.is_disabled_cfg(ncfg): LOG.debug("network config disabled by %s", loc) return (None, loc) -- cgit v1.2.3 From b89ac6b7caeeade5ad21137773ac4496cdaea2c5 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 9 Jun 2016 09:18:35 +0200 Subject: move gpg functions into gpg.py This helps for cleaner code structuring. ALong that makeing sure all these functions have a gpg_prefix. --- cloudinit/config/cc_apt_configure.py | 3 +- cloudinit/gpg.py | 64 ++++++++++++++++++++++ cloudinit/util.py | 38 ------------- .../test_handler/test_handler_apt_source.py | 4 +- 4 files changed, 68 insertions(+), 41 deletions(-) create mode 100644 cloudinit/gpg.py (limited to 'cloudinit') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index be6324a2..ba080930 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -24,6 +24,7 @@ import re from cloudinit import templater from cloudinit import util +from cloudinit import gpg distros = ['ubuntu', 'debian'] @@ -167,7 +168,7 @@ def add_apt_key(ent): keyserver = "keyserver.ubuntu.com" if 'keyserver' in ent: keyserver = ent['keyserver'] - ent['key'] = util.getkeybyid(ent['keyid'], keyserver) + ent['key'] = gpg.gpg_getkeybyid(ent['keyid'], keyserver) if 'key' in ent: add_apt_key_raw(ent['key']) diff --git a/cloudinit/gpg.py b/cloudinit/gpg.py new file mode 100644 index 00000000..620dfb19 --- /dev/null +++ b/cloudinit/gpg.py @@ -0,0 +1,64 @@ +"""gpg.py - Collection of gpg key related functions""" +# vi: ts=4 expandtab +# +# Copyright (C) 2016 Canonical Ltd. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# Author: Christian Ehrhardt +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from cloudinit import util +from cloudinit import log as logging + +LOG = logging.getLogger(__name__) + + +def gpg_export_armour(key): + """Export gpg key, armoured key gets returned""" + (armour, _) = util.subp(["gpg", "--export", "--armour", key], capture=True) + return armour + + +def gpg_recv_key(key, keyserver): + """Receive gpg key from the specified keyserver""" + try: + util.subp(["gpg", "--keyserver", keyserver, "--recv", key], + capture=True) + except util.ProcessExecutionError as error: + raise ValueError('Failed to import key %s from server %s - error %s' % + (key, keyserver, error)) + + +def gpg_delete_key(key): + """Delete the specified key from the local gpg ring""" + util.subp(["gpg", "--batch", "--yes", "--delete-keys", key], capture=True) + + +def gpg_getkeybyid(keyid, keyserver): + """get gpg keyid from keyserver""" + armour = gpg_export_armour(keyid) + if not armour: + try: + gpg_recv_key(keyid, keyserver=keyserver) + except ValueError: + LOG.exception('Failed to obtain gpg key %s', keyid) + raise + + armour = gpg_export_armour(keyid) + # delete just imported key to leave environment as it was before + gpg_delete_key(keyid) + + return armour.rstrip('\n') diff --git a/cloudinit/util.py b/cloudinit/util.py index 6d16532d..d6b80dbe 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2234,41 +2234,3 @@ def message_from_string(string): if sys.version_info[:2] < (2, 7): return email.message_from_file(six.StringIO(string)) return email.message_from_string(string) - - -def gpg_export_armour(key): - """Export gpg key, armoured key gets returned""" - (armour, _) = subp(["gpg", "--export", "--armour", key], capture=True) - return armour - - -def gpg_recv_key(key, keyserver): - """Receive gpg key from the specified keyserver""" - try: - subp(["gpg", "--keyserver", keyserver, "--recv", key], - capture=True) - except ProcessExecutionError as error: - raise ValueError('Failed to import key %s from server %s - error %s' % - (key, keyserver, error)) - - -def gpg_delete_key(key): - """Delete the specified key from the local gpg ring""" - subp(["gpg", "--batch", "--yes", "--delete-keys", key], capture=True) - - -def getkeybyid(keyid, keyserver): - """get gpg keyid from keyserver""" - armour = gpg_export_armour(keyid) - if not armour: - try: - gpg_recv_key(keyid, keyserver=keyserver) - except ValueError: - LOG.exception('Failed to obtain gpg key %s', keyid) - raise - - armour = gpg_export_armour(keyid) - # delete just imported key to leave environment as it was before - gpg_delete_key(keyid) - - return armour.rstrip('\n') diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 3f87fad3..b09dd8ab 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -14,6 +14,7 @@ from mock import call from cloudinit.config import cc_apt_configure from cloudinit import util +from cloudinit import gpg from ..helpers import TestCase @@ -58,7 +59,6 @@ class TestAptSourceConfig(TestCase): # mock fallback filename into writable tmp dir self.fallbackfn = os.path.join(self.tmp, "etc/apt/sources.list.d/", "cloud_config_sources.list") - self.orig_gpg_recv_key = util.gpg_recv_key patcher = mock.patch("cloudinit.config.cc_apt_configure.get_release") get_rel = patcher.start() @@ -407,7 +407,7 @@ class TestAptSourceConfig(TestCase): params = self._get_default_params() with mock.patch.object(cc_apt_configure, 'add_apt_key_raw') as mockkey: - with mock.patch.object(util, 'getkeybyid', + with mock.patch.object(gpg, 'gpg_getkeybyid', return_value=expectedkey) as mockgetkey: cc_apt_configure.add_apt_sources([cfg], params) -- cgit v1.2.3 From e6cfca0c898f0482f37216cf58e61ff1581aaaf6 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 9 Jun 2016 09:35:09 +0200 Subject: improve error handling and reporting in gpg functions --- cloudinit/gpg.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/gpg.py b/cloudinit/gpg.py index 620dfb19..baa8b534 100644 --- a/cloudinit/gpg.py +++ b/cloudinit/gpg.py @@ -28,23 +28,35 @@ LOG = logging.getLogger(__name__) def gpg_export_armour(key): """Export gpg key, armoured key gets returned""" - (armour, _) = util.subp(["gpg", "--export", "--armour", key], capture=True) + try: + (armour, _) = util.subp(["gpg", "--export", "--armour", key], + capture=True) + except util.ProcessExecutionError as error: + # debug, since it happens for any key not on the system initially + LOG.debug('Failed to export armoured key "%s": %s', key, error) + armour = None return armour def gpg_recv_key(key, keyserver): """Receive gpg key from the specified keyserver""" + LOG.debug('Receive gpg key "%s"', key) try: util.subp(["gpg", "--keyserver", keyserver, "--recv", key], capture=True) except util.ProcessExecutionError as error: - raise ValueError('Failed to import key %s from server %s - error %s' % + raise ValueError(('Failed to import key "%s" ' + 'from server "%s" - error %s') % (key, keyserver, error)) def gpg_delete_key(key): """Delete the specified key from the local gpg ring""" - util.subp(["gpg", "--batch", "--yes", "--delete-keys", key], capture=True) + try: + util.subp(["gpg", "--batch", "--yes", "--delete-keys", key], + capture=True) + except util.ProcessExecutionError as error: + LOG.warn('Failed delete key "%s": %s', key, error) def gpg_getkeybyid(keyid, keyserver): @@ -53,12 +65,12 @@ def gpg_getkeybyid(keyid, keyserver): if not armour: try: gpg_recv_key(keyid, keyserver=keyserver) + armour = gpg_export_armour(keyid) except ValueError: LOG.exception('Failed to obtain gpg key %s', keyid) raise - - armour = gpg_export_armour(keyid) - # delete just imported key to leave environment as it was before - gpg_delete_key(keyid) + finally: + # delete just imported key to leave environment as it was before + gpg_delete_key(keyid) return armour.rstrip('\n') -- cgit v1.2.3 From 7e527b1b2f3fda558fb0f3a6958c42dde4716079 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 10 Jun 2016 13:22:59 -0400 Subject: minor changes prior to merge a.) remove 'gpg_' from function names in new gpg module. b.) use --recv-keys rather than --recv --recv-keys is more obvious and works back to precise at least. c.) do not trim trailing '\n' from a armour'd key. --- cloudinit/config/cc_apt_configure.py | 4 ++-- cloudinit/gpg.py | 24 ++++++++++------------ .../test_handler/test_handler_apt_source.py | 4 ++-- 3 files changed, 15 insertions(+), 17 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index ba080930..96c4a43d 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -22,9 +22,9 @@ import glob import os import re +from cloudinit import gpg from cloudinit import templater from cloudinit import util -from cloudinit import gpg distros = ['ubuntu', 'debian'] @@ -168,7 +168,7 @@ def add_apt_key(ent): keyserver = "keyserver.ubuntu.com" if 'keyserver' in ent: keyserver = ent['keyserver'] - ent['key'] = gpg.gpg_getkeybyid(ent['keyid'], keyserver) + ent['key'] = gpg.get_key_by_id(ent['keyid'], keyserver) if 'key' in ent: add_apt_key_raw(ent['key']) diff --git a/cloudinit/gpg.py b/cloudinit/gpg.py index baa8b534..6a76d785 100644 --- a/cloudinit/gpg.py +++ b/cloudinit/gpg.py @@ -4,8 +4,6 @@ # Copyright (C) 2016 Canonical Ltd. # # Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow # Author: Christian Ehrhardt # # This program is free software: you can redistribute it and/or modify @@ -20,13 +18,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from cloudinit import util from cloudinit import log as logging +from cloudinit import util LOG = logging.getLogger(__name__) -def gpg_export_armour(key): +def export_armour(key): """Export gpg key, armoured key gets returned""" try: (armour, _) = util.subp(["gpg", "--export", "--armour", key], @@ -38,11 +36,11 @@ def gpg_export_armour(key): return armour -def gpg_recv_key(key, keyserver): +def receive_key(key, keyserver): """Receive gpg key from the specified keyserver""" LOG.debug('Receive gpg key "%s"', key) try: - util.subp(["gpg", "--keyserver", keyserver, "--recv", key], + util.subp(["gpg", "--keyserver", keyserver, "--recv-keys", key], capture=True) except util.ProcessExecutionError as error: raise ValueError(('Failed to import key "%s" ' @@ -50,7 +48,7 @@ def gpg_recv_key(key, keyserver): (key, keyserver, error)) -def gpg_delete_key(key): +def delete_key(key): """Delete the specified key from the local gpg ring""" try: util.subp(["gpg", "--batch", "--yes", "--delete-keys", key], @@ -59,18 +57,18 @@ def gpg_delete_key(key): LOG.warn('Failed delete key "%s": %s', key, error) -def gpg_getkeybyid(keyid, keyserver): +def get_key_by_id(keyid, keyserver="keyserver.ubuntu.com"): """get gpg keyid from keyserver""" - armour = gpg_export_armour(keyid) + armour = export_armour(keyid) if not armour: try: - gpg_recv_key(keyid, keyserver=keyserver) - armour = gpg_export_armour(keyid) + receive_key(keyid, keyserver=keyserver) + armour = export_armour(keyid) except ValueError: LOG.exception('Failed to obtain gpg key %s', keyid) raise finally: # delete just imported key to leave environment as it was before - gpg_delete_key(keyid) + delete_key(keyid) - return armour.rstrip('\n') + return armour diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 5b52f709..99a4d860 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -13,8 +13,8 @@ except ImportError: from mock import call from cloudinit.config import cc_apt_configure -from cloudinit import util from cloudinit import gpg +from cloudinit import util from ..helpers import TestCase @@ -405,7 +405,7 @@ class TestAptSourceConfig(TestCase): params = self._get_default_params() with mock.patch.object(cc_apt_configure, 'add_apt_key_raw') as mockkey: - with mock.patch.object(gpg, 'gpg_getkeybyid', + with mock.patch.object(gpg, 'get_key_by_id', return_value=expectedkey) as mockgetkey: cc_apt_configure.add_apt_sources([cfg], params) -- cgit v1.2.3 From 6ada153ebfd9483d76f904dafaa1fc80f61c9205 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 10 Jun 2016 14:16:51 -0700 Subject: Just mock 'on_first_boot' vs special argument --- cloudinit/sources/DataSourceConfigDrive.py | 5 ++--- tests/unittests/test_datasource/test_configdrive.py | 11 +++++++---- 2 files changed, 9 insertions(+), 7 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index cea08de2..3130e618 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -62,7 +62,7 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): mstr += "[source=%s]" % (self.source) return mstr - def get_data(self, skip_first_boot=False): + def get_data(self): found = None md = {} results = {} @@ -111,8 +111,7 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): # 'injected files' and apply legacy ENI network format. prev_iid = get_previous_iid(self.paths) cur_iid = md['instance-id'] - if (prev_iid != cur_iid and self.dsmode == sources.DSMODE_PASS - and not skip_first_boot): + if prev_iid != cur_iid and self.dsmode == sources.DSMODE_PASS: on_first_boot(results, distro=self.distro) LOG.debug("%s: not claiming datasource, dsmode=%s", self, self.dsmode) diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index b898e2bd..18551b92 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -370,7 +370,8 @@ class TestConfigDriveDataSource(TestCase): util.find_devs_with = orig_find_devs_with util.is_partition = orig_is_partition - def test_pubkeys_v2(self): + @mock.patch('cloudinit.sources.DataSourceConfigDrive.on_first_boot') + def test_pubkeys_v2(self, on_first_boot): """Verify that public-keys work in config-drive-v2.""" populate_dir(self.tmp, CFG_DRIVE_FILES_V2) myds = cfg_ds_from_dir(self.tmp) @@ -385,13 +386,15 @@ class TestNetJson(TestCase): self.addCleanup(shutil.rmtree, self.tmp) self.maxDiff = None - def test_network_data_is_found(self): + @mock.patch('cloudinit.sources.DataSourceConfigDrive.on_first_boot') + def test_network_data_is_found(self, on_first_boot): """Verify that network_data is present in ds in config-drive-v2.""" populate_dir(self.tmp, CFG_DRIVE_FILES_V2) myds = cfg_ds_from_dir(self.tmp) self.assertIsNotNone(myds.network_json) - def test_network_config_is_converted(self): + @mock.patch('cloudinit.sources.DataSourceConfigDrive.on_first_boot') + def test_network_config_is_converted(self, on_first_boot): """Verify that network_data is converted and present on ds object.""" populate_dir(self.tmp, CFG_DRIVE_FILES_V2) myds = cfg_ds_from_dir(self.tmp) @@ -558,7 +561,7 @@ def cfg_ds_from_dir(seed_d): helpers.Paths({})) cfg_ds.seed_dir = seed_d cfg_ds.known_macs = KNOWN_MACS.copy() - if not cfg_ds.get_data(skip_first_boot=True): + if not cfg_ds.get_data(): raise RuntimeError("Data source did not extract itself from" " seed directory %s" % seed_d) return cfg_ds -- cgit v1.2.3 From f47d811af4f2ae0977db076b3eedb27634da2132 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 10 Jun 2016 14:26:30 -0700 Subject: Add a sysconfig renderer --- cloudinit/net/sysconfig.py | 476 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 476 insertions(+) create mode 100644 cloudinit/net/sysconfig.py (limited to 'cloudinit') diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py new file mode 100644 index 00000000..ff7e467f --- /dev/null +++ b/cloudinit/net/sysconfig.py @@ -0,0 +1,476 @@ +# vi: ts=4 expandtab +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import re + +import six + +from cloudinit.distros.parsers import resolv_conf +from cloudinit import util + +from . import network_state +from .udev import generate_udev_rule + + +def _filter_by_type(match_type): + return lambda iface: match_type == iface['type'] + + +def _filter_by_name(match_name): + return lambda iface: match_name == iface['name'] + + +_filter_by_physical = _filter_by_type('physical') + + +def _is_default_route(route): + if route['network'] == '::' and route['netmask'] == 0: + return True + if route['network'] == '0.0.0.0' and route['netmask'] == '0.0.0.0': + return True + return False + + +def _quote_value(value): + if re.search(r"\s", value): + # This doesn't handle complex cases... + if value.startswith('"') and value.endswith('"'): + return value + else: + return '"%s"' % value + else: + return value + + +class NetworkStateHelper(object): + def __init__(self, network_state): + self._network_state = network_state.copy() + + @property + def dns_nameservers(self): + return self._network_state['dns']['nameservers'] + + @property + def dns_searchdomains(self): + return self._network_state['dns']['search'] + + def iter_interfaces(self, filter_func=None): + ifaces = self._network_state.get('interfaces') + if ifaces: + for iface in ifaces.values(): + if filter_func is None: + yield iface + else: + if filter_func(iface): + yield iface + + +class ConfigMap(object): + """Sysconfig like dictionary object.""" + + default_header = ('# Created by cloud-init on instance' + ' boot automatically, do not edit.\n#') + + # Why does redhat prefer yes/no to true/false?? + _bool_map = { + True: 'yes', + False: 'no', + } + + def __init__(self): + self._conf = {} + + def __setitem__(self, key, value): + self._conf[key] = value + + def drop(self, key): + self._conf.pop(key, None) + + def __len__(self): + return len(self._conf) + + def to_string(self): + buf = compat.StringIO() + buf.write(self.default_header) + if self._conf: + buf.write("\n") + for key in sorted(self._conf.keys()): + value = self._conf[key] + if isinstance(value, bool): + value = self._bool_map[value] + if not isinstance(value, compat.string_types): + value = str(value) + buf.write("%s=%s\n" % (key, _quote_value(value))) + return buf.getvalue() + + +class Route(ConfigMap): + """Represents a route configuration.""" + + route_fn_tpl = '%(base)s/network-scripts/route-%(name)s' + + def __init__(self, route_name, base_sysconf_dir): + super(Route, self).__init__() + self.last_idx = 1 + self.has_set_default = False + self._route_name = route_name + self._base_sysconf_dir = base_sysconf_dir + + def copy(self): + r = Route(self._route_name, self._base_sysconf_dir) + r._conf = self._conf.copy() + r.last_idx = self.last_idx + r.has_set_default = self.has_set_default + return r + + @property + def path(self): + return self.route_fn_tpl % ({'base': self._base_sysconf_dir, + 'name': self._route_name}) + + +class NetInterface(ConfigMap): + """Represents a sysconfig/networking-script (and its config + children).""" + + iface_fn_tpl = '%(base)s/network-scripts/ifcfg-%(name)s' + + iface_types = { + 'ethernet': 'Ethernet', + 'bond': 'Bond', + 'bridge': 'Bridge', + } + + def __init__(self, iface_name, base_sysconf_dir, kind='ethernet'): + super(NetInterface, self).__init__() + self.children = [] + self.routes = Route(iface_name, base_sysconf_dir) + self._kind = kind + self._iface_name = iface_name + self._conf['DEVICE'] = iface_name + self._conf['TYPE'] = self.iface_types[kind] + self._base_sysconf_dir = base_sysconf_dir + + @property + def name(self): + return self._iface_name + + @name.setter + def name(self, iface_name): + self._iface_name = iface_name + self._conf['DEVICE'] = iface_name + + @property + def kind(self): + return self._kind + + @kind.setter + def kind(self, kind): + self._kind = kind + self._conf['TYPE'] = self.iface_types[kind] + + @property + def path(self): + return self.iface_fn_tpl % ({'base': self._base_sysconf_dir, + 'name': self.name}) + + def copy(self, copy_children=False, copy_routes=False): + c = NetInterface(self.name, self._base_sysconf_dir, kind=self._kind) + c._conf = self._conf.copy() + if copy_children: + c.children = list(self.children) + if copy_routes: + c.routes = self.routes.copy() + return c + + +class Renderer(object): + """Renders network information in a /etc/sysconfig format.""" + + # See: https://access.redhat.com/documentation/en-US/\ + # Red_Hat_Enterprise_Linux/6/html/Deployment_Guide/\ + # s1-networkscripts-interfaces.html (or other docs for + # details about this) + + iface_defaults = tuple([ + ('ONBOOT', True), + ('USERCTL', False), + ('NM_CONTROLLED', False), + ('BOOTPROTO', 'none'), + ]) + + # If these keys exist, then there values will be used to form + # a BONDING_OPTS grouping; otherwise no grouping will be set. + bond_tpl_opts = tuple([ + ('bond_mode', "mode=%s"), + ('bond_xmit_hash_policy', "xmit_hash_policy=%s"), + ('bond_miimon', "miimon=%s"), + ]) + + bridge_opts_keys = tuple([ + ('bridge_stp', 'STP'), + ('bridge_ageing', 'AGEING'), + ('bridge_bridgeprio', 'PRIO'), + ]) + + @staticmethod + def _render_persistent_net(network_state): + """Given state, emit udev rules to map mac to ifname.""" + # TODO(harlowja): this seems shared between eni renderer and + # this, so move it to a shared location. + content = compat.StringIO() + for iface in network_state.iter_interfaces(_filter_by_physical): + # for physical interfaces write out a persist net udev rule + if 'name' in iface and iface.get('mac_address'): + content.write(generate_udev_rule(iface['name'], + iface['mac_address'])) + return content.getvalue() + + @classmethod + def _render_iface_shared(cls, iface, iface_cfg): + for k, v in cls.iface_defaults: + iface_cfg[k] = v + for (old_key, new_key) in [('mac_address', 'HWADDR'), ('mtu', 'MTU')]: + old_value = iface.get(old_key) + if old_value is not None: + iface_cfg[new_key] = old_value + + @classmethod + def _render_subnet(cls, iface_cfg, route_cfg, subnet): + subnet_type = subnet.get('type') + if subnet_type == 'dhcp6': + iface_cfg['DHCPV6C'] = True + iface_cfg['IPV6INIT'] = True + iface_cfg['BOOTPROTO'] = 'dhcp' + elif subnet_type == 'dhcp4': + iface_cfg['BOOTPROTO'] = 'dhcp' + elif subnet_type == 'static': + iface_cfg['BOOTPROTO'] = 'static' + if subnet.get('ipv6'): + iface_cfg['IPV6ADDR'] = subnet['address'] + iface_cfg['IPV6INIT'] = True + else: + iface_cfg['IPADDR'] = subnet['address'] + else: + raise ValueError("Unknown subnet type '%s' found" + " for interface '%s'" % (subnet_type, + iface_cfg.name)) + if 'netmask' in subnet: + iface_cfg['NETMASK'] = subnet['netmask'] + for route in subnet.get('routes', []): + if _is_default_route(route): + if route_cfg.has_set_default: + raise ValueError("Duplicate declaration of default" + " route found for interface '%s'" + % (iface_cfg.name)) + # NOTE(harlowja): ipv6 and ipv4 default gateways + gw_key = 'GATEWAY0' + nm_key = 'NETMASK0' + addr_key = 'ADDRESS0' + # The owning interface provides the default route. + # + # TODO(harlowja): add validation that no other iface has + # also provided the default route? + iface_cfg['DEFROUTE'] = True + if 'gateway' in route: + iface_cfg['GATEWAY'] = route['gateway'] + route_cfg.has_set_default = True + else: + gw_key = 'GATEWAY%s' % route_cfg.last_idx + nm_key = 'NETMASK%s' % route_cfg.last_idx + addr_key = 'ADDRESS%s' % route_cfg.last_idx + route_cfg.last_idx += 1 + for (old_key, new_key) in [('gateway', gw_key), + ('netmask', nm_key), + ('network', addr_key)]: + if old_key in route: + route_cfg[new_key] = route[old_key] + + @classmethod + def _render_bonding_opts(cls, iface_cfg, iface): + bond_opts = [] + for (bond_key, value_tpl) in cls.bond_tpl_opts: + # Seems like either dash or underscore is possible? + bond_keys = [bond_key, bond_key.replace("_", "-")] + for bond_key in bond_keys: + if bond_key in iface: + bond_value = iface[bond_key] + if isinstance(bond_value, (tuple, list)): + bond_value = " ".join(bond_value) + bond_opts.append(value_tpl % (bond_value)) + break + if bond_opts: + iface_cfg['BONDING_OPTS'] = " ".join(bond_opts) + + @classmethod + def _render_physical_interfaces(cls, network_state, iface_contents): + for iface in network_state.iter_interfaces(_filter_by_physical): + iface_name = iface['name'] + iface_subnets = iface.get("subnets", []) + iface_cfg = iface_contents[iface_name] + route_cfg = iface_cfg.routes + if len(iface_subnets) == 1: + cls._render_subnet(iface_cfg, route_cfg, iface_subnets[0]) + elif len(iface_subnets) > 1: + for i, iface_subnet in enumerate(iface_subnets, + start=len(iface.children)): + iface_sub_cfg = iface_cfg.copy() + iface_sub_cfg.name = "%s:%s" % (iface_name, i) + iface.children.append(iface_sub_cfg) + cls._render_subnet(iface_sub_cfg, route_cfg, iface_subnet) + + @classmethod + def _render_bond_interfaces(cls, network_state, iface_contents): + for iface in network_state.iter_interfaces(_filter_by_type('bond')): + iface_name = iface['name'] + iface_cfg = iface_contents[iface_name] + cls._render_bonding_opts(iface_cfg, iface) + iface_master_name = iface['bond-master'] + iface_cfg['MASTER'] = iface_master_name + iface_cfg['SLAVE'] = True + # Ensure that the master interface (and any of its children) + # are actually marked as being bond types... + master_cfg = iface_contents[iface_master_name] + master_cfgs = [master_cfg] + master_cfgs.extend(master_cfg.children) + for master_cfg in master_cfgs: + master_cfg['BONDING_MASTER'] = True + master_cfg.kind = 'bond' + + @staticmethod + def _render_vlan_interfaces(network_state, iface_contents): + for iface in network_state.iter_interfaces(_filter_by_type('vlan')): + iface_name = iface['name'] + iface_cfg = iface_contents[iface_name] + iface_cfg['VLAN'] = True + iface_cfg['PHYSDEV'] = iface_name[:iface_name.rfind('.')] + + @staticmethod + def _render_dns(network_state, existing_dns_path=None): + content = resolv_conf.ResolvConf("") + if existing_dns_path and os.path.isfile(existing_dns_path): + content = resolv_conf.ResolvConf(util.load_file(existing_dns_path)) + for ns in network_state.dns_nameservers: + content.add_nameserver(ns) + for d in network_state.dns_searchdomains: + content.add_search_domain(d) + return str(content) + + @classmethod + def _render_bridge_interfaces(cls, network_state, iface_contents): + for iface in network_state.iter_interfaces(_filter_by_type('bridge')): + iface_name = iface['name'] + iface_cfg = iface_contents[iface_name] + iface_cfg.kind = 'bridge' + for old_key, new_key in cls.bridge_opts_keys: + if old_key in iface: + iface_cfg[new_key] = iface[old_key] + # Is this the right key to get all the connected interfaces? + for bridged_iface_name in iface.get('bridge_ports', []): + # Ensure all bridged interfaces are correctly tagged + # as being bridged to this interface. + bridged_cfg = iface_contents[bridged_iface_name] + bridged_cfgs = [bridged_cfg] + bridged_cfgs.extend(bridged_cfg.children) + for bridge_cfg in bridged_cfgs: + bridge_cfg['BRIDGE'] = iface_name + + @classmethod + def _render_sysconfig(cls, base_sysconf_dir, network_state): + '''Given state, return /etc/sysconfig files + contents''' + iface_contents = {} + for iface in network_state.iter_interfaces(): + iface_name = iface['name'] + iface_cfg = NetInterface(iface_name, base_sysconf_dir) + cls._render_iface_shared(iface, iface_cfg) + iface_contents[iface_name] = iface_cfg + cls._render_physical_interfaces(network_state, iface_contents) + cls._render_bond_interfaces(network_state, iface_contents) + cls._render_vlan_interfaces(network_state, iface_contents) + cls._render_bridge_interfaces(network_state, iface_contents) + contents = {} + for iface_name, iface_cfg in iface_contents.items(): + if iface_cfg or iface_cfg.children: + contents[iface_cfg.path] = iface_cfg.to_string() + for iface_cfg in iface_cfg.children: + if iface_cfg: + contents[iface_cfg.path] = iface_cfg.to_string() + if iface_cfg.routes: + contents[iface_cfg.routes.path] = iface_cfg.routes.to_string() + return contents + + def render_network_state( + self, target, network_state, sysconf_dir="etc/sysconfig/", + netrules='etc/udev/rules.d/70-persistent-net.rules', + dns='etc/resolv.conf'): + if target: + base_sysconf_dir = os.path.join(target, sysconf_dir) + else: + base_sysconf_dir = sysconf_dir + for path, data in self._render_sysconfig(base_sysconf_dir, + network_state).items(): + if target: + util.write_file(path, data) + else: + print("File to be at: %s" % path) + print(data) + if dns: + if target: + dns_path = os.path.join(target, dns) + resolv_content = self._render_dns(network_state, + existing_dns_path=dns_path) + util.write_file(dns_path, resolv_content) + else: + resolv_content = self._render_dns(network_state) + dns_path = dns + print("File to be at: %s" % dns_path) + print(resolv_content) + if netrules: + netrules_content = self._render_persistent_net(network_state) + if target: + netrules_path = os.path.join(target, netrules) + util.write_file(netrules_path, netrules_content) + else: + netrules_path = netrules + print("File to be at: %s" % netrules_path) + print(netrules_content) + + +def main(): + """Reads a os network state json file and outputs what would be written.""" + from cloudinit.sources.helpers import openstack + + import argparse + import json + + parser = argparse.ArgumentParser() + parser.add_argument("-f", "--file", metavar="FILE", + help=("openstack network json file" + " to read (required)"), + required=True) + parser.add_argument("-d", "--dir", metavar="DIR", + help=("directory to write output into (if" + " not provided then written to stdout)"), + default=None) + args = parser.parse_args() + + network_json = json.loads(util.load_file(args.file)) + net_state = network_state.parse_net_config_data( + openstack.convert_net_json(network_json), skip_broken=False) + r = Renderer() + r.render_network_state(args.dir, NetworkStateHelper(net_state)) + + +if __name__ == '__main__': + main() -- cgit v1.2.3 From e5d4ee779981a0e572dc0ca4519b5bf770e3bcaa Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 10 Jun 2016 14:36:29 -0700 Subject: Ensure renderer is used in rhel --- cloudinit/distros/rhel.py | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'cloudinit') diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py index 812e7002..20525e47 100644 --- a/cloudinit/distros/rhel.py +++ b/cloudinit/distros/rhel.py @@ -23,6 +23,8 @@ from cloudinit import distros from cloudinit import helpers from cloudinit import log as logging +from cloudinit.net.network_state import parse_net_config_data +from cloudinit.net import sysconfig from cloudinit import util from cloudinit.distros import net_util @@ -59,10 +61,16 @@ class Distro(distros.Distro): # should only happen say once per instance...) self._runner = helpers.Runners(paths) self.osfamily = 'redhat' + self._net_renderer = sysconfig.Renderer() def install_packages(self, pkglist): self.package_command('install', pkgs=pkglist) + def _write_network_config(self, netconfig): + self._net_renderer.render_network_state( + target="/", parse_net_config_data(netconfig)) + return [] + def _write_network(self, settings): # TODO(harlowja) fix this... since this is the ubuntu format entries = net_util.translate_network(settings) -- cgit v1.2.3 From cf7ba5b30597660b7ab4fbdd92f0762e0011c1d6 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 10 Jun 2016 14:40:05 -0700 Subject: Fix the broken import and 'parse_net_config_data' function usage --- cloudinit/distros/debian.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index e71aaa97..603b0b61 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -25,8 +25,8 @@ import os from cloudinit import distros from cloudinit import helpers from cloudinit import log as logging -from cloudinit import net from cloudinit.net import eni +from cloudinit.net.network_state import parse_net_config_data from cloudinit import util from cloudinit.distros.parsers.hostname import HostnameConf @@ -81,7 +81,7 @@ class Distro(distros.Distro): return ['all'] def _write_network_config(self, netconfig): - ns = net.parse_net_config_data(netconfig) + ns = parse_net_config_data(netconfig) self._net_renderer.render_network_state( target="/", network_state=ns, eni=self.network_conf_fn, links_prefix=self.links_prefix, -- cgit v1.2.3 From cfcac2a70bc4dd9b47358261750ff535e78f4d5f Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 10 Jun 2016 15:02:21 -0700 Subject: Add a sysconfig rendering test --- cloudinit/net/sysconfig.py | 9 +++-- tests/unittests/test_net.py | 93 +++++++++++++++++++++++++++++++++------------ 2 files changed, 74 insertions(+), 28 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index ff7e467f..01770213 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -102,7 +102,7 @@ class ConfigMap(object): return len(self._conf) def to_string(self): - buf = compat.StringIO() + buf = six.StringIO() buf.write(self.default_header) if self._conf: buf.write("\n") @@ -110,7 +110,7 @@ class ConfigMap(object): value = self._conf[key] if isinstance(value, bool): value = self._bool_map[value] - if not isinstance(value, compat.string_types): + if not isinstance(value, six.string_types): value = str(value) buf.write("%s=%s\n" % (key, _quote_value(value))) return buf.getvalue() @@ -229,7 +229,7 @@ class Renderer(object): """Given state, emit udev rules to map mac to ifname.""" # TODO(harlowja): this seems shared between eni renderer and # this, so move it to a shared location. - content = compat.StringIO() + content = six.StringIO() for iface in network_state.iter_interfaces(_filter_by_physical): # for physical interfaces write out a persist net udev rule if 'name' in iface and iface.get('mac_address'): @@ -253,7 +253,7 @@ class Renderer(object): iface_cfg['DHCPV6C'] = True iface_cfg['IPV6INIT'] = True iface_cfg['BOOTPROTO'] = 'dhcp' - elif subnet_type == 'dhcp4': + elif subnet_type in ['dhcp4', 'dhcp']: iface_cfg['BOOTPROTO'] = 'dhcp' elif subnet_type == 'static': iface_cfg['BOOTPROTO'] = 'static' @@ -414,6 +414,7 @@ class Renderer(object): self, target, network_state, sysconf_dir="etc/sysconfig/", netrules='etc/udev/rules.d/70-persistent-net.rules', dns='etc/resolv.conf'): + network_state = NetworkStateHelper(network_state) if target: base_sysconf_dir = os.path.join(target, sysconf_dir) else: diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 7998111a..e899c749 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -2,6 +2,7 @@ from cloudinit import net from cloudinit.net import cmdline from cloudinit.net import eni from cloudinit.net import network_state +from cloudinit.net import sysconfig from cloudinit import util from .helpers import mock @@ -75,7 +76,36 @@ STATIC_EXPECTED_1 = { } -class TestEniNetRendering(TestCase): +def _setup_test(tmp_dir, mock_get_devicelist, mock_sys_netdev_info, + mock_sys_dev_path): + mock_get_devicelist.return_value = ['eth1000'] + dev_characteristics = { + 'eth1000': { + "bridge": False, + "carrier": False, + "dormant": False, + "operstate": "down", + "address": "07-1C-C6-75-A4-BE", + } + } + + def netdev_info(name, field): + return dev_characteristics[name][field] + + mock_sys_netdev_info.side_effect = netdev_info + + def sys_dev_path(devname, path=""): + return tmp_dir + devname + "/" + path + + for dev in dev_characteristics: + os.makedirs(os.path.join(tmp_dir, dev)) + with open(os.path.join(tmp_dir, dev, 'operstate'), 'w') as fh: + fh.write("down") + + mock_sys_dev_path.side_effect = sys_dev_path + + +class TestSysConfigRendering(TestCase): @mock.patch("cloudinit.net.sys_dev_path") @mock.patch("cloudinit.net.sys_netdev_info") @@ -83,35 +113,50 @@ class TestEniNetRendering(TestCase): def test_default_generation(self, mock_get_devicelist, mock_sys_netdev_info, mock_sys_dev_path): - mock_get_devicelist.return_value = ['eth1000', 'lo'] - - dev_characteristics = { - 'eth1000': { - "bridge": False, - "carrier": False, - "dormant": False, - "operstate": "down", - "address": "07-1C-C6-75-A4-BE", - } - } + tmp_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp_dir) + _setup_test(tmp_dir, mock_get_devicelist, + mock_sys_netdev_info, mock_sys_dev_path) - def netdev_info(name, field): - return dev_characteristics[name][field] + network_cfg = net.generate_fallback_config() + ns = network_state.parse_net_config_data(network_cfg, + skip_broken=False) - mock_sys_netdev_info.side_effect = netdev_info + render_dir = os.path.join(tmp_dir, "render") + os.makedirs(render_dir) - tmp_dir = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, tmp_dir) + renderer = sysconfig.Renderer() + renderer.render_network_state(render_dir, ns) + + render_file = 'etc/sysconfig/network-scripts/ifcfg-eth1000' + with open(os.path.join(render_dir, render_file)) as fh: + content = fh.read() + expected_content = """ +# Created by cloud-init on instance boot automatically, do not edit. +# +BOOTPROTO=dhcp +DEVICE=eth1000 +HWADDR=07-1C-C6-75-A4-BE +NM_CONTROLLED=no +ONBOOT=yes +TYPE=Ethernet +USERCTL=no +""".lstrip() + self.assertEqual(expected_content, content) - def sys_dev_path(devname, path=""): - return tmp_dir + devname + "/" + path - for dev in dev_characteristics: - os.makedirs(os.path.join(tmp_dir, dev)) - with open(os.path.join(tmp_dir, dev, 'operstate'), 'w') as fh: - fh.write("down") +class TestEniNetRendering(TestCase): - mock_sys_dev_path.side_effect = sys_dev_path + @mock.patch("cloudinit.net.sys_dev_path") + @mock.patch("cloudinit.net.sys_netdev_info") + @mock.patch("cloudinit.net.get_devicelist") + def test_default_generation(self, mock_get_devicelist, + mock_sys_netdev_info, + mock_sys_dev_path): + tmp_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp_dir) + _setup_test(tmp_dir, mock_get_devicelist, + mock_sys_netdev_info, mock_sys_dev_path) network_cfg = net.generate_fallback_config() ns = network_state.parse_net_config_data(network_cfg, -- cgit v1.2.3 From f666f6116be64cb1c033ed6d0f829bdb2f21ba69 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 10 Jun 2016 15:23:32 -0700 Subject: Add a bunch more sample tests for sysconfig --- cloudinit/net/sysconfig.py | 20 +++++++++--- tests/unittests/test_net.py | 80 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 5 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index 01770213..a4bcaaad 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -24,6 +24,19 @@ from . import network_state from .udev import generate_udev_rule +def _make_header(sep='#'): + lines = [ + "Created by cloud-init on instance boot automatically, do not edit.", + "", + ] + for i in range(0, len(lines)): + if lines[i]: + lines[i] = sep + " " + lines[i] + else: + lines[i] = sep + return "\n".join(lines) + + def _filter_by_type(match_type): return lambda iface: match_type == iface['type'] @@ -80,9 +93,6 @@ class NetworkStateHelper(object): class ConfigMap(object): """Sysconfig like dictionary object.""" - default_header = ('# Created by cloud-init on instance' - ' boot automatically, do not edit.\n#') - # Why does redhat prefer yes/no to true/false?? _bool_map = { True: 'yes', @@ -103,7 +113,7 @@ class ConfigMap(object): def to_string(self): buf = six.StringIO() - buf.write(self.default_header) + buf.write(_make_header()) if self._conf: buf.write("\n") for key in sorted(self._conf.keys()): @@ -365,7 +375,7 @@ class Renderer(object): content.add_nameserver(ns) for d in network_state.dns_searchdomains: content.add_search_domain(d) - return str(content) + return "\n".join([_make_header(';'), str(content)]) @classmethod def _render_bridge_interfaces(cls, network_state, iface_contents): diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index e899c749..4f4515a9 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -3,6 +3,7 @@ from cloudinit.net import cmdline from cloudinit.net import eni from cloudinit.net import network_state from cloudinit.net import sysconfig +from cloudinit.sources.helpers import openstack from cloudinit import util from .helpers import mock @@ -144,6 +145,85 @@ USERCTL=no """.lstrip() self.assertEqual(expected_content, content) + def test_openstack_rendering_sample(self): + ex_input = { + "services": [{"type": "dns", "address": "172.19.0.12"}], + "networks": [ + {"network_id": "dacd568d-5be6-4786-91fe-750c374b78b4", + "type": "ipv4", "netmask": "255.255.252.0", + "link": "tap1a81968a-79", + "routes": [{"netmask": "0.0.0.0", + "network": "0.0.0.0", + "gateway": "172.19.3.254"}], + "ip_address": "172.19.1.34", "id": "network0"}], + "links": [{"ethernet_mac_address": "fa:16:3e:ed:9a:59", + "mtu": None, "type": "bridge", "id": + "tap1a81968a-79", + "vif_id": "1a81968a-797a-400f-8a80-567f997eb93f"}]} + ex_mac_addrs = { + 'fa:16:3e:ed:9a:59': 'eth0', + } + network_cfg = openstack.convert_net_json( + ex_input, known_macs=ex_mac_addrs) + ns = network_state.parse_net_config_data(network_cfg, + skip_broken=False) + + tmp_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp_dir) + render_dir = os.path.join(tmp_dir, "render") + renderer = sysconfig.Renderer() + renderer.render_network_state(render_dir, ns) + + render_file = 'etc/sysconfig/network-scripts/ifcfg-eth0' + with open(os.path.join(render_dir, render_file)) as fh: + content = fh.read() + expected_content = """ +# Created by cloud-init on instance boot automatically, do not edit. +# +BOOTPROTO=static +DEFROUTE=yes +DEVICE=eth0 +GATEWAY=172.19.3.254 +HWADDR=fa:16:3e:ed:9a:59 +IPADDR=172.19.1.34 +NETMASK=255.255.252.0 +NM_CONTROLLED=no +ONBOOT=yes +TYPE=Ethernet +USERCTL=no +""".lstrip() + self.assertEqual(expected_content, content) + + route_render_file = 'etc/sysconfig/network-scripts/route-eth0' + with open(os.path.join(render_dir, route_render_file)) as fh: + content = fh.read() + expected_content = """ +# Created by cloud-init on instance boot automatically, do not edit. +# +ADDRESS0=0.0.0.0 +GATEWAY0=172.19.3.254 +NETMASK0=0.0.0.0 +""".lstrip() + self.assertEqual(expected_content, content) + + resolv_render_file = 'etc/resolv.conf' + with open(os.path.join(render_dir, resolv_render_file)) as fh: + content = fh.read() + expected_content = """ +; Created by cloud-init on instance boot automatically, do not edit. +; +nameserver 172.19.0.12 +""".lstrip() + self.assertEqual(expected_content, content) + + rules_render_file = 'etc/udev/rules.d/70-persistent-net.rules' + with open(os.path.join(render_dir, rules_render_file)) as fh: + content = fh.read() + expected_content = "".join([ + 'SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ', + 'ATTR{address}=="fa:16:3e:ed:9a:59", NAME="eth0"\n']) + self.assertEqual(expected_content, content) + class TestEniNetRendering(TestCase): -- cgit v1.2.3 From df583f83ac9d4869e0b3df5a904d13890d2f0986 Mon Sep 17 00:00:00 2001 From: Phil Roche Date: Mon, 13 Jun 2016 09:52:52 +0100 Subject: Removes trailing dot in metadata.google.internal GCE metadata lookup. A bug was reported (lp:1581200) where if there is no DNS server configured or it is not running then the metadata lookup on GCE will fail as it contains a trailing dot 'metadata.google.internal.'. As there is no DNS configured or running it will use the /etc/hosts file but the hosts file does not contain an entry with the trailing dot. One solution is to add an entry to the /etc/hosts file with the trailing dot but according to the manpage, /etc/hosts entries must end with an alphanumeric character and cannot end with a dot. The trailing dot was added to avoid MIM by dns search but we should probably assume the instance being started has no DNS and as such when querying metadata should use a URL that will resolve using /etc/hosts. LP: #1581200 --- cloudinit/sources/DataSourceGCE.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 9234d1f8..c660a350 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -25,7 +25,7 @@ from cloudinit import util LOG = logging.getLogger(__name__) BUILTIN_DS_CONFIG = { - 'metadata_url': 'http://metadata.google.internal./computeMetadata/v1/' + 'metadata_url': 'http://metadata.google.internal/computeMetadata/v1/' } REQUIRED_FIELDS = ('instance-id', 'availability-zone', 'local-hostname') -- cgit v1.2.3 From 6e19efb17609cd5c49941ea4f1608900b8459ca8 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 13 Jun 2016 13:10:07 -0700 Subject: Refactor some of sysconfig changes -> network_state module --- cloudinit/net/eni.py | 22 +++++---- cloudinit/net/network_state.py | 103 +++++++++++++++++++++++++++++------------ cloudinit/net/sysconfig.py | 26 +---------- 3 files changed, 87 insertions(+), 64 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index a695f5ed..df0df9b2 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -360,7 +360,15 @@ class Renderer(object): '''Given state, emit etc/network/interfaces content.''' content = "" - interfaces = network_state.get('interfaces') + content += "auto lo\niface lo inet loopback\n" + + nameservers = network_state.dns_nameservers + if nameservers: + content += " dns-nameservers %s\n" % (" ".join(nameservers)) + searchdomains = network_state.dns_searchdomains + if searchdomains: + content += " dns-search %s\n" % (" ".join(searchdomains)) + ''' Apply a sort order to ensure that we write out the physical interfaces first; this is critical for bonding @@ -371,12 +379,7 @@ class Renderer(object): 'bridge': 2, 'vlan': 3, } - content += "auto lo\niface lo inet loopback\n" - for dnskey, value in network_state.get('dns', {}).items(): - if len(value): - content += " dns-{} {}\n".format(dnskey, " ".join(value)) - - for iface in sorted(interfaces.values(), + for iface in sorted(network_state.iter_interfaces(), key=lambda k: (order[k['type']], k['name'])): if content[-2:] != "\n\n": @@ -409,7 +412,7 @@ class Renderer(object): content += "iface {name} {inet} {mode}\n".format(**iface) content += _iface_add_attrs(iface) - for route in network_state.get('routes'): + for route in network_state.iter_routes(): content += self._render_route(route) # global replacements until v2 format @@ -441,8 +444,7 @@ class Renderer(object): fp_prefix = os.path.sep.join((target, links_prefix)) for f in glob.glob(fp_prefix + "*"): os.unlink(f) - interfaces = network_state.get('interfaces') - for iface in interfaces.values(): + for iface in network_state.iter_interfaces(): if (iface['type'] == 'physical' and 'name' in iface and iface.get('mac_address')): fname = fp_prefix + iface['name'] + ".link" diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index a8be5e26..d75fc852 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -38,10 +38,10 @@ def parse_net_config_data(net_config, skip_broken=True): """ state = None if 'version' in net_config and 'config' in net_config: - ns = NetworkState(version=net_config.get('version'), - config=net_config.get('config')) - ns.parse_config(skip_broken=skip_broken) - state = ns.network_state + nsi = NetworkStateInterpreter(version=net_config.get('version'), + config=net_config.get('config')) + nsi.parse_config(skip_broken=skip_broken) + state = nsi.network_state return state @@ -57,11 +57,10 @@ def parse_net_config(path, skip_broken=True): def from_state_file(state_file): - network_state = None state = util.read_conf(state_file) - network_state = NetworkState() - network_state.load(state) - return network_state + nsi = NetworkStateInterpreter() + nsi.load(state) + return nsi def diff_keys(expected, actual): @@ -113,9 +112,51 @@ class CommandHandlerMeta(type): parents, dct) -@six.add_metaclass(CommandHandlerMeta) class NetworkState(object): + def __init__(self, network_state, version=NETWORK_STATE_VERSION): + self._network_state = copy.deepcopy(network_state) + self._version = version + + @property + def version(self): + return self._version + + def iter_routes(self, filter_func=None): + for route in self._network_state.get('routes', []): + if filter_func is not None: + if filter_func(route): + yield route + else: + yield route + + @property + def dns_nameservers(self): + try: + return self._network_state['dns']['nameservers'] + except KeyError: + return [] + + @property + def dns_searchdomains(self): + try: + return self._network_state['dns']['search'] + except KeyError: + return [] + + def iter_interfaces(self, filter_func=None): + ifaces = self._network_state.get('interfaces', {}) + for iface in six.itervalues(ifaces): + if filter_func is None: + yield iface + else: + if filter_func(iface): + yield iface + + +@six.add_metaclass(CommandHandlerMeta) +class NetworkStateInterpreter(object): + initial_network_state = { 'interfaces': {}, 'routes': [], @@ -126,22 +167,26 @@ class NetworkState(object): } def __init__(self, version=NETWORK_STATE_VERSION, config=None): - self.version = version - self.config = config - self.network_state = copy.deepcopy(self.initial_network_state) + self._version = version + self._config = config + self._network_state = copy.deepcopy(self.initial_network_state) + + @property + def network_state(self): + return NetworkState(self._network_state, version=self._version) def dump(self): state = { - 'version': self.version, - 'config': self.config, - 'network_state': self.network_state, + 'version': self._version, + 'config': self._config, + 'network_state': self._network_state, } return util.yaml_dumps(state) def load(self, state): if 'version' not in state: LOG.error('Invalid state, missing version field') - raise Exception('Invalid state, missing version field') + raise ValueError('Invalid state, missing version field') required_keys = NETWORK_STATE_REQUIRED_KEYS[state['version']] missing_keys = diff_keys(required_keys, state) @@ -155,11 +200,11 @@ class NetworkState(object): setattr(self, key, state[key]) def dump_network_state(self): - return util.yaml_dumps(self.network_state) + return util.yaml_dumps(self._network_state) def parse_config(self, skip_broken=True): # rebuild network state - for command in self.config: + for command in self._config: command_type = command['type'] try: handler = self.command_handlers[command_type] @@ -189,7 +234,7 @@ class NetworkState(object): } ''' - interfaces = self.network_state.get('interfaces') + interfaces = self._network_state.get('interfaces', {}) iface = interfaces.get(command['name'], {}) for param, val in command.get('params', {}).items(): iface.update({param: val}) @@ -215,7 +260,7 @@ class NetworkState(object): 'gateway': None, 'subnets': subnets, }) - self.network_state['interfaces'].update({command.get('name'): iface}) + self._network_state['interfaces'].update({command.get('name'): iface}) self.dump_network_state() @ensure_command_keys(['name', 'vlan_id', 'vlan_link']) @@ -228,7 +273,7 @@ class NetworkState(object): hwaddress ether BC:76:4E:06:96:B3 vlan-raw-device eth0 ''' - interfaces = self.network_state.get('interfaces') + interfaces = self._network_state.get('interfaces', {}) self.handle_physical(command) iface = interfaces.get(command.get('name'), {}) iface['vlan-raw-device'] = command.get('vlan_link') @@ -263,12 +308,12 @@ class NetworkState(object): ''' self.handle_physical(command) - interfaces = self.network_state.get('interfaces') + interfaces = self._network_state.get('interfaces') iface = interfaces.get(command.get('name'), {}) for param, val in command.get('params').items(): iface.update({param: val}) iface.update({'bond-slaves': 'none'}) - self.network_state['interfaces'].update({iface['name']: iface}) + self._network_state['interfaces'].update({iface['name']: iface}) # handle bond slaves for ifname in command.get('bond_interfaces'): @@ -280,13 +325,13 @@ class NetworkState(object): # inject placeholder self.handle_physical(cmd) - interfaces = self.network_state.get('interfaces') + interfaces = self._network_state.get('interfaces', {}) bond_if = interfaces.get(ifname) bond_if['bond-master'] = command.get('name') # copy in bond config into slave for param, val in command.get('params').items(): bond_if.update({param: val}) - self.network_state['interfaces'].update({ifname: bond_if}) + self._network_state['interfaces'].update({ifname: bond_if}) @ensure_command_keys(['name', 'bridge_interfaces', 'params']) def handle_bridge(self, command): @@ -319,7 +364,7 @@ class NetworkState(object): # find one of the bridge port ifaces to get mac_addr # handle bridge_slaves - interfaces = self.network_state.get('interfaces') + interfaces = self._network_state.get('interfaces', {}) for ifname in command.get('bridge_interfaces'): if ifname in interfaces: continue @@ -330,7 +375,7 @@ class NetworkState(object): # inject placeholder self.handle_physical(cmd) - interfaces = self.network_state.get('interfaces') + interfaces = self._network_state.get('interfaces', {}) self.handle_physical(command) iface = interfaces.get(command.get('name'), {}) iface['bridge_ports'] = command['bridge_interfaces'] @@ -341,7 +386,7 @@ class NetworkState(object): @ensure_command_keys(['address']) def handle_nameserver(self, command): - dns = self.network_state.get('dns') + dns = self._network_state.get('dns') if 'address' in command: addrs = command['address'] if not type(addrs) == list: @@ -357,7 +402,7 @@ class NetworkState(object): @ensure_command_keys(['destination']) def handle_route(self, command): - routes = self.network_state.get('routes') + routes = self._network_state.get('routes', []) network, cidr = command['destination'].split("/") netmask = cidr2mask(int(cidr)) route = { diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index a4bcaaad..647ca3ed 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -67,29 +67,6 @@ def _quote_value(value): return value -class NetworkStateHelper(object): - def __init__(self, network_state): - self._network_state = network_state.copy() - - @property - def dns_nameservers(self): - return self._network_state['dns']['nameservers'] - - @property - def dns_searchdomains(self): - return self._network_state['dns']['search'] - - def iter_interfaces(self, filter_func=None): - ifaces = self._network_state.get('interfaces') - if ifaces: - for iface in ifaces.values(): - if filter_func is None: - yield iface - else: - if filter_func(iface): - yield iface - - class ConfigMap(object): """Sysconfig like dictionary object.""" @@ -424,7 +401,6 @@ class Renderer(object): self, target, network_state, sysconf_dir="etc/sysconfig/", netrules='etc/udev/rules.d/70-persistent-net.rules', dns='etc/resolv.conf'): - network_state = NetworkStateHelper(network_state) if target: base_sysconf_dir = os.path.join(target, sysconf_dir) else: @@ -480,7 +456,7 @@ def main(): net_state = network_state.parse_net_config_data( openstack.convert_net_json(network_json), skip_broken=False) r = Renderer() - r.render_network_state(args.dir, NetworkStateHelper(net_state)) + r.render_network_state(args.dir, net_state) if __name__ == '__main__': -- cgit v1.2.3 From e0b1e3a8c925446ee9e6d15aae1783b49fb870f8 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 14 Jun 2016 10:58:18 -0700 Subject: Make the os samples easier to extend (for new samples) --- cloudinit/net/network_state.py | 1 + tests/unittests/test_net.py | 153 +++++++++++++++++++++-------------------- 2 files changed, 79 insertions(+), 75 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index d75fc852..8ca5106f 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -170,6 +170,7 @@ class NetworkStateInterpreter(object): self._version = version self._config = config self._network_state = copy.deepcopy(self.initial_network_state) + self._parsed = False @property def network_state(self): diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 4f4515a9..43602030 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -76,6 +76,71 @@ STATIC_EXPECTED_1 = { 'dns_nameservers': ['10.0.1.1']}], } +# Examples (and expected outputs for various renderers). +OS_SAMPLES = [ + { + 'in_data': { + "services": [{"type": "dns", "address": "172.19.0.12"}], + "networks": [{ + "network_id": "dacd568d-5be6-4786-91fe-750c374b78b4", + "type": "ipv4", "netmask": "255.255.252.0", + "link": "tap1a81968a-79", + "routes": [{ + "netmask": "0.0.0.0", + "network": "0.0.0.0", + "gateway": "172.19.3.254", + }], + "ip_address": "172.19.1.34", "id": "network0" + }], + "links": [ + { + "ethernet_mac_address": "fa:16:3e:ed:9a:59", + "mtu": None, "type": "bridge", "id": + "tap1a81968a-79", + "vif_id": "1a81968a-797a-400f-8a80-567f997eb93f" + }, + ], + }, + 'in_macs': { + 'fa:16:3e:ed:9a:59': 'eth0', + }, + 'out_sysconfig': [ + ('etc/sysconfig/network-scripts/ifcfg-eth0', + """ +# Created by cloud-init on instance boot automatically, do not edit. +# +BOOTPROTO=static +DEFROUTE=yes +DEVICE=eth0 +GATEWAY=172.19.3.254 +HWADDR=fa:16:3e:ed:9a:59 +IPADDR=172.19.1.34 +NETMASK=255.255.252.0 +NM_CONTROLLED=no +ONBOOT=yes +TYPE=Ethernet +USERCTL=no +""".lstrip()), + ('etc/sysconfig/network-scripts/route-eth0', + """ +# Created by cloud-init on instance boot automatically, do not edit. +# +ADDRESS0=0.0.0.0 +GATEWAY0=172.19.3.254 +NETMASK0=0.0.0.0 +""".lstrip()), + ('etc/resolv.conf', + """ +; Created by cloud-init on instance boot automatically, do not edit. +; +nameserver 172.19.0.12 +""".lstrip()), + ('etc/udev/rules.d/70-persistent-net.rules', + "".join(['SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ', + 'ATTR{address}=="fa:16:3e:ed:9a:59", NAME="eth0"\n']))] + } +] + def _setup_test(tmp_dir, mock_get_devicelist, mock_sys_netdev_info, mock_sys_dev_path): @@ -145,84 +210,22 @@ USERCTL=no """.lstrip() self.assertEqual(expected_content, content) - def test_openstack_rendering_sample(self): - ex_input = { - "services": [{"type": "dns", "address": "172.19.0.12"}], - "networks": [ - {"network_id": "dacd568d-5be6-4786-91fe-750c374b78b4", - "type": "ipv4", "netmask": "255.255.252.0", - "link": "tap1a81968a-79", - "routes": [{"netmask": "0.0.0.0", - "network": "0.0.0.0", - "gateway": "172.19.3.254"}], - "ip_address": "172.19.1.34", "id": "network0"}], - "links": [{"ethernet_mac_address": "fa:16:3e:ed:9a:59", - "mtu": None, "type": "bridge", "id": - "tap1a81968a-79", - "vif_id": "1a81968a-797a-400f-8a80-567f997eb93f"}]} - ex_mac_addrs = { - 'fa:16:3e:ed:9a:59': 'eth0', - } - network_cfg = openstack.convert_net_json( - ex_input, known_macs=ex_mac_addrs) - ns = network_state.parse_net_config_data(network_cfg, - skip_broken=False) - + def test_openstack_rendering_samples(self): tmp_dir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, tmp_dir) render_dir = os.path.join(tmp_dir, "render") - renderer = sysconfig.Renderer() - renderer.render_network_state(render_dir, ns) - - render_file = 'etc/sysconfig/network-scripts/ifcfg-eth0' - with open(os.path.join(render_dir, render_file)) as fh: - content = fh.read() - expected_content = """ -# Created by cloud-init on instance boot automatically, do not edit. -# -BOOTPROTO=static -DEFROUTE=yes -DEVICE=eth0 -GATEWAY=172.19.3.254 -HWADDR=fa:16:3e:ed:9a:59 -IPADDR=172.19.1.34 -NETMASK=255.255.252.0 -NM_CONTROLLED=no -ONBOOT=yes -TYPE=Ethernet -USERCTL=no -""".lstrip() - self.assertEqual(expected_content, content) - - route_render_file = 'etc/sysconfig/network-scripts/route-eth0' - with open(os.path.join(render_dir, route_render_file)) as fh: - content = fh.read() - expected_content = """ -# Created by cloud-init on instance boot automatically, do not edit. -# -ADDRESS0=0.0.0.0 -GATEWAY0=172.19.3.254 -NETMASK0=0.0.0.0 -""".lstrip() - self.assertEqual(expected_content, content) - - resolv_render_file = 'etc/resolv.conf' - with open(os.path.join(render_dir, resolv_render_file)) as fh: - content = fh.read() - expected_content = """ -; Created by cloud-init on instance boot automatically, do not edit. -; -nameserver 172.19.0.12 -""".lstrip() - self.assertEqual(expected_content, content) - - rules_render_file = 'etc/udev/rules.d/70-persistent-net.rules' - with open(os.path.join(render_dir, rules_render_file)) as fh: - content = fh.read() - expected_content = "".join([ - 'SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ', - 'ATTR{address}=="fa:16:3e:ed:9a:59", NAME="eth0"\n']) - self.assertEqual(expected_content, content) + for os_sample in OS_SAMPLES: + ex_input = os_sample['in_data'] + ex_mac_addrs = os_sample['in_macs'] + network_cfg = openstack.convert_net_json( + ex_input, known_macs=ex_mac_addrs) + ns = network_state.parse_net_config_data(network_cfg, + skip_broken=False) + renderer = sysconfig.Renderer() + renderer.render_network_state(render_dir, ns) + for fn, expected_content in os_sample.get('out_sysconfig', []): + with open(os.path.join(render_dir, fn)) as fh: + self.assertEqual(expected_content, fh.read()) class TestEniNetRendering(TestCase): -- cgit v1.2.3 From 780f88f4dd8ddd36c54e217ea5af3b18b7339758 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 14 Jun 2016 15:08:24 -0400 Subject: [Revert] Remove trailing dot from GCE metadata URL This change broke tox tests. --- ChangeLog | 1 - cloudinit/sources/DataSourceGCE.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) (limited to 'cloudinit') diff --git a/ChangeLog b/ChangeLog index 66539792..e2869522 100644 --- a/ChangeLog +++ b/ChangeLog @@ -122,7 +122,6 @@ - ConfigDrive: improved support for networking information from a network_data.json or older interfaces formated network_config. - Change missing Cheetah log warning to debug [Andrew Jorgensen] - - Remove trailing dot from GCE metadata URL (LP: #1581200) [Phil Roche] 0.7.6: - open 0.7.6 diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index c660a350..9234d1f8 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -25,7 +25,7 @@ from cloudinit import util LOG = logging.getLogger(__name__) BUILTIN_DS_CONFIG = { - 'metadata_url': 'http://metadata.google.internal/computeMetadata/v1/' + 'metadata_url': 'http://metadata.google.internal./computeMetadata/v1/' } REQUIRED_FIELDS = ('instance-id', 'availability-zone', 'local-hostname') -- cgit v1.2.3 From 632d56a23a947a23e18fa234d675e34a1a119593 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 14 Jun 2016 15:18:53 -0400 Subject: fix pep8 failure introduced in recent commit. The commit 1232 (Refactor a large part of the networking code) broke pep8. --- cloudinit/net/eni.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index a695f5ed..5a90eb32 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -391,8 +391,8 @@ class Renderer(object): iface['control'] = subnet.get('control', 'auto') if iface['mode'].endswith('6'): iface['inet'] += '6' - elif (iface['mode'] == 'static' - and ":" in subnet['address']): + elif (iface['mode'] == 'static' and + ":" in subnet['address']): iface['inet'] += '6' if iface['mode'].startswith('dhcp'): iface['mode'] = 'dhcp' -- cgit v1.2.3 From 93afde09d89e60c29dfd20790e30a06f031c82e1 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 14 Jun 2016 14:56:51 -0700 Subject: Make the bin/cloud-init an actual console entrypoint This allows for the test_cli test to be more sane. --- bin/cloud-init | 689 -------------------------------------------- cloudinit/cmd/__init__.py | 21 ++ cloudinit/cmd/main.py | 682 +++++++++++++++++++++++++++++++++++++++++++ setup.py | 8 +- tests/unittests/test_cli.py | 37 +-- 5 files changed, 721 insertions(+), 716 deletions(-) delete mode 100755 bin/cloud-init create mode 100644 cloudinit/cmd/__init__.py create mode 100755 cloudinit/cmd/main.py (limited to 'cloudinit') diff --git a/bin/cloud-init b/bin/cloud-init deleted file mode 100755 index 21c3a684..00000000 --- a/bin/cloud-init +++ /dev/null @@ -1,689 +0,0 @@ -#!/usr/bin/python -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import argparse -import json -import os -import sys -import time -import tempfile -import traceback - -# This is more just for running from the bin folder so that -# cloud-init binary can find the cloudinit module -possible_topdir = os.path.normpath(os.path.join(os.path.abspath( - sys.argv[0]), os.pardir, os.pardir)) -if os.path.exists(os.path.join(possible_topdir, "cloudinit", "__init__.py")): - sys.path.insert(0, possible_topdir) - -from cloudinit import patcher -patcher.patch() - -from cloudinit import log as logging -from cloudinit import netinfo -from cloudinit import signal_handler -from cloudinit import sources -from cloudinit import stages -from cloudinit import templater -from cloudinit import util -from cloudinit import reporting -from cloudinit.reporting import events -from cloudinit import version - -from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE, - CLOUD_CONFIG) - - -# Pretty little cheetah formatted welcome message template -WELCOME_MSG_TPL = ("Cloud-init v. ${version} running '${action}' at " - "${timestamp}. Up ${uptime} seconds.") - -# Module section template -MOD_SECTION_TPL = "cloud_%s_modules" - -# Things u can query on -QUERY_DATA_TYPES = [ - 'data', - 'data_raw', - 'instance_id', -] - -# Frequency shortname to full name -# (so users don't have to remember the full name...) -FREQ_SHORT_NAMES = { - 'instance': PER_INSTANCE, - 'always': PER_ALWAYS, - 'once': PER_ONCE, -} - -LOG = logging.getLogger() - - -# Used for when a logger may not be active -# and we still want to print exceptions... -def print_exc(msg=''): - if msg: - sys.stderr.write("%s\n" % (msg)) - sys.stderr.write('-' * 60) - sys.stderr.write("\n") - traceback.print_exc(file=sys.stderr) - sys.stderr.write('-' * 60) - sys.stderr.write("\n") - - -def welcome(action, msg=None): - if not msg: - msg = welcome_format(action) - util.multi_log("%s\n" % (msg), - console=False, stderr=True, log=LOG) - return msg - - -def welcome_format(action): - tpl_params = { - 'version': version.version_string(), - 'uptime': util.uptime(), - 'timestamp': util.time_rfc2822(), - 'action': action, - } - return templater.render_string(WELCOME_MSG_TPL, tpl_params) - - -def extract_fns(args): - # Files are already opened so lets just pass that along - # since it would of broke if it couldn't have - # read that file already... - fn_cfgs = [] - if args.files: - for fh in args.files: - # The realpath is more useful in logging - # so lets resolve to that... - fn_cfgs.append(os.path.realpath(fh.name)) - return fn_cfgs - - -def run_module_section(mods, action_name, section): - full_section_name = MOD_SECTION_TPL % (section) - (which_ran, failures) = mods.run_section(full_section_name) - total_attempted = len(which_ran) + len(failures) - if total_attempted == 0: - msg = ("No '%s' modules to run" - " under section '%s'") % (action_name, full_section_name) - sys.stderr.write("%s\n" % (msg)) - LOG.debug(msg) - return [] - else: - LOG.debug("Ran %s modules with %s failures", - len(which_ran), len(failures)) - return failures - - -def apply_reporting_cfg(cfg): - if cfg.get('reporting'): - reporting.update_configuration(cfg.get('reporting')) - - -def main_init(name, args): - deps = [sources.DEP_FILESYSTEM, sources.DEP_NETWORK] - if args.local: - deps = [sources.DEP_FILESYSTEM] - - if not args.local: - # See doc/kernel-cmdline.txt - # - # This is used in maas datasource, in "ephemeral" (read-only root) - # environment where the instance netboots to iscsi ro root. - # and the entity that controls the pxe config has to configure - # the maas datasource. - # - # Could be used elsewhere, only works on network based (not local). - root_name = "%s.d" % (CLOUD_CONFIG) - target_fn = os.path.join(root_name, "91_kernel_cmdline_url.cfg") - util.read_write_cmdline_url(target_fn) - - # Cloud-init 'init' stage is broken up into the following sub-stages - # 1. Ensure that the init object fetches its config without errors - # 2. Setup logging/output redirections with resultant config (if any) - # 3. Initialize the cloud-init filesystem - # 4. Check if we can stop early by looking for various files - # 5. Fetch the datasource - # 6. Connect to the current instance location + update the cache - # 7. Consume the userdata (handlers get activated here) - # 8. Construct the modules object - # 9. Adjust any subsequent logging/output redirections using the modules - # objects config as it may be different from init object - # 10. Run the modules for the 'init' stage - # 11. Done! - if not args.local: - w_msg = welcome_format(name) - else: - w_msg = welcome_format("%s-local" % (name)) - init = stages.Init(ds_deps=deps, reporter=args.reporter) - # Stage 1 - init.read_cfg(extract_fns(args)) - # Stage 2 - outfmt = None - errfmt = None - try: - LOG.debug("Closing stdin") - util.close_stdin() - (outfmt, errfmt) = util.fixup_output(init.cfg, name) - except: - util.logexc(LOG, "Failed to setup output redirection!") - print_exc("Failed to setup output redirection!") - if args.debug: - # Reset so that all the debug handlers are closed out - LOG.debug(("Logging being reset, this logger may no" - " longer be active shortly")) - logging.resetLogging() - logging.setupLogging(init.cfg) - apply_reporting_cfg(init.cfg) - - # Any log usage prior to setupLogging above did not have local user log - # config applied. We send the welcome message now, as stderr/out have - # been redirected and log now configured. - welcome(name, msg=w_msg) - - # Stage 3 - try: - init.initialize() - except Exception: - util.logexc(LOG, "Failed to initialize, likely bad things to come!") - # Stage 4 - path_helper = init.paths - mode = sources.DSMODE_LOCAL if args.local else sources.DSMODE_NETWORK - - if mode == sources.DSMODE_NETWORK: - existing = "trust" - sys.stderr.write("%s\n" % (netinfo.debug_info())) - LOG.debug(("Checking to see if files that we need already" - " exist from a previous run that would allow us" - " to stop early.")) - # no-net is written by upstart cloud-init-nonet when network failed - # to come up - stop_files = [ - os.path.join(path_helper.get_cpath("data"), "no-net"), - ] - existing_files = [] - for fn in stop_files: - if os.path.isfile(fn): - existing_files.append(fn) - - if existing_files: - LOG.debug("[%s] Exiting. stop file %s existed", - mode, existing_files) - return (None, []) - else: - LOG.debug("Execution continuing, no previous run detected that" - " would allow us to stop early.") - else: - existing = "check" - if util.get_cfg_option_bool(init.cfg, 'manual_cache_clean', False): - existing = "trust" - - init.purge_cache() - # Delete the non-net file as well - util.del_file(os.path.join(path_helper.get_cpath("data"), "no-net")) - - # Stage 5 - try: - init.fetch(existing=existing) - # if in network mode, and the datasource is local - # then work was done at that stage. - if mode == sources.DSMODE_NETWORK and init.datasource.dsmode != mode: - LOG.debug("[%s] Exiting. datasource %s in local mode", - mode, init.datasource) - return (None, []) - except sources.DataSourceNotFoundException: - # In the case of 'cloud-init init' without '--local' it is a bit - # more likely that the user would consider it failure if nothing was - # found. When using upstart it will also mentions job failure - # in console log if exit code is != 0. - if mode == sources.DSMODE_LOCAL: - LOG.debug("No local datasource found") - else: - util.logexc(LOG, ("No instance datasource found!" - " Likely bad things to come!")) - if not args.force: - init.apply_network_config(bring_up=not args.local) - LOG.debug("[%s] Exiting without datasource in local mode", mode) - if mode == sources.DSMODE_LOCAL: - return (None, []) - else: - return (None, ["No instance datasource found."]) - else: - LOG.debug("[%s] barreling on in force mode without datasource", - mode) - - # Stage 6 - iid = init.instancify() - LOG.debug("[%s] %s will now be targeting instance id: %s. new=%s", - mode, name, iid, init.is_new_instance()) - - init.apply_network_config(bring_up=bool(mode != sources.DSMODE_LOCAL)) - - if mode == sources.DSMODE_LOCAL: - if init.datasource.dsmode != mode: - LOG.debug("[%s] Exiting. datasource %s not in local mode.", - mode, init.datasource) - return (init.datasource, []) - else: - LOG.debug("[%s] %s is in local mode, will apply init modules now.", - mode, init.datasource) - - # update fully realizes user-data (pulling in #include if necessary) - init.update() - # Stage 7 - try: - # Attempt to consume the data per instance. - # This may run user-data handlers and/or perform - # url downloads and such as needed. - (ran, _results) = init.cloudify().run('consume_data', - init.consume_data, - args=[PER_INSTANCE], - freq=PER_INSTANCE) - if not ran: - # Just consume anything that is set to run per-always - # if nothing ran in the per-instance code - # - # See: https://bugs.launchpad.net/bugs/819507 for a little - # reason behind this... - init.consume_data(PER_ALWAYS) - except Exception: - util.logexc(LOG, "Consuming user data failed!") - return (init.datasource, ["Consuming user data failed!"]) - - apply_reporting_cfg(init.cfg) - - # Stage 8 - re-read and apply relevant cloud-config to include user-data - mods = stages.Modules(init, extract_fns(args), reporter=args.reporter) - # Stage 9 - try: - outfmt_orig = outfmt - errfmt_orig = errfmt - (outfmt, errfmt) = util.get_output_cfg(mods.cfg, name) - if outfmt_orig != outfmt or errfmt_orig != errfmt: - LOG.warn("Stdout, stderr changing to (%s, %s)", outfmt, errfmt) - (outfmt, errfmt) = util.fixup_output(mods.cfg, name) - except: - util.logexc(LOG, "Failed to re-adjust output redirection!") - logging.setupLogging(mods.cfg) - - # Stage 10 - return (init.datasource, run_module_section(mods, name, name)) - - -def main_modules(action_name, args): - name = args.mode - # Cloud-init 'modules' stages are broken up into the following sub-stages - # 1. Ensure that the init object fetches its config without errors - # 2. Get the datasource from the init object, if it does - # not exist then that means the main_init stage never - # worked, and thus this stage can not run. - # 3. Construct the modules object - # 4. Adjust any subsequent logging/output redirections using - # the modules objects configuration - # 5. Run the modules for the given stage name - # 6. Done! - w_msg = welcome_format("%s:%s" % (action_name, name)) - init = stages.Init(ds_deps=[], reporter=args.reporter) - # Stage 1 - init.read_cfg(extract_fns(args)) - # Stage 2 - try: - init.fetch(existing="trust") - except sources.DataSourceNotFoundException: - # There was no datasource found, theres nothing to do - msg = ('Can not apply stage %s, no datasource found! Likely bad ' - 'things to come!' % name) - util.logexc(LOG, msg) - print_exc(msg) - if not args.force: - return [(msg)] - # Stage 3 - mods = stages.Modules(init, extract_fns(args), reporter=args.reporter) - # Stage 4 - try: - LOG.debug("Closing stdin") - util.close_stdin() - util.fixup_output(mods.cfg, name) - except: - util.logexc(LOG, "Failed to setup output redirection!") - if args.debug: - # Reset so that all the debug handlers are closed out - LOG.debug(("Logging being reset, this logger may no" - " longer be active shortly")) - logging.resetLogging() - logging.setupLogging(mods.cfg) - apply_reporting_cfg(init.cfg) - - # now that logging is setup and stdout redirected, send welcome - welcome(name, msg=w_msg) - - # Stage 5 - return run_module_section(mods, name, name) - - -def main_query(name, _args): - raise NotImplementedError(("Action '%s' is not" - " currently implemented") % (name)) - - -def main_single(name, args): - # Cloud-init single stage is broken up into the following sub-stages - # 1. Ensure that the init object fetches its config without errors - # 2. Attempt to fetch the datasource (warn if it doesn't work) - # 3. Construct the modules object - # 4. Adjust any subsequent logging/output redirections using - # the modules objects configuration - # 5. Run the single module - # 6. Done! - mod_name = args.name - w_msg = welcome_format(name) - init = stages.Init(ds_deps=[], reporter=args.reporter) - # Stage 1 - init.read_cfg(extract_fns(args)) - # Stage 2 - try: - init.fetch(existing="trust") - except sources.DataSourceNotFoundException: - # There was no datasource found, - # that might be bad (or ok) depending on - # the module being ran (so continue on) - util.logexc(LOG, ("Failed to fetch your datasource," - " likely bad things to come!")) - print_exc(("Failed to fetch your datasource," - " likely bad things to come!")) - if not args.force: - return 1 - # Stage 3 - mods = stages.Modules(init, extract_fns(args), reporter=args.reporter) - mod_args = args.module_args - if mod_args: - LOG.debug("Using passed in arguments %s", mod_args) - mod_freq = args.frequency - if mod_freq: - LOG.debug("Using passed in frequency %s", mod_freq) - mod_freq = FREQ_SHORT_NAMES.get(mod_freq) - # Stage 4 - try: - LOG.debug("Closing stdin") - util.close_stdin() - util.fixup_output(mods.cfg, None) - except: - util.logexc(LOG, "Failed to setup output redirection!") - if args.debug: - # Reset so that all the debug handlers are closed out - LOG.debug(("Logging being reset, this logger may no" - " longer be active shortly")) - logging.resetLogging() - logging.setupLogging(mods.cfg) - apply_reporting_cfg(init.cfg) - - # now that logging is setup and stdout redirected, send welcome - welcome(name, msg=w_msg) - - # Stage 5 - (which_ran, failures) = mods.run_single(mod_name, - mod_args, - mod_freq) - if failures: - LOG.warn("Ran %s but it failed!", mod_name) - return 1 - elif not which_ran: - LOG.warn("Did not run %s, does it exist?", mod_name) - return 1 - else: - # Guess it worked - return 0 - - -def atomic_write_file(path, content, mode='w'): - tf = None - try: - tf = tempfile.NamedTemporaryFile(dir=os.path.dirname(path), - delete=False, mode=mode) - tf.write(content) - tf.close() - os.rename(tf.name, path) - except Exception as e: - if tf is not None: - os.unlink(tf.name) - raise e - - -def atomic_write_json(path, data): - return atomic_write_file(path, json.dumps(data, indent=1) + "\n") - - -def status_wrapper(name, args, data_d=None, link_d=None): - if data_d is None: - data_d = os.path.normpath("/var/lib/cloud/data") - if link_d is None: - link_d = os.path.normpath("/run/cloud-init") - - status_path = os.path.join(data_d, "status.json") - status_link = os.path.join(link_d, "status.json") - result_path = os.path.join(data_d, "result.json") - result_link = os.path.join(link_d, "result.json") - - util.ensure_dirs((data_d, link_d,)) - - (_name, functor) = args.action - - if name == "init": - if args.local: - mode = "init-local" - else: - mode = "init" - elif name == "modules": - mode = "modules-%s" % args.mode - else: - raise ValueError("unknown name: %s" % name) - - modes = ('init', 'init-local', 'modules-config', 'modules-final') - - status = None - if mode == 'init-local': - for f in (status_link, result_link, status_path, result_path): - util.del_file(f) - else: - try: - status = json.loads(util.load_file(status_path)) - except: - pass - - if status is None: - nullstatus = { - 'errors': [], - 'start': None, - 'finished': None, - } - status = {'v1': {}} - for m in modes: - status['v1'][m] = nullstatus.copy() - status['v1']['datasource'] = None - - v1 = status['v1'] - v1['stage'] = mode - v1[mode]['start'] = time.time() - - atomic_write_json(status_path, status) - util.sym_link(os.path.relpath(status_path, link_d), status_link, - force=True) - - try: - ret = functor(name, args) - if mode in ('init', 'init-local'): - (datasource, errors) = ret - if datasource is not None: - v1['datasource'] = str(datasource) - else: - errors = ret - - v1[mode]['errors'] = [str(e) for e in errors] - - except Exception as e: - util.logexc(LOG, "failed stage %s", mode) - print_exc("failed run of stage %s" % mode) - v1[mode]['errors'] = [str(e)] - - v1[mode]['finished'] = time.time() - v1['stage'] = None - - atomic_write_json(status_path, status) - - if mode == "modules-final": - # write the 'finished' file - errors = [] - for m in modes: - if v1[m]['errors']: - errors.extend(v1[m].get('errors', [])) - - atomic_write_json(result_path, - {'v1': {'datasource': v1['datasource'], - 'errors': errors}}) - util.sym_link(os.path.relpath(result_path, link_d), result_link, - force=True) - - return len(v1[mode]['errors']) - - -def main(): - parser = argparse.ArgumentParser() - - # Top level args - parser.add_argument('--version', '-v', action='version', - version='%(prog)s ' + (version.version_string())) - parser.add_argument('--file', '-f', action='append', - dest='files', - help=('additional yaml configuration' - ' files to use'), - type=argparse.FileType('rb')) - parser.add_argument('--debug', '-d', action='store_true', - help=('show additional pre-action' - ' logging (default: %(default)s)'), - default=False) - parser.add_argument('--force', action='store_true', - help=('force running even if no datasource is' - ' found (use at your own risk)'), - dest='force', - default=False) - - parser.set_defaults(reporter=None) - subparsers = parser.add_subparsers() - - # Each action and its sub-options (if any) - parser_init = subparsers.add_parser('init', - help=('initializes cloud-init and' - ' performs initial modules')) - parser_init.add_argument("--local", '-l', action='store_true', - help="start in local mode (default: %(default)s)", - default=False) - # This is used so that we can know which action is selected + - # the functor to use to run this subcommand - parser_init.set_defaults(action=('init', main_init)) - - # These settings are used for the 'config' and 'final' stages - parser_mod = subparsers.add_parser('modules', - help=('activates modules using ' - 'a given configuration key')) - parser_mod.add_argument("--mode", '-m', action='store', - help=("module configuration name " - "to use (default: %(default)s)"), - default='config', - choices=('init', 'config', 'final')) - parser_mod.set_defaults(action=('modules', main_modules)) - - # These settings are used when you want to query information - # stored in the cloud-init data objects/directories/files - parser_query = subparsers.add_parser('query', - help=('query information stored ' - 'in cloud-init')) - parser_query.add_argument("--name", '-n', action="store", - help="item name to query on", - required=True, - choices=QUERY_DATA_TYPES) - parser_query.set_defaults(action=('query', main_query)) - - # This subcommand allows you to run a single module - parser_single = subparsers.add_parser('single', - help=('run a single module ')) - parser_single.set_defaults(action=('single', main_single)) - parser_single.add_argument("--name", '-n', action="store", - help="module name to run", - required=True) - parser_single.add_argument("--frequency", action="store", - help=("frequency of the module"), - required=False, - choices=list(FREQ_SHORT_NAMES.keys())) - parser_single.add_argument("--report", action="store_true", - help="enable reporting", - required=False) - parser_single.add_argument("module_args", nargs="*", - metavar='argument', - help=('any additional arguments to' - ' pass to this module')) - parser_single.set_defaults(action=('single', main_single)) - - args = parser.parse_args() - - # Setup basic logging to start (until reinitialized) - # iff in debug mode... - if args.debug: - logging.setupBasicLogging() - - # Setup signal handlers before running - signal_handler.attach_handlers() - - if not hasattr(args, 'action'): - parser.error('too few arguments') - (name, functor) = args.action - if name in ("modules", "init"): - functor = status_wrapper - - report_on = True - if name == "init": - if args.local: - rname, rdesc = ("init-local", "searching for local datasources") - else: - rname, rdesc = ("init-network", - "searching for network datasources") - elif name == "modules": - rname, rdesc = ("modules-%s" % args.mode, - "running modules for %s" % args.mode) - elif name == "single": - rname, rdesc = ("single/%s" % args.name, - "running single module %s" % args.name) - report_on = args.report - - args.reporter = events.ReportEventStack( - rname, rdesc, reporting_enabled=report_on) - with args.reporter: - return util.log_time( - logfunc=LOG.debug, msg="cloud-init mode '%s'" % name, - get_uptime=True, func=functor, args=(name, args)) - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/cloudinit/cmd/__init__.py b/cloudinit/cmd/__init__.py new file mode 100644 index 00000000..da124641 --- /dev/null +++ b/cloudinit/cmd/__init__.py @@ -0,0 +1,21 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py new file mode 100755 index 00000000..27b7b56d --- /dev/null +++ b/cloudinit/cmd/main.py @@ -0,0 +1,682 @@ +#!/usr/bin/python +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import argparse +import json +import os +import sys +import tempfile +import time +import traceback + +from cloudinit import patcher +patcher.patch() + +from cloudinit import log as logging +from cloudinit import netinfo +from cloudinit import signal_handler +from cloudinit import sources +from cloudinit import stages +from cloudinit import templater +from cloudinit import util +from cloudinit import version + +from cloudinit import reporting +from cloudinit.reporting import events + +from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE, + CLOUD_CONFIG) + + +# Pretty little cheetah formatted welcome message template +WELCOME_MSG_TPL = ("Cloud-init v. ${version} running '${action}' at " + "${timestamp}. Up ${uptime} seconds.") + +# Module section template +MOD_SECTION_TPL = "cloud_%s_modules" + +# Things u can query on +QUERY_DATA_TYPES = [ + 'data', + 'data_raw', + 'instance_id', +] + +# Frequency shortname to full name +# (so users don't have to remember the full name...) +FREQ_SHORT_NAMES = { + 'instance': PER_INSTANCE, + 'always': PER_ALWAYS, + 'once': PER_ONCE, +} + +LOG = logging.getLogger() + + +# Used for when a logger may not be active +# and we still want to print exceptions... +def print_exc(msg=''): + if msg: + sys.stderr.write("%s\n" % (msg)) + sys.stderr.write('-' * 60) + sys.stderr.write("\n") + traceback.print_exc(file=sys.stderr) + sys.stderr.write('-' * 60) + sys.stderr.write("\n") + + +def welcome(action, msg=None): + if not msg: + msg = welcome_format(action) + util.multi_log("%s\n" % (msg), + console=False, stderr=True, log=LOG) + return msg + + +def welcome_format(action): + tpl_params = { + 'version': version.version_string(), + 'uptime': util.uptime(), + 'timestamp': util.time_rfc2822(), + 'action': action, + } + return templater.render_string(WELCOME_MSG_TPL, tpl_params) + + +def extract_fns(args): + # Files are already opened so lets just pass that along + # since it would of broke if it couldn't have + # read that file already... + fn_cfgs = [] + if args.files: + for fh in args.files: + # The realpath is more useful in logging + # so lets resolve to that... + fn_cfgs.append(os.path.realpath(fh.name)) + return fn_cfgs + + +def run_module_section(mods, action_name, section): + full_section_name = MOD_SECTION_TPL % (section) + (which_ran, failures) = mods.run_section(full_section_name) + total_attempted = len(which_ran) + len(failures) + if total_attempted == 0: + msg = ("No '%s' modules to run" + " under section '%s'") % (action_name, full_section_name) + sys.stderr.write("%s\n" % (msg)) + LOG.debug(msg) + return [] + else: + LOG.debug("Ran %s modules with %s failures", + len(which_ran), len(failures)) + return failures + + +def apply_reporting_cfg(cfg): + if cfg.get('reporting'): + reporting.update_configuration(cfg.get('reporting')) + + +def main_init(name, args): + deps = [sources.DEP_FILESYSTEM, sources.DEP_NETWORK] + if args.local: + deps = [sources.DEP_FILESYSTEM] + + if not args.local: + # See doc/kernel-cmdline.txt + # + # This is used in maas datasource, in "ephemeral" (read-only root) + # environment where the instance netboots to iscsi ro root. + # and the entity that controls the pxe config has to configure + # the maas datasource. + # + # Could be used elsewhere, only works on network based (not local). + root_name = "%s.d" % (CLOUD_CONFIG) + target_fn = os.path.join(root_name, "91_kernel_cmdline_url.cfg") + util.read_write_cmdline_url(target_fn) + + # Cloud-init 'init' stage is broken up into the following sub-stages + # 1. Ensure that the init object fetches its config without errors + # 2. Setup logging/output redirections with resultant config (if any) + # 3. Initialize the cloud-init filesystem + # 4. Check if we can stop early by looking for various files + # 5. Fetch the datasource + # 6. Connect to the current instance location + update the cache + # 7. Consume the userdata (handlers get activated here) + # 8. Construct the modules object + # 9. Adjust any subsequent logging/output redirections using the modules + # objects config as it may be different from init object + # 10. Run the modules for the 'init' stage + # 11. Done! + if not args.local: + w_msg = welcome_format(name) + else: + w_msg = welcome_format("%s-local" % (name)) + init = stages.Init(ds_deps=deps, reporter=args.reporter) + # Stage 1 + init.read_cfg(extract_fns(args)) + # Stage 2 + outfmt = None + errfmt = None + try: + LOG.debug("Closing stdin") + util.close_stdin() + (outfmt, errfmt) = util.fixup_output(init.cfg, name) + except Exception: + util.logexc(LOG, "Failed to setup output redirection!") + print_exc("Failed to setup output redirection!") + if args.debug: + # Reset so that all the debug handlers are closed out + LOG.debug(("Logging being reset, this logger may no" + " longer be active shortly")) + logging.resetLogging() + logging.setupLogging(init.cfg) + apply_reporting_cfg(init.cfg) + + # Any log usage prior to setupLogging above did not have local user log + # config applied. We send the welcome message now, as stderr/out have + # been redirected and log now configured. + welcome(name, msg=w_msg) + + # Stage 3 + try: + init.initialize() + except Exception: + util.logexc(LOG, "Failed to initialize, likely bad things to come!") + # Stage 4 + path_helper = init.paths + mode = sources.DSMODE_LOCAL if args.local else sources.DSMODE_NETWORK + + if mode == sources.DSMODE_NETWORK: + existing = "trust" + sys.stderr.write("%s\n" % (netinfo.debug_info())) + LOG.debug(("Checking to see if files that we need already" + " exist from a previous run that would allow us" + " to stop early.")) + # no-net is written by upstart cloud-init-nonet when network failed + # to come up + stop_files = [ + os.path.join(path_helper.get_cpath("data"), "no-net"), + ] + existing_files = [] + for fn in stop_files: + if os.path.isfile(fn): + existing_files.append(fn) + + if existing_files: + LOG.debug("[%s] Exiting. stop file %s existed", + mode, existing_files) + return (None, []) + else: + LOG.debug("Execution continuing, no previous run detected that" + " would allow us to stop early.") + else: + existing = "check" + if util.get_cfg_option_bool(init.cfg, 'manual_cache_clean', False): + existing = "trust" + + init.purge_cache() + # Delete the non-net file as well + util.del_file(os.path.join(path_helper.get_cpath("data"), "no-net")) + + # Stage 5 + try: + init.fetch(existing=existing) + # if in network mode, and the datasource is local + # then work was done at that stage. + if mode == sources.DSMODE_NETWORK and init.datasource.dsmode != mode: + LOG.debug("[%s] Exiting. datasource %s in local mode", + mode, init.datasource) + return (None, []) + except sources.DataSourceNotFoundException: + # In the case of 'cloud-init init' without '--local' it is a bit + # more likely that the user would consider it failure if nothing was + # found. When using upstart it will also mentions job failure + # in console log if exit code is != 0. + if mode == sources.DSMODE_LOCAL: + LOG.debug("No local datasource found") + else: + util.logexc(LOG, ("No instance datasource found!" + " Likely bad things to come!")) + if not args.force: + init.apply_network_config(bring_up=not args.local) + LOG.debug("[%s] Exiting without datasource in local mode", mode) + if mode == sources.DSMODE_LOCAL: + return (None, []) + else: + return (None, ["No instance datasource found."]) + else: + LOG.debug("[%s] barreling on in force mode without datasource", + mode) + + # Stage 6 + iid = init.instancify() + LOG.debug("[%s] %s will now be targeting instance id: %s. new=%s", + mode, name, iid, init.is_new_instance()) + + init.apply_network_config(bring_up=bool(mode != sources.DSMODE_LOCAL)) + + if mode == sources.DSMODE_LOCAL: + if init.datasource.dsmode != mode: + LOG.debug("[%s] Exiting. datasource %s not in local mode.", + mode, init.datasource) + return (init.datasource, []) + else: + LOG.debug("[%s] %s is in local mode, will apply init modules now.", + mode, init.datasource) + + # update fully realizes user-data (pulling in #include if necessary) + init.update() + # Stage 7 + try: + # Attempt to consume the data per instance. + # This may run user-data handlers and/or perform + # url downloads and such as needed. + (ran, _results) = init.cloudify().run('consume_data', + init.consume_data, + args=[PER_INSTANCE], + freq=PER_INSTANCE) + if not ran: + # Just consume anything that is set to run per-always + # if nothing ran in the per-instance code + # + # See: https://bugs.launchpad.net/bugs/819507 for a little + # reason behind this... + init.consume_data(PER_ALWAYS) + except Exception: + util.logexc(LOG, "Consuming user data failed!") + return (init.datasource, ["Consuming user data failed!"]) + + apply_reporting_cfg(init.cfg) + + # Stage 8 - re-read and apply relevant cloud-config to include user-data + mods = stages.Modules(init, extract_fns(args), reporter=args.reporter) + # Stage 9 + try: + outfmt_orig = outfmt + errfmt_orig = errfmt + (outfmt, errfmt) = util.get_output_cfg(mods.cfg, name) + if outfmt_orig != outfmt or errfmt_orig != errfmt: + LOG.warn("Stdout, stderr changing to (%s, %s)", outfmt, errfmt) + (outfmt, errfmt) = util.fixup_output(mods.cfg, name) + except Exception: + util.logexc(LOG, "Failed to re-adjust output redirection!") + logging.setupLogging(mods.cfg) + + # Stage 10 + return (init.datasource, run_module_section(mods, name, name)) + + +def main_modules(action_name, args): + name = args.mode + # Cloud-init 'modules' stages are broken up into the following sub-stages + # 1. Ensure that the init object fetches its config without errors + # 2. Get the datasource from the init object, if it does + # not exist then that means the main_init stage never + # worked, and thus this stage can not run. + # 3. Construct the modules object + # 4. Adjust any subsequent logging/output redirections using + # the modules objects configuration + # 5. Run the modules for the given stage name + # 6. Done! + w_msg = welcome_format("%s:%s" % (action_name, name)) + init = stages.Init(ds_deps=[], reporter=args.reporter) + # Stage 1 + init.read_cfg(extract_fns(args)) + # Stage 2 + try: + init.fetch(existing="trust") + except sources.DataSourceNotFoundException: + # There was no datasource found, theres nothing to do + msg = ('Can not apply stage %s, no datasource found! Likely bad ' + 'things to come!' % name) + util.logexc(LOG, msg) + print_exc(msg) + if not args.force: + return [(msg)] + # Stage 3 + mods = stages.Modules(init, extract_fns(args), reporter=args.reporter) + # Stage 4 + try: + LOG.debug("Closing stdin") + util.close_stdin() + util.fixup_output(mods.cfg, name) + except Exception: + util.logexc(LOG, "Failed to setup output redirection!") + if args.debug: + # Reset so that all the debug handlers are closed out + LOG.debug(("Logging being reset, this logger may no" + " longer be active shortly")) + logging.resetLogging() + logging.setupLogging(mods.cfg) + apply_reporting_cfg(init.cfg) + + # now that logging is setup and stdout redirected, send welcome + welcome(name, msg=w_msg) + + # Stage 5 + return run_module_section(mods, name, name) + + +def main_query(name, _args): + raise NotImplementedError(("Action '%s' is not" + " currently implemented") % (name)) + + +def main_single(name, args): + # Cloud-init single stage is broken up into the following sub-stages + # 1. Ensure that the init object fetches its config without errors + # 2. Attempt to fetch the datasource (warn if it doesn't work) + # 3. Construct the modules object + # 4. Adjust any subsequent logging/output redirections using + # the modules objects configuration + # 5. Run the single module + # 6. Done! + mod_name = args.name + w_msg = welcome_format(name) + init = stages.Init(ds_deps=[], reporter=args.reporter) + # Stage 1 + init.read_cfg(extract_fns(args)) + # Stage 2 + try: + init.fetch(existing="trust") + except sources.DataSourceNotFoundException: + # There was no datasource found, + # that might be bad (or ok) depending on + # the module being ran (so continue on) + util.logexc(LOG, ("Failed to fetch your datasource," + " likely bad things to come!")) + print_exc(("Failed to fetch your datasource," + " likely bad things to come!")) + if not args.force: + return 1 + # Stage 3 + mods = stages.Modules(init, extract_fns(args), reporter=args.reporter) + mod_args = args.module_args + if mod_args: + LOG.debug("Using passed in arguments %s", mod_args) + mod_freq = args.frequency + if mod_freq: + LOG.debug("Using passed in frequency %s", mod_freq) + mod_freq = FREQ_SHORT_NAMES.get(mod_freq) + # Stage 4 + try: + LOG.debug("Closing stdin") + util.close_stdin() + util.fixup_output(mods.cfg, None) + except Exception: + util.logexc(LOG, "Failed to setup output redirection!") + if args.debug: + # Reset so that all the debug handlers are closed out + LOG.debug(("Logging being reset, this logger may no" + " longer be active shortly")) + logging.resetLogging() + logging.setupLogging(mods.cfg) + apply_reporting_cfg(init.cfg) + + # now that logging is setup and stdout redirected, send welcome + welcome(name, msg=w_msg) + + # Stage 5 + (which_ran, failures) = mods.run_single(mod_name, + mod_args, + mod_freq) + if failures: + LOG.warn("Ran %s but it failed!", mod_name) + return 1 + elif not which_ran: + LOG.warn("Did not run %s, does it exist?", mod_name) + return 1 + else: + # Guess it worked + return 0 + + +def atomic_write_file(path, content, mode='w'): + tf = None + try: + tf = tempfile.NamedTemporaryFile(dir=os.path.dirname(path), + delete=False, mode=mode) + tf.write(content) + tf.close() + os.rename(tf.name, path) + except Exception as e: + if tf is not None: + os.unlink(tf.name) + raise e + + +def atomic_write_json(path, data): + return atomic_write_file(path, json.dumps(data, indent=1) + "\n") + + +def status_wrapper(name, args, data_d=None, link_d=None): + if data_d is None: + data_d = os.path.normpath("/var/lib/cloud/data") + if link_d is None: + link_d = os.path.normpath("/run/cloud-init") + + status_path = os.path.join(data_d, "status.json") + status_link = os.path.join(link_d, "status.json") + result_path = os.path.join(data_d, "result.json") + result_link = os.path.join(link_d, "result.json") + + util.ensure_dirs((data_d, link_d,)) + + (_name, functor) = args.action + + if name == "init": + if args.local: + mode = "init-local" + else: + mode = "init" + elif name == "modules": + mode = "modules-%s" % args.mode + else: + raise ValueError("unknown name: %s" % name) + + modes = ('init', 'init-local', 'modules-config', 'modules-final') + + status = None + if mode == 'init-local': + for f in (status_link, result_link, status_path, result_path): + util.del_file(f) + else: + try: + status = json.loads(util.load_file(status_path)) + except Exception: + pass + + if status is None: + nullstatus = { + 'errors': [], + 'start': None, + 'finished': None, + } + status = {'v1': {}} + for m in modes: + status['v1'][m] = nullstatus.copy() + status['v1']['datasource'] = None + + v1 = status['v1'] + v1['stage'] = mode + v1[mode]['start'] = time.time() + + atomic_write_json(status_path, status) + util.sym_link(os.path.relpath(status_path, link_d), status_link, + force=True) + + try: + ret = functor(name, args) + if mode in ('init', 'init-local'): + (datasource, errors) = ret + if datasource is not None: + v1['datasource'] = str(datasource) + else: + errors = ret + + v1[mode]['errors'] = [str(e) for e in errors] + + except Exception as e: + util.logexc(LOG, "failed stage %s", mode) + print_exc("failed run of stage %s" % mode) + v1[mode]['errors'] = [str(e)] + + v1[mode]['finished'] = time.time() + v1['stage'] = None + + atomic_write_json(status_path, status) + + if mode == "modules-final": + # write the 'finished' file + errors = [] + for m in modes: + if v1[m]['errors']: + errors.extend(v1[m].get('errors', [])) + + atomic_write_json(result_path, + {'v1': {'datasource': v1['datasource'], + 'errors': errors}}) + util.sym_link(os.path.relpath(result_path, link_d), result_link, + force=True) + + return len(v1[mode]['errors']) + + +def main(sysv_args=None): + if sysv_args is not None: + parser = argparse.ArgumentParser(prog=sysv_args[0]) + sysv_args = sysv_args[1:] + else: + parser = argparse.ArgumentParser() + + # Top level args + parser.add_argument('--version', '-v', action='version', + version='%(prog)s ' + (version.version_string())) + parser.add_argument('--file', '-f', action='append', + dest='files', + help=('additional yaml configuration' + ' files to use'), + type=argparse.FileType('rb')) + parser.add_argument('--debug', '-d', action='store_true', + help=('show additional pre-action' + ' logging (default: %(default)s)'), + default=False) + parser.add_argument('--force', action='store_true', + help=('force running even if no datasource is' + ' found (use at your own risk)'), + dest='force', + default=False) + + parser.set_defaults(reporter=None) + subparsers = parser.add_subparsers() + + # Each action and its sub-options (if any) + parser_init = subparsers.add_parser('init', + help=('initializes cloud-init and' + ' performs initial modules')) + parser_init.add_argument("--local", '-l', action='store_true', + help="start in local mode (default: %(default)s)", + default=False) + # This is used so that we can know which action is selected + + # the functor to use to run this subcommand + parser_init.set_defaults(action=('init', main_init)) + + # These settings are used for the 'config' and 'final' stages + parser_mod = subparsers.add_parser('modules', + help=('activates modules using ' + 'a given configuration key')) + parser_mod.add_argument("--mode", '-m', action='store', + help=("module configuration name " + "to use (default: %(default)s)"), + default='config', + choices=('init', 'config', 'final')) + parser_mod.set_defaults(action=('modules', main_modules)) + + # These settings are used when you want to query information + # stored in the cloud-init data objects/directories/files + parser_query = subparsers.add_parser('query', + help=('query information stored ' + 'in cloud-init')) + parser_query.add_argument("--name", '-n', action="store", + help="item name to query on", + required=True, + choices=QUERY_DATA_TYPES) + parser_query.set_defaults(action=('query', main_query)) + + # This subcommand allows you to run a single module + parser_single = subparsers.add_parser('single', + help=('run a single module ')) + parser_single.set_defaults(action=('single', main_single)) + parser_single.add_argument("--name", '-n', action="store", + help="module name to run", + required=True) + parser_single.add_argument("--frequency", action="store", + help=("frequency of the module"), + required=False, + choices=list(FREQ_SHORT_NAMES.keys())) + parser_single.add_argument("--report", action="store_true", + help="enable reporting", + required=False) + parser_single.add_argument("module_args", nargs="*", + metavar='argument', + help=('any additional arguments to' + ' pass to this module')) + parser_single.set_defaults(action=('single', main_single)) + + args = parser.parse_args(args=sysv_args) + + # Setup basic logging to start (until reinitialized) + # iff in debug mode... + if args.debug: + logging.setupBasicLogging() + + # Setup signal handlers before running + signal_handler.attach_handlers() + + (name, functor) = args.action + + if name in ("modules", "init"): + functor = status_wrapper + + report_on = True + if name == "init": + if args.local: + rname, rdesc = ("init-local", "searching for local datasources") + else: + rname, rdesc = ("init-network", + "searching for network datasources") + elif name == "modules": + rname, rdesc = ("modules-%s" % args.mode, + "running modules for %s" % args.mode) + elif name == "single": + rname, rdesc = ("single/%s" % args.name, + "running single module %s" % args.name) + report_on = args.report + + args.reporter = events.ReportEventStack( + rname, rdesc, reporting_enabled=report_on) + with args.reporter: + return util.log_time( + logfunc=LOG.debug, msg="cloud-init mode '%s'" % name, + get_uptime=True, func=functor, args=(name, args)) diff --git a/setup.py b/setup.py index 8c8f2402..0af576a9 100755 --- a/setup.py +++ b/setup.py @@ -204,10 +204,14 @@ setuptools.setup( author_email='scott.moser@canonical.com', url='http://launchpad.net/cloud-init/', packages=setuptools.find_packages(exclude=['tests']), - scripts=['bin/cloud-init', - 'tools/cloud-init-per'], + scripts=['tools/cloud-init-per'], license='GPLv3', data_files=data_files, install_requires=requirements, cmdclass=cmdclass, + entry_points={ + 'console_scripts': [ + 'cloud-init = cloudinit.cmd.main:main' + ], + } ) diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py index 163ce64c..8268cd5e 100644 --- a/tests/unittests/test_cli.py +++ b/tests/unittests/test_cli.py @@ -6,7 +6,7 @@ import sys from . import helpers as test_helpers mock = test_helpers.mock -BIN_CLOUDINIT = "bin/cloud-init" +from cloudinit.cmd import main as cli class TestCLI(test_helpers.FilesystemMockingTestCase): @@ -15,35 +15,22 @@ class TestCLI(test_helpers.FilesystemMockingTestCase): super(TestCLI, self).setUp() self.stderr = six.StringIO() self.patchStdoutAndStderr(stderr=self.stderr) - self.sys_exit = mock.MagicMock() - self.patched_funcs.enter_context( - mock.patch.object(sys, 'exit', self.sys_exit)) - - def _call_main(self): - self.patched_funcs.enter_context( - mock.patch.object(sys, 'argv', ['cloud-init'])) - cli = imp.load_module( - 'cli', open(BIN_CLOUDINIT), '', ('', 'r', imp.PY_SOURCE)) + + def _call_main(self, sysv_args=None): + if not sysv_args: + sysv_args = ['cloud-init'] try: - return cli.main() - except Exception: - pass + return cli.main(sysv_args=sysv_args) + except SystemExit as e: + return e.code - @test_helpers.skipIf(not os.path.isfile(BIN_CLOUDINIT), "no bin/cloudinit") def test_no_arguments_shows_usage(self): - self._call_main() - self.assertIn('usage: cloud-init', self.stderr.getvalue()) - - @test_helpers.skipIf(not os.path.isfile(BIN_CLOUDINIT), "no bin/cloudinit") - def test_no_arguments_exits_2(self): exit_code = self._call_main() - if self.sys_exit.call_count: - self.assertEqual(mock.call(2), self.sys_exit.call_args) - else: - self.assertEqual(2, exit_code) + self.assertIn('usage: cloud-init', self.stderr.getvalue()) + self.assertEqual(2, exit_code) - @test_helpers.skipIf(not os.path.isfile(BIN_CLOUDINIT), "no bin/cloudinit") def test_no_arguments_shows_error_message(self): - self._call_main() + exit_code = self._call_main() self.assertIn('cloud-init: error: too few arguments', self.stderr.getvalue()) + self.assertEqual(2, exit_code) -- cgit v1.2.3 From 628eaedba94dae953f6e0c0469881d9eb417cafe Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 14 Jun 2016 15:00:05 -0700 Subject: Retain the prior attribute missing handling --- cloudinit/cmd/main.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py index 27b7b56d..c8916e03 100755 --- a/cloudinit/cmd/main.py +++ b/cloudinit/cmd/main.py @@ -654,7 +654,10 @@ def main(sysv_args=None): # Setup signal handlers before running signal_handler.attach_handlers() - (name, functor) = args.action + try: + (name, functor) = args.action + except AttributeError: + parser.error('too few arguments') if name in ("modules", "init"): functor = status_wrapper -- cgit v1.2.3 From 4dbfc55a6c90aee390ab981b0d4775d8b94e4803 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 14 Jun 2016 15:00:40 -0700 Subject: Don't continue running with no action --- cloudinit/cmd/main.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py index c8916e03..2bd75328 100755 --- a/cloudinit/cmd/main.py +++ b/cloudinit/cmd/main.py @@ -646,6 +646,11 @@ def main(sysv_args=None): args = parser.parse_args(args=sysv_args) + try: + (name, functor) = args.action + except AttributeError: + parser.error('too few arguments') + # Setup basic logging to start (until reinitialized) # iff in debug mode... if args.debug: @@ -654,11 +659,6 @@ def main(sysv_args=None): # Setup signal handlers before running signal_handler.attach_handlers() - try: - (name, functor) = args.action - except AttributeError: - parser.error('too few arguments') - if name in ("modules", "init"): functor = status_wrapper -- cgit v1.2.3 From 946b0f5542ffb43149f60b5dd50b2fb450d23713 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Wed, 15 Jun 2016 10:31:44 +0100 Subject: Re-apply "Remove trailing dot from GCE metadata URL (LP: #1581200) [Phil Roche]" This commit includes the content of that commit, plus a fix for the tests (provided by Phil). --- ChangeLog | 1 + cloudinit/sources/DataSourceGCE.py | 2 +- tests/unittests/test_datasource/test_gce.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) (limited to 'cloudinit') diff --git a/ChangeLog b/ChangeLog index e2869522..66539792 100644 --- a/ChangeLog +++ b/ChangeLog @@ -122,6 +122,7 @@ - ConfigDrive: improved support for networking information from a network_data.json or older interfaces formated network_config. - Change missing Cheetah log warning to debug [Andrew Jorgensen] + - Remove trailing dot from GCE metadata URL (LP: #1581200) [Phil Roche] 0.7.6: - open 0.7.6 diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 9234d1f8..c660a350 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -25,7 +25,7 @@ from cloudinit import util LOG = logging.getLogger(__name__) BUILTIN_DS_CONFIG = { - 'metadata_url': 'http://metadata.google.internal./computeMetadata/v1/' + 'metadata_url': 'http://metadata.google.internal/computeMetadata/v1/' } REQUIRED_FIELDS = ('instance-id', 'availability-zone', 'local-hostname') diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py index 1f7eb99e..6e62a4d2 100644 --- a/tests/unittests/test_datasource/test_gce.py +++ b/tests/unittests/test_datasource/test_gce.py @@ -52,7 +52,7 @@ GCE_META_ENCODING = { HEADERS = {'X-Google-Metadata-Request': 'True'} MD_URL_RE = re.compile( - r'http://metadata.google.internal./computeMetadata/v1/.*') + r'http://metadata.google.internal/computeMetadata/v1/.*') def _set_mock_metadata(gce_meta=None): -- cgit v1.2.3 From 2c5c2d7a500ccb88eee245a1d5f1ca1f9a2156e6 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 15 Jun 2016 12:14:27 -0700 Subject: Silence pep8 warnings due to patcher activation --- cloudinit/cmd/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py index 2bd75328..63621c1d 100755 --- a/cloudinit/cmd/main.py +++ b/cloudinit/cmd/main.py @@ -30,7 +30,7 @@ import time import traceback from cloudinit import patcher -patcher.patch() +patcher.patch() # noqa from cloudinit import log as logging from cloudinit import netinfo -- cgit v1.2.3 From 459de83024b41c32823b75cf483da994fb1388b7 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 15 Jun 2016 16:11:24 -0700 Subject: Fixup code review comments --- cloudinit/distros/debian.py | 12 ++--- cloudinit/distros/rhel.py | 4 +- cloudinit/net/__init__.py | 1 - cloudinit/net/eni.py | 36 +++++++------ cloudinit/net/renderer.py | 48 +++++++++++++++++ cloudinit/net/sysconfig.py | 123 +++++++++++--------------------------------- tests/unittests/test_net.py | 10 ++-- 7 files changed, 109 insertions(+), 125 deletions(-) create mode 100644 cloudinit/net/renderer.py (limited to 'cloudinit') diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index e71aaa97..511ed4fe 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -57,7 +57,11 @@ class Distro(distros.Distro): # should only happen say once per instance...) self._runner = helpers.Runners(paths) self.osfamily = 'debian' - self._net_renderer = eni.Renderer() + self._net_renderer = eni.Renderer({ + 'eni_path': self.network_conf_fn, + 'links_prefix_path': self.links_prefix, + 'netrules_path': None, + }) def apply_locale(self, locale, out_fn=None): if not out_fn: @@ -82,12 +86,8 @@ class Distro(distros.Distro): def _write_network_config(self, netconfig): ns = net.parse_net_config_data(netconfig) - self._net_renderer.render_network_state( - target="/", network_state=ns, - eni=self.network_conf_fn, links_prefix=self.links_prefix, - netrules=None) + self._net_renderer.render_network_state("/", ns) _maybe_remove_legacy_eth0() - return [] def _bring_up_interfaces(self, device_names): diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py index 20525e47..1aa42d75 100644 --- a/cloudinit/distros/rhel.py +++ b/cloudinit/distros/rhel.py @@ -67,8 +67,8 @@ class Distro(distros.Distro): self.package_command('install', pkgs=pkglist) def _write_network_config(self, netconfig): - self._net_renderer.render_network_state( - target="/", parse_net_config_data(netconfig)) + ns = parse_net_config_data(netconfig) + self._net_renderer.render_network_state("/", ns) return [] def _write_network(self, settings): diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index f5668fff..6959ad34 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -26,7 +26,6 @@ from cloudinit import util LOG = logging.getLogger(__name__) SYS_CLASS_NET = "/sys/class/net/" DEFAULT_PRIMARY_INTERFACE = 'eth0' -LINKS_FNAME_PREFIX = "etc/systemd/network/50-cloud-init-" def sys_dev_path(devname, path=""): diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index df0df9b2..352f7dd5 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -16,10 +16,9 @@ import glob import os import re -from . import LINKS_FNAME_PREFIX from . import ParserError -from .udev import generate_udev_rule +from . import renderer from cloudinit import util @@ -297,9 +296,18 @@ def _ifaces_to_net_config_data(ifaces): 'config': [devs[d] for d in sorted(devs)]} -class Renderer(object): +class Renderer(renderer.Renderer): """Renders network information in a /etc/network/interfaces format.""" + def __init__(self, config=None): + if not config: + config = {} + self.eni_path = config.get('eni_path', 'etc/network/interfaces') + self.links_path_prefix = config.get( + 'links_path_prefix', 'etc/systemd/network/50-cloud-init-') + self.netrules_path = config.get( + 'netrules_path', 'etc/udev/rules.d/70-persistent-net.rules') + def _render_persistent_net(self, network_state): """Given state, emit udev rules to map mac to ifname.""" content = "" @@ -419,29 +427,23 @@ class Renderer(object): content = content.replace('mac_address', 'hwaddress') return content - def render_network_state( - self, target, network_state, eni="etc/network/interfaces", - links_prefix=LINKS_FNAME_PREFIX, - netrules='etc/udev/rules.d/70-persistent-net.rules', - writer=None): - - fpeni = os.path.sep.join((target, eni,)) + def render_network_state(self, target, network_state): + fpeni = os.path.join(target, self.eni_path) util.ensure_dir(os.path.dirname(fpeni)) util.write_file(fpeni, self._render_interfaces(network_state)) - if netrules: - netrules = os.path.sep.join((target, netrules,)) + if self.netrules_path: + netrules = os.path.join(target, self.netrules_path) util.ensure_dir(os.path.dirname(netrules)) util.write_file(netrules, self._render_persistent_net(network_state)) - if links_prefix: + if self.links_path_prefix: self._render_systemd_links(target, network_state, - links_prefix=links_prefix) + links_prefix=self.links_path_prefix) - def _render_systemd_links(self, target, network_state, - links_prefix=LINKS_FNAME_PREFIX): - fp_prefix = os.path.sep.join((target, links_prefix)) + def _render_systemd_links(self, target, network_state, links_prefix): + fp_prefix = os.path.join(target, links_prefix) for f in glob.glob(fp_prefix + "*"): os.unlink(f) for iface in network_state.iter_interfaces(): diff --git a/cloudinit/net/renderer.py b/cloudinit/net/renderer.py new file mode 100644 index 00000000..310cbe0d --- /dev/null +++ b/cloudinit/net/renderer.py @@ -0,0 +1,48 @@ +# Copyright (C) 2013-2014 Canonical Ltd. +# +# Author: Scott Moser +# Author: Blake Rouse +# +# Curtin is free software: you can redistribute it and/or modify it under +# the terms of the GNU Affero General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# Curtin is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for +# more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Curtin. If not, see . + +import six + +from .udev import generate_udev_rule + + +def filter_by_type(match_type): + return lambda iface: match_type == iface['type'] + + +def filter_by_name(match_name): + return lambda iface: match_name == iface['name'] + + +filter_by_physical = filter_by_type('physical') + + +class Renderer(object): + + @staticmethod + def _render_persistent_net(network_state): + """Given state, emit udev rules to map mac to ifname.""" + # TODO(harlowja): this seems shared between eni renderer and + # this, so move it to a shared location. + content = six.StringIO() + for iface in network_state.iter_interfaces(filter_by_physical): + # for physical interfaces write out a persist net udev rule + if 'name' in iface and iface.get('mac_address'): + content.write(generate_udev_rule(iface['name'], + iface['mac_address'])) + return content.getvalue() diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index 647ca3ed..dd005796 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -20,9 +20,10 @@ import six from cloudinit.distros.parsers import resolv_conf from cloudinit import util -from . import network_state from .udev import generate_udev_rule +from . import renderer + def _make_header(sep='#'): lines = [ @@ -37,17 +38,6 @@ def _make_header(sep='#'): return "\n".join(lines) -def _filter_by_type(match_type): - return lambda iface: match_type == iface['type'] - - -def _filter_by_name(match_name): - return lambda iface: match_name == iface['name'] - - -_filter_by_physical = _filter_by_type('physical') - - def _is_default_route(route): if route['network'] == '::' and route['netmask'] == 0: return True @@ -182,7 +172,7 @@ class NetInterface(ConfigMap): return c -class Renderer(object): +class Renderer(renderer.Renderer): """Renders network information in a /etc/sysconfig format.""" # See: https://access.redhat.com/documentation/en-US/\ @@ -211,18 +201,13 @@ class Renderer(object): ('bridge_bridgeprio', 'PRIO'), ]) - @staticmethod - def _render_persistent_net(network_state): - """Given state, emit udev rules to map mac to ifname.""" - # TODO(harlowja): this seems shared between eni renderer and - # this, so move it to a shared location. - content = six.StringIO() - for iface in network_state.iter_interfaces(_filter_by_physical): - # for physical interfaces write out a persist net udev rule - if 'name' in iface and iface.get('mac_address'): - content.write(generate_udev_rule(iface['name'], - iface['mac_address'])) - return content.getvalue() + def __init__(self, config=None): + if not config: + config = {} + self.sysconf_dir = config.get('sysconf_dir', 'etc/sysconfig/') + self.netrules_path = config.get( + 'netrules_path', 'etc/udev/rules.d/70-persistent-net.rules') + self.dns_path = config.get('dns_path', 'etc/resolv.conf') @classmethod def _render_iface_shared(cls, iface, iface_cfg): @@ -302,7 +287,7 @@ class Renderer(object): @classmethod def _render_physical_interfaces(cls, network_state, iface_contents): - for iface in network_state.iter_interfaces(_filter_by_physical): + for iface in network_state.iter_interfaces(renderer.filter_by_physical): iface_name = iface['name'] iface_subnets = iface.get("subnets", []) iface_cfg = iface_contents[iface_name] @@ -319,7 +304,7 @@ class Renderer(object): @classmethod def _render_bond_interfaces(cls, network_state, iface_contents): - for iface in network_state.iter_interfaces(_filter_by_type('bond')): + for iface in network_state.iter_interfaces(renderer.filter_by_type('bond')): iface_name = iface['name'] iface_cfg = iface_contents[iface_name] cls._render_bonding_opts(iface_cfg, iface) @@ -337,7 +322,7 @@ class Renderer(object): @staticmethod def _render_vlan_interfaces(network_state, iface_contents): - for iface in network_state.iter_interfaces(_filter_by_type('vlan')): + for iface in network_state.iter_interfaces(renderer.filter_by_type('vlan')): iface_name = iface['name'] iface_cfg = iface_contents[iface_name] iface_cfg['VLAN'] = True @@ -348,15 +333,15 @@ class Renderer(object): content = resolv_conf.ResolvConf("") if existing_dns_path and os.path.isfile(existing_dns_path): content = resolv_conf.ResolvConf(util.load_file(existing_dns_path)) - for ns in network_state.dns_nameservers: - content.add_nameserver(ns) - for d in network_state.dns_searchdomains: - content.add_search_domain(d) + for nameserver in network_state.dns_nameservers: + content.add_nameserver(nameserver) + for searchdomain in network_state.dns_searchdomains: + content.add_search_domain(searchdomain) return "\n".join([_make_header(';'), str(content)]) @classmethod def _render_bridge_interfaces(cls, network_state, iface_contents): - for iface in network_state.iter_interfaces(_filter_by_type('bridge')): + for iface in network_state.iter_interfaces(renderer.filter_by_type('bridge')): iface_name = iface['name'] iface_cfg = iface_contents[iface_name] iface_cfg.kind = 'bridge' @@ -397,67 +382,17 @@ class Renderer(object): contents[iface_cfg.routes.path] = iface_cfg.routes.to_string() return contents - def render_network_state( - self, target, network_state, sysconf_dir="etc/sysconfig/", - netrules='etc/udev/rules.d/70-persistent-net.rules', - dns='etc/resolv.conf'): - if target: - base_sysconf_dir = os.path.join(target, sysconf_dir) - else: - base_sysconf_dir = sysconf_dir + def render_network_state(self, target, network_state): + base_sysconf_dir = os.path.join(target, self.sysconf_dir) for path, data in self._render_sysconfig(base_sysconf_dir, network_state).items(): - if target: - util.write_file(path, data) - else: - print("File to be at: %s" % path) - print(data) - if dns: - if target: - dns_path = os.path.join(target, dns) - resolv_content = self._render_dns(network_state, - existing_dns_path=dns_path) - util.write_file(dns_path, resolv_content) - else: - resolv_content = self._render_dns(network_state) - dns_path = dns - print("File to be at: %s" % dns_path) - print(resolv_content) - if netrules: + util.write_file(path, data) + if self.dns_path: + dns_path = os.path.join(target, self.dns_path) + resolv_content = self._render_dns(network_state, + existing_dns_path=dns_path) + util.write_file(dns_path, resolv_content) + if self.netrules_path: netrules_content = self._render_persistent_net(network_state) - if target: - netrules_path = os.path.join(target, netrules) - util.write_file(netrules_path, netrules_content) - else: - netrules_path = netrules - print("File to be at: %s" % netrules_path) - print(netrules_content) - - -def main(): - """Reads a os network state json file and outputs what would be written.""" - from cloudinit.sources.helpers import openstack - - import argparse - import json - - parser = argparse.ArgumentParser() - parser.add_argument("-f", "--file", metavar="FILE", - help=("openstack network json file" - " to read (required)"), - required=True) - parser.add_argument("-d", "--dir", metavar="DIR", - help=("directory to write output into (if" - " not provided then written to stdout)"), - default=None) - args = parser.parse_args() - - network_json = json.loads(util.load_file(args.file)) - net_state = network_state.parse_net_config_data( - openstack.convert_net_json(network_json), skip_broken=False) - r = Renderer() - r.render_network_state(args.dir, net_state) - - -if __name__ == '__main__': - main() + netrules_path = os.path.join(target, self.netrules_path) + util.write_file(netrules_path, netrules_content) diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 43602030..3ae00fc6 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -248,11 +248,11 @@ class TestEniNetRendering(TestCase): render_dir = os.path.join(tmp_dir, "render") os.makedirs(render_dir) - renderer = eni.Renderer() - renderer.render_network_state(render_dir, ns, - eni="interfaces", - links_prefix=None, - netrules=None) + renderer = eni.Renderer( + {'links_path_prefix': None, + 'eni_path': 'interfaces', 'netrules_path': None, + }) + renderer.render_network_state(render_dir, ns) self.assertTrue(os.path.exists(os.path.join(render_dir, 'interfaces'))) -- cgit v1.2.3 From a454a6408b3c7ecc816073049d060632191099bb Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 15 Jun 2016 16:15:28 -0700 Subject: Fix line length issues --- cloudinit/net/eni.py | 13 ------------- cloudinit/net/sysconfig.py | 14 ++++++++------ 2 files changed, 8 insertions(+), 19 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index 352f7dd5..c8adb1ae 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -308,19 +308,6 @@ class Renderer(renderer.Renderer): self.netrules_path = config.get( 'netrules_path', 'etc/udev/rules.d/70-persistent-net.rules') - def _render_persistent_net(self, network_state): - """Given state, emit udev rules to map mac to ifname.""" - content = "" - interfaces = network_state.get('interfaces') - for iface in interfaces.values(): - # for physical interfaces write out a persist net udev rule - if iface['type'] == 'physical' and \ - 'name' in iface and iface.get('mac_address'): - content += generate_udev_rule(iface['name'], - iface['mac_address']) - - return content - def _render_route(self, route, indent=""): """When rendering routes for an iface, in some cases applying a route may result in the route command returning non-zero which produces diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index dd005796..c53acf71 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -20,8 +20,6 @@ import six from cloudinit.distros.parsers import resolv_conf from cloudinit import util -from .udev import generate_udev_rule - from . import renderer @@ -287,7 +285,8 @@ class Renderer(renderer.Renderer): @classmethod def _render_physical_interfaces(cls, network_state, iface_contents): - for iface in network_state.iter_interfaces(renderer.filter_by_physical): + physical_filter = renderer.filter_by_physical + for iface in network_state.iter_interfaces(physical_filter): iface_name = iface['name'] iface_subnets = iface.get("subnets", []) iface_cfg = iface_contents[iface_name] @@ -304,7 +303,8 @@ class Renderer(renderer.Renderer): @classmethod def _render_bond_interfaces(cls, network_state, iface_contents): - for iface in network_state.iter_interfaces(renderer.filter_by_type('bond')): + bond_filter = renderer.filter_by_type('bond') + for iface in network_state.iter_interfaces(bond_filter): iface_name = iface['name'] iface_cfg = iface_contents[iface_name] cls._render_bonding_opts(iface_cfg, iface) @@ -322,7 +322,8 @@ class Renderer(renderer.Renderer): @staticmethod def _render_vlan_interfaces(network_state, iface_contents): - for iface in network_state.iter_interfaces(renderer.filter_by_type('vlan')): + vlan_filter = renderer.filter_by_type('vlan') + for iface in network_state.iter_interfaces(vlan_filter): iface_name = iface['name'] iface_cfg = iface_contents[iface_name] iface_cfg['VLAN'] = True @@ -341,7 +342,8 @@ class Renderer(renderer.Renderer): @classmethod def _render_bridge_interfaces(cls, network_state, iface_contents): - for iface in network_state.iter_interfaces(renderer.filter_by_type('bridge')): + bridge_filter = renderer.filter_by_type('bridge') + for iface in network_state.iter_interfaces(bridge_filter): iface_name = iface['name'] iface_cfg = iface_contents[iface_name] iface_cfg.kind = 'bridge' -- cgit v1.2.3 From 4f989cc595775ab6b06fac604218d77910f3e4f4 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 15 Jun 2016 21:08:04 -0400 Subject: fis some Datasourcenocloud issues LP: #1592505 --- cloudinit/sources/DataSourceNoCloud.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 7e30118c..19d63950 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -24,7 +24,7 @@ import errno import os from cloudinit import log as logging -from cloudinit import net +from cloudinit.net import eni from cloudinit import sources from cloudinit import util @@ -194,8 +194,7 @@ class DataSourceNoCloud(sources.DataSource): # LP: #1568150 need getattr in the case that an old class object # has been loaded from a pickled file and now executing new source. dirs = getattr(self, 'seed_dirs', [self.seed_dir]) - quick_id = _quick_read_instance_id(cmdline_id=self.cmdline_id, - dirs=dirs) + quick_id = _quick_read_instance_id(dirs=dirs) if not quick_id: return None return quick_id == current @@ -203,20 +202,19 @@ class DataSourceNoCloud(sources.DataSource): @property def network_config(self): if self._network_config is None: - if self.network_eni is not None: - self._network_config = net.convert_eni_data(self.network_eni) + if self._network_eni is not None: + self._network_config = eni.convert_eni_data(self._network_eni) return self._network_config -def _quick_read_instance_id(cmdline_id, dirs=None): +def _quick_read_instance_id(dirs=None): if dirs is None: dirs = [] iid_key = 'instance-id' - if cmdline_id is None: - fill = {} - if parse_cmdline_data(cmdline_id, fill) and iid_key in fill: - return fill[iid_key] + fill = {} + if load_cmdline_data(fill) and iid_key in fill: + return fill[iid_key] for d in dirs: if d is None: -- cgit v1.2.3 From d4b587ebf500ddc2259fffc94a3c69c199c9a427 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 15 Jun 2016 22:50:12 -0400 Subject: fix some errors reported by pylint pylint --errors-only found several errors. Some of the changes here represent real errors, others just code that pylint did not like. --- cloudinit/config/cc_apt_configure.py | 3 ++- cloudinit/config/cc_growpart.py | 10 +++++----- cloudinit/distros/__init__.py | 2 +- cloudinit/distros/arch.py | 4 ++-- cloudinit/sources/DataSourceOVF.py | 10 +++++----- cloudinit/sources/DataSourceOpenNebula.py | 2 +- cloudinit/sources/helpers/openstack.py | 2 +- 7 files changed, 17 insertions(+), 16 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 96c4a43d..05ad4b03 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -208,8 +208,9 @@ def add_apt_sources(srclist, template_params=None, aa_repo_match=None): template_params = {} if aa_repo_match is None: - def aa_repo_match(x): + def _aa_repo_match(x): return False + aa_repo_match = _aa_repo_match errorlist = [] srcdict = convert_to_new_format(srclist) diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index 859d69f1..40560f11 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -36,13 +36,13 @@ DEFAULT_CONFIG = { } -def enum(**enums): - return type('Enum', (), enums) +class RESIZE(object): + SKIPPED = "SKIPPED" + CHANGED = "CHANGED" + NOCHANGE = "NOCHANGE" + FAILED = "FAILED" -RESIZE = enum(SKIPPED="SKIPPED", CHANGED="CHANGED", NOCHANGE="NOCHANGE", - FAILED="FAILED") - LOG = logging.getLogger(__name__) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 5c29c804..14b500f8 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -454,7 +454,7 @@ class Distro(object): keys = kwargs['ssh_authorized_keys'] if isinstance(keys, six.string_types): keys = [keys] - if isinstance(keys, dict): + elif isinstance(keys, dict): keys = list(keys.values()) if keys is not None: if not isinstance(keys, (tuple, list, set)): diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py index 93a2e008..66209f22 100644 --- a/cloudinit/distros/arch.py +++ b/cloudinit/distros/arch.py @@ -196,6 +196,6 @@ def convert_resolv_conf(settings): """Returns a settings string formatted for resolv.conf.""" result = '' if isinstance(settings, list): - for ns in list: + for ns in settings: result = result + 'nameserver %s\n' % ns - return result + return result diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index f2bb9366..43347cfb 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -37,16 +37,16 @@ from cloudinit.sources.helpers.vmware.imc.config_file \ import ConfigFile from cloudinit.sources.helpers.vmware.imc.config_nic \ import NicConfigurator +from cloudinit.sources.helpers.vmware.imc.guestcust_error \ + import GuestCustErrorEnum from cloudinit.sources.helpers.vmware.imc.guestcust_event \ import GuestCustEventEnum from cloudinit.sources.helpers.vmware.imc.guestcust_state \ import GuestCustStateEnum -from cloudinit.sourceshelpers.vmware.imc.guestcust_error \ - import GuestCustErrorEnum -from cloudinit.sourceshelpers.vmware.imc.guestcust_util import ( - set_customization_status, +from cloudinit.sources.helpers.vmware.imc.guestcust_util import ( + enable_nics, get_nics_to_enable, - enable_nics + set_customization_status ) LOG = logging.getLogger(__name__) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 8f85b115..7b3a76b9 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -104,7 +104,7 @@ class DataSourceOpenNebula(sources.DataSource): def get_hostname(self, fqdn=False, resolve_ip=None): if resolve_ip is None: - if self.dsmode == sources.DSMODE_NET: + if self.dsmode == sources.DSMODE_NETWORK: resolve_ip = True else: resolve_ip = False diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 494335b3..d52cb56a 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -157,7 +157,7 @@ class BaseReader(object): pass @abc.abstractmethod - def _path_read(self, path): + def _path_read(self, path, decode=False): pass @abc.abstractmethod -- cgit v1.2.3 From b2fd5b65dc89c3fd5dbc1414968505367fc7126a Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 15 Jun 2016 23:03:11 -0400 Subject: fix usage of OSError.message that will not work in python3 python3's OSError does not have a .message attribute. --- cloudinit/sources/DataSourceAltCloud.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py index cd61df31..a3529609 100644 --- a/cloudinit/sources/DataSourceAltCloud.py +++ b/cloudinit/sources/DataSourceAltCloud.py @@ -205,8 +205,7 @@ class DataSourceAltCloud(sources.DataSource): _err.message) return False except OSError as _err: - util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), - _err.message) + util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err) return False floppy_dev = '/dev/fd0' -- cgit v1.2.3 From 8e5f63cb2f11ce08f7de1df23b8937305b91b707 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 15 Jun 2016 23:16:35 -0400 Subject: remove declaration of dsmode as local in DataSourceNoCloud this would cause the datasource to operate in local mode by default. --- cloudinit/sources/DataSourceNoCloud.py | 1 - 1 file changed, 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 19d63950..cdc9eef5 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -34,7 +34,6 @@ LOG = logging.getLogger(__name__) class DataSourceNoCloud(sources.DataSource): def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) - self.dsmode = 'local' self.seed = None self.seed_dirs = [os.path.join(paths.seed_dir, 'nocloud'), os.path.join(paths.seed_dir, 'nocloud-net')] -- cgit v1.2.3 From 922562bccbc6b6c7f3309ecd36f1835b2ad817da Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 15 Jun 2016 23:32:22 -0400 Subject: python3 OSError does not have a .message --- cloudinit/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/util.py b/cloudinit/util.py index 8873264d..e5dd61a0 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2210,7 +2210,7 @@ def _call_dmidecode(key, dmidecode_path): return "" return result except (IOError, OSError) as _err: - LOG.debug('failed dmidecode cmd: %s\n%s', cmd, _err.message) + LOG.debug('failed dmidecode cmd: %s\n%s', cmd, _err) return None -- cgit v1.2.3 From 51a4781e868627592cc95f4e5ee9e485afa6c1cf Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 20 Jun 2016 14:54:12 -0400 Subject: leave early if no interfaces to rename --- cloudinit/net/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'cloudinit') diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 6959ad34..63e54f91 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -237,6 +237,11 @@ def _get_current_rename_info(check_downable=True): def _rename_interfaces(renames, strict_present=True, strict_busy=True, current_info=None): + + if not len(renames): + LOG.debug("no interfaces to rename") + return + if current_info is None: current_info = _get_current_rename_info() -- cgit v1.2.3 From 62b0593341e1a9cc1b37a00dd92c3bbf41dc3db7 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 20 Jun 2016 14:54:37 -0400 Subject: do not render systemd.link files on debian. These are unnecessary, as cloud-init is renaming the nics that should be renamed itself. --- cloudinit/distros/debian.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 53f3aa4d..244af349 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -59,7 +59,7 @@ class Distro(distros.Distro): self.osfamily = 'debian' self._net_renderer = eni.Renderer({ 'eni_path': self.network_conf_fn, - 'links_prefix_path': self.links_prefix, + 'links_prefix_path': None, 'netrules_path': None, }) -- cgit v1.2.3 From 4fc10ef3f349e6280e2002d980c4c63e6a741d2f Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 20 Jun 2016 17:07:10 -0400 Subject: add a header to rendered ENI file describing its content --- cloudinit/distros/debian.py | 8 ++++++++ cloudinit/net/eni.py | 4 +++- 2 files changed, 11 insertions(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 244af349..5ae9a509 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -43,6 +43,13 @@ APT_GET_WRAPPER = { 'enabled': 'auto', } +ENI_HEADER = """# This file is generated from information provided by +# the datasource. Changes to it will not persist across an instance. +# To disable cloud-init's network configuration capabilities, write a file +# /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg with the following: +# network: {config: disabled} +""" + class Distro(distros.Distro): hostname_conf_fn = "/etc/hostname" @@ -59,6 +66,7 @@ class Distro(distros.Distro): self.osfamily = 'debian' self._net_renderer = eni.Renderer({ 'eni_path': self.network_conf_fn, + 'eni_header': ENI_HEADER, 'links_prefix_path': None, 'netrules_path': None, }) diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index ccd16ba7..e5ed10fd 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -303,6 +303,7 @@ class Renderer(renderer.Renderer): if not config: config = {} self.eni_path = config.get('eni_path', 'etc/network/interfaces') + self.eni_header = config.get('eni_header', None) self.links_path_prefix = config.get( 'links_path_prefix', 'etc/systemd/network/50-cloud-init-') self.netrules_path = config.get( @@ -417,7 +418,8 @@ class Renderer(renderer.Renderer): def render_network_state(self, target, network_state): fpeni = os.path.join(target, self.eni_path) util.ensure_dir(os.path.dirname(fpeni)) - util.write_file(fpeni, self._render_interfaces(network_state)) + header = self.eni_header if self.eni_header else "" + util.write_file(fpeni, header + self._render_interfaces(network_state)) if self.netrules_path: netrules = os.path.join(target, self.netrules_path) -- cgit v1.2.3 From a919f6e6ee3bfaf489411a4452fc708061b1239f Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 21 Jun 2016 14:37:23 -0400 Subject: write_files: if no permissions are given, just use default without warn. if no permissions were given in a write_files stanza, then a warning would be emitted. The fix here is just to special case handling of None. --- ChangeLog | 1 + cloudinit/config/cc_write_files.py | 2 ++ 2 files changed, 3 insertions(+) (limited to 'cloudinit') diff --git a/ChangeLog b/ChangeLog index 10bd58b8..8edf3249 100644 --- a/ChangeLog +++ b/ChangeLog @@ -124,6 +124,7 @@ - Change missing Cheetah log warning to debug [Andrew Jorgensen] - Remove trailing dot from GCE metadata URL (LP: #1581200) [Phil Roche] - support network rendering to sysconfig (for centos and RHEL) + - write_files: if no permissions are given, just use default without warn. 0.7.6: - open 0.7.6 diff --git a/cloudinit/config/cc_write_files.py b/cloudinit/config/cc_write_files.py index 351cfc8c..b1096b9b 100644 --- a/cloudinit/config/cc_write_files.py +++ b/cloudinit/config/cc_write_files.py @@ -79,6 +79,8 @@ def write_files(name, files, log): def decode_perms(perm, default, log): + if perm is None: + return default try: if isinstance(perm, six.integer_types + (float,)): # Just 'downcast' it (if a float) -- cgit v1.2.3 From 145256f964388423965bc77c2a780a89ed62dc2e Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 21 Jun 2016 16:27:02 -0400 Subject: user_data: fix error when user-data is not utf-8 decodable when user-data was not decodable, cloud-init would raise exception. LP: #1532072 --- ChangeLog | 1 + cloudinit/user_data.py | 28 ++++++++++++++++------------ tests/unittests/test_data.py | 12 ++++++++++++ 3 files changed, 29 insertions(+), 12 deletions(-) (limited to 'cloudinit') diff --git a/ChangeLog b/ChangeLog index 8edf3249..fa5e7df4 100644 --- a/ChangeLog +++ b/ChangeLog @@ -125,6 +125,7 @@ - Remove trailing dot from GCE metadata URL (LP: #1581200) [Phil Roche] - support network rendering to sysconfig (for centos and RHEL) - write_files: if no permissions are given, just use default without warn. + - user_data: fix error when user-data is not utf-8 decodable (LP: #1532072) 0.7.6: - open 0.7.6 diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index f0631906..393bf0bb 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -334,19 +334,23 @@ def is_skippable(part): # Coverts a raw string into a mime message -def convert_string(raw_data, headers=None): +def convert_string(raw_data, content_type=NOT_MULTIPART_TYPE): if not raw_data: raw_data = '' - if not headers: - headers = {} - data = util.decode_binary(util.decomp_gzip(raw_data)) - if "mime-version:" in data[0:4096].lower(): - msg = util.message_from_string(data) - for (key, val) in headers.items(): - _replace_header(msg, key, val) - else: - mtype = headers.get(CONTENT_TYPE, NOT_MULTIPART_TYPE) - maintype, subtype = mtype.split("/", 1) - msg = MIMEBase(maintype, subtype, *headers) + + def create_binmsg(data, content_type): + maintype, subtype = content_type.split("/", 1) + msg = MIMEBase(maintype, subtype) msg.set_payload(data) + return msg + + try: + data = util.decode_binary(util.decomp_gzip(raw_data)) + if "mime-version:" in data[0:4096].lower(): + msg = util.message_from_string(data) + else: + msg = create_binmsg(data, content_type) + except UnicodeDecodeError: + msg = create_binmsg(raw_data, content_type) + return msg diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py index 1923e2af..17951117 100644 --- a/tests/unittests/test_data.py +++ b/tests/unittests/test_data.py @@ -557,3 +557,15 @@ class TestUDProcess(helpers.ResourceUsingTestCase): ud_proc = ud.UserDataProcessor(self.getCloudPaths()) message = ud_proc.process(msg) self.assertTrue(count_messages(message) == 1) + + +class TestConvert(helpers.TestCase): + def test_handles_binary(self): + blob = b'\x32\x99' + msg = ud.convert_string(blob) + self.assertEqual(blob, msg.get_payload(decode=True)) + + def test_handle_headers(self): + text = "hi mom" + msg = ud.convert_string(text) + self.assertEqual(text, msg.get_payload(decode=False)) -- cgit v1.2.3