From 41900b72f31a1bd0eebe2f58a8598bfab25f0003 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 9 Oct 2015 14:01:11 +0100 Subject: Handle escaped quotes in WALinuxAgentShim.find_endpoint. This fixes bug 1488891. --- cloudinit/sources/helpers/azure.py | 2 +- tests/unittests/test_datasource/test_azure_helper.py | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 281d733e..33003da0 100644 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -233,7 +233,7 @@ class WALinuxAgentShim(object): hex_string += hex_pair value = struct.pack('>L', int(hex_string.replace(':', ''), 16)) else: - value = value.encode('utf-8') + value = value.replace('\\', '').encode('utf-8') endpoint_ip_address = socket.inet_ntoa(value) LOG.debug('Azure endpoint found at %s', endpoint_ip_address) return endpoint_ip_address diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index a5228870..68af31cd 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -97,7 +97,8 @@ class TestFindEndpoint(TestCase): if not use_hex: ip_address_repr = struct.pack( '>L', int(ip_address_repr.replace(':', ''), 16)) - ip_address_repr = '"{0}"'.format(ip_address_repr.decode('utf-8')) + ip_address_repr = '"{0}"'.format( + ip_address_repr.decode('utf-8').replace('"', '\\"')) return '\n'.join([ 'lease {', ' interface "eth0";', @@ -125,6 +126,13 @@ class TestFindEndpoint(TestCase): self.assertEqual(ip_address, azure_helper.WALinuxAgentShim.find_endpoint()) + def test_packed_string_with_escaped_quote(self): + ip_address = '100.72.34.108' + file_content = self._build_lease_content(ip_address, use_hex=False) + self.load_file.return_value = file_content + self.assertEqual(ip_address, + azure_helper.WALinuxAgentShim.find_endpoint()) + def test_latest_lease_used(self): ip_addresses = ['4.3.2.1', '98.76.54.32'] file_content = '\n'.join([self._build_lease_content(ip_address) -- cgit v1.2.3 From 20dc4190e27c7778cfa6c2943961f2ad27e14b48 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 9 Oct 2015 14:01:11 +0100 Subject: Handle colons in packed strings in WALinuxAgentShim.find_endpoint. This fixes bug 1488896. --- cloudinit/sources/helpers/azure.py | 12 +++++++----- tests/unittests/test_datasource/test_azure_helper.py | 7 +++++++ 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 33003da0..21b4cd21 100644 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -225,16 +225,18 @@ class WALinuxAgentShim(object): value = line.strip(' ').split(' ', 2)[-1].strip(';\n"') if value is None: raise Exception('No endpoint found in DHCP config.') - if ':' in value: + unescaped_value = value.replace('\\', '') + if len(unescaped_value) > 4: hex_string = '' - for hex_pair in value.split(':'): + for hex_pair in unescaped_value.split(':'): if len(hex_pair) == 1: hex_pair = '0' + hex_pair hex_string += hex_pair - value = struct.pack('>L', int(hex_string.replace(':', ''), 16)) + packed_bytes = struct.pack( + '>L', int(hex_string.replace(':', ''), 16)) else: - value = value.replace('\\', '').encode('utf-8') - endpoint_ip_address = socket.inet_ntoa(value) + packed_bytes = unescaped_value.encode('utf-8') + endpoint_ip_address = socket.inet_ntoa(packed_bytes) LOG.debug('Azure endpoint found at %s', endpoint_ip_address) return endpoint_ip_address diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 68af31cd..5f906837 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -133,6 +133,13 @@ class TestFindEndpoint(TestCase): self.assertEqual(ip_address, azure_helper.WALinuxAgentShim.find_endpoint()) + def test_packed_string_containing_a_colon(self): + ip_address = '100.72.58.108' + file_content = self._build_lease_content(ip_address, use_hex=False) + self.load_file.return_value = file_content + self.assertEqual(ip_address, + azure_helper.WALinuxAgentShim.find_endpoint()) + def test_latest_lease_used(self): ip_addresses = ['4.3.2.1', '98.76.54.32'] file_content = '\n'.join([self._build_lease_content(ip_address) -- cgit v1.2.3 From a9f5d20bc55abc5ff12a5ce1cec1125173f11c7b Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 9 Oct 2015 14:01:11 +0100 Subject: Convert test helper to staticmethod. --- tests/unittests/test_datasource/test_azure_helper.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 5f906837..1746f6b8 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -90,7 +90,8 @@ class TestFindEndpoint(TestCase): self.assertRaises(Exception, azure_helper.WALinuxAgentShim.find_endpoint) - def _build_lease_content(self, ip_address, use_hex=True): + @staticmethod + def _build_lease_content(ip_address, use_hex=True): ip_address_repr = ':'.join( [hex(int(part)).replace('0x', '') for part in ip_address.split('.')]) -- cgit v1.2.3 From d78ea2f8191847242b639f23fe085a5dd8b36014 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 9 Oct 2015 14:01:11 +0100 Subject: Refactor WALinuxAgentShim.find_endpoint to use a helper method for IP address unpacking. --- cloudinit/sources/helpers/azure.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 21b4cd21..fd08be16 100644 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -216,16 +216,8 @@ class WALinuxAgentShim(object): self.openssl_manager.clean_up() @staticmethod - def find_endpoint(): - LOG.debug('Finding Azure endpoint...') - content = util.load_file('/var/lib/dhcp/dhclient.eth0.leases') - value = None - for line in content.splitlines(): - if 'unknown-245' in line: - value = line.strip(' ').split(' ', 2)[-1].strip(';\n"') - if value is None: - raise Exception('No endpoint found in DHCP config.') - unescaped_value = value.replace('\\', '') + def get_ip_from_lease_value(lease_value): + unescaped_value = lease_value.replace('\\', '') if len(unescaped_value) > 4: hex_string = '' for hex_pair in unescaped_value.split(':'): @@ -236,7 +228,19 @@ class WALinuxAgentShim(object): '>L', int(hex_string.replace(':', ''), 16)) else: packed_bytes = unescaped_value.encode('utf-8') - endpoint_ip_address = socket.inet_ntoa(packed_bytes) + return socket.inet_ntoa(packed_bytes) + + @staticmethod + def find_endpoint(): + LOG.debug('Finding Azure endpoint...') + content = util.load_file('/var/lib/dhcp/dhclient.eth0.leases') + value = None + for line in content.splitlines(): + if 'unknown-245' in line: + value = line.strip(' ').split(' ', 2)[-1].strip(';\n"') + if value is None: + raise Exception('No endpoint found in DHCP config.') + endpoint_ip_address = WALinuxAgentShim.get_ip_from_lease_value(value) LOG.debug('Azure endpoint found at %s', endpoint_ip_address) return endpoint_ip_address -- cgit v1.2.3 From 2f5a5ed9f9c5ba390a693bc441ab18cbcdff9623 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 9 Oct 2015 14:01:11 +0100 Subject: Refactor tests to test helper method directly, and remove need for test helper. --- .../unittests/test_datasource/test_azure_helper.py | 84 +++++++++++----------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 1746f6b8..fed2a7d8 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -91,63 +91,63 @@ class TestFindEndpoint(TestCase): azure_helper.WALinuxAgentShim.find_endpoint) @staticmethod - def _build_lease_content(ip_address, use_hex=True): - ip_address_repr = ':'.join( - [hex(int(part)).replace('0x', '') - for part in ip_address.split('.')]) - if not use_hex: - ip_address_repr = struct.pack( - '>L', int(ip_address_repr.replace(':', ''), 16)) - ip_address_repr = '"{0}"'.format( - ip_address_repr.decode('utf-8').replace('"', '\\"')) + def _build_lease_content(encoded_address): return '\n'.join([ 'lease {', ' interface "eth0";', - ' option unknown-245 {0};'.format(ip_address_repr), + ' option unknown-245 {0};'.format(encoded_address), '}']) - def test_hex_string(self): - ip_address = '98.76.54.32' - file_content = self._build_lease_content(ip_address) + def test_latest_lease_used(self): + encoded_addresses = ['5:4:3:2', '4:3:2:1'] + file_content = '\n'.join([self._build_lease_content(encoded_address) + for encoded_address in encoded_addresses]) self.load_file.return_value = file_content - self.assertEqual(ip_address, + self.assertEqual(encoded_addresses[-1].replace(':', '.'), azure_helper.WALinuxAgentShim.find_endpoint()) + +class TestExtractIpAddressFromLeaseValue(TestCase): + + def test_hex_string(self): + ip_address, encoded_address = '98.76.54.32', '62:4c:36:20' + self.assertEqual( + ip_address, + azure_helper.WALinuxAgentShim.get_ip_from_lease_value( + encoded_address + )) + def test_hex_string_with_single_character_part(self): - ip_address = '4.3.2.1' - file_content = self._build_lease_content(ip_address) - self.load_file.return_value = file_content - self.assertEqual(ip_address, - azure_helper.WALinuxAgentShim.find_endpoint()) + ip_address, encoded_address = '4.3.2.1', '4:3:2:1' + self.assertEqual( + ip_address, + azure_helper.WALinuxAgentShim.get_ip_from_lease_value( + encoded_address + )) def test_packed_string(self): - ip_address = '98.76.54.32' - file_content = self._build_lease_content(ip_address, use_hex=False) - self.load_file.return_value = file_content - self.assertEqual(ip_address, - azure_helper.WALinuxAgentShim.find_endpoint()) + ip_address, encoded_address = '98.76.54.32', 'bL6 ' + self.assertEqual( + ip_address, + azure_helper.WALinuxAgentShim.get_ip_from_lease_value( + encoded_address + )) def test_packed_string_with_escaped_quote(self): - ip_address = '100.72.34.108' - file_content = self._build_lease_content(ip_address, use_hex=False) - self.load_file.return_value = file_content - self.assertEqual(ip_address, - azure_helper.WALinuxAgentShim.find_endpoint()) + ip_address, encoded_address = '100.72.34.108', 'dH\\"l' + self.assertEqual( + ip_address, + azure_helper.WALinuxAgentShim.get_ip_from_lease_value( + encoded_address + )) def test_packed_string_containing_a_colon(self): - ip_address = '100.72.58.108' - file_content = self._build_lease_content(ip_address, use_hex=False) - self.load_file.return_value = file_content - self.assertEqual(ip_address, - azure_helper.WALinuxAgentShim.find_endpoint()) - - def test_latest_lease_used(self): - ip_addresses = ['4.3.2.1', '98.76.54.32'] - file_content = '\n'.join([self._build_lease_content(ip_address) - for ip_address in ip_addresses]) - self.load_file.return_value = file_content - self.assertEqual(ip_addresses[-1], - azure_helper.WALinuxAgentShim.find_endpoint()) + ip_address, encoded_address = '100.72.58.108', 'dH:l' + self.assertEqual( + ip_address, + azure_helper.WALinuxAgentShim.get_ip_from_lease_value( + encoded_address + )) class TestGoalStateParsing(TestCase): -- cgit v1.2.3 From 8844ffb5988bcfbb8cfbe57d9139c3dcb8b429cc Mon Sep 17 00:00:00 2001 From: Sankar Tanguturi Date: Wed, 18 Nov 2015 16:03:15 -0800 Subject: Add Image Customization Parser for VMware vSphere Hypervisor Support. This is the first changeset submitted as a part of project to add cloud-init support for VMware vSphere Hypervisor. This changeset contains _only_ the changes for a simple python parser for a Image Customization Specification file pushed by VMware vSphere hypervisor into the guest VMs. In a later changeset, will be submitting another patch to actually detect the underlying VMware vSphere hypervisor and do the necessary customization. --- cloudinit/sources/helpers/vmware/__init__.py | 13 ++ cloudinit/sources/helpers/vmware/imc/__init__.py | 13 ++ cloudinit/sources/helpers/vmware/imc/boot_proto.py | 11 + cloudinit/sources/helpers/vmware/imc/config.py | 125 ++++++++++++ .../sources/helpers/vmware/imc/config_file.py | 221 +++++++++++++++++++++ .../sources/helpers/vmware/imc/config_namespace.py | 5 + .../sources/helpers/vmware/imc/config_source.py | 2 + cloudinit/sources/helpers/vmware/imc/ipv4_mode.py | 29 +++ cloudinit/sources/helpers/vmware/imc/nic.py | 107 ++++++++++ 9 files changed, 526 insertions(+) create mode 100644 cloudinit/sources/helpers/vmware/__init__.py create mode 100644 cloudinit/sources/helpers/vmware/imc/__init__.py create mode 100644 cloudinit/sources/helpers/vmware/imc/boot_proto.py create mode 100644 cloudinit/sources/helpers/vmware/imc/config.py create mode 100644 cloudinit/sources/helpers/vmware/imc/config_file.py create mode 100644 cloudinit/sources/helpers/vmware/imc/config_namespace.py create mode 100644 cloudinit/sources/helpers/vmware/imc/config_source.py create mode 100644 cloudinit/sources/helpers/vmware/imc/ipv4_mode.py create mode 100644 cloudinit/sources/helpers/vmware/imc/nic.py diff --git a/cloudinit/sources/helpers/vmware/__init__.py b/cloudinit/sources/helpers/vmware/__init__.py new file mode 100644 index 00000000..386225d5 --- /dev/null +++ b/cloudinit/sources/helpers/vmware/__init__.py @@ -0,0 +1,13 @@ +# vi: ts=4 expandtab +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . diff --git a/cloudinit/sources/helpers/vmware/imc/__init__.py b/cloudinit/sources/helpers/vmware/imc/__init__.py new file mode 100644 index 00000000..386225d5 --- /dev/null +++ b/cloudinit/sources/helpers/vmware/imc/__init__.py @@ -0,0 +1,13 @@ +# vi: ts=4 expandtab +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . diff --git a/cloudinit/sources/helpers/vmware/imc/boot_proto.py b/cloudinit/sources/helpers/vmware/imc/boot_proto.py new file mode 100644 index 00000000..6c3b070a --- /dev/null +++ b/cloudinit/sources/helpers/vmware/imc/boot_proto.py @@ -0,0 +1,11 @@ +# from enum import Enum + +class BootProto: + DHCP = 'dhcp' + STATIC = 'static' + +# def __eq__(self, other): +# return self.name == other.name and self.value == other.value +# +# def __ne__(self, other): +# return not self.__eq__(other) diff --git a/cloudinit/sources/helpers/vmware/imc/config.py b/cloudinit/sources/helpers/vmware/imc/config.py new file mode 100644 index 00000000..ea0873fb --- /dev/null +++ b/cloudinit/sources/helpers/vmware/imc/config.py @@ -0,0 +1,125 @@ +from cloudinit.sources.helpers.vmware.imc.nic import Nic + + +class Config: + DNS = 'DNS|NAMESERVER|' + SUFFIX = 'DNS|SUFFIX|' + PASS = 'PASSWORD|-PASS' + TIMEZONE = 'DATETIME|TIMEZONE' + UTC = 'DATETIME|UTC' + HOSTNAME = 'NETWORK|HOSTNAME' + OMAINNAME = 'NETWORK|DOMAINNAME' + + def __init__(self, configFile): + self._configFile = configFile + + # Retrieves hostname. + # + # Args: + # None + # Results: + # string: hostname + # Throws: + # None + @property + def hostName(self): + return self._configFile.get(Config.HOSTNAME, None) + + # Retrieves domainName. + # + # Args: + # None + # Results: + # string: domainName + # Throws: + # None + @property + def domainName(self): + return self._configFile.get(Config.DOMAINNAME, None) + + # Retrieves timezone. + # + # Args: + # None + # Results: + # string: timezone + # Throws: + # None + @property + def timeZone(self): + return self._configFile.get(Config.TIMEZONE, None) + + # Retrieves whether to set time to UTC or Local. + # + # Args: + # None + # Results: + # boolean: True for yes/YES, True for no/NO, otherwise - None + # Throws: + # None + @property + def utc(self): + return self._configFile.get(Config.UTC, None) + + # Retrieves root password to be set. + # + # Args: + # None + # Results: + # string: base64-encoded root password or None + # Throws: + # None + @property + def adminPassword(self): + return self._configFile.get(Config.PASS, None) + + # Retrieves DNS Servers. + # + # Args: + # None + # Results: + # integer: count or 0 + # Throws: + # None + @property + def nameServers(self): + res = [] + for i in range(1, self._configFile.getCnt(Config.DNS) + 1): + key = Config.DNS + str(i) + res.append(self._configFile[key]) + + return res + + # Retrieves DNS Suffixes. + # + # Args: + # None + # Results: + # integer: count or 0 + # Throws: + # None + @property + def dnsSuffixes(self): + res = [] + for i in range(1, self._configFile.getCnt(Config.SUFFIX) + 1): + key = Config.SUFFIX + str(i) + res.append(self._configFile[key]) + + return res + + # Retrieves NICs. + # + # Args: + # None + # Results: + # integer: count + # Throws: + # None + @property + def nics(self): + res = [] + nics = self._configFile['NIC-CONFIG|NICS'] + for nic in nics.split(','): + res.append(Nic(nic, self._configFile)) + + return res diff --git a/cloudinit/sources/helpers/vmware/imc/config_file.py b/cloudinit/sources/helpers/vmware/imc/config_file.py new file mode 100644 index 00000000..3f9938da --- /dev/null +++ b/cloudinit/sources/helpers/vmware/imc/config_file.py @@ -0,0 +1,221 @@ +import logging +import re + +from cloudinit.sources.helpers.vmware.imc.config_source import ConfigSource + +logger = logging.getLogger(__name__) + + +class ConfigFile(ConfigSource): + def __init__(self): + self._configData = {} + + def __getitem__(self, key): + return self._configData[key] + + def get(self, key, default=None): + return self._configData.get(key, default) + + # Removes all the properties. + # + # Args: + # None + # Results: + # None + # Throws: + # None + def clear(self): + self._configData.clear() + + # Inserts k/v pair. + # + # Does not do any key/cross-key validation. + # + # Args: + # key: string: key + # val: string: value + # Results: + # None + # Throws: + # None + def _insertKey(self, key, val): + # cleaning up on all "input" path + + # remove end char \n (chomp) + key = key.strip() + val = val.strip() + + if key.startswith('-') or '|-' in key: + canLog = 0 + else: + canLog = 1 + + # "sensitive" settings shall not be logged + if canLog: + logger.debug("ADDED KEY-VAL :: '%s' = '%s'" % (key, val)) + else: + logger.debug("ADDED KEY-VAL :: '%s' = '*****************'" % key) + + self._configData[key] = val + + # Determines properties count. + # + # Args: + # None + # Results: + # integer: properties count + # Throws: + # None + def size(self): + return len(self._configData) + + # Parses properties from a .cfg file content. + # + # Any previously available properties will be removed. + # + # Sensitive data will not be logged in case key starts from '-'. + # + # Args: + # content: string: e.g. content of config/cust.cfg + # Results: + # None + # Throws: + # None + def loadConfigContent(self, content): + self.clear() + + # remove end char \n (chomp) + for line in content.split('\n'): + # TODO validate against allowed characters (not done in Perl) + + # spaces at the end are not allowed, things like passwords must be + # at least base64-encoded + line = line.strip() + + # "sensitive" settings shall not be logged + if line.startswith('-'): + canLog = 0 + else: + canLog = 1 + + if canLog: + logger.debug("Processing line: '%s'" % line) + else: + logger.debug("Processing line: '***********************'") + + if not line: + logger.debug("Empty line. Ignored.") + continue + + if line.startswith('#'): + logger.debug("Comment found. Line ignored.") + continue + + matchObj = re.match(r'\[(.+)\]', line) + if matchObj: + category = matchObj.group(1) + logger.debug("FOUND CATEGORY = '%s'" % category) + else: + # POSIX.2 regex doesn't support non-greedy like in (.+?)=(.*) + # key value pair (non-eager '=' for base64) + matchObj = re.match(r'([^=]+)=(.*)', line) + if matchObj: + # cleaning up on all "input" paths + key = category + "|" + matchObj.group(1).strip() + val = matchObj.group(2).strip() + + self._insertKey(key, val) + else: + # TODO document + raise Exception("Unrecognizable line: '%s'" % line) + + self.validate() + + # Parses properties from a .cfg file + # + # Any previously available properties will be removed. + # + # Sensitive data will not be logged in case key starts from '-'. + # + # Args: + # filename: string: full path to a .cfg file + # Results: + # None + # Throws: + # None + def loadConfigFile(self, filename): + logger.info("Opening file name %s." % filename) + # TODO what throws? + with open(filename, "r") as myfile: + self.loadConfigContent(myfile.read()) + + # Determines whether a property with a given key exists. + # + # Args: + # key: string: key + # Results: + # boolean: True if such property exists, otherwise - False. + # Throws: + # None + def hasKey(self, key): + return key in self._configData + + # Determines whether a value for a property must be kept. + # + # If the property is missing, it's treated as it should be not changed by + # the engine. + # + # Args: + # key: string: key + # Results: + # boolean: True if property must be kept, otherwise - False. + # Throws: + # None + def keepCurrentValue(self, key): + # helps to distinguish from "empty" value which is used to indicate + # "removal" + return not self.hasKey(key) + + # Determines whether a value for a property must be removed. + # + # If the property is empty, it's treated as it should be removed by the + # engine. + # + # Args: + # key: string: key + # Results: + # boolean: True if property must be removed, otherwise - False. + # Throws: + # None + def removeCurrentValue(self, key): + # helps to distinguish from "missing" value which is used to indicate + # "keeping unchanged" + if self.hasKey(key): + return not bool(self._configData[key]) + else: + return False + + # TODO + def getCnt(self, prefix): + res = 0 + for key in self._configData.keys(): + if key.startswith(prefix): + res += 1 + + return res + + # TODO + # TODO pass base64 + # Throws: + # Dies in case timezone is present but empty. + # Dies in case password is present but empty. + # Dies in case hostname is present but empty or greater than 63 chars. + # Dies in case UTC is present, but is not yes/YES or no/NO. + # Dies in case NICS is not present. + def validate(self): + # TODO must log all the errors + keyValidators = {'NIC1|IPv6GATEWAY|': None} + crossValidators = {} + + for key in self._configData.keys(): + pass diff --git a/cloudinit/sources/helpers/vmware/imc/config_namespace.py b/cloudinit/sources/helpers/vmware/imc/config_namespace.py new file mode 100644 index 00000000..7f76ac8b --- /dev/null +++ b/cloudinit/sources/helpers/vmware/imc/config_namespace.py @@ -0,0 +1,5 @@ +from cloudinit.sources.helpers.vmware.imc.config_source import ConfigSource + + +class ConfigNamespace(ConfigSource): + pass diff --git a/cloudinit/sources/helpers/vmware/imc/config_source.py b/cloudinit/sources/helpers/vmware/imc/config_source.py new file mode 100644 index 00000000..fad3a389 --- /dev/null +++ b/cloudinit/sources/helpers/vmware/imc/config_source.py @@ -0,0 +1,2 @@ +class ConfigSource: + pass diff --git a/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py b/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py new file mode 100644 index 00000000..66b4fad7 --- /dev/null +++ b/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py @@ -0,0 +1,29 @@ +# from enum import Enum + + +# The IPv4 configuration mode which directly represents the user's goal. +# +# This mode effectively acts as a contract of the inguest customization engine. +# It must be set based on what the user has requested via VMODL/generators API +# and should not be changed by those layers. It's up to the in-guest engine to +# interpret and materialize the user's request. +# +# Also defined in linuxconfiggenerator.h. +class Ipv4Mode: + # The legacy mode which only allows dhcp/static based on whether IPv4 + # addresses list is empty or not + IPV4_MODE_BACKWARDS_COMPATIBLE = 'BACKWARDS_COMPATIBLE' + # IPv4 must use static address. Reserved for future use + IPV4_MODE_STATIC = 'STATIC' + # IPv4 must use DHCPv4. Reserved for future use + IPV4_MODE_DHCP = 'DHCP' + # IPv4 must be disabled + IPV4_MODE_DISABLED = 'DISABLED' + # IPv4 settings should be left untouched. Reserved for future use + IPV4_MODE_AS_IS = 'AS_IS' + + # def __eq__(self, other): + # return self.name == other.name and self.value == other.value + # + # def __ne__(self, other): + # return not self.__eq__(other) diff --git a/cloudinit/sources/helpers/vmware/imc/nic.py b/cloudinit/sources/helpers/vmware/imc/nic.py new file mode 100644 index 00000000..b90a5640 --- /dev/null +++ b/cloudinit/sources/helpers/vmware/imc/nic.py @@ -0,0 +1,107 @@ +from cloudinit.sources.helpers.vmware.imc.boot_proto import BootProto + + +class Nic: + def __init__(self, name, configFile): + self._name = name + self._configFile = configFile + + def _get(self, what): + return self._configFile.get(self.name + what, None) + + def _getCnt(self, prefix): + return self._configFile.getCnt(self.name + prefix) + + @property + def name(self): + return self._name + + @property + def mac(self): + return self._get('|MACADDR').lower() + + @property + def bootProto(self): + return self._get('|BOOTPROTO').lower() + + @property + def ipv4(self): + # TODO implement NONE + if self.bootProto == BootProto.STATIC: + return StaticIpv4Conf(self) + + return DhcpIpv4Conf(self) + + @property + def ipv6(self): + # TODO implement NONE + cnt = self._getCnt("|IPv6ADDR|") + + if cnt != 0: + return StaticIpv6Conf(self) + + return DhcpIpv6Conf(self) + + +class DhcpIpv4Conf: + def __init__(self, nic): + self._nic = nic + + +class StaticIpv4Addr: + def __init__(self, nic): + self._nic = nic + + @property + def ip(self): + return self._nic._get('|IPADDR') + + @property + def netmask(self): + return self._nic._get('|NETMASK') + + @property + def gateway(self): + return self._nic._get('|GATEWAY') + + +class StaticIpv4Conf(DhcpIpv4Conf): + @property + def addrs(self): + return [StaticIpv4Addr(self._nic)] + + +class DhcpIpv6Conf: + def __init__(self, nic): + self._nic = nic + + +class StaticIpv6Addr: + def __init__(self, nic, index): + self._nic = nic + self._index = index + + @property + def ip(self): + return self._nic._get("|IPv6ADDR|" + str(self._index)) + + @property + def prefix(self): + return self._nic._get("|IPv6NETMASK|" + str(self._index)) + + @property + def gateway(self): + return self._nic._get("|IPv6GATEWAY|" + str(self._index)) + + +class StaticIpv6Conf(DhcpIpv6Conf): + @property + def addrs(self): + cnt = self._nic._getCnt("|IPv6ADDR|") + + res = [] + + for i in range(1, cnt + 1): + res.append(StaticIpv6Addr(self._nic, i)) + + return res -- cgit v1.2.3 From 8d9e5bd7fcda8f56a4fe087150db1456af738335 Mon Sep 17 00:00:00 2001 From: Sankar Tanguturi Date: Tue, 5 Jan 2016 12:05:11 -0800 Subject: Fixed all the styling nits. Used proper naming convention for the methods. Added proper documentation. Checked pep8 and flake8 output and no issues were reported. --- cloudinit/sources/helpers/vmware/imc/boot_proto.py | 28 +- cloudinit/sources/helpers/vmware/imc/config.py | 116 +++---- .../sources/helpers/vmware/imc/config_file.py | 372 +++++++++------------ .../sources/helpers/vmware/imc/config_namespace.py | 22 +- .../sources/helpers/vmware/imc/config_source.py | 21 ++ cloudinit/sources/helpers/vmware/imc/ipv4_mode.py | 74 ++-- cloudinit/sources/helpers/vmware/imc/nic.py | 254 ++++++++------ 7 files changed, 448 insertions(+), 439 deletions(-) diff --git a/cloudinit/sources/helpers/vmware/imc/boot_proto.py b/cloudinit/sources/helpers/vmware/imc/boot_proto.py index 6c3b070a..abfffd75 100644 --- a/cloudinit/sources/helpers/vmware/imc/boot_proto.py +++ b/cloudinit/sources/helpers/vmware/imc/boot_proto.py @@ -1,11 +1,25 @@ -# from enum import Enum +# vi: ts=4 expandtab +# +# Copyright (C) 2015 Canonical Ltd. +# Copyright (C) 2015 VMware Inc. +# +# Author: Sankar Tanguturi +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + class BootProto: + """Specifies the NIC Boot Settings.""" + DHCP = 'dhcp' STATIC = 'static' - -# def __eq__(self, other): -# return self.name == other.name and self.value == other.value -# -# def __ne__(self, other): -# return not self.__eq__(other) diff --git a/cloudinit/sources/helpers/vmware/imc/config.py b/cloudinit/sources/helpers/vmware/imc/config.py index ea0873fb..7eee47a5 100644 --- a/cloudinit/sources/helpers/vmware/imc/config.py +++ b/cloudinit/sources/helpers/vmware/imc/config.py @@ -1,122 +1,90 @@ -from cloudinit.sources.helpers.vmware.imc.nic import Nic +# vi: ts=4 expandtab +# +# Copyright (C) 2015 Canonical Ltd. +# Copyright (C) 2015 VMware Inc. +# +# Author: Sankar Tanguturi +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from .nic import Nic class Config: + """ + Stores the Contents specified in the Customization + Specification file. + """ + DNS = 'DNS|NAMESERVER|' SUFFIX = 'DNS|SUFFIX|' PASS = 'PASSWORD|-PASS' TIMEZONE = 'DATETIME|TIMEZONE' UTC = 'DATETIME|UTC' HOSTNAME = 'NETWORK|HOSTNAME' - OMAINNAME = 'NETWORK|DOMAINNAME' + DOMAINNAME = 'NETWORK|DOMAINNAME' def __init__(self, configFile): self._configFile = configFile - # Retrieves hostname. - # - # Args: - # None - # Results: - # string: hostname - # Throws: - # None @property - def hostName(self): + def host_name(self): + """Return the hostname.""" return self._configFile.get(Config.HOSTNAME, None) - # Retrieves domainName. - # - # Args: - # None - # Results: - # string: domainName - # Throws: - # None @property - def domainName(self): + def domain_name(self): + """Return the domain name.""" return self._configFile.get(Config.DOMAINNAME, None) - # Retrieves timezone. - # - # Args: - # None - # Results: - # string: timezone - # Throws: - # None @property - def timeZone(self): + def timezone(self): + """Return the timezone.""" return self._configFile.get(Config.TIMEZONE, None) - # Retrieves whether to set time to UTC or Local. - # - # Args: - # None - # Results: - # boolean: True for yes/YES, True for no/NO, otherwise - None - # Throws: - # None @property def utc(self): + """Retrieves whether to set time to UTC or Local.""" return self._configFile.get(Config.UTC, None) - # Retrieves root password to be set. - # - # Args: - # None - # Results: - # string: base64-encoded root password or None - # Throws: - # None @property - def adminPassword(self): + def admin_password(self): + """Return the root password to be set.""" return self._configFile.get(Config.PASS, None) - # Retrieves DNS Servers. - # - # Args: - # None - # Results: - # integer: count or 0 - # Throws: - # None @property - def nameServers(self): + def name_servers(self): + """Return the list of DNS servers.""" res = [] - for i in range(1, self._configFile.getCnt(Config.DNS) + 1): + for i in range(1, self._configFile.get_count(Config.DNS) + 1): key = Config.DNS + str(i) res.append(self._configFile[key]) return res - # Retrieves DNS Suffixes. - # - # Args: - # None - # Results: - # integer: count or 0 - # Throws: - # None @property - def dnsSuffixes(self): + def dns_suffixes(self): + """Return the list of DNS Suffixes.""" res = [] - for i in range(1, self._configFile.getCnt(Config.SUFFIX) + 1): + for i in range(1, self._configFile.get_count(Config.SUFFIX) + 1): key = Config.SUFFIX + str(i) res.append(self._configFile[key]) return res - # Retrieves NICs. - # - # Args: - # None - # Results: - # integer: count - # Throws: - # None @property def nics(self): + """Return the list of associated NICs.""" res = [] nics = self._configFile['NIC-CONFIG|NICS'] for nic in nics.split(','): diff --git a/cloudinit/sources/helpers/vmware/imc/config_file.py b/cloudinit/sources/helpers/vmware/imc/config_file.py index 3f9938da..e08a2a9a 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_file.py +++ b/cloudinit/sources/helpers/vmware/imc/config_file.py @@ -1,221 +1,151 @@ -import logging -import re - -from cloudinit.sources.helpers.vmware.imc.config_source import ConfigSource - -logger = logging.getLogger(__name__) - - -class ConfigFile(ConfigSource): - def __init__(self): - self._configData = {} - - def __getitem__(self, key): - return self._configData[key] - - def get(self, key, default=None): - return self._configData.get(key, default) - - # Removes all the properties. - # - # Args: - # None - # Results: - # None - # Throws: - # None - def clear(self): - self._configData.clear() - - # Inserts k/v pair. - # - # Does not do any key/cross-key validation. - # - # Args: - # key: string: key - # val: string: value - # Results: - # None - # Throws: - # None - def _insertKey(self, key, val): - # cleaning up on all "input" path - - # remove end char \n (chomp) - key = key.strip() - val = val.strip() - - if key.startswith('-') or '|-' in key: - canLog = 0 - else: - canLog = 1 - - # "sensitive" settings shall not be logged - if canLog: - logger.debug("ADDED KEY-VAL :: '%s' = '%s'" % (key, val)) - else: - logger.debug("ADDED KEY-VAL :: '%s' = '*****************'" % key) - - self._configData[key] = val - - # Determines properties count. - # - # Args: - # None - # Results: - # integer: properties count - # Throws: - # None - def size(self): - return len(self._configData) - - # Parses properties from a .cfg file content. - # - # Any previously available properties will be removed. - # - # Sensitive data will not be logged in case key starts from '-'. - # - # Args: - # content: string: e.g. content of config/cust.cfg - # Results: - # None - # Throws: - # None - def loadConfigContent(self, content): - self.clear() - - # remove end char \n (chomp) - for line in content.split('\n'): - # TODO validate against allowed characters (not done in Perl) - - # spaces at the end are not allowed, things like passwords must be - # at least base64-encoded - line = line.strip() - - # "sensitive" settings shall not be logged - if line.startswith('-'): - canLog = 0 - else: - canLog = 1 - - if canLog: - logger.debug("Processing line: '%s'" % line) - else: - logger.debug("Processing line: '***********************'") - - if not line: - logger.debug("Empty line. Ignored.") - continue - - if line.startswith('#'): - logger.debug("Comment found. Line ignored.") - continue - - matchObj = re.match(r'\[(.+)\]', line) - if matchObj: - category = matchObj.group(1) - logger.debug("FOUND CATEGORY = '%s'" % category) - else: - # POSIX.2 regex doesn't support non-greedy like in (.+?)=(.*) - # key value pair (non-eager '=' for base64) - matchObj = re.match(r'([^=]+)=(.*)', line) - if matchObj: - # cleaning up on all "input" paths - key = category + "|" + matchObj.group(1).strip() - val = matchObj.group(2).strip() - - self._insertKey(key, val) - else: - # TODO document - raise Exception("Unrecognizable line: '%s'" % line) - - self.validate() - - # Parses properties from a .cfg file - # - # Any previously available properties will be removed. - # - # Sensitive data will not be logged in case key starts from '-'. - # - # Args: - # filename: string: full path to a .cfg file - # Results: - # None - # Throws: - # None - def loadConfigFile(self, filename): - logger.info("Opening file name %s." % filename) - # TODO what throws? - with open(filename, "r") as myfile: - self.loadConfigContent(myfile.read()) - - # Determines whether a property with a given key exists. - # - # Args: - # key: string: key - # Results: - # boolean: True if such property exists, otherwise - False. - # Throws: - # None - def hasKey(self, key): - return key in self._configData - - # Determines whether a value for a property must be kept. - # - # If the property is missing, it's treated as it should be not changed by - # the engine. - # - # Args: - # key: string: key - # Results: - # boolean: True if property must be kept, otherwise - False. - # Throws: - # None - def keepCurrentValue(self, key): - # helps to distinguish from "empty" value which is used to indicate - # "removal" - return not self.hasKey(key) - - # Determines whether a value for a property must be removed. - # - # If the property is empty, it's treated as it should be removed by the - # engine. - # - # Args: - # key: string: key - # Results: - # boolean: True if property must be removed, otherwise - False. - # Throws: - # None - def removeCurrentValue(self, key): - # helps to distinguish from "missing" value which is used to indicate - # "keeping unchanged" - if self.hasKey(key): - return not bool(self._configData[key]) - else: - return False - - # TODO - def getCnt(self, prefix): - res = 0 - for key in self._configData.keys(): - if key.startswith(prefix): - res += 1 - - return res - - # TODO - # TODO pass base64 - # Throws: - # Dies in case timezone is present but empty. - # Dies in case password is present but empty. - # Dies in case hostname is present but empty or greater than 63 chars. - # Dies in case UTC is present, but is not yes/YES or no/NO. - # Dies in case NICS is not present. - def validate(self): - # TODO must log all the errors - keyValidators = {'NIC1|IPv6GATEWAY|': None} - crossValidators = {} - - for key in self._configData.keys(): - pass +# vi: ts=4 expandtab +# +# Copyright (C) 2015 Canonical Ltd. +# Copyright (C) 2015 VMware Inc. +# +# Author: Sankar Tanguturi +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import logging + +try: + import configparser +except ImportError: + import ConfigParser as configparser + +from .config_source import ConfigSource + +logger = logging.getLogger(__name__) + + +class ConfigFile(ConfigSource, dict): + """ConfigFile module to load the content from a specified source.""" + + def __init__(self): + pass + + def _insertKey(self, key, val): + """ + Inserts a Key Value pair. + + Keyword arguments: + key -- The key to insert + val -- The value to insert for the key + + """ + key = key.strip() + val = val.strip() + + if key.startswith('-') or '|-' in key: + canLog = 0 + else: + canLog = 1 + + # "sensitive" settings shall not be logged + if canLog: + logger.debug("ADDED KEY-VAL :: '%s' = '%s'" % (key, val)) + else: + logger.debug("ADDED KEY-VAL :: '%s' = '*****************'" % key) + + self[key] = val + + def size(self): + """Return the number of properties present.""" + return len(self) + + def loadConfigFile(self, filename): + """ + Parses properties from the specified config file. + + Any previously available properties will be removed. + Sensitive data will not be logged in case the key starts + from '-'. + + Keyword arguments: + filename - The full path to the config file. + """ + logger.info('Parsing the config file %s.' % filename) + + config = configparser.ConfigParser() + config.optionxform = str + config.read(filename) + + self.clear() + + for category in config.sections(): + logger.debug("FOUND CATEGORY = '%s'" % category) + + for (key, value) in config.items(category): + # "sensitive" settings shall not be logged + if key.startswith('-'): + canLog = 0 + else: + canLog = 1 + + if canLog: + logger.debug("Processing key, value: '%s':'%s'" % + (key, value)) + else: + logger.debug("Processing key, value : " + "'*********************'") + + self._insertKey(category + '|' + key, value) + + def keep_current_value(self, key): + """ + Determines whether a value for a property must be kept. + + If the propery is missing, it is treated as it should be not + changed by the engine. + + Keyword arguments: + key -- The key to search for. + """ + # helps to distinguish from "empty" value which is used to indicate + # "removal" + return not key in self + + def remove_current_value(self, key): + """ + Determines whether a value for the property must be removed. + + If the specified key is empty, it is treated as it should be + removed by the engine. + + Return true if the value can be removed, false otherwise. + + Keyword arguments: + key -- The key to search for. + """ + # helps to distinguish from "missing" value which is used to indicate + # "keeping unchanged" + if key in self: + return not bool(self[key]) + else: + return False + + def get_count(self, prefix): + """ + Return the total number of keys that start with the + specified prefix. + + Keyword arguments: + prefix -- prefix of the key + """ + res = 0 + for key in self.keys(): + if key.startswith(prefix): + res += 1 + + return res diff --git a/cloudinit/sources/helpers/vmware/imc/config_namespace.py b/cloudinit/sources/helpers/vmware/imc/config_namespace.py index 7f76ac8b..7266b699 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_namespace.py +++ b/cloudinit/sources/helpers/vmware/imc/config_namespace.py @@ -1,5 +1,25 @@ -from cloudinit.sources.helpers.vmware.imc.config_source import ConfigSource +# vi: ts=4 expandtab +# +# Copyright (C) 2015 Canonical Ltd. +# Copyright (C) 2015 VMware Inc. +# +# Author: Sankar Tanguturi +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from .config_source import ConfigSource class ConfigNamespace(ConfigSource): + """Specifies the Config Namespace.""" pass diff --git a/cloudinit/sources/helpers/vmware/imc/config_source.py b/cloudinit/sources/helpers/vmware/imc/config_source.py index fad3a389..a367e476 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_source.py +++ b/cloudinit/sources/helpers/vmware/imc/config_source.py @@ -1,2 +1,23 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2015 Canonical Ltd. +# Copyright (C) 2015 VMware Inc. +# +# Author: Sankar Tanguturi +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + class ConfigSource: + """Specifies a source for the Config Content.""" pass diff --git a/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py b/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py index 66b4fad7..28544e4f 100644 --- a/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py +++ b/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py @@ -1,29 +1,45 @@ -# from enum import Enum - - -# The IPv4 configuration mode which directly represents the user's goal. -# -# This mode effectively acts as a contract of the inguest customization engine. -# It must be set based on what the user has requested via VMODL/generators API -# and should not be changed by those layers. It's up to the in-guest engine to -# interpret and materialize the user's request. -# -# Also defined in linuxconfiggenerator.h. -class Ipv4Mode: - # The legacy mode which only allows dhcp/static based on whether IPv4 - # addresses list is empty or not - IPV4_MODE_BACKWARDS_COMPATIBLE = 'BACKWARDS_COMPATIBLE' - # IPv4 must use static address. Reserved for future use - IPV4_MODE_STATIC = 'STATIC' - # IPv4 must use DHCPv4. Reserved for future use - IPV4_MODE_DHCP = 'DHCP' - # IPv4 must be disabled - IPV4_MODE_DISABLED = 'DISABLED' - # IPv4 settings should be left untouched. Reserved for future use - IPV4_MODE_AS_IS = 'AS_IS' - - # def __eq__(self, other): - # return self.name == other.name and self.value == other.value - # - # def __ne__(self, other): - # return not self.__eq__(other) +# vi: ts=4 expandtab +# +# Copyright (C) 2015 Canonical Ltd. +# Copyright (C) 2015 VMware Inc. +# +# Author: Sankar Tanguturi +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +class Ipv4Mode: + """ + The IPv4 configuration mode which directly represents the user's goal. + + This mode effectively acts as a contract of the in-guest customization + engine. It must be set based on what the user has requested and should + not be changed by those layers. It's up to the in-guest engine to + interpret and materialize the user's request. + """ + + # The legacy mode which only allows dhcp/static based on whether IPv4 + # addresses list is empty or not + IPV4_MODE_BACKWARDS_COMPATIBLE = 'BACKWARDS_COMPATIBLE' + + # IPv4 must use static address. Reserved for future use + IPV4_MODE_STATIC = 'STATIC' + + # IPv4 must use DHCPv4. Reserved for future use + IPV4_MODE_DHCP = 'DHCP' + + # IPv4 must be disabled + IPV4_MODE_DISABLED = 'DISABLED' + + # IPv4 settings should be left untouched. Reserved for future use + IPV4_MODE_AS_IS = 'AS_IS' diff --git a/cloudinit/sources/helpers/vmware/imc/nic.py b/cloudinit/sources/helpers/vmware/imc/nic.py index b90a5640..bb45a9e6 100644 --- a/cloudinit/sources/helpers/vmware/imc/nic.py +++ b/cloudinit/sources/helpers/vmware/imc/nic.py @@ -1,107 +1,147 @@ -from cloudinit.sources.helpers.vmware.imc.boot_proto import BootProto - - -class Nic: - def __init__(self, name, configFile): - self._name = name - self._configFile = configFile - - def _get(self, what): - return self._configFile.get(self.name + what, None) - - def _getCnt(self, prefix): - return self._configFile.getCnt(self.name + prefix) - - @property - def name(self): - return self._name - - @property - def mac(self): - return self._get('|MACADDR').lower() - - @property - def bootProto(self): - return self._get('|BOOTPROTO').lower() - - @property - def ipv4(self): - # TODO implement NONE - if self.bootProto == BootProto.STATIC: - return StaticIpv4Conf(self) - - return DhcpIpv4Conf(self) - - @property - def ipv6(self): - # TODO implement NONE - cnt = self._getCnt("|IPv6ADDR|") - - if cnt != 0: - return StaticIpv6Conf(self) - - return DhcpIpv6Conf(self) - - -class DhcpIpv4Conf: - def __init__(self, nic): - self._nic = nic - - -class StaticIpv4Addr: - def __init__(self, nic): - self._nic = nic - - @property - def ip(self): - return self._nic._get('|IPADDR') - - @property - def netmask(self): - return self._nic._get('|NETMASK') - - @property - def gateway(self): - return self._nic._get('|GATEWAY') - - -class StaticIpv4Conf(DhcpIpv4Conf): - @property - def addrs(self): - return [StaticIpv4Addr(self._nic)] - - -class DhcpIpv6Conf: - def __init__(self, nic): - self._nic = nic - - -class StaticIpv6Addr: - def __init__(self, nic, index): - self._nic = nic - self._index = index - - @property - def ip(self): - return self._nic._get("|IPv6ADDR|" + str(self._index)) - - @property - def prefix(self): - return self._nic._get("|IPv6NETMASK|" + str(self._index)) - - @property - def gateway(self): - return self._nic._get("|IPv6GATEWAY|" + str(self._index)) - - -class StaticIpv6Conf(DhcpIpv6Conf): - @property - def addrs(self): - cnt = self._nic._getCnt("|IPv6ADDR|") - - res = [] - - for i in range(1, cnt + 1): - res.append(StaticIpv6Addr(self._nic, i)) - - return res +# vi: ts=4 expandtab +# +# Copyright (C) 2015 Canonical Ltd. +# Copyright (C) 2015 VMware Inc. +# +# Author: Sankar Tanguturi +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from .boot_proto import BootProto + + +class Nic: + """ + Holds the information about each NIC specified + in the customization specification file + """ + + def __init__(self, name, configFile): + self._name = name + self._configFile = configFile + + def _get(self, what): + return self._configFile.get(self.name + what, None) + + def _get_count(self, prefix): + return self._configFile.get_count(self.name + prefix) + + @property + def name(self): + return self._name + + @property + def mac(self): + return self._get('|MACADDR').lower() + + @property + def bootProto(self): + return self._get('|BOOTPROTO').lower() + + @property + def ipv4(self): + """ + Retrieves the DHCP or Static IPv6 configuration + based on the BOOTPROTO property associated with the NIC + """ + if self.bootProto == BootProto.STATIC: + return StaticIpv4Conf(self) + + return DhcpIpv4Conf(self) + + @property + def ipv6(self): + cnt = self._get_count("|IPv6ADDR|") + + if cnt != 0: + return StaticIpv6Conf(self) + + return DhcpIpv6Conf(self) + + +class DhcpIpv4Conf: + """DHCP Configuration Setting.""" + + def __init__(self, nic): + self._nic = nic + + +class StaticIpv4Addr: + """Static IPV4 Setting.""" + + def __init__(self, nic): + self._nic = nic + + @property + def ip(self): + return self._nic._get('|IPADDR') + + @property + def netmask(self): + return self._nic._get('|NETMASK') + + @property + def gateway(self): + return self._nic._get('|GATEWAY') + + +class StaticIpv4Conf(DhcpIpv4Conf): + """Static IPV4 Configuration.""" + + @property + def addrs(self): + """Return the list of associated IPv4 addresses.""" + return [StaticIpv4Addr(self._nic)] + + +class DhcpIpv6Conf: + """DHCP IPV6 Configuration.""" + + def __init__(self, nic): + self._nic = nic + + +class StaticIpv6Addr: + """Static IPV6 Address.""" + + def __init__(self, nic, index): + self._nic = nic + self._index = index + + @property + def ip(self): + return self._nic._get("|IPv6ADDR|" + str(self._index)) + + @property + def prefix(self): + return self._nic._get("|IPv6NETMASK|" + str(self._index)) + + @property + def gateway(self): + return self._nic._get("|IPv6GATEWAY|" + str(self._index)) + + +class StaticIpv6Conf(DhcpIpv6Conf): + """Static IPV6 Configuration.""" + + @property + def addrs(self): + """Return the list Associated IPV6 addresses.""" + cnt = self._nic._get_count("|IPv6ADDR|") + + res = [] + + for i in range(1, cnt + 1): + res.append(StaticIpv6Addr(self._nic, i)) + + return res -- cgit v1.2.3 From 415c45a2b9b66603e672e8ea54cee8f40a19abd1 Mon Sep 17 00:00:00 2001 From: Sankar Tanguturi Date: Tue, 19 Jan 2016 18:24:54 -0800 Subject: Fixed all the review comments from Daniel. Added a new file i.e. nic_base.py which will be used a base calls for all NIC related configuration. Modified some code in nic.py. --- cloudinit/sources/helpers/vmware/imc/boot_proto.py | 2 +- cloudinit/sources/helpers/vmware/imc/config.py | 6 +- .../sources/helpers/vmware/imc/config_file.py | 40 ++---- cloudinit/sources/helpers/vmware/imc/ipv4_mode.py | 2 +- cloudinit/sources/helpers/vmware/imc/nic.py | 118 +++++++--------- cloudinit/sources/helpers/vmware/imc/nic_base.py | 154 +++++++++++++++++++++ 6 files changed, 222 insertions(+), 100 deletions(-) create mode 100644 cloudinit/sources/helpers/vmware/imc/nic_base.py diff --git a/cloudinit/sources/helpers/vmware/imc/boot_proto.py b/cloudinit/sources/helpers/vmware/imc/boot_proto.py index abfffd75..faba5887 100644 --- a/cloudinit/sources/helpers/vmware/imc/boot_proto.py +++ b/cloudinit/sources/helpers/vmware/imc/boot_proto.py @@ -18,7 +18,7 @@ # along with this program. If not, see . -class BootProto: +class BootProtoEnum: """Specifies the NIC Boot Settings.""" DHCP = 'dhcp' diff --git a/cloudinit/sources/helpers/vmware/imc/config.py b/cloudinit/sources/helpers/vmware/imc/config.py index 7eee47a5..aebc12a0 100644 --- a/cloudinit/sources/helpers/vmware/imc/config.py +++ b/cloudinit/sources/helpers/vmware/imc/config.py @@ -66,7 +66,8 @@ class Config: def name_servers(self): """Return the list of DNS servers.""" res = [] - for i in range(1, self._configFile.get_count(Config.DNS) + 1): + cnt = self._configFile.get_count_with_prefix(Config.DNS) + for i in range(1, cnt + 1): key = Config.DNS + str(i) res.append(self._configFile[key]) @@ -76,7 +77,8 @@ class Config: def dns_suffixes(self): """Return the list of DNS Suffixes.""" res = [] - for i in range(1, self._configFile.get_count(Config.SUFFIX) + 1): + cnt = self._configFile.get_count_with_prefix(Config.SUFFIX) + for i in range(1, cnt + 1): key = Config.SUFFIX + str(i) res.append(self._configFile[key]) diff --git a/cloudinit/sources/helpers/vmware/imc/config_file.py b/cloudinit/sources/helpers/vmware/imc/config_file.py index e08a2a9a..7c47d14c 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_file.py +++ b/cloudinit/sources/helpers/vmware/imc/config_file.py @@ -32,7 +32,8 @@ logger = logging.getLogger(__name__) class ConfigFile(ConfigSource, dict): """ConfigFile module to load the content from a specified source.""" - def __init__(self): + def __init__(self, filename): + self._loadConfigFile(filename) pass def _insertKey(self, key, val): @@ -48,9 +49,9 @@ class ConfigFile(ConfigSource, dict): val = val.strip() if key.startswith('-') or '|-' in key: - canLog = 0 + canLog = False else: - canLog = 1 + canLog = True # "sensitive" settings shall not be logged if canLog: @@ -64,7 +65,7 @@ class ConfigFile(ConfigSource, dict): """Return the number of properties present.""" return len(self) - def loadConfigFile(self, filename): + def _loadConfigFile(self, filename): """ Parses properties from the specified config file. @@ -87,22 +88,9 @@ class ConfigFile(ConfigSource, dict): logger.debug("FOUND CATEGORY = '%s'" % category) for (key, value) in config.items(category): - # "sensitive" settings shall not be logged - if key.startswith('-'): - canLog = 0 - else: - canLog = 1 - - if canLog: - logger.debug("Processing key, value: '%s':'%s'" % - (key, value)) - else: - logger.debug("Processing key, value : " - "'*********************'") - self._insertKey(category + '|' + key, value) - def keep_current_value(self, key): + def should_keep_current_value(self, key): """ Determines whether a value for a property must be kept. @@ -114,9 +102,9 @@ class ConfigFile(ConfigSource, dict): """ # helps to distinguish from "empty" value which is used to indicate # "removal" - return not key in self + return key not in self - def remove_current_value(self, key): + def should_remove_current_value(self, key): """ Determines whether a value for the property must be removed. @@ -135,17 +123,11 @@ class ConfigFile(ConfigSource, dict): else: return False - def get_count(self, prefix): + def get_count_with_prefix(self, prefix): """ - Return the total number of keys that start with the - specified prefix. + Return the total count of keys that start with the specified prefix. Keyword arguments: prefix -- prefix of the key """ - res = 0 - for key in self.keys(): - if key.startswith(prefix): - res += 1 - - return res + return len([key for key in self if key.startswith(prefix)]) diff --git a/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py b/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py index 28544e4f..33f88726 100644 --- a/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py +++ b/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py @@ -18,7 +18,7 @@ # along with this program. If not, see . -class Ipv4Mode: +class Ipv4ModeEnum: """ The IPv4 configuration mode which directly represents the user's goal. diff --git a/cloudinit/sources/helpers/vmware/imc/nic.py b/cloudinit/sources/helpers/vmware/imc/nic.py index bb45a9e6..a7594874 100644 --- a/cloudinit/sources/helpers/vmware/imc/nic.py +++ b/cloudinit/sources/helpers/vmware/imc/nic.py @@ -17,10 +17,11 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from .boot_proto import BootProto +from .boot_proto import BootProtoEnum +from .nic_base import NicBase, StaticIpv4Base, StaticIpv6Base -class Nic: +class Nic(NicBase): """ Holds the information about each NIC specified in the customization specification file @@ -31,10 +32,10 @@ class Nic: self._configFile = configFile def _get(self, what): - return self._configFile.get(self.name + what, None) + return self._configFile.get(self.name + '|' + what, None) - def _get_count(self, prefix): - return self._configFile.get_count(self.name + prefix) + def _get_count_with_prefix(self, prefix): + return self._configFile.get_count_with_prefix(self.name + prefix) @property def name(self): @@ -42,41 +43,52 @@ class Nic: @property def mac(self): - return self._get('|MACADDR').lower() + return self._get('MACADDR').lower() @property - def bootProto(self): - return self._get('|BOOTPROTO').lower() + def primary(self): + value = self._get('PRIMARY').lower() + return value == 'yes' or value == 'true' @property - def ipv4(self): - """ - Retrieves the DHCP or Static IPv6 configuration - based on the BOOTPROTO property associated with the NIC - """ - if self.bootProto == BootProto.STATIC: - return StaticIpv4Conf(self) + def onboot(self): + value = self._get('ONBOOT').lower() + return value == 'yes' or value == 'true' - return DhcpIpv4Conf(self) + @property + def bootProto(self): + return self._get('BOOTPROTO').lower() @property - def ipv6(self): - cnt = self._get_count("|IPv6ADDR|") + def ipv4_mode(self): + return self._get('IPv4_MODE').lower() - if cnt != 0: - return StaticIpv6Conf(self) + @property + def staticIpv4(self): + """ + Checks the BOOTPROTO property and returns StaticIPv4Addr + configuration object if STATIC configuration is set. + """ + if self.bootProto == BootProtoEnum.STATIC: + return [StaticIpv4Addr(self)] + else: + return None - return DhcpIpv6Conf(self) + @property + def staticIpv6(self): + cnt = self._get_count_with_prefix('|IPv6ADDR|') + if not cnt: + return None -class DhcpIpv4Conf: - """DHCP Configuration Setting.""" + result = [] + for index in range(1, cnt + 1): + result.append(StaticIpv6Addr(self, index)) - def __init__(self, nic): - self._nic = nic + return result -class StaticIpv4Addr: +class StaticIpv4Addr(StaticIpv4Base): """Static IPV4 Setting.""" def __init__(self, nic): @@ -84,34 +96,22 @@ class StaticIpv4Addr: @property def ip(self): - return self._nic._get('|IPADDR') + return self._nic._get('IPADDR') @property def netmask(self): - return self._nic._get('|NETMASK') + return self._nic._get('NETMASK') @property - def gateway(self): - return self._nic._get('|GATEWAY') + def gateways(self): + value = self._nic._get('GATEWAY') + if value: + return [x.strip() for x in value.split(',')] + else: + return None -class StaticIpv4Conf(DhcpIpv4Conf): - """Static IPV4 Configuration.""" - - @property - def addrs(self): - """Return the list of associated IPv4 addresses.""" - return [StaticIpv4Addr(self._nic)] - - -class DhcpIpv6Conf: - """DHCP IPV6 Configuration.""" - - def __init__(self, nic): - self._nic = nic - - -class StaticIpv6Addr: +class StaticIpv6Addr(StaticIpv6Base): """Static IPV6 Address.""" def __init__(self, nic, index): @@ -120,28 +120,12 @@ class StaticIpv6Addr: @property def ip(self): - return self._nic._get("|IPv6ADDR|" + str(self._index)) + return self._nic._get('IPv6ADDR|' + str(self._index)) @property - def prefix(self): - return self._nic._get("|IPv6NETMASK|" + str(self._index)) + def netmask(self): + return self._nic._get('IPv6NETMASK|' + str(self._index)) @property def gateway(self): - return self._nic._get("|IPv6GATEWAY|" + str(self._index)) - - -class StaticIpv6Conf(DhcpIpv6Conf): - """Static IPV6 Configuration.""" - - @property - def addrs(self): - """Return the list Associated IPV6 addresses.""" - cnt = self._nic._get_count("|IPv6ADDR|") - - res = [] - - for i in range(1, cnt + 1): - res.append(StaticIpv6Addr(self._nic, i)) - - return res + return self._nic._get('IPv6GATEWAY|' + str(self._index)) diff --git a/cloudinit/sources/helpers/vmware/imc/nic_base.py b/cloudinit/sources/helpers/vmware/imc/nic_base.py new file mode 100644 index 00000000..030ba311 --- /dev/null +++ b/cloudinit/sources/helpers/vmware/imc/nic_base.py @@ -0,0 +1,154 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2015 Canonical Ltd. +# Copyright (C) 2015 VMware Inc. +# +# Author: Sankar Tanguturi +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +class NicBase: + """ + Define what are expected of each nic. + The following properties should be provided in an implementation class. + """ + + @property + def mac(self): + """ + Retrieves the mac address of the nic + @return (str) : the MACADDR setting + """ + raise NotImplementedError('MACADDR') + + @property + def primary(self): + """ + Retrieves whether the nic is the primary nic + Indicates whether NIC will be used to define the default gateway. + If none of the NICs is configured to be primary, default gateway won't + be set. + @return (bool): the PRIMARY setting + """ + raise NotImplementedError('PRIMARY') + + @property + def onboot(self): + """ + Retrieves whether the nic should be up at the boot time + @return (bool) : the ONBOOT setting + """ + raise NotImplementedError('ONBOOT') + + @property + def bootProto(self): + """ + Retrieves the boot protocol of the nic + @return (str): the BOOTPROTO setting, valid values: dhcp and static. + """ + raise NotImplementedError('BOOTPROTO') + + @property + def ipv4_mode(self): + """ + Retrieves the IPv4_MODE + @return (str): the IPv4_MODE setting, valid values: + backwards_compatible, static, dhcp, disabled, as_is + """ + raise NotImplementedError('IPv4_MODE') + + @property + def staticIpv4(self): + """ + Retrieves the static IPv4 configuration of the nic + @return (StaticIpv4Base list): the static ipv4 setting + """ + raise NotImplementedError('Static IPv4') + + @property + def staticIpv6(self): + """ + Retrieves the IPv6 configuration of the nic + @return (StaticIpv6Base list): the static ipv6 setting + """ + raise NotImplementedError('Static Ipv6') + + def validate(self): + """ + Validate the object + For example, the staticIpv4 property is required and should not be + empty when ipv4Mode is STATIC + """ + raise NotImplementedError('Check constraints on properties') + + +class StaticIpv4Base: + """ + Define what are expected of a static IPv4 setting + The following properties should be provided in an implementation class. + """ + + @property + def ip(self): + """ + Retrieves the Ipv4 address + @return (str): the IPADDR setting + """ + raise NotImplementedError('Ipv4 Address') + + @property + def netmask(self): + """ + Retrieves the Ipv4 NETMASK setting + @return (str): the NETMASK setting + """ + raise NotImplementedError('Ipv4 NETMASK') + + @property + def gateways(self): + """ + Retrieves the gateways on this Ipv4 subnet + @return (str list): the GATEWAY setting + """ + raise NotImplementedError('Ipv4 GATEWAY') + + +class StaticIpv6Base: + """Define what are expected of a static IPv6 setting + The following properties should be provided in an implementation class. + """ + + @property + def ip(self): + """ + Retrieves the Ipv6 address + @return (str): the IPv6ADDR setting + """ + raise NotImplementedError('Ipv6 Address') + + @property + def netmask(self): + """ + Retrieves the Ipv6 NETMASK setting + @return (str): the IPv6NETMASK setting + """ + raise NotImplementedError('Ipv6 NETMASK') + + @property + def gateway(self): + """ + Retrieves the Ipv6 GATEWAY setting + @return (str): the IPv6GATEWAY setting + """ + raise NotImplementedError('Ipv6 GATEWAY') -- cgit v1.2.3 From 601535f5cb457a50180c99010fa6406b99da2b13 Mon Sep 17 00:00:00 2001 From: Robert Jennings Date: Fri, 29 Jan 2016 11:47:48 -0600 Subject: Correct lock_passwd in docs --- doc/examples/cloud-config-user-groups.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/examples/cloud-config-user-groups.txt b/doc/examples/cloud-config-user-groups.txt index 31491faf..0e8ed243 100644 --- a/doc/examples/cloud-config-user-groups.txt +++ b/doc/examples/cloud-config-user-groups.txt @@ -15,14 +15,14 @@ users: selinux-user: staff_u expiredate: 2012-09-01 ssh-import-id: foobar - lock-passwd: false + lock_passwd: false passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/ - name: barfoo gecos: Bar B. Foo sudo: ALL=(ALL) NOPASSWD:ALL groups: users, admin ssh-import-id: None - lock-passwd: true + lock_passwd: true ssh-authorized-keys: - - @@ -42,7 +42,7 @@ users: # selinux-user: Optional. The SELinux user for the user's login, such as # "staff_u". When this is omitted the system will select the default # SELinux user. -# lock-passwd: Defaults to true. Lock the password to disable password login +# lock_passwd: Defaults to true. Lock the password to disable password login # inactive: Create the user as inactive # passwd: The hash -- not the password itself -- of the password you want # to use for this user. You can generate a safe hash via: -- cgit v1.2.3 From ee40614b0a34a110265493c176c64db823aa34b3 Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Wed, 3 Feb 2016 22:21:40 -0600 Subject: lxd: add support for setting up lxd using 'lxd init' If lxd key is present in cfg, then run 'lxd init' with values from the 'init' entry in lxd configuration as flags. --- ChangeLog | 1 + cloudinit/config/cc_lxd.py | 50 +++++++++++++++++++ config/cloud.cfg | 1 + tests/unittests/test_handler/test_handler_lxd.py | 62 ++++++++++++++++++++++++ 4 files changed, 114 insertions(+) create mode 100644 cloudinit/config/cc_lxd.py create mode 100644 tests/unittests/test_handler/test_handler_lxd.py diff --git a/ChangeLog b/ChangeLog index 0ba16492..9fbc920d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -71,6 +71,7 @@ - Azure: get instance id from dmi instead of SharedConfig (LP: #1506187) - systemd/power_state: fix power_state to work even if cloud-final exited non-zero (LP: #1449318) + - lxd: add support for setting up lxd using 'lxd init' 0.7.6: - open 0.7.6 - Enable vendordata on CloudSigma datasource (LP: #1303986) diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py new file mode 100644 index 00000000..0db8356b --- /dev/null +++ b/cloudinit/config/cc_lxd.py @@ -0,0 +1,50 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2016 Canonical Ltd. +# +# Author: Wesley Wiedenmeier +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +This module initializes lxd using 'lxd init' + +Example config: + #cloud-config + lxd: + init: + network_address: + network_port: + storage_backend: + storage_create_device: + storage_create_loop: + storage_pool: + trust_password: +""" + +from cloudinit import util + + +def handle(name, cfg, cloud, log, args): + if not cfg.get('lxd') and cfg['lxd'].get('init'): + log.debug("Skipping module named %s, not present or disabled by cfg") + return + lxd_conf = cfg['lxd']['init'] + keys = ('network_address', 'network_port', 'storage_backend', + 'storage_create_device', 'storage_create_loop', 'storage_pool', + 'trust_password') + cmd = ['lxd', 'init', '--auto'] + for k in keys: + if lxd_conf.get(k): + cmd.extend(["--%s" % k.replace('_', '-'), lxd_conf[k]]) + util.subp(cmd) diff --git a/config/cloud.cfg b/config/cloud.cfg index 74794ab0..795df19f 100644 --- a/config/cloud.cfg +++ b/config/cloud.cfg @@ -56,6 +56,7 @@ cloud_config_modules: - fan - landscape - timezone + - lxd - puppet - chef - salt-minion diff --git a/tests/unittests/test_handler/test_handler_lxd.py b/tests/unittests/test_handler/test_handler_lxd.py new file mode 100644 index 00000000..89863d52 --- /dev/null +++ b/tests/unittests/test_handler/test_handler_lxd.py @@ -0,0 +1,62 @@ +from cloudinit.config import cc_lxd +from cloudinit import (util, distros, helpers, cloud) +from cloudinit.sources import DataSourceNoCloud +from .. import helpers as t_help + +import logging + +LOG = logging.getLogger(__name__) + + +class TestLxd(t_help.TestCase): + def setUp(self): + super(TestLxd, self).setUp() + self.unapply = [] + apply_patches([(util, 'subp', self._mock_subp)]) + self.subp_called = [] + + def tearDown(self): + apply_patches([i for i in reversed(self.unapply)]) + + def _mock_subp(self, *args, **kwargs): + if 'args' not in kwargs: + kwargs['args'] = args[0] + self.subp_called.append(kwargs) + return + + def _get_cloud(self, distro): + cls = distros.fetch(distro) + paths = helpers.Paths({}) + d = cls(distro, {}, paths) + ds = DataSourceNoCloud.DataSourceNoCloud({}, d, paths) + cc = cloud.Cloud(ds, paths, {}, d, None) + return cc + + def test_lxd_init(self): + cfg = { + 'lxd': { + 'init': { + 'network_address': '0.0.0.0', + 'storage_backend': 'zfs', + 'storage_pool': 'poolname', + } + } + } + cc = self._get_cloud('ubuntu') + cc_lxd.handle('cc_lxd', cfg, cc, LOG, []) + + self.assertEqual( + self.subp_called[0].get('args'), + ['lxd', 'init', '--auto', '--network-address', '0.0.0.0', + '--storage-backend', 'zfs', '--storage-pool', 'poolname']) + + +def apply_patches(patches): + ret = [] + for (ref, name, replace) in patches: + if replace is None: + continue + orig = getattr(ref, name) + setattr(ref, name, replace) + ret.append((ref, name, orig)) + return ret -- cgit v1.2.3 From 814e04654736db6b7443d72d5daa62b56f7a61b9 Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Thu, 4 Feb 2016 16:14:15 -0600 Subject: Added example cc_lxd config --- doc/examples/cloud-config-lxd.txt | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 doc/examples/cloud-config-lxd.txt diff --git a/doc/examples/cloud-config-lxd.txt b/doc/examples/cloud-config-lxd.txt new file mode 100644 index 00000000..f66da4c3 --- /dev/null +++ b/doc/examples/cloud-config-lxd.txt @@ -0,0 +1,21 @@ +#cloud-config + +# configure lxd +# default: none +# all options default to none if not specified +# lxd: config sections for lxd +# init: dict of options for lxd init, see 'man lxd' +# network_address: address for lxd to listen on +# network_port: port for lxd to listen on +# storage_backend: either 'zfs' or 'dir' +# storage_create_device: device based storage using specified device +# storage_create_loop: set up loop based storage with size in GB +# storage_pool: name of storage pool to use or create +# trust_password: password required to add new clients + +lxd: + init: + network_address: 0.0.0.0 + network_port: 8443 + storage_backend: zfs + storage_pool: datapool -- cgit v1.2.3 From a2e251c46307fed0b91e34084c361816829f251d Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Thu, 4 Feb 2016 19:09:05 -0600 Subject: - Ensure that lxd is installed before running lxd init. - Handle init cfg separately from main cfg to allow multiple sections under lxd config to be handled independantly. - Check for properly formatted lxd init cfg --- cloudinit/config/cc_lxd.py | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py index 0db8356b..c9cf8704 100644 --- a/cloudinit/config/cc_lxd.py +++ b/cloudinit/config/cc_lxd.py @@ -36,15 +36,31 @@ from cloudinit import util def handle(name, cfg, cloud, log, args): - if not cfg.get('lxd') and cfg['lxd'].get('init'): + # Get config + lxd_cfg = cfg.get('lxd') + if not lxd_cfg and isinstance(lxd_cfg, dict): log.debug("Skipping module named %s, not present or disabled by cfg") return - lxd_conf = cfg['lxd']['init'] - keys = ('network_address', 'network_port', 'storage_backend', - 'storage_create_device', 'storage_create_loop', 'storage_pool', - 'trust_password') - cmd = ['lxd', 'init', '--auto'] - for k in keys: - if lxd_conf.get(k): - cmd.extend(["--%s" % k.replace('_', '-'), lxd_conf[k]]) - util.subp(cmd) + + # Ensure lxd is installed + if not util.which("lxd"): + try: + cloud.distro.install_packages(("lxd",)) + except util.ProcessExecutionError as e: + log.warn("no lxd executable and could not install lxd: '%s'" % e) + return + + # Set up lxd if init config is given + init_cfg = lxd_cfg.get('init') + if init_cfg: + if not isinstance(init_cfg, dict): + log.warn("lxd init config must be a dict of flag: val pairs") + return + init_keys = ('network_address', 'network_port', 'storage_backend', + 'storage_create_device', 'storage_create_loop', + 'storage_pool', 'trust_password') + cmd = ['lxd', 'init', '--auto'] + for k in init_keys: + if init_cfg.get(k): + cmd.extend(["--%s" % k.replace('_', '-'), init_cfg[k]]) + util.subp(cmd) -- cgit v1.2.3 From 1f24e50c655ccc0c49afcd3225ce8f0e0bce05b9 Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Thu, 4 Feb 2016 19:11:02 -0600 Subject: Use mock in test_handler_lxd.py and add test for lxd installation --- tests/unittests/test_handler/test_handler_lxd.py | 76 +++++++++++------------- 1 file changed, 36 insertions(+), 40 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_lxd.py b/tests/unittests/test_handler/test_handler_lxd.py index 89863d52..4d858b8f 100644 --- a/tests/unittests/test_handler/test_handler_lxd.py +++ b/tests/unittests/test_handler/test_handler_lxd.py @@ -1,28 +1,31 @@ from cloudinit.config import cc_lxd -from cloudinit import (util, distros, helpers, cloud) +from cloudinit import (distros, helpers, cloud) from cloudinit.sources import DataSourceNoCloud from .. import helpers as t_help import logging +try: + from unittest import mock +except ImportError: + import mock + LOG = logging.getLogger(__name__) class TestLxd(t_help.TestCase): + lxd_cfg = { + 'lxd': { + 'init': { + 'network_address': '0.0.0.0', + 'storage_backend': 'zfs', + 'storage_pool': 'poolname', + } + } + } + def setUp(self): super(TestLxd, self).setUp() - self.unapply = [] - apply_patches([(util, 'subp', self._mock_subp)]) - self.subp_called = [] - - def tearDown(self): - apply_patches([i for i in reversed(self.unapply)]) - - def _mock_subp(self, *args, **kwargs): - if 'args' not in kwargs: - kwargs['args'] = args[0] - self.subp_called.append(kwargs) - return def _get_cloud(self, distro): cls = distros.fetch(distro) @@ -32,31 +35,24 @@ class TestLxd(t_help.TestCase): cc = cloud.Cloud(ds, paths, {}, d, None) return cc - def test_lxd_init(self): - cfg = { - 'lxd': { - 'init': { - 'network_address': '0.0.0.0', - 'storage_backend': 'zfs', - 'storage_pool': 'poolname', - } - } - } + @mock.patch("cloudinit.config.cc_lxd.util") + def test_lxd_init(self, mock_util): + cc = self._get_cloud('ubuntu') + mock_util.which.return_value = True + cc_lxd.handle('cc_lxd', self.lxd_cfg, cc, LOG, []) + self.assertTrue(mock_util.which.called) + init_call = mock_util.subp.call_args_list[0][0][0] + self.assertEquals(init_call, + ['lxd', 'init', '--auto', '--network-address', + '0.0.0.0', '--storage-backend', 'zfs', + '--storage-pool', 'poolname']) + + @mock.patch("cloudinit.config.cc_lxd.util") + def test_lxd_install(self, mock_util): cc = self._get_cloud('ubuntu') - cc_lxd.handle('cc_lxd', cfg, cc, LOG, []) - - self.assertEqual( - self.subp_called[0].get('args'), - ['lxd', 'init', '--auto', '--network-address', '0.0.0.0', - '--storage-backend', 'zfs', '--storage-pool', 'poolname']) - - -def apply_patches(patches): - ret = [] - for (ref, name, replace) in patches: - if replace is None: - continue - orig = getattr(ref, name) - setattr(ref, name, replace) - ret.append((ref, name, orig)) - return ret + cc.distro = mock.MagicMock() + mock_util.which.return_value = None + cc_lxd.handle('cc_lxd', self.lxd_cfg, cc, LOG, []) + self.assertTrue(cc.distro.install_packages.called) + install_pkg = cc.distro.install_packages.call_args_list[0][0][0] + self.assertEquals(install_pkg, ('lxd',)) -- cgit v1.2.3 From 9ba841efc4263fcd1ec8eb266a3afa83b525ba49 Mon Sep 17 00:00:00 2001 From: Sankar Tanguturi Date: Mon, 8 Feb 2016 17:06:39 -0800 Subject: - Added new Unittests for VMware support. --- tests/data/vmware/cust-dhcp-2nic.cfg | 34 ++++++++++ tests/data/vmware/cust-static-2nic.cfg | 39 +++++++++++ tests/unittests/test_vmware_config_file.py | 103 +++++++++++++++++++++++++++++ 3 files changed, 176 insertions(+) create mode 100644 tests/data/vmware/cust-dhcp-2nic.cfg create mode 100644 tests/data/vmware/cust-static-2nic.cfg create mode 100644 tests/unittests/test_vmware_config_file.py diff --git a/tests/data/vmware/cust-dhcp-2nic.cfg b/tests/data/vmware/cust-dhcp-2nic.cfg new file mode 100644 index 00000000..f687311a --- /dev/null +++ b/tests/data/vmware/cust-dhcp-2nic.cfg @@ -0,0 +1,34 @@ +[NETWORK] +NETWORKING = yes +BOOTPROTO = dhcp +HOSTNAME = myhost1 +DOMAINNAME = eng.vmware.com + +[NIC-CONFIG] +NICS = NIC1,NIC2 + +[NIC1] +MACADDR = 00:50:56:a6:8c:08 +ONBOOT = yes +IPv4_MODE = BACKWARDS_COMPATIBLE +BOOTPROTO = dhcp + +[NIC2] +MACADDR = 00:50:56:a6:5a:de +ONBOOT = yes +IPv4_MODE = BACKWARDS_COMPATIBLE +BOOTPROTO = dhcp + +# some random comment + +[PASSWORD] +# secret +-PASS = c2VjcmV0Cg== + +[DNS] +DNSFROMDHCP=yes +SUFFIX|1 = eng.vmware.com + +[DATETIME] +TIMEZONE = Africa/Abidjan +UTC = yes diff --git a/tests/data/vmware/cust-static-2nic.cfg b/tests/data/vmware/cust-static-2nic.cfg new file mode 100644 index 00000000..0d80c2c4 --- /dev/null +++ b/tests/data/vmware/cust-static-2nic.cfg @@ -0,0 +1,39 @@ +[NETWORK] +NETWORKING = yes +BOOTPROTO = dhcp +HOSTNAME = myhost1 +DOMAINNAME = eng.vmware.com + +[NIC-CONFIG] +NICS = NIC1,NIC2 + +[NIC1] +MACADDR = 00:50:56:a6:8c:08 +ONBOOT = yes +IPv4_MODE = BACKWARDS_COMPATIBLE +BOOTPROTO = static +IPADDR = 10.20.87.154 +NETMASK = 255.255.252.0 +GATEWAY = 10.20.87.253, 10.20.87.105 +IPv6ADDR|1 = fc00:10:20:87::154 +IPv6NETMASK|1 = 64 +IPv6GATEWAY|1 = fc00:10:20:87::253 +[NIC2] +MACADDR = 00:50:56:a6:ef:7d +ONBOOT = yes +IPv4_MODE = BACKWARDS_COMPATIBLE +BOOTPROTO = static +IPADDR = 192.168.6.102 +NETMASK = 255.255.0.0 +GATEWAY = 192.168.0.10 + +[DNS] +DNSFROMDHCP=no +SUFFIX|1 = eng.vmware.com +SUFFIX|2 = proxy.vmware.com +NAMESERVER|1 = 10.20.145.1 +NAMESERVER|2 = 10.20.145.2 + +[DATETIME] +TIMEZONE = Africa/Abidjan +UTC = yes diff --git a/tests/unittests/test_vmware_config_file.py b/tests/unittests/test_vmware_config_file.py new file mode 100644 index 00000000..51166dd7 --- /dev/null +++ b/tests/unittests/test_vmware_config_file.py @@ -0,0 +1,103 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2015 Canonical Ltd. +# Copyright (C) 2016 VMware INC. +# +# Author: Sankar Tanguturi +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import logging +import sys +import unittest + +from cloudinit.sources.helpers.vmware.imc.boot_proto import BootProtoEnum +from cloudinit.sources.helpers.vmware.imc.config import Config +from cloudinit.sources.helpers.vmware.imc.config_file import ConfigFile + +logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) +logger = logging.getLogger(__name__) + + +class TestVmwareConfigFile(unittest.TestCase): + + def test_utility_methods(self): + cf = ConfigFile("tests/data/vmware/cust-dhcp-2nic.cfg") + + cf.clear() + + self.assertEqual(0, cf.size(), "clear size") + + cf._insertKey(" PASSWORD|-PASS ", " foo ") + cf._insertKey("BAR", " ") + + self.assertEqual(2, cf.size(), "insert size") + self.assertEqual('foo', cf["PASSWORD|-PASS"], "password") + self.assertTrue("PASSWORD|-PASS" in cf, "hasPassword") + self.assertFalse(cf.should_keep_current_value("PASSWORD|-PASS"), + "keepPassword") + self.assertFalse(cf.should_remove_current_value("PASSWORD|-PASS"), + "removePassword") + self.assertFalse("FOO" in cf, "hasFoo") + self.assertTrue(cf.should_keep_current_value("FOO"), "keepFoo") + self.assertFalse(cf.should_remove_current_value("FOO"), "removeFoo") + self.assertTrue("BAR" in cf, "hasBar") + self.assertFalse(cf.should_keep_current_value("BAR"), "keepBar") + self.assertTrue(cf.should_remove_current_value("BAR"), "removeBar") + + def test_configfile_static_2nics(self): + cf = ConfigFile("tests/data/vmware/cust-static-2nic.cfg") + + conf = Config(cf) + + self.assertEqual('myhost1', conf.host_name, "hostName") + self.assertEqual('Africa/Abidjan', conf.timezone, "tz") + self.assertTrue(conf.utc, "utc") + + self.assertEqual(['10.20.145.1', '10.20.145.2'], + conf.name_servers, + "dns") + self.assertEqual(['eng.vmware.com', 'proxy.vmware.com'], + conf.dns_suffixes, + "suffixes") + + nics = conf.nics + ipv40 = nics[0].staticIpv4 + + self.assertEqual(2, len(nics), "nics") + self.assertEqual('NIC1', nics[0].name, "nic0") + self.assertEqual('00:50:56:a6:8c:08', nics[0].mac, "mac0") + self.assertEqual(BootProtoEnum.STATIC, nics[0].bootProto, "bootproto0") + self.assertEqual('10.20.87.154', ipv40[0].ip, "ipv4Addr0") + self.assertEqual('255.255.252.0', ipv40[0].netmask, "ipv4Mask0") + self.assertEqual(2, len(ipv40[0].gateways), "ipv4Gw0") + self.assertEqual('10.20.87.253', ipv40[0].gateways[0], "ipv4Gw0_0") + self.assertEqual('10.20.87.105', ipv40[0].gateways[1], "ipv4Gw0_1") + + self.assertEqual(1, len(nics[0].staticIpv6), "ipv6Cnt0") + self.assertEqual('fc00:10:20:87::154', + nics[0].staticIpv6[0].ip, + "ipv6Addr0") + + self.assertEqual('NIC2', nics[1].name, "nic1") + self.assertTrue(not nics[1].staticIpv6, "ipv61 dhcp") + + def test_config_file_dhcp_2nics(self): + cf = ConfigFile("tests/data/vmware/cust-dhcp-2nic.cfg") + + conf = Config(cf) + nics = conf.nics + self.assertEqual(2, len(nics), "nics") + self.assertEqual('NIC1', nics[0].name, "nic0") + self.assertEqual('00:50:56:a6:8c:08', nics[0].mac, "mac0") + self.assertEqual(BootProtoEnum.DHCP, nics[0].bootProto, "bootproto0") -- cgit v1.2.3 From 39f668e5db8d09c46eee3a5df73a69f8d85ba489 Mon Sep 17 00:00:00 2001 From: Sankar Tanguturi Date: Tue, 9 Feb 2016 17:54:07 -0800 Subject: - Added the code to configure the NICs. - Added the code to detect VMware Virtual Platform and apply the customization based on the 'Customization Specification File' Pushed into the guest VM. --- cloudinit/sources/DataSourceOVF.py | 107 ++++++++- cloudinit/sources/helpers/vmware/imc/config_nic.py | 246 +++++++++++++++++++++ cloudinit/sources/helpers/vmware/imc/nic.py | 28 ++- 3 files changed, 372 insertions(+), 9 deletions(-) create mode 100644 cloudinit/sources/helpers/vmware/imc/config_nic.py diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 58a4b2a2..add7d243 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -24,11 +24,16 @@ from xml.dom import minidom import base64 import os +import shutil import re +import time from cloudinit import log as logging from cloudinit import sources from cloudinit import util +from cloudinit.sources.helpers.vmware.imc.config import Config +from cloudinit.sources.helpers.vmware.imc.config_file import ConfigFile +from cloudinit.sources.helpers.vmware.imc.config_nic import NicConfigurator LOG = logging.getLogger(__name__) @@ -50,13 +55,51 @@ class DataSourceOVF(sources.DataSource): found = [] md = {} ud = "" + vmwarePlatformFound = False + vmwareImcConfigFilePath = '' defaults = { "instance-id": "iid-dsovf", } (seedfile, contents) = get_ovf_env(self.paths.seed_dir) - if seedfile: + dmi_info = dmi_data() + system_uuid = "" + system_type = "" + + if dmi_info is False: + LOG.debug("No dmidata utility found") + else: + system_uuid, system_type = tuple(dmi_info) + + if 'vmware' in system_type.lower(): + LOG.debug("VMware Virtual Platform found") + deployPkgPluginPath = search_file("/usr/lib/vmware-tools", "libdeployPkgPlugin.so") + if deployPkgPluginPath: + vmwareImcConfigFilePath = util.log_time(logfunc=LOG.debug, + msg="waiting for configuration file", + func=wait_for_imc_cfg_file, + args=("/tmp", "cust.cfg")) + + if vmwareImcConfigFilePath: + LOG.debug("Found VMware DeployPkg Config File Path at %s" % vmwareImcConfigFilePath) + else: + LOG.debug("Didn't find VMware DeployPkg Config File Path") + + if vmwareImcConfigFilePath: + try: + cf = ConfigFile(vmwareImcConfigFilePath) + conf = Config(cf) + (md, ud, cfg) = read_vmware_imc(conf) + nicConfigurator = NicConfigurator(conf.nics) + nicConfigurator.configure() + vmwarePlatformFound = True + except Exception as inst: + LOG.debug("Error while parsing the Customization Config File") + finally: + dirPath = os.path.dirname(vmwareImcConfigFilePath) + shutil.rmtree(dirPath) + elif seedfile: # Found a seed dir seed = os.path.join(self.paths.seed_dir, seedfile) (md, ud, cfg) = read_ovf_environment(contents) @@ -76,7 +119,7 @@ class DataSourceOVF(sources.DataSource): found.append(name) # There was no OVF transports found - if len(found) == 0: + if len(found) == 0 and not vmwarePlatformFound: return False if 'seedfrom' in md and md['seedfrom']: @@ -108,7 +151,7 @@ class DataSourceOVF(sources.DataSource): def get_public_ssh_keys(self): if 'public-keys' not in self.metadata: - return [] + return [] pks = self.metadata['public-keys'] if isinstance(pks, (list)): return pks @@ -129,6 +172,31 @@ class DataSourceOVFNet(DataSourceOVF): self.supported_seed_starts = ("http://", "https://", "ftp://") +def wait_for_imc_cfg_file(directoryPath, filename, maxwait=180, naplen=5): + waited = 0 + + while waited < maxwait: + fileFullPath = search_file(directoryPath, filename) + if fileFullPath: + return fileFullPath + time.sleep(naplen) + waited += naplen + return None + +# This will return a dict with some content +# meta-data, user-data, some config +def read_vmware_imc(config): + md = {} + cfg = {} + ud = "" + if config.host_name: + if config.domain_name: + md['local-hostname'] = config.host_name + "." + config.domain_name + else: + md['local-hostname'] = config.host_name + + return (md, ud, cfg) + # This will return a dict with some content # meta-data, user-data, some config def read_ovf_environment(contents): @@ -280,6 +348,39 @@ def get_properties(contents): return props +def dmi_data(): + sys_uuid = util.read_dmi_data("system-uuid") + sys_type = util.read_dmi_data("system-product-name") + + if not sys_uuid or not sys_type: + return None + + return (sys_uuid.lower(), sys_type) + +def search_file(directoryPath, filename): + if not directoryPath or not filename: + return None + + dirs = [] + + if os.path.isdir(directoryPath): + dirs.append(directoryPath) + + while dirs: + dir = dirs.pop() + children = [] + try: + children.extend(os.listdir(dir)) + except: + LOG.debug("Ignoring the error while searching the directory %s" % dir) + for child in children: + childFullPath = os.path.join(dir, child) + if os.path.isdir(childFullPath): + dirs.append(childFullPath) + elif child == filename: + return childFullPath + + return None class XmlError(Exception): pass diff --git a/cloudinit/sources/helpers/vmware/imc/config_nic.py b/cloudinit/sources/helpers/vmware/imc/config_nic.py new file mode 100644 index 00000000..8e2fc5d3 --- /dev/null +++ b/cloudinit/sources/helpers/vmware/imc/config_nic.py @@ -0,0 +1,246 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2015 Canonical Ltd. +# Copyright (C) 2016 VMware INC. +# +# Author: Sankar Tanguturi +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import logging +import os +import subprocess +import re + +logger = logging.getLogger(__name__) + + +class NicConfigurator: + def __init__(self, nics): + """ + Initialize the Nic Configurator + @param nics (list) an array of nics to configure + """ + self.nics = nics + self.mac2Name = {} + self.ipv4PrimaryGateway = None + self.ipv6PrimaryGateway = None + self.find_devices() + self._primaryNic = self.get_primary_nic() + + def get_primary_nic(self): + """ + Retrieve the primary nic if it exists + @return (NicBase): the primary nic if exists, None otherwise + """ + primaryNic = None + + for nic in self.nics: + if nic.primary: + if primaryNic: + raise Exception('There can only be one primary nic', + primaryNic.mac, nic.mac) + primaryNic = nic + + return primaryNic + + def find_devices(self): + """ + Create the mac2Name dictionary + The mac address(es) are in the lower case + """ + cmd = 'ip addr show' + outText = subprocess.check_output(cmd, shell=True).decode() + sections = re.split(r'\n\d+: ', '\n' + outText)[1:] + + macPat = r'link/ether (([0-9A-Fa-f]{2}[:]){5}([0-9A-Fa-f]{2}))' + for section in sections: + matcher = re.search(macPat, section) + if not matcher: # Only keep info about nics + continue + mac = matcher.group(1).lower() + name = section.split(':', 1)[0] + self.mac2Name[mac] = name + + def gen_one_nic(self, nic): + """ + Return the lines needed to configure a nic + @return (str list): the string list to configure the nic + @param nic (NicBase): the nic to configure + """ + lines = [] + name = self.mac2Name.get(nic.mac.lower()) + if not name: + raise ValueError('No known device has MACADDR: %s' % nic.mac) + + if nic.onboot: + lines.append('auto %s' % name) + + # Customize IPv4 + lines.extend(self.gen_ipv4(name, nic)) + + # Customize IPv6 + lines.extend(self.gen_ipv6(name, nic)) + + lines.append('') + + return lines + + def gen_ipv4(self, name, nic): + """ + Return the lines needed to configure the IPv4 setting of a nic + @return (str list): the string list to configure the gateways + @param name (str): name of the nic + @param nic (NicBase): the nic to configure + """ + lines = [] + + bootproto = nic.bootProto.lower() + if nic.ipv4_mode.lower() == 'disabled': + bootproto = 'manual' + lines.append('iface %s inet %s' % (name, bootproto)) + + if bootproto != 'static': + return lines + + # Static Ipv4 + v4 = nic.staticIpv4 + if v4.ip: + lines.append(' address %s' % v4.ip) + if v4.netmask: + lines.append(' netmask %s' % v4.netmask) + + # Add the primary gateway + if nic.primary and v4.gateways: + self.ipv4PrimaryGateway = v4.gateways[0] + lines.append(' gateway %s metric 0' % self.ipv4PrimaryGateway) + return lines + + # Add routes if there is no primary nic + if not self._primaryNic: + lines.extend(self.gen_ipv4_route(nic, v4.gateways)) + + return lines + + def gen_ipv4_route(self, nic, gateways): + """ + Return the lines needed to configure additional Ipv4 route + @return (str list): the string list to configure the gateways + @param nic (NicBase): the nic to configure + @param gateways (str list): the list of gateways + """ + lines = [] + + for gateway in gateways: + lines.append(' up route add default gw %s metric 10000' % gateway) + + return lines + + def gen_ipv6(self, name, nic): + """ + Return the lines needed to configure the gateways for a nic + @return (str list): the string list to configure the gateways + @param name (str): name of the nic + @param nic (NicBase): the nic to configure + """ + lines = [] + + if not nic.staticIpv6: + return lines + + # Static Ipv6 + addrs = nic.staticIpv6 + lines.append('iface %s inet6 static' % name) + lines.append(' address %s' % addrs[0].ip) + lines.append(' netmask %s' % addrs[0].netmask) + + for addr in addrs[1:]: + lines.append(' up ifconfig %s inet6 add %s/%s' % (name, addr.ip, + addr.netmask)) + # Add the primary gateway + if nic.primary: + for addr in addrs: + if addr.gateway: + self.ipv6PrimaryGateway = addr.gateway + lines.append(' gateway %s' % self.ipv6PrimaryGateway) + return lines + + # Add routes if there is no primary nic + if not self._primaryNic: + lines.extend(self._genIpv6Route(name, nic, addrs)) + + return lines + + def _genIpv6Route(self, name, nic, addrs): + lines = [] + + for addr in addrs: + lines.append(' up route -A inet6 add default gw %s metric 10000' % + addr.gateway) + + return lines + + def generate(self): + """Return the lines that is needed to configure the nics""" + lines = [] + lines.append('iface lo inet loopback') + lines.append('auto lo') + lines.append('') + + for nic in self.nics: + lines.extend(self.gen_one_nic(nic)) + + return lines + + def clear_dhcp(self): + logger.info('Clearing DHCP leases') + + subprocess.call('pkill dhclient', shell=True) + subprocess.check_call('rm -f /var/lib/dhcp/*', shell=True) + + def if_down_up(self): + names = [] + for nic in self.nics: + name = self.mac2Name.get(nic.mac.lower()) + names.append(name) + + for name in names: + logger.info('Bring down interface %s' % name) + subprocess.check_call('ifdown %s' % name, shell=True) + + self.clear_dhcp() + + for name in names: + logger.info('Bring up interface %s' % name) + subprocess.check_call('ifup %s' % name, shell=True) + + def configure(self): + """ + Configure the /etc/network/intefaces + Make a back up of the original + """ + containingDir = '/etc/network' + + interfaceFile = os.path.join(containingDir, 'interfaces') + originalFile = os.path.join(containingDir, + 'interfaces.before_vmware_customization') + + if not os.path.exists(originalFile) and os.path.exists(interfaceFile): + os.rename(interfaceFile, originalFile) + + lines = self.generate() + with open(interfaceFile, 'w') as fp: + for line in lines: + fp.write('%s\n' % line) + + self.if_down_up() diff --git a/cloudinit/sources/helpers/vmware/imc/nic.py b/cloudinit/sources/helpers/vmware/imc/nic.py index a7594874..6628a3ec 100644 --- a/cloudinit/sources/helpers/vmware/imc/nic.py +++ b/cloudinit/sources/helpers/vmware/imc/nic.py @@ -47,21 +47,37 @@ class Nic(NicBase): @property def primary(self): - value = self._get('PRIMARY').lower() - return value == 'yes' or value == 'true' + value = self._get('PRIMARY') + if value: + value = value.lower() + return value == 'yes' or value == 'true' + else: + return False @property def onboot(self): - value = self._get('ONBOOT').lower() - return value == 'yes' or value == 'true' + value = self._get('ONBOOT') + if value: + value = value.lower() + return value == 'yes' or value == 'true' + else: + return False @property def bootProto(self): - return self._get('BOOTPROTO').lower() + value = self._get('BOOTPROTO') + if value: + return value.lower() + else: + return "" @property def ipv4_mode(self): - return self._get('IPv4_MODE').lower() + value = self._get('IPv4_MODE') + if value: + return value.lower() + else: + return "" @property def staticIpv4(self): -- cgit v1.2.3 From 0ce71cb8975e19677eea415101e15da5f4095cd5 Mon Sep 17 00:00:00 2001 From: Sankar Tanguturi Date: Tue, 16 Feb 2016 17:34:24 -0800 Subject: - Used proper 4 space indentations for config_nic.py and nic.py - Implemented the 'search_file' function using 'os.walk()' - Fixed few variable names. - Removed size() function in config_file.py - Updated the test_config_file.py to use len() instead of .size() --- cloudinit/sources/DataSourceOVF.py | 34 +- .../sources/helpers/vmware/imc/config_file.py | 4 - cloudinit/sources/helpers/vmware/imc/config_nic.py | 433 +++++++++++---------- cloudinit/sources/helpers/vmware/imc/nic.py | 20 +- tests/unittests/test_vmware_config_file.py | 4 +- 5 files changed, 238 insertions(+), 257 deletions(-) diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index add7d243..6d3bf7bb 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -64,13 +64,12 @@ class DataSourceOVF(sources.DataSource): (seedfile, contents) = get_ovf_env(self.paths.seed_dir) dmi_info = dmi_data() - system_uuid = "" system_type = "" - if dmi_info is False: + if dmi_info is None: LOG.debug("No dmidata utility found") else: - system_uuid, system_type = tuple(dmi_info) + (_, system_type) = dmi_info if 'vmware' in system_type.lower(): LOG.debug("VMware Virtual Platform found") @@ -172,11 +171,11 @@ class DataSourceOVFNet(DataSourceOVF): self.supported_seed_starts = ("http://", "https://", "ftp://") -def wait_for_imc_cfg_file(directoryPath, filename, maxwait=180, naplen=5): +def wait_for_imc_cfg_file(dirpath, filename, maxwait=180, naplen=5): waited = 0 while waited < maxwait: - fileFullPath = search_file(directoryPath, filename) + fileFullPath = search_file(dirpath, filename) if fileFullPath: return fileFullPath time.sleep(naplen) @@ -357,28 +356,13 @@ def dmi_data(): return (sys_uuid.lower(), sys_type) -def search_file(directoryPath, filename): - if not directoryPath or not filename: +def search_file(dirpath, filename): + if not dirpath or not filename: return None - dirs = [] - - if os.path.isdir(directoryPath): - dirs.append(directoryPath) - - while dirs: - dir = dirs.pop() - children = [] - try: - children.extend(os.listdir(dir)) - except: - LOG.debug("Ignoring the error while searching the directory %s" % dir) - for child in children: - childFullPath = os.path.join(dir, child) - if os.path.isdir(childFullPath): - dirs.append(childFullPath) - elif child == filename: - return childFullPath + for root, dirs, files in os.walk(dirpath): + if filename in files: + return os.path.join(root, filename) return None diff --git a/cloudinit/sources/helpers/vmware/imc/config_file.py b/cloudinit/sources/helpers/vmware/imc/config_file.py index 7c47d14c..bb9fb7dc 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_file.py +++ b/cloudinit/sources/helpers/vmware/imc/config_file.py @@ -61,10 +61,6 @@ class ConfigFile(ConfigSource, dict): self[key] = val - def size(self): - """Return the number of properties present.""" - return len(self) - def _loadConfigFile(self, filename): """ Parses properties from the specified config file. diff --git a/cloudinit/sources/helpers/vmware/imc/config_nic.py b/cloudinit/sources/helpers/vmware/imc/config_nic.py index 8e2fc5d3..d79e6936 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_nic.py +++ b/cloudinit/sources/helpers/vmware/imc/config_nic.py @@ -26,221 +26,222 @@ logger = logging.getLogger(__name__) class NicConfigurator: - def __init__(self, nics): - """ - Initialize the Nic Configurator - @param nics (list) an array of nics to configure - """ - self.nics = nics - self.mac2Name = {} - self.ipv4PrimaryGateway = None - self.ipv6PrimaryGateway = None - self.find_devices() - self._primaryNic = self.get_primary_nic() - - def get_primary_nic(self): - """ - Retrieve the primary nic if it exists - @return (NicBase): the primary nic if exists, None otherwise - """ - primaryNic = None - - for nic in self.nics: - if nic.primary: - if primaryNic: - raise Exception('There can only be one primary nic', - primaryNic.mac, nic.mac) + def __init__(self, nics): + """ + Initialize the Nic Configurator + @param nics (list) an array of nics to configure + """ + self.nics = nics + self.mac2Name = {} + self.ipv4PrimaryGateway = None + self.ipv6PrimaryGateway = None + self.find_devices() + self._primaryNic = self.get_primary_nic() + + def get_primary_nic(self): + """ + Retrieve the primary nic if it exists + @return (NicBase): the primary nic if exists, None otherwise + """ + primaryNic = None + + for nic in self.nics: + if nic.primary: + if primaryNic: + raise Exception('There can only be one primary nic', + primaryNic.mac, nic.mac) primaryNic = nic - return primaryNic - - def find_devices(self): - """ - Create the mac2Name dictionary - The mac address(es) are in the lower case - """ - cmd = 'ip addr show' - outText = subprocess.check_output(cmd, shell=True).decode() - sections = re.split(r'\n\d+: ', '\n' + outText)[1:] - - macPat = r'link/ether (([0-9A-Fa-f]{2}[:]){5}([0-9A-Fa-f]{2}))' - for section in sections: - matcher = re.search(macPat, section) - if not matcher: # Only keep info about nics - continue - mac = matcher.group(1).lower() - name = section.split(':', 1)[0] - self.mac2Name[mac] = name - - def gen_one_nic(self, nic): - """ - Return the lines needed to configure a nic - @return (str list): the string list to configure the nic - @param nic (NicBase): the nic to configure - """ - lines = [] - name = self.mac2Name.get(nic.mac.lower()) - if not name: - raise ValueError('No known device has MACADDR: %s' % nic.mac) - - if nic.onboot: - lines.append('auto %s' % name) - - # Customize IPv4 - lines.extend(self.gen_ipv4(name, nic)) - - # Customize IPv6 - lines.extend(self.gen_ipv6(name, nic)) - - lines.append('') - - return lines - - def gen_ipv4(self, name, nic): - """ - Return the lines needed to configure the IPv4 setting of a nic - @return (str list): the string list to configure the gateways - @param name (str): name of the nic - @param nic (NicBase): the nic to configure - """ - lines = [] - - bootproto = nic.bootProto.lower() - if nic.ipv4_mode.lower() == 'disabled': - bootproto = 'manual' - lines.append('iface %s inet %s' % (name, bootproto)) - - if bootproto != 'static': - return lines - - # Static Ipv4 - v4 = nic.staticIpv4 - if v4.ip: - lines.append(' address %s' % v4.ip) - if v4.netmask: - lines.append(' netmask %s' % v4.netmask) - - # Add the primary gateway - if nic.primary and v4.gateways: - self.ipv4PrimaryGateway = v4.gateways[0] - lines.append(' gateway %s metric 0' % self.ipv4PrimaryGateway) - return lines - - # Add routes if there is no primary nic - if not self._primaryNic: - lines.extend(self.gen_ipv4_route(nic, v4.gateways)) - - return lines - - def gen_ipv4_route(self, nic, gateways): - """ - Return the lines needed to configure additional Ipv4 route - @return (str list): the string list to configure the gateways - @param nic (NicBase): the nic to configure - @param gateways (str list): the list of gateways - """ - lines = [] - - for gateway in gateways: - lines.append(' up route add default gw %s metric 10000' % gateway) - - return lines - - def gen_ipv6(self, name, nic): - """ - Return the lines needed to configure the gateways for a nic - @return (str list): the string list to configure the gateways - @param name (str): name of the nic - @param nic (NicBase): the nic to configure - """ - lines = [] - - if not nic.staticIpv6: - return lines - - # Static Ipv6 - addrs = nic.staticIpv6 - lines.append('iface %s inet6 static' % name) - lines.append(' address %s' % addrs[0].ip) - lines.append(' netmask %s' % addrs[0].netmask) - - for addr in addrs[1:]: - lines.append(' up ifconfig %s inet6 add %s/%s' % (name, addr.ip, - addr.netmask)) - # Add the primary gateway - if nic.primary: - for addr in addrs: - if addr.gateway: - self.ipv6PrimaryGateway = addr.gateway - lines.append(' gateway %s' % self.ipv6PrimaryGateway) - return lines - - # Add routes if there is no primary nic - if not self._primaryNic: - lines.extend(self._genIpv6Route(name, nic, addrs)) - - return lines - - def _genIpv6Route(self, name, nic, addrs): - lines = [] - - for addr in addrs: - lines.append(' up route -A inet6 add default gw %s metric 10000' % - addr.gateway) - - return lines - - def generate(self): - """Return the lines that is needed to configure the nics""" - lines = [] - lines.append('iface lo inet loopback') - lines.append('auto lo') - lines.append('') - - for nic in self.nics: - lines.extend(self.gen_one_nic(nic)) - - return lines - - def clear_dhcp(self): - logger.info('Clearing DHCP leases') - - subprocess.call('pkill dhclient', shell=True) - subprocess.check_call('rm -f /var/lib/dhcp/*', shell=True) - - def if_down_up(self): - names = [] - for nic in self.nics: - name = self.mac2Name.get(nic.mac.lower()) - names.append(name) - - for name in names: - logger.info('Bring down interface %s' % name) - subprocess.check_call('ifdown %s' % name, shell=True) - - self.clear_dhcp() - - for name in names: - logger.info('Bring up interface %s' % name) - subprocess.check_call('ifup %s' % name, shell=True) - - def configure(self): - """ - Configure the /etc/network/intefaces - Make a back up of the original - """ - containingDir = '/etc/network' - - interfaceFile = os.path.join(containingDir, 'interfaces') - originalFile = os.path.join(containingDir, - 'interfaces.before_vmware_customization') - - if not os.path.exists(originalFile) and os.path.exists(interfaceFile): - os.rename(interfaceFile, originalFile) - - lines = self.generate() - with open(interfaceFile, 'w') as fp: - for line in lines: - fp.write('%s\n' % line) - - self.if_down_up() + return primaryNic + + def find_devices(self): + """ + Create the mac2Name dictionary + The mac address(es) are in the lower case + """ + cmd = 'ip addr show' + outText = subprocess.check_output(cmd, shell=True).decode() + sections = re.split(r'\n\d+: ', '\n' + outText)[1:] + + macPat = r'link/ether (([0-9A-Fa-f]{2}[:]){5}([0-9A-Fa-f]{2}))' + for section in sections: + matcher = re.search(macPat, section) + if not matcher: # Only keep info about nics + continue + mac = matcher.group(1).lower() + name = section.split(':', 1)[0] + self.mac2Name[mac] = name + + def gen_one_nic(self, nic): + """ + Return the lines needed to configure a nic + @return (str list): the string list to configure the nic + @param nic (NicBase): the nic to configure + """ + lines = [] + name = self.mac2Name.get(nic.mac.lower()) + if not name: + raise ValueError('No known device has MACADDR: %s' % nic.mac) + + if nic.onboot: + lines.append('auto %s' % name) + + # Customize IPv4 + lines.extend(self.gen_ipv4(name, nic)) + + # Customize IPv6 + lines.extend(self.gen_ipv6(name, nic)) + + lines.append('') + + return lines + + def gen_ipv4(self, name, nic): + """ + Return the lines needed to configure the IPv4 setting of a nic + @return (str list): the string list to configure the gateways + @param name (str): name of the nic + @param nic (NicBase): the nic to configure + """ + lines = [] + + bootproto = nic.bootProto.lower() + if nic.ipv4_mode.lower() == 'disabled': + bootproto = 'manual' + lines.append('iface %s inet %s' % (name, bootproto)) + + if bootproto != 'static': + return lines + + # Static Ipv4 + v4 = nic.staticIpv4 + if v4.ip: + lines.append(' address %s' % v4.ip) + if v4.netmask: + lines.append(' netmask %s' % v4.netmask) + + # Add the primary gateway + if nic.primary and v4.gateways: + self.ipv4PrimaryGateway = v4.gateways[0] + lines.append(' gateway %s metric 0' % self.ipv4PrimaryGateway) + return lines + + # Add routes if there is no primary nic + if not self._primaryNic: + lines.extend(self.gen_ipv4_route(nic, v4.gateways)) + + return lines + + def gen_ipv4_route(self, nic, gateways): + """ + Return the lines needed to configure additional Ipv4 route + @return (str list): the string list to configure the gateways + @param nic (NicBase): the nic to configure + @param gateways (str list): the list of gateways + """ + lines = [] + + for gateway in gateways: + lines.append(' up route add default gw %s metric 10000' % + gateway) + + return lines + + def gen_ipv6(self, name, nic): + """ + Return the lines needed to configure the gateways for a nic + @return (str list): the string list to configure the gateways + @param name (str): name of the nic + @param nic (NicBase): the nic to configure + """ + lines = [] + + if not nic.staticIpv6: + return lines + + # Static Ipv6 + addrs = nic.staticIpv6 + lines.append('iface %s inet6 static' % name) + lines.append(' address %s' % addrs[0].ip) + lines.append(' netmask %s' % addrs[0].netmask) + + for addr in addrs[1:]: + lines.append(' up ifconfig %s inet6 add %s/%s' % (name, addr.ip, + addr.netmask)) + # Add the primary gateway + if nic.primary: + for addr in addrs: + if addr.gateway: + self.ipv6PrimaryGateway = addr.gateway + lines.append(' gateway %s' % self.ipv6PrimaryGateway) + return lines + + # Add routes if there is no primary nic + if not self._primaryNic: + lines.extend(self._genIpv6Route(name, nic, addrs)) + + return lines + + def _genIpv6Route(self, name, nic, addrs): + lines = [] + + for addr in addrs: + lines.append(' up route -A inet6 add default gw %s metric 10000' % + addr.gateway) + + return lines + + def generate(self): + """Return the lines that is needed to configure the nics""" + lines = [] + lines.append('iface lo inet loopback') + lines.append('auto lo') + lines.append('') + + for nic in self.nics: + lines.extend(self.gen_one_nic(nic)) + + return lines + + def clear_dhcp(self): + logger.info('Clearing DHCP leases') + + subprocess.call('pkill dhclient', shell=True) + subprocess.check_call('rm -f /var/lib/dhcp/*', shell=True) + + def if_down_up(self): + names = [] + for nic in self.nics: + name = self.mac2Name.get(nic.mac.lower()) + names.append(name) + + for name in names: + logger.info('Bring down interface %s' % name) + subprocess.check_call('ifdown %s' % name, shell=True) + + self.clear_dhcp() + + for name in names: + logger.info('Bring up interface %s' % name) + subprocess.check_call('ifup %s' % name, shell=True) + + def configure(self): + """ + Configure the /etc/network/intefaces + Make a back up of the original + """ + containingDir = '/etc/network' + + interfaceFile = os.path.join(containingDir, 'interfaces') + originalFile = os.path.join(containingDir, + 'interfaces.before_vmware_customization') + + if not os.path.exists(originalFile) and os.path.exists(interfaceFile): + os.rename(interfaceFile, originalFile) + + lines = self.generate() + with open(interfaceFile, 'w') as fp: + for line in lines: + fp.write('%s\n' % line) + + self.if_down_up() diff --git a/cloudinit/sources/helpers/vmware/imc/nic.py b/cloudinit/sources/helpers/vmware/imc/nic.py index 6628a3ec..b5d704ea 100644 --- a/cloudinit/sources/helpers/vmware/imc/nic.py +++ b/cloudinit/sources/helpers/vmware/imc/nic.py @@ -49,35 +49,35 @@ class Nic(NicBase): def primary(self): value = self._get('PRIMARY') if value: - value = value.lower() - return value == 'yes' or value == 'true' + value = value.lower() + return value == 'yes' or value == 'true' else: - return False + return False @property def onboot(self): value = self._get('ONBOOT') if value: - value = value.lower() - return value == 'yes' or value == 'true' + value = value.lower() + return value == 'yes' or value == 'true' else: - return False + return False @property def bootProto(self): value = self._get('BOOTPROTO') if value: - return value.lower() + return value.lower() else: - return "" + return "" @property def ipv4_mode(self): value = self._get('IPv4_MODE') if value: - return value.lower() + return value.lower() else: - return "" + return "" @property def staticIpv4(self): diff --git a/tests/unittests/test_vmware_config_file.py b/tests/unittests/test_vmware_config_file.py index 51166dd7..d5c7367b 100644 --- a/tests/unittests/test_vmware_config_file.py +++ b/tests/unittests/test_vmware_config_file.py @@ -36,12 +36,12 @@ class TestVmwareConfigFile(unittest.TestCase): cf.clear() - self.assertEqual(0, cf.size(), "clear size") + self.assertEqual(0, len(cf), "clear size") cf._insertKey(" PASSWORD|-PASS ", " foo ") cf._insertKey("BAR", " ") - self.assertEqual(2, cf.size(), "insert size") + self.assertEqual(2, len(cf), "insert size") self.assertEqual('foo', cf["PASSWORD|-PASS"], "password") self.assertTrue("PASSWORD|-PASS" in cf, "hasPassword") self.assertFalse(cf.should_keep_current_value("PASSWORD|-PASS"), -- cgit v1.2.3 From c5d2f79a982258d86181368b25ce6bc6638ef645 Mon Sep 17 00:00:00 2001 From: Sankar Tanguturi Date: Thu, 18 Feb 2016 18:31:07 -0800 Subject: - Removed dmi_data function. - Fixed few variable names. - Used util.subp methods for process related manipulations. --- cloudinit/sources/DataSourceOVF.py | 20 +++-------- cloudinit/sources/helpers/vmware/imc/config_nic.py | 40 +++++++++++----------- 2 files changed, 24 insertions(+), 36 deletions(-) diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 6d3bf7bb..72ba5aba 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -63,15 +63,11 @@ class DataSourceOVF(sources.DataSource): } (seedfile, contents) = get_ovf_env(self.paths.seed_dir) - dmi_info = dmi_data() - system_type = "" - if dmi_info is None: - LOG.debug("No dmidata utility found") - else: - (_, system_type) = dmi_info - - if 'vmware' in system_type.lower(): + system_type = util.read_dmi_data("system-product-name") + if system_type is None: + LOG.debug("No system-product-name found") + elif 'vmware' in system_type.lower(): LOG.debug("VMware Virtual Platform found") deployPkgPluginPath = search_file("/usr/lib/vmware-tools", "libdeployPkgPlugin.so") if deployPkgPluginPath: @@ -347,14 +343,6 @@ def get_properties(contents): return props -def dmi_data(): - sys_uuid = util.read_dmi_data("system-uuid") - sys_type = util.read_dmi_data("system-product-name") - - if not sys_uuid or not sys_type: - return None - - return (sys_uuid.lower(), sys_type) def search_file(dirpath, filename): if not dirpath or not filename: diff --git a/cloudinit/sources/helpers/vmware/imc/config_nic.py b/cloudinit/sources/helpers/vmware/imc/config_nic.py index d79e6936..172a1649 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_nic.py +++ b/cloudinit/sources/helpers/vmware/imc/config_nic.py @@ -22,6 +22,8 @@ import os import subprocess import re +from cloudinit import util + logger = logging.getLogger(__name__) @@ -43,32 +45,30 @@ class NicConfigurator: Retrieve the primary nic if it exists @return (NicBase): the primary nic if exists, None otherwise """ - primaryNic = None - - for nic in self.nics: - if nic.primary: - if primaryNic: - raise Exception('There can only be one primary nic', - primaryNic.mac, nic.mac) - primaryNic = nic - - return primaryNic + primary_nics = [nic for nic in self.nics if nic.primary] + if not primary_nics: + return None + elif len(primary_nics) > 1: + raise Exception('There can only be one primary nic', + [nic.mac for nic in primary_nics]) + else: + return primary_nics[0] def find_devices(self): """ Create the mac2Name dictionary The mac address(es) are in the lower case """ - cmd = 'ip addr show' - outText = subprocess.check_output(cmd, shell=True).decode() - sections = re.split(r'\n\d+: ', '\n' + outText)[1:] + cmd = ['ip', 'addr', 'show'] + (output, err) = util.subp(cmd) + sections = re.split(r'\n\d+: ', '\n' + output)[1:] macPat = r'link/ether (([0-9A-Fa-f]{2}[:]){5}([0-9A-Fa-f]{2}))' for section in sections: - matcher = re.search(macPat, section) - if not matcher: # Only keep info about nics + match = re.search(macPat, section) + if not match: # Only keep info about nics continue - mac = matcher.group(1).lower() + mac = match.group(1).lower() name = section.split(':', 1)[0] self.mac2Name[mac] = name @@ -206,8 +206,8 @@ class NicConfigurator: def clear_dhcp(self): logger.info('Clearing DHCP leases') - subprocess.call('pkill dhclient', shell=True) - subprocess.check_call('rm -f /var/lib/dhcp/*', shell=True) + util.subp(["pkill", "dhclient"]) + util.subp(["rm", "-f", "/var/lib/dhcp/*"]) def if_down_up(self): names = [] @@ -217,13 +217,13 @@ class NicConfigurator: for name in names: logger.info('Bring down interface %s' % name) - subprocess.check_call('ifdown %s' % name, shell=True) + util.subp(["ifdown", "%s" % name]) self.clear_dhcp() for name in names: logger.info('Bring up interface %s' % name) - subprocess.check_call('ifup %s' % name, shell=True) + util.subp(["ifup", "%s" % name]) def configure(self): """ -- cgit v1.2.3 From b20191f04c586147165a304b88a2b89c89f79225 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 25 Feb 2016 14:32:14 -0500 Subject: minor cleanups --- cloudinit/config/cc_lxd.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py index c9cf8704..aaafb643 100644 --- a/cloudinit/config/cc_lxd.py +++ b/cloudinit/config/cc_lxd.py @@ -47,18 +47,20 @@ def handle(name, cfg, cloud, log, args): try: cloud.distro.install_packages(("lxd",)) except util.ProcessExecutionError as e: - log.warn("no lxd executable and could not install lxd: '%s'" % e) + log.warn("no lxd executable and could not install lxd:", e) return # Set up lxd if init config is given + init_keys = ( + 'network_address', 'network_port', 'storage_backend', + 'storage_create_device', 'storage_create_loop', + 'storage_pool', 'trust_password') init_cfg = lxd_cfg.get('init') if init_cfg: if not isinstance(init_cfg, dict): - log.warn("lxd init config must be a dict of flag: val pairs") + log.warn("lxd/init config must be a dictionary. found a '%s'", + type(f)) return - init_keys = ('network_address', 'network_port', 'storage_backend', - 'storage_create_device', 'storage_create_loop', - 'storage_pool', 'trust_password') cmd = ['lxd', 'init', '--auto'] for k in init_keys: if init_cfg.get(k): -- cgit v1.2.3 From b7a6e92274bb0d146c4637a78128bc771f2612e9 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 29 Feb 2016 22:57:37 -0500 Subject: tox.ini: only specify py3 not specific py34 This makes tox work on xenial where python3 is python3.5 and on older (trusty) where python3 is python3.4. --- tox.ini | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 849481f0..b72df0c9 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py34 +envlist = py27,py3 recreate = True [testenv] @@ -7,6 +7,9 @@ commands = python -m nose {posargs:tests} deps = -r{toxinidir}/test-requirements.txt -r{toxinidir}/requirements.txt +[testenv:py3] +basepython = python3 + # https://github.com/gabrielfalcao/HTTPretty/issues/223 setenv = LC_ALL = en_US.utf-8 -- cgit v1.2.3 From 316bdf523b01241036335b27a0afd89f6d3207c2 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 29 Feb 2016 23:02:55 -0500 Subject: setup.py: pep8/flake8 changes only --- setup.py | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/setup.py b/setup.py index 5f61681b..6ff6dde9 100755 --- a/setup.py +++ b/setup.py @@ -45,13 +45,13 @@ def tiny_p(cmd, capture=True): stdout = None stderr = None sp = subprocess.Popen(cmd, stdout=stdout, - stderr=stderr, stdin=None, - universal_newlines=True) + stderr=stderr, stdin=None, + universal_newlines=True) (out, err) = sp.communicate() ret = sp.returncode if ret not in [0]: - raise RuntimeError("Failed running %s [rc=%s] (%s, %s)" - % (cmd, ret, out, err)) + raise RuntimeError("Failed running %s [rc=%s] (%s, %s)" % + (cmd, ret, out, err)) return (out, err) @@ -122,9 +122,8 @@ class InitsysInstallData(install): user_options = install.user_options + [ # This will magically show up in member variable 'init_sys' ('init-system=', None, - ('init system(s) to configure (%s) [default: None]') % - (", ".join(INITSYS_TYPES)) - ), + ('init system(s) to configure (%s) [default: None]' % + (", ".join(INITSYS_TYPES)))), ] def initialize_options(self): @@ -138,7 +137,8 @@ class InitsysInstallData(install): self.init_system = self.init_system.split(",") if len(self.init_system) == 0: - raise DistutilsArgError(("You must specify one of (%s) when" + raise DistutilsArgError( + ("You must specify one of (%s) when" " specifying init system(s)!") % (", ".join(INITSYS_TYPES))) bad = [f for f in self.init_system if f not in INITSYS_TYPES] @@ -182,18 +182,18 @@ if sys.version_info < (3,): requirements.append('cheetah') -setuptools.setup(name='cloud-init', - version=get_version(), - description='EC2 initialisation magic', - author='Scott Moser', - author_email='scott.moser@canonical.com', - url='http://launchpad.net/cloud-init/', - packages=setuptools.find_packages(exclude=['tests']), - scripts=['bin/cloud-init', - 'tools/cloud-init-per', - ], - license='GPLv3', - data_files=data_files, - install_requires=requirements, - cmdclass=cmdclass, - ) +setuptools.setup( + name='cloud-init', + version=get_version(), + description='EC2 initialisation magic', + author='Scott Moser', + author_email='scott.moser@canonical.com', + url='http://launchpad.net/cloud-init/', + packages=setuptools.find_packages(exclude=['tests']), + scripts=['bin/cloud-init', + 'tools/cloud-init-per'], + license='GPLv3', + data_files=data_files, + install_requires=requirements, + cmdclass=cmdclass, + ) -- cgit v1.2.3 From 2a73e674605916077cf6a87410f4ad0829938400 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 29 Feb 2016 23:10:44 -0500 Subject: add cloud-init.target, update service files accordingly This adds cloud-init.target which is a single target to encompass all the services of cloud-init. --- systemd/cloud-config.service | 2 +- systemd/cloud-final.service | 2 +- systemd/cloud-init-local.service | 2 +- systemd/cloud-init.service | 2 +- systemd/cloud-init.target | 16 ++++++++++++++++ 5 files changed, 20 insertions(+), 4 deletions(-) create mode 100644 systemd/cloud-init.target diff --git a/systemd/cloud-config.service b/systemd/cloud-config.service index f9f1996e..45d2a63b 100644 --- a/systemd/cloud-config.service +++ b/systemd/cloud-config.service @@ -13,4 +13,4 @@ TimeoutSec=0 StandardOutput=journal+console [Install] -WantedBy=multi-user.target +WantedBy=cloud-init.target diff --git a/systemd/cloud-final.service b/systemd/cloud-final.service index bcbdd36f..bfb08d4a 100644 --- a/systemd/cloud-final.service +++ b/systemd/cloud-final.service @@ -14,4 +14,4 @@ KillMode=process StandardOutput=journal+console [Install] -WantedBy=multi-user.target +WantedBy=cloud-init.target diff --git a/systemd/cloud-init-local.service b/systemd/cloud-init-local.service index a31985c6..73aa46f6 100644 --- a/systemd/cloud-init-local.service +++ b/systemd/cloud-init-local.service @@ -13,4 +13,4 @@ TimeoutSec=0 StandardOutput=journal+console [Install] -WantedBy=multi-user.target +WantedBy=cloud-init.target diff --git a/systemd/cloud-init.service b/systemd/cloud-init.service index 48920283..1f656f7f 100644 --- a/systemd/cloud-init.service +++ b/systemd/cloud-init.service @@ -15,4 +15,4 @@ TimeoutSec=0 StandardOutput=journal+console [Install] -WantedBy=multi-user.target +WantedBy=cloud-init.target diff --git a/systemd/cloud-init.target b/systemd/cloud-init.target new file mode 100644 index 00000000..03f61002 --- /dev/null +++ b/systemd/cloud-init.target @@ -0,0 +1,16 @@ +# cloud-init target is enabled by cloud-init-generator +# To disable it you can either: +# a.) boot with kernel cmdline of 'cloudinit=disabled' +# b.) touch a file /etc/cloud/cloud-init.disabled +# cloud-init normally emits a "cloud-config" upstart event to inform third +# parties that cloud-config is available, which does us no good when we're +# using systemd. cloud-config.target serves as this synchronization point +# instead. Services that would "start on cloud-config" with upstart can +# instead use "After=cloud-config.target" and "Wants=cloud-config.target" +# as appropriate. + +[Unit] +Description=Cloud-init target +Wants=cloud-init-local.service cloud-init.service +After=cloud-init-local.service cloud-init.service + -- cgit v1.2.3 From df2d690341ba3e5e1c1ed87f12c206f3cfa0ab45 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 29 Feb 2016 23:12:19 -0500 Subject: systemd/cloud-init-generator: add a generator to support disabling This gets installed in /lib/systemd/system/cloud-init.target and then is called to generate the symlink (or not generate the symlink) for the cloud-init target. The end result is cloud-init can be completely disabled by: touch /etc/cloud/cloud-init.disabled or a kernel command line with 'cloud-init=disabled' --- systemd/cloud-init-generator | 123 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100755 systemd/cloud-init-generator diff --git a/systemd/cloud-init-generator b/systemd/cloud-init-generator new file mode 100755 index 00000000..0c698d6b --- /dev/null +++ b/systemd/cloud-init-generator @@ -0,0 +1,123 @@ +#!/bin/sh +set -f + +LOG="" +DEBUG_LEVEL=2 +LOG_D="/run" +ENABLE="enabled" +DISABLE="disabled" +CLOUD_SYSTEM_TARGET="/lib/systemd/system/cloud-init.target" +CLOUD_TARGET_NAME="cloud-init.target" + +debug() { + local lvl="$1" + shift + [ "$lvl" -gt "$DEBUG_LEVEL" ] && return + if [ -z "$LOG" ]; then + local log="$LOG_D/${0##*/}-generator.log" + { : > "$log"; } >/dev/null 2>&1 && LOG="$log" || + LOG="/dev/kmsg" + fi + echo "$@" >> "$LOG" +} + +etc_file() { + local pprefix="${1:-/etc/cloud/cloud-init.}" + _RET="unset" + [ -f "${pprefix}.$ENABLE" ] && _RET="$ENABLE" && return 0 + [ -f "${pprefix}.$DISABLE" ] && _RET="$DISABLE" && return 0 + return 0 +} + +read_proc_cmdline() { + local out="" + if [ "$container" = "lxc" ]; then + _RET="" + return 0 + fi + + if systemd-detect-virt --container --quiet; then + _RET="" + return 0 + fi + + read _RET < /proc/cmdline +} + +kernel_cmdline() { + local cmdline="" tok="" + if [ -n "${KERNEL_CMDLINE+$KERNEL_CMDLINE}" ]; then + cmdline=${KERNEL_CMDLINE} + debug 1 "kernel command line from env KERNEL_CMDLINE: $cmdline" + elif read_proc_cmdline; then + read_proc_cmdline && cmdline="$_RET" + debug 1 "kernel command line from /proc/cmdline: $cmdline" + fi + _RET="unset" + cmdline=" $cmdline " + tok=${cmdline##* cloud-init=} + [ "$tok" = "$cmdline" ] && _RET="unset" + tok=${tok%% *} + [ "$tok" = "$ENABLE" -o "$tok" = "$DISABLE" ] && _RET="$tok" + return 0 +} + +default() { + _RET="$ENABLE" +} + +main() { + local normal_d="$1" early_d="$2" late_d="$3" + local target_name="multi-user.target" gen_d="$early_d" + local link_path="$gen_d/${target_name}.wants/${CLOUD_TARGET_NAME}" + + #LOG_D="${gen_d:-${LOG_D}}" + debug 1 "$0 normal=$normal_d early=$early_d late=$late_d" + debug 2 "$0 $*" + + local search result="error" ret="" + for search in kernel_cmdline etc_file default; do + if $search; then + debug 1 "$search found $_RET" + [ "$_RET" = "$ENABLE" -o "$_RET" = "$DISABLE" ] && result=$_RET + else + ret=$? + debug 0 "search $search returned $ret" + fi + done + + if [ "$result" = "$ENABLE" ]; then + if [ -e "$link_path" ]; then + debug 1 "already enabled: no change needed" + else + [ -d "${link_path%/*}" ] || mkdir -p "${link_path%/*}" || + debug 0 "failed to make dir $link_path" + if ln -snf "$CLOUD_SYSTEM_TARGET" "$link_path"; then + debug 1 "enabled via $link_path -> $CLOUD_SYSTEM_TARGET" + else + ret=$? + debug 0 "[$ret] enable failed:" \ + "ln $CLOUD_SYSTEM_TARGET $link_path" + fi + fi + elif [ "$result" = "$DISABLE" ]; then + if [ -f "$link_path" ]; then + if rm -f "$link_path"; then + debug 1 "disabled. removed existing $link_path" + else + ret=$? + debug 0 "[$ret] disable failed, remove $link_path" + fi + else + debug 1 "already disabled: no change needed" + fi + else + debug 0 "unexpected result '$result'" + ret=3 + fi + return $ret +} + +main "$@" + +# vi: ts=4 expandtab -- cgit v1.2.3 From 0a89426da655b2bf7badf54b38b3b4be2b70e738 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 29 Feb 2016 23:14:39 -0500 Subject: update setup.py to install generator --- setup.py | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index 6ff6dde9..0b261dfe 100755 --- a/setup.py +++ b/setup.py @@ -55,29 +55,40 @@ def tiny_p(cmd, capture=True): return (out, err) -def systemd_unitdir(): - cmd = ['pkg-config', '--variable=systemdsystemunitdir', 'systemd'] +def pkg_config_read(library, var): + fallbacks = { + 'systemd': { + 'systemdsystemunitdir': '/lib/systemd/system', + 'systemdsystemgeneratordir': '/lib/systemd/system-generators', + } + } + cmd = ['pkg-config', '--variable=%s' % var, library] try: (path, err) = tiny_p(cmd) except: - return '/lib/systemd/system' + return fallbacks[library][var] return str(path).strip() + INITSYS_FILES = { 'sysvinit': [f for f in glob('sysvinit/redhat/*') if is_f(f)], 'sysvinit_freebsd': [f for f in glob('sysvinit/freebsd/*') if is_f(f)], 'sysvinit_deb': [f for f in glob('sysvinit/debian/*') if is_f(f)], - 'systemd': [f for f in glob('systemd/*') if is_f(f)], + 'systemd': [f for f in (glob('systemd/*.service') + + glob('systemd/*.target')) if is_f(f)], + 'systemd.generators': [f for f in glob('systemd/*-generator') if is_f(f)], 'upstart': [f for f in glob('upstart/*') if is_f(f)], } INITSYS_ROOTS = { 'sysvinit': '/etc/rc.d/init.d', 'sysvinit_freebsd': '/usr/local/etc/rc.d', 'sysvinit_deb': '/etc/init.d', - 'systemd': systemd_unitdir(), + 'systemd': pkg_config_read('systemd', 'systemdsystemunitdir'), + 'systemd.generators': pkg_config_read('systemd', + 'systemdsystemgeneratordir'), 'upstart': '/etc/init/', } -INITSYS_TYPES = sorted(list(INITSYS_ROOTS.keys())) +INITSYS_TYPES = sorted([f.partition(".")[0] for f in INITSYS_ROOTS.keys()]) # Install everything in the right location and take care of Linux (default) and # FreeBSD systems. @@ -147,8 +158,12 @@ class InitsysInstallData(install): "Invalid --init-system: %s" % (','.join(bad))) for system in self.init_system: - self.distribution.data_files.append( - (INITSYS_ROOTS[system], INITSYS_FILES[system])) + # add data files for anything that starts with '.' + datakeys = [k for k in INITSYS_ROOTS + if k.partition(".")[0] == system] + for k in datakeys: + self.distribution.data_files.append( + (INITSYS_ROOTS[k], INITSYS_FILES[k])) # Force that command to reinitalize (with new file list) self.distribution.reinitialize_command('install_data', True) -- cgit v1.2.3 From 4b22f00f02c6cccd88a81f05db8cc33a4e9a7419 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 29 Feb 2016 23:20:01 -0500 Subject: be less verbose by default --- systemd/cloud-init-generator | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/systemd/cloud-init-generator b/systemd/cloud-init-generator index 0c698d6b..c9900394 100755 --- a/systemd/cloud-init-generator +++ b/systemd/cloud-init-generator @@ -2,7 +2,7 @@ set -f LOG="" -DEBUG_LEVEL=2 +DEBUG_LEVEL=1 LOG_D="/run" ENABLE="enabled" DISABLE="disabled" @@ -71,7 +71,6 @@ main() { local target_name="multi-user.target" gen_d="$early_d" local link_path="$gen_d/${target_name}.wants/${CLOUD_TARGET_NAME}" - #LOG_D="${gen_d:-${LOG_D}}" debug 1 "$0 normal=$normal_d early=$early_d late=$late_d" debug 2 "$0 $*" -- cgit v1.2.3 From af5d2a102c029da4b853a3bc42a6e23ce3a40fe6 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 29 Feb 2016 23:38:40 -0500 Subject: remove unused var --- systemd/cloud-init-generator | 1 - 1 file changed, 1 deletion(-) diff --git a/systemd/cloud-init-generator b/systemd/cloud-init-generator index c9900394..d97b58d9 100755 --- a/systemd/cloud-init-generator +++ b/systemd/cloud-init-generator @@ -30,7 +30,6 @@ etc_file() { } read_proc_cmdline() { - local out="" if [ "$container" = "lxc" ]; then _RET="" return 0 -- cgit v1.2.3 From 14915526ca67bbf7842028d48170015b85f87469 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 1 Mar 2016 00:19:55 -0500 Subject: lxd: general fix after testing A few changes: a.) change to using '--name=value' rather than '--name' 'value' b.) make sure only strings are passed to command (useful for storage_create_loop: which is likely an integer) c.) document simple working example d.) support installing zfs if not present and storage_backedn has it. --- cloudinit/config/cc_lxd.py | 35 ++++++++++++++++++------ doc/examples/cloud-config-lxd.txt | 7 +++++ tests/unittests/test_handler/test_handler_lxd.py | 9 +++--- 3 files changed, 38 insertions(+), 13 deletions(-) diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py index aaafb643..84eec7a5 100644 --- a/cloudinit/config/cc_lxd.py +++ b/cloudinit/config/cc_lxd.py @@ -38,16 +38,36 @@ from cloudinit import util def handle(name, cfg, cloud, log, args): # Get config lxd_cfg = cfg.get('lxd') - if not lxd_cfg and isinstance(lxd_cfg, dict): + if not lxd_cfg: log.debug("Skipping module named %s, not present or disabled by cfg") return + if not isinstance(lxd_cfg, dict): + log.warn("lxd config must be a dictionary. found a '%s'", + type(lxd_cfg)) + return + + init_cfg = lxd_cfg.get('init') + if not init_cfg: + init_cfg = {} + + if not isinstance(init_cfg, dict): + log.warn("lxd/init config must be a dictionary. found a '%s'", + type(init_cfg)) + init_cfg = {} + + packages = [] + if (init_cfg.get("storage_backend") == "zfs" and not util.which('zfs')): + packages.append('zfs') # Ensure lxd is installed if not util.which("lxd"): + packages.append('lxd') + + if len(packages): try: - cloud.distro.install_packages(("lxd",)) + cloud.distro.install_packages(packages) except util.ProcessExecutionError as e: - log.warn("no lxd executable and could not install lxd:", e) + log.warn("failed to install packages %s: %s", packages, e) return # Set up lxd if init config is given @@ -55,14 +75,11 @@ def handle(name, cfg, cloud, log, args): 'network_address', 'network_port', 'storage_backend', 'storage_create_device', 'storage_create_loop', 'storage_pool', 'trust_password') - init_cfg = lxd_cfg.get('init') + if init_cfg: - if not isinstance(init_cfg, dict): - log.warn("lxd/init config must be a dictionary. found a '%s'", - type(f)) - return cmd = ['lxd', 'init', '--auto'] for k in init_keys: if init_cfg.get(k): - cmd.extend(["--%s" % k.replace('_', '-'), init_cfg[k]]) + cmd.extend(["--%s=%s" % + (k.replace('_', '-'), str(init_cfg[k]))]) util.subp(cmd) diff --git a/doc/examples/cloud-config-lxd.txt b/doc/examples/cloud-config-lxd.txt index f66da4c3..b9bb4aa5 100644 --- a/doc/examples/cloud-config-lxd.txt +++ b/doc/examples/cloud-config-lxd.txt @@ -19,3 +19,10 @@ lxd: network_port: 8443 storage_backend: zfs storage_pool: datapool + storage_create_loop: 10 + + +# The simplist working configuration is +# lxd: +# init: +# storage_backend: dir diff --git a/tests/unittests/test_handler/test_handler_lxd.py b/tests/unittests/test_handler/test_handler_lxd.py index 4d858b8f..65794a41 100644 --- a/tests/unittests/test_handler/test_handler_lxd.py +++ b/tests/unittests/test_handler/test_handler_lxd.py @@ -43,9 +43,10 @@ class TestLxd(t_help.TestCase): self.assertTrue(mock_util.which.called) init_call = mock_util.subp.call_args_list[0][0][0] self.assertEquals(init_call, - ['lxd', 'init', '--auto', '--network-address', - '0.0.0.0', '--storage-backend', 'zfs', - '--storage-pool', 'poolname']) + ['lxd', 'init', '--auto', + '--network-address=0.0.0.0', + '--storage-backend=zfs', + '--storage-pool=poolname']) @mock.patch("cloudinit.config.cc_lxd.util") def test_lxd_install(self, mock_util): @@ -55,4 +56,4 @@ class TestLxd(t_help.TestCase): cc_lxd.handle('cc_lxd', self.lxd_cfg, cc, LOG, []) self.assertTrue(cc.distro.install_packages.called) install_pkg = cc.distro.install_packages.call_args_list[0][0][0] - self.assertEquals(install_pkg, ('lxd',)) + self.assertEquals(sorted(install_pkg), ['lxd', 'zfs']) -- cgit v1.2.3 From af3653032704f79f418e976d02994c273e25f87f Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 1 Mar 2016 00:28:52 -0500 Subject: 2 fixups --- systemd/cloud-init-generator | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/systemd/cloud-init-generator b/systemd/cloud-init-generator index d97b58d9..d976ae0b 100755 --- a/systemd/cloud-init-generator +++ b/systemd/cloud-init-generator @@ -24,8 +24,8 @@ debug() { etc_file() { local pprefix="${1:-/etc/cloud/cloud-init.}" _RET="unset" - [ -f "${pprefix}.$ENABLE" ] && _RET="$ENABLE" && return 0 - [ -f "${pprefix}.$DISABLE" ] && _RET="$DISABLE" && return 0 + [ -f "${pprefix}$ENABLE" ] && _RET="$ENABLE" && return 0 + [ -f "${pprefix}$DISABLE" ] && _RET="$DISABLE" && return 0 return 0 } @@ -77,7 +77,8 @@ main() { for search in kernel_cmdline etc_file default; do if $search; then debug 1 "$search found $_RET" - [ "$_RET" = "$ENABLE" -o "$_RET" = "$DISABLE" ] && result=$_RET + [ "$_RET" = "$ENABLE" -o "$_RET" = "$DISABLE" ] && + result=$_RET && break else ret=$? debug 0 "search $search returned $ret" -- cgit v1.2.3 From 290afe72d53b5e38c3781934e23a676a3c1986e5 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 1 Mar 2016 12:30:31 -0500 Subject: timezone: use a symlink when updating /etc/localtime Unless /etc/localtime is an existing file and not a symlink, then we will symlink instead of copying the tz_file to /etc/localtime. The copy was due to an old bug in Ubuntu, symlink should be preferred. LP: #1543025 --- ChangeLog | 2 ++ cloudinit/distros/__init__.py | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index b31148ac..2f1f9f87 100644 --- a/ChangeLog +++ b/ChangeLog @@ -81,6 +81,8 @@ - lxd: add support for setting up lxd using 'lxd init' (LP: #1522879) - Add Image Customization Parser for VMware vSphere Hypervisor Support. [Sankar Tanguturi] + - timezone: use a symlink rather than copy for /etc/localtime + unless it is already a file (LP: #1543025). 0.7.6: - open 0.7.6 diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 71884b32..8167c594 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -897,5 +897,9 @@ def set_etc_timezone(tz, tz_file=None, tz_conf="/etc/timezone", util.write_file(tz_conf, str(tz).rstrip() + "\n") # This ensures that the correct tz will be used for the system if tz_local and tz_file: - util.copy(tz_file, tz_local) + # use a symlink if there exists a symlink or tz_local is not present + if os.path.islink(tz_local) or not os.path.exists(tz_local): + os.symlink(tz_file, tz_local) + else: + util.copy(tz_file, tz_local) return -- cgit v1.2.3 From 51a27968ae9805c747cdc27d35a31c49df6d2217 Mon Sep 17 00:00:00 2001 From: Sankar Tanguturi Date: Tue, 1 Mar 2016 16:43:50 -0800 Subject: Added a kill switch for customization on VMware platform. The customization is set to False by default and is triggered only when the option disable_vmware_customization is set to false in /etc/cloud/cloud.cfg --- cloudinit/sources/DataSourceOVF.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 72ba5aba..d92c128c 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -68,18 +68,23 @@ class DataSourceOVF(sources.DataSource): if system_type is None: LOG.debug("No system-product-name found") elif 'vmware' in system_type.lower(): - LOG.debug("VMware Virtual Platform found") - deployPkgPluginPath = search_file("/usr/lib/vmware-tools", "libdeployPkgPlugin.so") - if deployPkgPluginPath: - vmwareImcConfigFilePath = util.log_time(logfunc=LOG.debug, - msg="waiting for configuration file", - func=wait_for_imc_cfg_file, - args=("/tmp", "cust.cfg")) - - if vmwareImcConfigFilePath: - LOG.debug("Found VMware DeployPkg Config File Path at %s" % vmwareImcConfigFilePath) + LOG.debug("VMware Virtualization Platform found") + if not util.get_cfg_option_bool(self.sys_cfg, + "disable_vmware_customization", + True): + deployPkgPluginPath = search_file("/usr/lib/vmware-tools", "libdeployPkgPlugin.so") + if deployPkgPluginPath: + vmwareImcConfigFilePath = util.log_time(logfunc=LOG.debug, + msg="waiting for configuration file", + func=wait_for_imc_cfg_file, + args=("/tmp", "cust.cfg")) + + if vmwareImcConfigFilePath: + LOG.debug("Found VMware DeployPkg Config File Path at %s" % vmwareImcConfigFilePath) + else: + LOG.debug("Did not find VMware DeployPkg Config File Path") else: - LOG.debug("Didn't find VMware DeployPkg Config File Path") + LOG.debug("Customization for VMware platform is disabled.") if vmwareImcConfigFilePath: try: -- cgit v1.2.3 From ab6f166da7290928d56ff3c62a5280536e1d241f Mon Sep 17 00:00:00 2001 From: root Date: Wed, 2 Mar 2016 08:51:16 +0000 Subject: Added Bigstep datasource. --- cloudinit/sources/DataSourceBigstep.py | 48 ++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 cloudinit/sources/DataSourceBigstep.py diff --git a/cloudinit/sources/DataSourceBigstep.py b/cloudinit/sources/DataSourceBigstep.py new file mode 100644 index 00000000..67d43eb3 --- /dev/null +++ b/cloudinit/sources/DataSourceBigstep.py @@ -0,0 +1,48 @@ +# +# Copyright (C) 2015-2016 Bigstep Cloud Ltd. +# +# Author: Alexandru Sirbu +# + +import json + +from cloudinit import log as logging +from cloudinit import sources +from cloudinit import util +from cloudinit import url_helper + +LOG = logging.getLogger(__name__) + + +class DataSourceBigstep(sources.DataSource): + def __init__(self, sys_cfg, distro, paths): + sources.DataSource.__init__(self, sys_cfg, distro, paths) + self.metadata = {} + self.vendordata_raw = "" + self.userdata_raw = "" + + + def get_data(self, apply_filter=False): + url = get_url_from_file() + response = url_helper.readurl(url) + decoded = json.loads(response.contents) + self.metadata = decoded["metadata"] + self.vendordata_raw = decoded["vendordata_raw"] + self.userdata_raw = decoded["userdata_raw"] + return True + + +def get_url_from_file(): + content = util.load_file("/var/lib/cloud/data/seed/bigstep/url") + return content + +# Used to match classes to dependencies +datasources = [ + (DataSourceBigstep, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), +] + + +# Return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return sources.list_from_depends(depends, datasources) + -- cgit v1.2.3 From d5d89cfb1e61e6cc3f732a18ec1aa4d2b288489d Mon Sep 17 00:00:00 2001 From: root Date: Wed, 2 Mar 2016 08:53:47 +0000 Subject: Pep8 changes to Bigstep datasource. --- cloudinit/sources/DataSourceBigstep.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/cloudinit/sources/DataSourceBigstep.py b/cloudinit/sources/DataSourceBigstep.py index 67d43eb3..c22ffdb6 100644 --- a/cloudinit/sources/DataSourceBigstep.py +++ b/cloudinit/sources/DataSourceBigstep.py @@ -21,7 +21,6 @@ class DataSourceBigstep(sources.DataSource): self.vendordata_raw = "" self.userdata_raw = "" - def get_data(self, apply_filter=False): url = get_url_from_file() response = url_helper.readurl(url) @@ -45,4 +44,3 @@ datasources = [ # Return a list of data sources that match this set of dependencies def get_datasource_list(depends): return sources.list_from_depends(depends, datasources) - -- cgit v1.2.3 From 603bdecc5aaa34043379aa4311c271d52dfe61e8 Mon Sep 17 00:00:00 2001 From: Alex Sirbu Date: Wed, 2 Mar 2016 09:15:42 +0000 Subject: Added the hashed_passwd argument for the function create_user, which uses the already implemented functionality of changing the password with a hashed string, but which wasn't used anywhere. --- cloudinit/distros/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 8167c594..6778c93a 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -393,6 +393,10 @@ class Distro(object): if 'plain_text_passwd' in kwargs and kwargs['plain_text_passwd']: self.set_passwd(name, kwargs['plain_text_passwd']) + # Set password if hashed password is provided and non-empty + if 'hashed_passwd' in kwargs and kwargs['hashed_passwd']: + self.set_passwd(name, kwargs['hashed_passwd'], True) + # Default locking down the account. 'lock_passwd' defaults to True. # lock account unless lock_password is False. if kwargs.get('lock_passwd', True): -- cgit v1.2.3 From 921728d42731091f849d21dbef0920b84c559480 Mon Sep 17 00:00:00 2001 From: Alex Sirbu Date: Wed, 2 Mar 2016 09:57:05 +0000 Subject: Used keyword for parameter in order to make it clearer what it represents. --- cloudinit/distros/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 6778c93a..fec18cd2 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -395,7 +395,7 @@ class Distro(object): # Set password if hashed password is provided and non-empty if 'hashed_passwd' in kwargs and kwargs['hashed_passwd']: - self.set_passwd(name, kwargs['hashed_passwd'], True) + self.set_passwd(name, kwargs['hashed_passwd'], hashed=True) # Default locking down the account. 'lock_passwd' defaults to True. # lock account unless lock_password is False. -- cgit v1.2.3 From 568d15d7fb239e609fb70cc7c7a08205e640bf25 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Wed, 2 Mar 2016 13:23:55 -0600 Subject: Fix logic error in lxd config check If the cloud-config does not contain and lxd dictionary then we should not attempt to install the package. Change the latter half of the check to negate the dictionary type check. This fix prevents us from always installing lxd, rather than only installing when we have a config. Fix pyflakes check on init_cfg dict error message. --- cloudinit/config/cc_lxd.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py index aaafb643..7d8a0202 100644 --- a/cloudinit/config/cc_lxd.py +++ b/cloudinit/config/cc_lxd.py @@ -38,7 +38,7 @@ from cloudinit import util def handle(name, cfg, cloud, log, args): # Get config lxd_cfg = cfg.get('lxd') - if not lxd_cfg and isinstance(lxd_cfg, dict): + if not lxd_cfg and not isinstance(lxd_cfg, dict): log.debug("Skipping module named %s, not present or disabled by cfg") return @@ -59,7 +59,7 @@ def handle(name, cfg, cloud, log, args): if init_cfg: if not isinstance(init_cfg, dict): log.warn("lxd/init config must be a dictionary. found a '%s'", - type(f)) + type(init_cfg)) return cmd = ['lxd', 'init', '--auto'] for k in init_keys: -- cgit v1.2.3 From c496b6a11d504ef62371cb5e03ac80b4ceb37540 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 3 Mar 2016 12:20:48 -0500 Subject: run pyflakes in more places, fix fallout this makes 'make' run pyflakes, so failures there will stop a build. also adds it to tox. --- Makefile | 6 ++++-- cloudinit/sources/DataSourceOVF.py | 3 ++- cloudinit/sources/helpers/vmware/imc/config_nic.py | 1 - cloudinit/util.py | 2 +- tests/unittests/test_datasource/test_azure_helper.py | 2 -- tests/unittests/test_datasource/test_smartos.py | 1 - tests/unittests/test_handler/test_handler_power_state.py | 2 +- tox.ini | 6 +++++- 8 files changed, 13 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index bb0c5253..8987d51c 100644 --- a/Makefile +++ b/Makefile @@ -14,13 +14,15 @@ ifeq ($(distro),) distro = redhat endif -all: test check_version +all: check + +check: test check_version pyflakes pep8: @$(CWD)/tools/run-pep8 $(PY_FILES) pyflakes: - @$(CWD)/tools/tox-venv py34 pyflakes $(PY_FILES) + @pyflakes $(PY_FILES) pip-requirements: @echo "Installing cloud-init dependencies..." diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 72ba5aba..d12601a4 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -90,7 +90,8 @@ class DataSourceOVF(sources.DataSource): nicConfigurator.configure() vmwarePlatformFound = True except Exception as inst: - LOG.debug("Error while parsing the Customization Config File") + LOG.debug("Error while parsing the Customization " + "Config File: %s", inst) finally: dirPath = os.path.dirname(vmwareImcConfigFilePath) shutil.rmtree(dirPath) diff --git a/cloudinit/sources/helpers/vmware/imc/config_nic.py b/cloudinit/sources/helpers/vmware/imc/config_nic.py index 172a1649..6d721134 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_nic.py +++ b/cloudinit/sources/helpers/vmware/imc/config_nic.py @@ -19,7 +19,6 @@ import logging import os -import subprocess import re from cloudinit import util diff --git a/cloudinit/util.py b/cloudinit/util.py index 45d49e66..0a639bb9 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2147,7 +2147,7 @@ def _read_dmi_syspath(key): LOG.debug("dmi data %s returned %s", dmi_key_path, key_data) return key_data.strip() - except Exception as e: + except Exception: logexc(LOG, "failed read of %s", dmi_key_path) return None diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 8dbdfb0b..1134199b 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -1,6 +1,4 @@ import os -import struct -import unittest from cloudinit.sources.helpers import azure as azure_helper from ..helpers import TestCase diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 1235436d..ccb9f080 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -31,7 +31,6 @@ import shutil import stat import tempfile import uuid -import unittest from binascii import crc32 import serial diff --git a/tests/unittests/test_handler/test_handler_power_state.py b/tests/unittests/test_handler/test_handler_power_state.py index 5687b10d..cd376e9c 100644 --- a/tests/unittests/test_handler/test_handler_power_state.py +++ b/tests/unittests/test_handler/test_handler_power_state.py @@ -107,7 +107,7 @@ def check_lps_ret(psc_return, mode=None): if 'shutdown' not in psc_return[0][0]: errs.append("string 'shutdown' not in cmd") - if 'condition' is None: + if condition is None: errs.append("condition was not returned") if mode is not None: diff --git a/tox.ini b/tox.ini index b72df0c9..fd65f6ef 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py3 +envlist = py27,py3,pyflakes recreate = True [testenv] @@ -10,6 +10,10 @@ deps = -r{toxinidir}/test-requirements.txt [testenv:py3] basepython = python3 +[testenv:pyflakes] +basepython = python3 +commands = {envpython} -m pyflakes {posargs:cloudinit/ tests/ tools/} + # https://github.com/gabrielfalcao/HTTPretty/issues/223 setenv = LC_ALL = en_US.utf-8 -- cgit v1.2.3 From 112d25e6b2a20e8e39c1f22f6a30e0130616c64f Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 3 Mar 2016 12:53:41 -0500 Subject: make package build run tests --- Makefile | 4 ++-- packages/debian/rules.in | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 8987d51c..a7fa5a3b 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ CWD=$(shell pwd) PY_FILES=$(shell find cloudinit bin tests tools -name "*.py" -type f ) PY_FILES+="bin/cloud-init" +noseopts ?= -v YAML_FILES=$(shell find cloudinit bin tests tools -name "*.yaml" -type f ) YAML_FILES+=$(shell find doc/examples -name "cloud-config*.txt" -type f ) @@ -33,8 +34,7 @@ pip-test-requirements: $(PIP_INSTALL) -r "$@.txt" -q test: clean_pyc - @echo "Running tests..." - @nosetests $(noseopts) tests/ + @n=$$(which nosetests3) || n=nosetests; set -- $$n $(noseopts) tests/; echo "Running $$*"; "$$@" check_version: @if [ "$(CHANGELOG_VERSION)" != "$(CODE_VERSION)" ]; then \ diff --git a/packages/debian/rules.in b/packages/debian/rules.in index bb2e1d5c..d6cd23ae 100755 --- a/packages/debian/rules.in +++ b/packages/debian/rules.in @@ -16,4 +16,4 @@ override_dh_install: override_dh_auto_test: # Because setup tools didn't copy data... [ ! -d .pybuild/pythonX.Y_?.?/build/tests ] || cp -r tests/data .pybuild/pythonX.Y_?.?/build/tests - http_proxy= dh_auto_test -- --test-nose + http_proxy= make check -- cgit v1.2.3 From 9a3a490fadc5b61e0fa14cde2d4f79164115ae25 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Thu, 3 Mar 2016 13:30:30 -0600 Subject: Makefile: add make check target to run pep8, pyflakes, unittests Run a check of pep8, pyflakes (py27), pyflakes3 (py34), and use nosetest and nosetest3 to run unittests. We should pass all of these. --- Makefile | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Makefile b/Makefile index bb0c5253..058ac199 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,7 @@ YAML_FILES+=$(shell find doc/examples -name "cloud-config*.txt" -type f ) CHANGELOG_VERSION=$(shell $(CWD)/tools/read-version) CODE_VERSION=$(shell python -c "from cloudinit import version; print version.version_string()") +noseopts ?= -vv --nologcapture PIP_INSTALL := pip install @@ -16,12 +17,21 @@ endif all: test check_version +check: pep8 pyflakes pyflakes3 unittest + pep8: @$(CWD)/tools/run-pep8 $(PY_FILES) +pyflakes: + @$(CWD)/tools/tox-venv py27 pyflakes $(PY_FILES) + pyflakes: @$(CWD)/tools/tox-venv py34 pyflakes $(PY_FILES) +unittest: + nosetests $(noseopts) tests/unittests + nosetests3 $(noseopts) tests/unittests + pip-requirements: @echo "Installing cloud-init dependencies..." $(PIP_INSTALL) -r "$@.txt" -q -- cgit v1.2.3 From 96f1742b36241cee152aa2cb5b4a5e1a267a4770 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 3 Mar 2016 15:17:24 -0500 Subject: fix lxd module to not do anything unless config provided --- cloudinit/config/cc_lxd.py | 30 ++++++++++++------------ tests/unittests/test_handler/test_handler_lxd.py | 16 +++++++++++++ 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py index 84eec7a5..80a4d219 100644 --- a/cloudinit/config/cc_lxd.py +++ b/cloudinit/config/cc_lxd.py @@ -47,22 +47,24 @@ def handle(name, cfg, cloud, log, args): return init_cfg = lxd_cfg.get('init') - if not init_cfg: - init_cfg = {} - if not isinstance(init_cfg, dict): log.warn("lxd/init config must be a dictionary. found a '%s'", type(init_cfg)) init_cfg = {} - packages = [] - if (init_cfg.get("storage_backend") == "zfs" and not util.which('zfs')): - packages.append('zfs') + if not init_cfg: + log.debug("no lxd/init config. disabled.") + return + packages = [] # Ensure lxd is installed if not util.which("lxd"): packages.append('lxd') - + + # if using zfs, get the utils + if (init_cfg.get("storage_backend") == "zfs" and not util.which('zfs')): + packages.append('zfs') + if len(packages): try: cloud.distro.install_packages(packages) @@ -75,11 +77,9 @@ def handle(name, cfg, cloud, log, args): 'network_address', 'network_port', 'storage_backend', 'storage_create_device', 'storage_create_loop', 'storage_pool', 'trust_password') - - if init_cfg: - cmd = ['lxd', 'init', '--auto'] - for k in init_keys: - if init_cfg.get(k): - cmd.extend(["--%s=%s" % - (k.replace('_', '-'), str(init_cfg[k]))]) - util.subp(cmd) + cmd = ['lxd', 'init', '--auto'] + for k in init_keys: + if init_cfg.get(k): + cmd.extend(["--%s=%s" % + (k.replace('_', '-'), str(init_cfg[k]))]) + util.subp(cmd) diff --git a/tests/unittests/test_handler/test_handler_lxd.py b/tests/unittests/test_handler/test_handler_lxd.py index 65794a41..7ffa2a53 100644 --- a/tests/unittests/test_handler/test_handler_lxd.py +++ b/tests/unittests/test_handler/test_handler_lxd.py @@ -57,3 +57,19 @@ class TestLxd(t_help.TestCase): self.assertTrue(cc.distro.install_packages.called) install_pkg = cc.distro.install_packages.call_args_list[0][0][0] self.assertEquals(sorted(install_pkg), ['lxd', 'zfs']) + + @mock.patch("cloudinit.config.cc_lxd.util") + def test_no_init_does_nothing(self, mock_util): + cc = self._get_cloud('ubuntu') + cc.distro = mock.MagicMock() + cc_lxd.handle('cc_lxd', {'lxd': {}}, cc, LOG, []) + self.assertFalse(cc.distro.install_packages.called) + self.assertFalse(mock_util.subp.called) + + @mock.patch("cloudinit.config.cc_lxd.util") + def test_no_lxd_does_nothing(self, mock_util): + cc = self._get_cloud('ubuntu') + cc.distro = mock.MagicMock() + cc_lxd.handle('cc_lxd', {'package_update': True}, cc, LOG, []) + self.assertFalse(cc.distro.install_packages.called) + self.assertFalse(mock_util.subp.called) -- cgit v1.2.3 From cb64cf1e14a474794654f5d1586b117912bed4f9 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 3 Mar 2016 15:21:48 -0500 Subject: fix some of pylints complaints --- cloudinit/config/cc_lxd.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py index 80a4d219..63b8fb63 100644 --- a/cloudinit/config/cc_lxd.py +++ b/cloudinit/config/cc_lxd.py @@ -49,7 +49,7 @@ def handle(name, cfg, cloud, log, args): init_cfg = lxd_cfg.get('init') if not isinstance(init_cfg, dict): log.warn("lxd/init config must be a dictionary. found a '%s'", - type(init_cfg)) + type(init_cfg)) init_cfg = {} if not init_cfg: @@ -62,14 +62,14 @@ def handle(name, cfg, cloud, log, args): packages.append('lxd') # if using zfs, get the utils - if (init_cfg.get("storage_backend") == "zfs" and not util.which('zfs')): + if init_cfg.get("storage_backend") == "zfs" and not util.which('zfs'): packages.append('zfs') if len(packages): try: cloud.distro.install_packages(packages) - except util.ProcessExecutionError as e: - log.warn("failed to install packages %s: %s", packages, e) + except util.ProcessExecutionError as exc: + log.warn("failed to install packages %s: %s", packages, exc) return # Set up lxd if init config is given -- cgit v1.2.3 From f4c25ab96c572e0a503bb211a11cd2641ac321a3 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 3 Mar 2016 16:53:49 -0500 Subject: consume KERNEL_CMDLINE even if set to "". explain 'container' --- systemd/cloud-init-generator | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/systemd/cloud-init-generator b/systemd/cloud-init-generator index d976ae0b..b09924ac 100755 --- a/systemd/cloud-init-generator +++ b/systemd/cloud-init-generator @@ -8,6 +8,8 @@ ENABLE="enabled" DISABLE="disabled" CLOUD_SYSTEM_TARGET="/lib/systemd/system/cloud-init.target" CLOUD_TARGET_NAME="cloud-init.target" +# lxc sets 'container', but lets make that explicitly a global +CONTAINER="${container}" debug() { local lvl="$1" @@ -30,7 +32,7 @@ etc_file() { } read_proc_cmdline() { - if [ "$container" = "lxc" ]; then + if [ "$CONTAINER" = "lxc" ]; then _RET="" return 0 fi @@ -45,7 +47,8 @@ read_proc_cmdline() { kernel_cmdline() { local cmdline="" tok="" - if [ -n "${KERNEL_CMDLINE+$KERNEL_CMDLINE}" ]; then + if [ -n "${KERNEL_CMDLINE+x}" ]; then + # use KERNEL_CMDLINE if present in environment even if empty cmdline=${KERNEL_CMDLINE} debug 1 "kernel command line from env KERNEL_CMDLINE: $cmdline" elif read_proc_cmdline; then -- cgit v1.2.3 From 8cb7c3f7b5339e686bfbf95996b51afafeaf9c9e Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Thu, 3 Mar 2016 16:20:10 -0600 Subject: Update pep8 runner and fix pep8 issues --- Makefile | 9 ++-- bin/cloud-init | 43 +++++++++--------- cloudinit/config/cc_apt_configure.py | 6 ++- cloudinit/config/cc_disk_setup.py | 31 +++++++------ cloudinit/config/cc_grub_dpkg.py | 8 ++-- cloudinit/config/cc_keys_to_console.py | 2 +- cloudinit/config/cc_lxd.py | 2 +- cloudinit/config/cc_mounts.py | 12 ++--- cloudinit/config/cc_power_state_change.py | 2 +- cloudinit/config/cc_puppet.py | 6 +-- cloudinit/config/cc_resizefs.py | 2 +- cloudinit/config/cc_rh_subscription.py | 4 +- cloudinit/config/cc_set_hostname.py | 2 +- cloudinit/config/cc_ssh.py | 7 +-- cloudinit/config/cc_update_etc_hosts.py | 6 +-- cloudinit/config/cc_update_hostname.py | 2 +- cloudinit/config/cc_yum_add_repo.py | 2 +- cloudinit/distros/__init__.py | 12 ++--- cloudinit/distros/arch.py | 6 +-- cloudinit/distros/debian.py | 5 ++- cloudinit/distros/freebsd.py | 4 +- cloudinit/distros/gentoo.py | 4 +- cloudinit/distros/parsers/hostname.py | 2 +- cloudinit/distros/parsers/resolv_conf.py | 2 +- cloudinit/distros/parsers/sys_conf.py | 7 ++- cloudinit/filters/launch_index.py | 2 +- cloudinit/helpers.py | 7 +-- cloudinit/sources/DataSourceAzure.py | 21 +++++---- cloudinit/sources/DataSourceConfigDrive.py | 2 +- cloudinit/sources/DataSourceEc2.py | 10 ++--- cloudinit/sources/DataSourceMAAS.py | 15 ++++--- cloudinit/sources/DataSourceOVF.py | 4 +- cloudinit/sources/DataSourceOpenNebula.py | 3 +- cloudinit/sources/DataSourceSmartOS.py | 7 ++- cloudinit/ssh_util.py | 3 +- cloudinit/stages.py | 18 ++++---- cloudinit/url_helper.py | 6 +-- cloudinit/util.py | 15 ++++--- tests/unittests/test_data.py | 5 ++- tests/unittests/test_datasource/test_altcloud.py | 23 +++++----- tests/unittests/test_datasource/test_azure.py | 15 ++++--- .../unittests/test_datasource/test_configdrive.py | 12 ++--- tests/unittests/test_datasource/test_maas.py | 16 +++---- tests/unittests/test_datasource/test_smartos.py | 6 +-- .../test_handler/test_handler_power_state.py | 3 +- .../test_handler/test_handler_seed_random.py | 3 +- .../unittests/test_handler/test_handler_snappy.py | 3 +- tests/unittests/test_sshutil.py | 3 +- tests/unittests/test_templating.py | 3 +- tools/hacking.py | 16 +++---- tools/mock-meta.py | 27 +++++++----- tools/run-pep8 | 51 ++++++++-------------- 52 files changed, 244 insertions(+), 243 deletions(-) diff --git a/Makefile b/Makefile index 058ac199..fb65b70b 100644 --- a/Makefile +++ b/Makefile @@ -20,13 +20,14 @@ all: test check_version check: pep8 pyflakes pyflakes3 unittest pep8: - @$(CWD)/tools/run-pep8 $(PY_FILES) + @$(CWD)/tools/run-pep8 pyflakes: - @$(CWD)/tools/tox-venv py27 pyflakes $(PY_FILES) + @$(CWD)/tools/run-pyflakes -pyflakes: - @$(CWD)/tools/tox-venv py34 pyflakes $(PY_FILES) +pyflakes3: + @$(CWD)/tools/run-pyflakes3 + unittest: nosetests $(noseopts) tests/unittests diff --git a/bin/cloud-init b/bin/cloud-init index 9b90c45e..7f665e7e 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -194,7 +194,7 @@ def main_init(name, args): if args.debug: # Reset so that all the debug handlers are closed out LOG.debug(("Logging being reset, this logger may no" - " longer be active shortly")) + " longer be active shortly")) logging.resetLogging() logging.setupLogging(init.cfg) apply_reporting_cfg(init.cfg) @@ -276,9 +276,9 @@ def main_init(name, args): # This may run user-data handlers and/or perform # url downloads and such as needed. (ran, _results) = init.cloudify().run('consume_data', - init.consume_data, - args=[PER_INSTANCE], - freq=PER_INSTANCE) + init.consume_data, + args=[PER_INSTANCE], + freq=PER_INSTANCE) if not ran: # Just consume anything that is set to run per-always # if nothing ran in the per-instance code @@ -349,7 +349,7 @@ def main_modules(action_name, args): if args.debug: # Reset so that all the debug handlers are closed out LOG.debug(("Logging being reset, this logger may no" - " longer be active shortly")) + " longer be active shortly")) logging.resetLogging() logging.setupLogging(mods.cfg) apply_reporting_cfg(init.cfg) @@ -534,7 +534,8 @@ def status_wrapper(name, args, data_d=None, link_d=None): errors.extend(v1[m].get('errors', [])) atomic_write_json(result_path, - {'v1': {'datasource': v1['datasource'], 'errors': errors}}) + {'v1': {'datasource': v1['datasource'], + 'errors': errors}}) util.sym_link(os.path.relpath(result_path, link_d), result_link, force=True) @@ -578,13 +579,13 @@ def main(): # These settings are used for the 'config' and 'final' stages parser_mod = subparsers.add_parser('modules', - help=('activates modules ' - 'using a given configuration key')) + help=('activates modules using ' + 'a given configuration key')) parser_mod.add_argument("--mode", '-m', action='store', - help=("module configuration name " - "to use (default: %(default)s)"), - default='config', - choices=('init', 'config', 'final')) + help=("module configuration name " + "to use (default: %(default)s)"), + default='config', + choices=('init', 'config', 'final')) parser_mod.set_defaults(action=('modules', main_modules)) # These settings are used when you want to query information @@ -600,22 +601,22 @@ def main(): # This subcommand allows you to run a single module parser_single = subparsers.add_parser('single', - help=('run a single module ')) + help=('run a single module ')) parser_single.set_defaults(action=('single', main_single)) parser_single.add_argument("--name", '-n', action="store", - help="module name to run", - required=True) + help="module name to run", + required=True) parser_single.add_argument("--frequency", action="store", - help=("frequency of the module"), - required=False, - choices=list(FREQ_SHORT_NAMES.keys())) + help=("frequency of the module"), + required=False, + choices=list(FREQ_SHORT_NAMES.keys())) parser_single.add_argument("--report", action="store_true", help="enable reporting", required=False) parser_single.add_argument("module_args", nargs="*", - metavar='argument', - help=('any additional arguments to' - ' pass to this module')) + metavar='argument', + help=('any additional arguments to' + ' pass to this module')) parser_single.set_defaults(action=('single', main_single)) args = parser.parse_args() diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 9e9e9e26..702977cb 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -91,7 +91,8 @@ def handle(name, cfg, cloud, log, _args): if matchcfg: matcher = re.compile(matchcfg).search else: - matcher = lambda f: False + def matcher(x): + return False errors = add_sources(cfg['apt_sources'], params, aa_repo_match=matcher) @@ -173,7 +174,8 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): template_params = {} if aa_repo_match is None: - aa_repo_match = lambda f: False + def aa_repo_match(x): + return False errorlist = [] for ent in srclist: diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index d5b0d1d7..0ecc2e4c 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -167,11 +167,12 @@ def enumerate_disk(device, nodeps=False): parts = [x for x in (info.strip()).splitlines() if len(x.split()) > 0] for part in parts: - d = {'name': None, - 'type': None, - 'fstype': None, - 'label': None, - } + d = { + 'name': None, + 'type': None, + 'fstype': None, + 'label': None, + } for key, value in value_splitter(part): d[key.lower()] = value @@ -701,11 +702,12 @@ def lookup_force_flag(fs): """ A force flag might be -F or -F, this look it up """ - flags = {'ext': '-F', - 'btrfs': '-f', - 'xfs': '-f', - 'reiserfs': '-f', - } + flags = { + 'ext': '-F', + 'btrfs': '-f', + 'xfs': '-f', + 'reiserfs': '-f', + } if 'ext' in fs.lower(): fs = 'ext' @@ -824,10 +826,11 @@ def mkfs(fs_cfg): # Create the commands if fs_cmd: - fs_cmd = fs_cfg['cmd'] % {'label': label, - 'filesystem': fs_type, - 'device': device, - } + fs_cmd = fs_cfg['cmd'] % { + 'label': label, + 'filesystem': fs_type, + 'device': device, + } else: # Find the mkfs command mkfs_cmd = util.which("mkfs.%s" % fs_type) diff --git a/cloudinit/config/cc_grub_dpkg.py b/cloudinit/config/cc_grub_dpkg.py index 456597af..acd3e60a 100644 --- a/cloudinit/config/cc_grub_dpkg.py +++ b/cloudinit/config/cc_grub_dpkg.py @@ -38,11 +38,11 @@ def handle(name, cfg, _cloud, log, _args): idevs = util.get_cfg_option_str(mycfg, "grub-pc/install_devices", None) idevs_empty = util.get_cfg_option_str(mycfg, - "grub-pc/install_devices_empty", None) + "grub-pc/install_devices_empty", + None) if ((os.path.exists("/dev/sda1") and not os.path.exists("/dev/sda")) or - (os.path.exists("/dev/xvda1") - and not os.path.exists("/dev/xvda"))): + (os.path.exists("/dev/xvda1") and not os.path.exists("/dev/xvda"))): if idevs is None: idevs = "" if idevs_empty is None: @@ -66,7 +66,7 @@ def handle(name, cfg, _cloud, log, _args): (idevs, idevs_empty)) log.debug("Setting grub debconf-set-selections with '%s','%s'" % - (idevs, idevs_empty)) + (idevs, idevs_empty)) try: util.subp(['debconf-set-selections'], dconf_sel) diff --git a/cloudinit/config/cc_keys_to_console.py b/cloudinit/config/cc_keys_to_console.py index f1c1adff..aa844ee9 100644 --- a/cloudinit/config/cc_keys_to_console.py +++ b/cloudinit/config/cc_keys_to_console.py @@ -48,7 +48,7 @@ def handle(name, cfg, cloud, log, _args): "ssh_fp_console_blacklist", []) key_blacklist = util.get_cfg_option_list(cfg, "ssh_key_console_blacklist", - ["ssh-dss"]) + ["ssh-dss"]) try: cmd = [helper_path] diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py index 7d8a0202..e2fdf68e 100644 --- a/cloudinit/config/cc_lxd.py +++ b/cloudinit/config/cc_lxd.py @@ -59,7 +59,7 @@ def handle(name, cfg, cloud, log, args): if init_cfg: if not isinstance(init_cfg, dict): log.warn("lxd/init config must be a dictionary. found a '%s'", - type(init_cfg)) + type(init_cfg)) return cmd = ['lxd', 'init', '--auto'] for k in init_keys: diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 11089d8d..4fe3ee21 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -204,12 +204,12 @@ def setup_swapfile(fname, size=None, maxsize=None): try: util.ensure_dir(tdir) util.log_time(LOG.debug, msg, func=util.subp, - args=[['sh', '-c', - ('rm -f "$1" && umask 0066 && ' - '{ fallocate -l "${2}M" "$1" || ' - ' dd if=/dev/zero "of=$1" bs=1M "count=$2"; } && ' - 'mkswap "$1" || { r=$?; rm -f "$1"; exit $r; }'), - 'setup_swap', fname, mbsize]]) + args=[['sh', '-c', + ('rm -f "$1" && umask 0066 && ' + '{ fallocate -l "${2}M" "$1" || ' + ' dd if=/dev/zero "of=$1" bs=1M "count=$2"; } && ' + 'mkswap "$1" || { r=$?; rm -f "$1"; exit $r; }'), + 'setup_swap', fname, mbsize]]) except Exception as e: raise IOError("Failed %s: %s" % (msg, e)) diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py index 7d9567e3..cc3f7f70 100644 --- a/cloudinit/config/cc_power_state_change.py +++ b/cloudinit/config/cc_power_state_change.py @@ -105,7 +105,7 @@ def handle(_name, cfg, _cloud, log, _args): log.debug("After pid %s ends, will execute: %s" % (mypid, ' '.join(args))) - util.fork_cb(run_after_pid_gone, mypid, cmdline, timeout, log, + util.fork_cb(run_after_pid_gone, mypid, cmdline, timeout, log, condition, execmd, [args, devnull_fp]) diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py index 4501598e..774d3322 100644 --- a/cloudinit/config/cc_puppet.py +++ b/cloudinit/config/cc_puppet.py @@ -36,8 +36,8 @@ def _autostart_puppet(log): # Set puppet to automatically start if os.path.exists('/etc/default/puppet'): util.subp(['sed', '-i', - '-e', 's/^START=.*/START=yes/', - '/etc/default/puppet'], capture=False) + '-e', 's/^START=.*/START=yes/', + '/etc/default/puppet'], capture=False) elif os.path.exists('/bin/systemctl'): util.subp(['/bin/systemctl', 'enable', 'puppet.service'], capture=False) @@ -65,7 +65,7 @@ def handle(name, cfg, cloud, log, _args): " doing nothing.")) elif install: log.debug(("Attempting to install puppet %s,"), - version if version else 'latest') + version if version else 'latest') cloud.distro.install_packages(('puppet', version)) # ... and then update the puppet configuration diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index cbc07853..2a2a9f59 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -166,7 +166,7 @@ def handle(name, cfg, _cloud, log, args): func=do_resize, args=(resize_cmd, log)) else: util.log_time(logfunc=log.debug, msg="Resizing", - func=do_resize, args=(resize_cmd, log)) + func=do_resize, args=(resize_cmd, log)) action = 'Resized' if resize_root == NOBLOCK: diff --git a/cloudinit/config/cc_rh_subscription.py b/cloudinit/config/cc_rh_subscription.py index 3b30c47e..6f474aed 100644 --- a/cloudinit/config/cc_rh_subscription.py +++ b/cloudinit/config/cc_rh_subscription.py @@ -127,8 +127,8 @@ class SubscriptionManager(object): return False, not_bool if (self.servicelevel is not None) and \ - ((not self.auto_attach) - or (util.is_false(str(self.auto_attach)))): + ((not self.auto_attach) or + (util.is_false(str(self.auto_attach)))): no_auto = ("The service-level key must be used in conjunction " "with the auto-attach key. Please re-run with " diff --git a/cloudinit/config/cc_set_hostname.py b/cloudinit/config/cc_set_hostname.py index 5d7f4331..f43d8d5a 100644 --- a/cloudinit/config/cc_set_hostname.py +++ b/cloudinit/config/cc_set_hostname.py @@ -24,7 +24,7 @@ from cloudinit import util def handle(name, cfg, cloud, log, _args): if util.get_cfg_option_bool(cfg, "preserve_hostname", False): log.debug(("Configuration option 'preserve_hostname' is set," - " not setting the hostname in module %s"), name) + " not setting the hostname in module %s"), name) return (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py index 5bd2dec6..d24e43c0 100644 --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -30,9 +30,10 @@ from cloudinit import distros as ds from cloudinit import ssh_util from cloudinit import util -DISABLE_ROOT_OPTS = ("no-port-forwarding,no-agent-forwarding," -"no-X11-forwarding,command=\"echo \'Please login as the user \\\"$USER\\\" " -"rather than the user \\\"root\\\".\';echo;sleep 10\"") +DISABLE_ROOT_OPTS = ( + "no-port-forwarding,no-agent-forwarding," + "no-X11-forwarding,command=\"echo \'Please login as the user \\\"$USER\\\"" + " rather than the user \\\"root\\\".\';echo;sleep 10\"") GENERATE_KEY_NAMES = ['rsa', 'dsa', 'ecdsa', 'ed25519'] KEY_FILE_TPL = '/etc/ssh/ssh_host_%s_key' diff --git a/cloudinit/config/cc_update_etc_hosts.py b/cloudinit/config/cc_update_etc_hosts.py index d3dd1f32..15703efe 100644 --- a/cloudinit/config/cc_update_etc_hosts.py +++ b/cloudinit/config/cc_update_etc_hosts.py @@ -41,10 +41,10 @@ def handle(name, cfg, cloud, log, _args): if not tpl_fn_name: raise RuntimeError(("No hosts template could be" " found for distro %s") % - (cloud.distro.osfamily)) + (cloud.distro.osfamily)) templater.render_to_file(tpl_fn_name, '/etc/hosts', - {'hostname': hostname, 'fqdn': fqdn}) + {'hostname': hostname, 'fqdn': fqdn}) elif manage_hosts == "localhost": (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) @@ -57,4 +57,4 @@ def handle(name, cfg, cloud, log, _args): cloud.distro.update_etc_hosts(hostname, fqdn) else: log.debug(("Configuration option 'manage_etc_hosts' is not set," - " not managing /etc/hosts in module %s"), name) + " not managing /etc/hosts in module %s"), name) diff --git a/cloudinit/config/cc_update_hostname.py b/cloudinit/config/cc_update_hostname.py index e396ba13..5b78afe1 100644 --- a/cloudinit/config/cc_update_hostname.py +++ b/cloudinit/config/cc_update_hostname.py @@ -29,7 +29,7 @@ frequency = PER_ALWAYS def handle(name, cfg, cloud, log, _args): if util.get_cfg_option_bool(cfg, "preserve_hostname", False): log.debug(("Configuration option 'preserve_hostname' is set," - " not updating the hostname in module %s"), name) + " not updating the hostname in module %s"), name) return (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) diff --git a/cloudinit/config/cc_yum_add_repo.py b/cloudinit/config/cc_yum_add_repo.py index 3b821af9..64fba869 100644 --- a/cloudinit/config/cc_yum_add_repo.py +++ b/cloudinit/config/cc_yum_add_repo.py @@ -92,7 +92,7 @@ def handle(name, cfg, _cloud, log, _args): for req_field in ['baseurl']: if req_field not in repo_config: log.warn(("Repository %s does not contain a %s" - " configuration 'required' entry"), + " configuration 'required' entry"), repo_id, req_field) missing_required += 1 if not missing_required: diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 71884b32..661a9fd2 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -211,8 +211,8 @@ class Distro(object): # If the system hostname is different than the previous # one or the desired one lets update it as well - if (not sys_hostname) or (sys_hostname == prev_hostname - and sys_hostname != hostname): + if ((not sys_hostname) or (sys_hostname == prev_hostname and + sys_hostname != hostname)): update_files.append(sys_fn) # If something else has changed the hostname after we set it @@ -221,7 +221,7 @@ class Distro(object): if (sys_hostname and prev_hostname and sys_hostname != prev_hostname): LOG.info("%s differs from %s, assuming user maintained hostname.", - prev_hostname_fn, sys_fn) + prev_hostname_fn, sys_fn) return # Remove duplicates (incase the previous config filename) @@ -289,7 +289,7 @@ class Distro(object): def _bring_up_interface(self, device_name): cmd = ['ifup', device_name] LOG.debug("Attempting to run bring up interface %s using command %s", - device_name, cmd) + device_name, cmd) try: (_out, err) = util.subp(cmd) if len(err): @@ -548,7 +548,7 @@ class Distro(object): for member in members: if not util.is_user(member): LOG.warn("Unable to add group member '%s' to group '%s'" - "; user does not exist.", member, name) + "; user does not exist.", member, name) continue util.subp(['usermod', '-a', '-G', name, member]) @@ -886,7 +886,7 @@ def fetch(name): locs, looked_locs = importer.find_module(name, ['', __name__], ['Distro']) if not locs: raise ImportError("No distribution found for distro %s (searched %s)" - % (name, looked_locs)) + % (name, looked_locs)) mod = importer.import_module(locs[0]) cls = getattr(mod, 'Distro') return cls diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py index 45fcf26f..93a2e008 100644 --- a/cloudinit/distros/arch.py +++ b/cloudinit/distros/arch.py @@ -74,7 +74,7 @@ class Distro(distros.Distro): 'Interface': dev, 'IP': info.get('bootproto'), 'Address': "('%s/%s')" % (info.get('address'), - info.get('netmask')), + info.get('netmask')), 'Gateway': info.get('gateway'), 'DNS': str(tuple(info.get('dns-nameservers'))).replace(',', '') } @@ -86,7 +86,7 @@ class Distro(distros.Distro): if nameservers: util.write_file(self.resolve_conf_fn, - convert_resolv_conf(nameservers)) + convert_resolv_conf(nameservers)) return dev_names @@ -102,7 +102,7 @@ class Distro(distros.Distro): def _bring_up_interface(self, device_name): cmd = ['netctl', 'restart', device_name] LOG.debug("Attempting to run bring up interface %s using command %s", - device_name, cmd) + device_name, cmd) try: (_out, err) = util.subp(cmd) if len(err): diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 6d3a82bf..db5890b1 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -159,8 +159,9 @@ class Distro(distros.Distro): # Allow the output of this to flow outwards (ie not be captured) util.log_time(logfunc=LOG.debug, - msg="apt-%s [%s]" % (command, ' '.join(cmd)), func=util.subp, - args=(cmd,), kwargs={'env': e, 'capture': False}) + msg="apt-%s [%s]" % (command, ' '.join(cmd)), + func=util.subp, + args=(cmd,), kwargs={'env': e, 'capture': False}) def update_package_sources(self): self._runner.run("update-sources", self.package_command, diff --git a/cloudinit/distros/freebsd.py b/cloudinit/distros/freebsd.py index 4c484639..72012056 100644 --- a/cloudinit/distros/freebsd.py +++ b/cloudinit/distros/freebsd.py @@ -205,8 +205,8 @@ class Distro(distros.Distro): redact_opts = ['passwd'] for key, val in kwargs.items(): - if (key in adduser_opts and val - and isinstance(val, six.string_types)): + if (key in adduser_opts and val and + isinstance(val, six.string_types)): adduser_cmd.extend([adduser_opts[key], val]) # Redact certain fields from the logs diff --git a/cloudinit/distros/gentoo.py b/cloudinit/distros/gentoo.py index 9e80583c..6267dd6e 100644 --- a/cloudinit/distros/gentoo.py +++ b/cloudinit/distros/gentoo.py @@ -66,7 +66,7 @@ class Distro(distros.Distro): def _bring_up_interface(self, device_name): cmd = ['/etc/init.d/net.%s' % device_name, 'restart'] LOG.debug("Attempting to run bring up interface %s using command %s", - device_name, cmd) + device_name, cmd) try: (_out, err) = util.subp(cmd) if len(err): @@ -88,7 +88,7 @@ class Distro(distros.Distro): (_out, err) = util.subp(cmd) if len(err): LOG.warn("Running %s resulted in stderr output: %s", cmd, - err) + err) except util.ProcessExecutionError: util.logexc(LOG, "Running interface command %s failed", cmd) return False diff --git a/cloudinit/distros/parsers/hostname.py b/cloudinit/distros/parsers/hostname.py index 84a1de42..efb185d4 100644 --- a/cloudinit/distros/parsers/hostname.py +++ b/cloudinit/distros/parsers/hostname.py @@ -84,5 +84,5 @@ class HostnameConf(object): hostnames_found.add(head) if len(hostnames_found) > 1: raise IOError("Multiple hostnames (%s) found!" - % (hostnames_found)) + % (hostnames_found)) return entries diff --git a/cloudinit/distros/parsers/resolv_conf.py b/cloudinit/distros/parsers/resolv_conf.py index 8aee03a4..2ed13d9c 100644 --- a/cloudinit/distros/parsers/resolv_conf.py +++ b/cloudinit/distros/parsers/resolv_conf.py @@ -132,7 +132,7 @@ class ResolvConf(object): # Some hard limit on 256 chars total raise ValueError(("Adding %r would go beyond the " "256 maximum search list character limit") - % (search_domain)) + % (search_domain)) self._remove_option('search') self._contents.append(('option', ['search', s_list, ''])) return flat_sds diff --git a/cloudinit/distros/parsers/sys_conf.py b/cloudinit/distros/parsers/sys_conf.py index d795e12f..6157cf32 100644 --- a/cloudinit/distros/parsers/sys_conf.py +++ b/cloudinit/distros/parsers/sys_conf.py @@ -77,8 +77,7 @@ class SysConf(configobj.ConfigObj): quot_func = None if value[0] in ['"', "'"] and value[-1] in ['"', "'"]: if len(value) == 1: - quot_func = (lambda x: - self._get_single_quote(x) % x) + quot_func = (lambda x: self._get_single_quote(x) % x) else: # Quote whitespace if it isn't the start + end of a shell command if value.strip().startswith("$(") and value.strip().endswith(")"): @@ -91,10 +90,10 @@ class SysConf(configobj.ConfigObj): # to use single quotes which won't get expanded... if re.search(r"[\n\"']", value): quot_func = (lambda x: - self._get_triple_quote(x) % x) + self._get_triple_quote(x) % x) else: quot_func = (lambda x: - self._get_single_quote(x) % x) + self._get_single_quote(x) % x) else: quot_func = pipes.quote if not quot_func: diff --git a/cloudinit/filters/launch_index.py b/cloudinit/filters/launch_index.py index 5bebd318..baecdac9 100644 --- a/cloudinit/filters/launch_index.py +++ b/cloudinit/filters/launch_index.py @@ -61,7 +61,7 @@ class Filter(object): discarded += 1 LOG.debug(("Discarding %s multipart messages " "which do not match launch index %s"), - discarded, self.wanted_idx) + discarded, self.wanted_idx) new_message = copy.copy(message) new_message.set_payload(new_msgs) new_message[ud.ATTACHMENT_FIELD] = str(len(new_msgs)) diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index 5e99d185..a6eb20fe 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -139,9 +139,10 @@ class FileSemaphores(object): # but the item had run before we did canon_sem_name. if cname != name and os.path.exists(self._get_path(name, freq)): LOG.warn("%s has run without canonicalized name [%s].\n" - "likely the migrator has not yet run. It will run next boot.\n" - "run manually with: cloud-init single --name=migrator" - % (name, cname)) + "likely the migrator has not yet run. " + "It will run next boot.\n" + "run manually with: cloud-init single --name=migrator" + % (name, cname)) return True return False diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index bd80a8a6..b03ab895 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -38,7 +38,8 @@ LOG = logging.getLogger(__name__) DS_NAME = 'Azure' DEFAULT_METADATA = {"instance-id": "iid-AZURE-NODE"} AGENT_START = ['service', 'walinuxagent', 'start'] -BOUNCE_COMMAND = ['sh', '-xc', +BOUNCE_COMMAND = [ + 'sh', '-xc', "i=$interface; x=0; ifdown $i || x=$?; ifup $i || x=$?; exit $x"] BUILTIN_DS_CONFIG = { @@ -91,9 +92,9 @@ def temporary_hostname(temp_hostname, cfg, hostname_command='hostname'): """ policy = cfg['hostname_bounce']['policy'] previous_hostname = get_hostname(hostname_command) - if (not util.is_true(cfg.get('set_hostname')) - or util.is_false(policy) - or (previous_hostname == temp_hostname and policy != 'force')): + if (not util.is_true(cfg.get('set_hostname')) or + util.is_false(policy) or + (previous_hostname == temp_hostname and policy != 'force')): yield None return set_hostname(temp_hostname, hostname_command) @@ -123,8 +124,8 @@ class DataSourceAzureNet(sources.DataSource): with temporary_hostname(temp_hostname, self.ds_cfg, hostname_command=hostname_command) \ as previous_hostname: - if (previous_hostname is not None - and util.is_true(self.ds_cfg.get('set_hostname'))): + if (previous_hostname is not None and + util.is_true(self.ds_cfg.get('set_hostname'))): cfg = self.ds_cfg['hostname_bounce'] try: perform_hostname_bounce(hostname=temp_hostname, @@ -152,7 +153,8 @@ class DataSourceAzureNet(sources.DataSource): else: bname = str(pk['fingerprint'] + ".crt") fp_files += [os.path.join(ddir, bname)] - LOG.debug("ssh authentication: using fingerprint from fabirc") + LOG.debug("ssh authentication: " + "using fingerprint from fabirc") missing = util.log_time(logfunc=LOG.debug, msg="waiting for files", func=wait_for_files, @@ -506,7 +508,7 @@ def read_azure_ovf(contents): raise BrokenAzureDataSource("invalid xml: %s" % e) results = find_child(dom.documentElement, - lambda n: n.localName == "ProvisioningSection") + lambda n: n.localName == "ProvisioningSection") if len(results) == 0: raise NonAzureDataSource("No ProvisioningSection") @@ -516,7 +518,8 @@ def read_azure_ovf(contents): provSection = results[0] lpcs_nodes = find_child(provSection, - lambda n: n.localName == "LinuxProvisioningConfigurationSet") + lambda n: + n.localName == "LinuxProvisioningConfigurationSet") if len(results) == 0: raise NonAzureDataSource("No LinuxProvisioningConfigurationSet") diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index eb474079..e3916208 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -39,7 +39,7 @@ FS_TYPES = ('vfat', 'iso9660') LABEL_TYPES = ('config-2',) POSSIBLE_MOUNTS = ('sr', 'cd') OPTICAL_DEVICES = tuple(('/dev/%s%s' % (z, i) for z in POSSIBLE_MOUNTS - for i in range(0, 2))) + for i in range(0, 2))) class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 0032d06c..6a897f7d 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -61,12 +61,12 @@ class DataSourceEc2(sources.DataSource): if not self.wait_for_metadata_service(): return False start_time = time.time() - self.userdata_raw = ec2.get_instance_userdata(self.api_ver, - self.metadata_address) + self.userdata_raw = \ + ec2.get_instance_userdata(self.api_ver, self.metadata_address) self.metadata = ec2.get_instance_metadata(self.api_ver, self.metadata_address) LOG.debug("Crawl of metadata service took %s seconds", - int(time.time() - start_time)) + int(time.time() - start_time)) return True except Exception: util.logexc(LOG, "Failed reading from metadata address %s", @@ -132,13 +132,13 @@ class DataSourceEc2(sources.DataSource): start_time = time.time() url = uhelp.wait_for_url(urls=urls, max_wait=max_wait, - timeout=timeout, status_cb=LOG.warn) + timeout=timeout, status_cb=LOG.warn) if url: LOG.debug("Using metadata source: '%s'", url2base[url]) else: LOG.critical("Giving up on md from %s after %s seconds", - urls, int(time.time() - start_time)) + urls, int(time.time() - start_time)) self.metadata_address = url2base.get(url) return bool(url) diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index cfc59ca5..f18c4cee 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -275,17 +275,18 @@ if __name__ == "__main__": parser = argparse.ArgumentParser(description='Interact with MAAS DS') parser.add_argument("--config", metavar="file", - help="specify DS config file", default=None) + help="specify DS config file", default=None) parser.add_argument("--ckey", metavar="key", - help="the consumer key to auth with", default=None) + help="the consumer key to auth with", default=None) parser.add_argument("--tkey", metavar="key", - help="the token key to auth with", default=None) + help="the token key to auth with", default=None) parser.add_argument("--csec", metavar="secret", - help="the consumer secret (likely '')", default="") + help="the consumer secret (likely '')", default="") parser.add_argument("--tsec", metavar="secret", - help="the token secret to auth with", default=None) + help="the token secret to auth with", default=None) parser.add_argument("--apiver", metavar="version", - help="the apiver to use ("" can be used)", default=MD_VERSION) + help="the apiver to use ("" can be used)", + default=MD_VERSION) subcmds = parser.add_subparsers(title="subcommands", dest="subcmd") subcmds.add_parser('crawl', help="crawl the datasource") @@ -297,7 +298,7 @@ if __name__ == "__main__": args = parser.parse_args() creds = {'consumer_key': args.ckey, 'token_key': args.tkey, - 'token_secret': args.tsec, 'consumer_secret': args.csec} + 'token_secret': args.tsec, 'consumer_secret': args.csec} if args.config: cfg = util.read_conf(args.config) diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 58a4b2a2..adf9b12e 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -264,14 +264,14 @@ def get_properties(contents): # could also check here that elem.namespaceURI == # "http://schemas.dmtf.org/ovf/environment/1" propSections = find_child(dom.documentElement, - lambda n: n.localName == "PropertySection") + lambda n: n.localName == "PropertySection") if len(propSections) == 0: raise XmlError("No 'PropertySection's") props = {} propElems = find_child(propSections[0], - (lambda n: n.localName == "Property")) + (lambda n: n.localName == "Property")) for elem in propElems: key = elem.attributes.getNamedItemNS(envNsURI, "key").value diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index ac2c3b45..b26940d1 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -404,7 +404,8 @@ def read_context_disk_dir(source_dir, asuser=None): if ssh_key_var: lines = context.get(ssh_key_var).splitlines() results['metadata']['public-keys'] = [l for l in lines - if len(l) and not l.startswith("#")] + if len(l) and not + l.startswith("#")] # custom hostname -- try hostname or leave cloud-init # itself create hostname from IP address later diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 7453379a..139ee52c 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -90,8 +90,7 @@ BUILTIN_DS_CONFIG = { 'user-data', 'user-script', 'sdc:datacenter_name', - 'sdc:uuid', - ], + 'sdc:uuid'], 'base64_keys': [], 'base64_all': False, 'disk_aliases': {'ephemeral0': '/dev/vdb'}, @@ -450,7 +449,7 @@ class JoyentMetadataClient(object): response = bytearray() response.extend(self.metasource.read(1)) - while response[-1:] != b'\n': + while response[-1:] != b'\n': response.extend(self.metasource.read(1)) response = response.rstrip().decode('ascii') LOG.debug('Read "%s" from metadata transport.', response) @@ -513,7 +512,7 @@ def write_boot_content(content, content_f, link=None, shebang=False, except Exception as e: util.logexc(LOG, ("Failed to identify script type for %s" % - content_f, e)) + content_f, e)) if link: try: diff --git a/cloudinit/ssh_util.py b/cloudinit/ssh_util.py index 9b2f5ed5..c74a7ae2 100644 --- a/cloudinit/ssh_util.py +++ b/cloudinit/ssh_util.py @@ -31,7 +31,8 @@ LOG = logging.getLogger(__name__) DEF_SSHD_CFG = "/etc/ssh/sshd_config" # taken from openssh source key.c/key_type_from_name -VALID_KEY_TYPES = ("rsa", "dsa", "ssh-rsa", "ssh-dss", "ecdsa", +VALID_KEY_TYPES = ( + "rsa", "dsa", "ssh-rsa", "ssh-dss", "ecdsa", "ssh-rsa-cert-v00@openssh.com", "ssh-dss-cert-v00@openssh.com", "ssh-rsa-cert-v00@openssh.com", "ssh-dss-cert-v00@openssh.com", "ssh-rsa-cert-v01@openssh.com", "ssh-dss-cert-v01@openssh.com", diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 9f192c8d..dbcf3d55 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -509,13 +509,13 @@ class Init(object): def consume_data(self, frequency=PER_INSTANCE): # Consume the userdata first, because we need want to let the part # handlers run first (for merging stuff) - with events.ReportEventStack( - "consume-user-data", "reading and applying user-data", - parent=self.reporter): + with events.ReportEventStack("consume-user-data", + "reading and applying user-data", + parent=self.reporter): self._consume_userdata(frequency) - with events.ReportEventStack( - "consume-vendor-data", "reading and applying vendor-data", - parent=self.reporter): + with events.ReportEventStack("consume-vendor-data", + "reading and applying vendor-data", + parent=self.reporter): self._consume_vendordata(frequency) # Perform post-consumption adjustments so that @@ -655,7 +655,7 @@ class Modules(object): else: raise TypeError(("Failed to read '%s' item in config," " unknown type %s") % - (item, type_utils.obj_name(item))) + (item, type_utils.obj_name(item))) return module_list def _fixup_modules(self, raw_mods): @@ -762,8 +762,8 @@ class Modules(object): if skipped: LOG.info("Skipping modules %s because they are not verified " - "on distro '%s'. To run anyway, add them to " - "'unverified_modules' in config.", skipped, d_name) + "on distro '%s'. To run anyway, add them to " + "'unverified_modules' in config.", skipped, d_name) if forced: LOG.info("running unverified_modules: %s", forced) diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index f2e1390e..936f7da5 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -252,9 +252,9 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, # attrs return UrlResponse(r) except exceptions.RequestException as e: - if (isinstance(e, (exceptions.HTTPError)) - and hasattr(e, 'response') # This appeared in v 0.10.8 - and hasattr(e.response, 'status_code')): + if (isinstance(e, (exceptions.HTTPError)) and + hasattr(e, 'response') and # This appeared in v 0.10.8 + hasattr(e.response, 'status_code')): excps.append(UrlError(e, code=e.response.status_code, headers=e.response.headers, url=url)) diff --git a/cloudinit/util.py b/cloudinit/util.py index 45d49e66..de37b0f5 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -612,7 +612,7 @@ def redirect_output(outfmt, errfmt, o_out=None, o_err=None): def make_url(scheme, host, port=None, - path='', params='', query='', fragment=''): + path='', params='', query='', fragment=''): pieces = [] pieces.append(scheme or '') @@ -804,8 +804,8 @@ def load_yaml(blob, default=None, allowed=(dict,)): blob = decode_binary(blob) try: LOG.debug("Attempting to load yaml from string " - "of length %s with allowed root types %s", - len(blob), allowed) + "of length %s with allowed root types %s", + len(blob), allowed) converted = safeyaml.load(blob) if not isinstance(converted, allowed): # Yes this will just be caught, but thats ok for now... @@ -878,7 +878,7 @@ def read_conf_with_confd(cfgfile): if not isinstance(confd, six.string_types): raise TypeError(("Config file %s contains 'conf_d' " "with non-string type %s") % - (cfgfile, type_utils.obj_name(confd))) + (cfgfile, type_utils.obj_name(confd))) else: confd = str(confd).strip() elif os.path.isdir("%s.d" % cfgfile): @@ -1041,7 +1041,8 @@ def is_resolvable(name): for iname in badnames: try: result = socket.getaddrinfo(iname, None, 0, 0, - socket.SOCK_STREAM, socket.AI_CANONNAME) + socket.SOCK_STREAM, + socket.AI_CANONNAME) badresults[iname] = [] for (_fam, _stype, _proto, cname, sockaddr) in result: badresults[iname].append("%s: %s" % (cname, sockaddr[0])) @@ -1109,7 +1110,7 @@ def close_stdin(): def find_devs_with(criteria=None, oformat='device', - tag=None, no_cache=False, path=None): + tag=None, no_cache=False, path=None): """ find devices matching given criteria (via blkid) criteria can be *one* of: @@ -1628,7 +1629,7 @@ def write_file(filename, content, mode=0o644, omode="wb"): content = decode_binary(content) write_type = 'characters' LOG.debug("Writing to %s - %s: [%s] %s %s", - filename, omode, mode, len(content), write_type) + filename, omode, mode, len(content), write_type) with SeLinuxGuard(path=filename): with open(filename, omode) as fh: fh.write(content) diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py index c603bfdb..9c1ec1d4 100644 --- a/tests/unittests/test_data.py +++ b/tests/unittests/test_data.py @@ -27,11 +27,12 @@ from cloudinit import stages from cloudinit import user_data as ud from cloudinit import util -INSTANCE_ID = "i-testing" - from . import helpers +INSTANCE_ID = "i-testing" + + class FakeDataSource(sources.DataSource): def __init__(self, userdata=None, vendordata=None): diff --git a/tests/unittests/test_datasource/test_altcloud.py b/tests/unittests/test_datasource/test_altcloud.py index e9cd2fa5..85759c68 100644 --- a/tests/unittests/test_datasource/test_altcloud.py +++ b/tests/unittests/test_datasource/test_altcloud.py @@ -134,8 +134,7 @@ class TestGetCloudType(TestCase): ''' util.read_dmi_data = _dmi_data('RHEV') dsrc = DataSourceAltCloud({}, None, self.paths) - self.assertEquals('RHEV', \ - dsrc.get_cloud_type()) + self.assertEquals('RHEV', dsrc.get_cloud_type()) def test_vsphere(self): ''' @@ -144,8 +143,7 @@ class TestGetCloudType(TestCase): ''' util.read_dmi_data = _dmi_data('VMware Virtual Platform') dsrc = DataSourceAltCloud({}, None, self.paths) - self.assertEquals('VSPHERE', \ - dsrc.get_cloud_type()) + self.assertEquals('VSPHERE', dsrc.get_cloud_type()) def test_unknown(self): ''' @@ -154,8 +152,7 @@ class TestGetCloudType(TestCase): ''' util.read_dmi_data = _dmi_data('Unrecognized Platform') dsrc = DataSourceAltCloud({}, None, self.paths) - self.assertEquals('UNKNOWN', \ - dsrc.get_cloud_type()) + self.assertEquals('UNKNOWN', dsrc.get_cloud_type()) class TestGetDataCloudInfoFile(TestCase): @@ -412,27 +409,27 @@ class TestReadUserDataCallback(TestCase): '''Test read_user_data_callback() with both files.''' self.assertEquals('test user data', - read_user_data_callback(self.mount_dir)) + read_user_data_callback(self.mount_dir)) def test_callback_dc(self): '''Test read_user_data_callback() with only DC file.''' _remove_user_data_files(self.mount_dir, - dc_file=False, - non_dc_file=True) + dc_file=False, + non_dc_file=True) self.assertEquals('test user data', - read_user_data_callback(self.mount_dir)) + read_user_data_callback(self.mount_dir)) def test_callback_non_dc(self): '''Test read_user_data_callback() with only non-DC file.''' _remove_user_data_files(self.mount_dir, - dc_file=True, - non_dc_file=False) + dc_file=True, + non_dc_file=False) self.assertEquals('test user data', - read_user_data_callback(self.mount_dir)) + read_user_data_callback(self.mount_dir)) def test_callback_none(self): '''Test read_user_data_callback() no files are found.''' diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 3933794f..4c9c7d8b 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -207,7 +207,7 @@ class TestAzureDataSource(TestCase): yaml_cfg = "{agent_command: my_command}\n" cfg = yaml.safe_load(yaml_cfg) odata = {'HostName': "myhost", 'UserName': "myuser", - 'dscfg': {'text': yaml_cfg, 'encoding': 'plain'}} + 'dscfg': {'text': yaml_cfg, 'encoding': 'plain'}} data = {'ovfcontent': construct_valid_ovf_env(data=odata)} dsrc = self._get_ds(data) @@ -219,8 +219,8 @@ class TestAzureDataSource(TestCase): # set dscfg in via base64 encoded yaml cfg = {'agent_command': "my_command"} odata = {'HostName': "myhost", 'UserName': "myuser", - 'dscfg': {'text': b64e(yaml.dump(cfg)), - 'encoding': 'base64'}} + 'dscfg': {'text': b64e(yaml.dump(cfg)), + 'encoding': 'base64'}} data = {'ovfcontent': construct_valid_ovf_env(data=odata)} dsrc = self._get_ds(data) @@ -267,7 +267,8 @@ class TestAzureDataSource(TestCase): # should equal that after the '$' pos = defuser['passwd'].rfind("$") + 1 self.assertEqual(defuser['passwd'], - crypt.crypt(odata['UserPassword'], defuser['passwd'][0:pos])) + crypt.crypt(odata['UserPassword'], + defuser['passwd'][0:pos])) def test_userdata_plain(self): mydata = "FOOBAR" @@ -364,8 +365,8 @@ class TestAzureDataSource(TestCase): # Make sure that user can affect disk aliases dscfg = {'disk_aliases': {'ephemeral0': '/dev/sdc'}} odata = {'HostName': "myhost", 'UserName': "myuser", - 'dscfg': {'text': b64e(yaml.dump(dscfg)), - 'encoding': 'base64'}} + 'dscfg': {'text': b64e(yaml.dump(dscfg)), + 'encoding': 'base64'}} usercfg = {'disk_setup': {'/dev/sdc': {'something': '...'}, 'ephemeral0': False}} userdata = '#cloud-config' + yaml.dump(usercfg) + "\n" @@ -634,7 +635,7 @@ class TestReadAzureOvf(TestCase): def test_invalid_xml_raises_non_azure_ds(self): invalid_xml = "" + construct_valid_ovf_env(data={}) self.assertRaises(DataSourceAzure.BrokenAzureDataSource, - DataSourceAzure.read_azure_ovf, invalid_xml) + DataSourceAzure.read_azure_ovf, invalid_xml) def test_load_with_pubkeys(self): mypklist = [{'fingerprint': 'fp1', 'path': 'path1', 'value': ''}] diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index 83aca505..3954ceb3 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -293,9 +293,8 @@ class TestConfigDriveDataSource(TestCase): util.is_partition = my_is_partition devs_with_answers = {"TYPE=vfat": [], - "TYPE=iso9660": ["/dev/vdb"], - "LABEL=config-2": ["/dev/vdb"], - } + "TYPE=iso9660": ["/dev/vdb"], + "LABEL=config-2": ["/dev/vdb"]} self.assertEqual(["/dev/vdb"], ds.find_candidate_devs()) # add a vfat item @@ -306,9 +305,10 @@ class TestConfigDriveDataSource(TestCase): # verify that partitions are considered, that have correct label. devs_with_answers = {"TYPE=vfat": ["/dev/sda1"], - "TYPE=iso9660": [], "LABEL=config-2": ["/dev/vdb3"]} + "TYPE=iso9660": [], + "LABEL=config-2": ["/dev/vdb3"]} self.assertEqual(["/dev/vdb3"], - ds.find_candidate_devs()) + ds.find_candidate_devs()) finally: util.find_devs_with = orig_find_devs_with @@ -319,7 +319,7 @@ class TestConfigDriveDataSource(TestCase): populate_dir(self.tmp, CFG_DRIVE_FILES_V2) myds = cfg_ds_from_dir(self.tmp) self.assertEqual(myds.get_public_ssh_keys(), - [OSTACK_META['public_keys']['mykey']]) + [OSTACK_META['public_keys']['mykey']]) def cfg_ds_from_dir(seed_d): diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index eb97b692..77d15cac 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -25,9 +25,9 @@ class TestMAASDataSource(TestCase): """Verify a valid seeddir is read as such.""" data = {'instance-id': 'i-valid01', - 'local-hostname': 'valid01-hostname', - 'user-data': b'valid01-userdata', - 'public-keys': 'ssh-rsa AAAAB3Nz...aC1yc2E= keyname'} + 'local-hostname': 'valid01-hostname', + 'user-data': b'valid01-userdata', + 'public-keys': 'ssh-rsa AAAAB3Nz...aC1yc2E= keyname'} my_d = os.path.join(self.tmp, "valid") populate_dir(my_d, data) @@ -45,8 +45,8 @@ class TestMAASDataSource(TestCase): """Verify extra files do not affect seed_dir validity.""" data = {'instance-id': 'i-valid-extra', - 'local-hostname': 'valid-extra-hostname', - 'user-data': b'valid-extra-userdata', 'foo': 'bar'} + 'local-hostname': 'valid-extra-hostname', + 'user-data': b'valid-extra-userdata', 'foo': 'bar'} my_d = os.path.join(self.tmp, "valid_extra") populate_dir(my_d, data) @@ -64,7 +64,7 @@ class TestMAASDataSource(TestCase): """Verify that invalid seed_dir raises MAASSeedDirMalformed.""" valid = {'instance-id': 'i-instanceid', - 'local-hostname': 'test-hostname', 'user-data': ''} + 'local-hostname': 'test-hostname', 'user-data': ''} my_based = os.path.join(self.tmp, "valid_extra") @@ -94,8 +94,8 @@ class TestMAASDataSource(TestCase): def test_seed_dir_missing(self): """Verify that missing seed_dir raises MAASSeedDirNone.""" self.assertRaises(DataSourceMAAS.MAASSeedDirNone, - DataSourceMAAS.read_maas_seed_dir, - os.path.join(self.tmp, "nonexistantdirectory")) + DataSourceMAAS.read_maas_seed_dir, + os.path.join(self.tmp, "nonexistantdirectory")) def test_seed_url_valid(self): """Verify that valid seed_url is read as such.""" diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 1235436d..5e617b83 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -463,8 +463,8 @@ class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase): payloadstr = ' {0}'.format(self.response_parts['payload']) return ('V2 {length} {crc} {request_id} ' '{command}{payloadstr}\n'.format( - payloadstr=payloadstr, - **self.response_parts).encode('ascii')) + payloadstr=payloadstr, + **self.response_parts).encode('ascii')) self.metasource_data = None @@ -501,7 +501,7 @@ class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase): written_line = self.serial.write.call_args[0][0] print(type(written_line)) self.assertEndsWith(written_line.decode('ascii'), - b'\n'.decode('ascii')) + b'\n'.decode('ascii')) self.assertEqual(1, written_line.count(b'\n')) def _get_written_line(self, key='some_key'): diff --git a/tests/unittests/test_handler/test_handler_power_state.py b/tests/unittests/test_handler/test_handler_power_state.py index 5687b10d..f9660ff6 100644 --- a/tests/unittests/test_handler/test_handler_power_state.py +++ b/tests/unittests/test_handler/test_handler_power_state.py @@ -74,7 +74,7 @@ class TestLoadPowerState(t_help.TestCase): class TestCheckCondition(t_help.TestCase): def cmd_with_exit(self, rc): return([sys.executable, '-c', 'import sys; sys.exit(%s)' % rc]) - + def test_true_is_true(self): self.assertEqual(psc.check_condition(True), True) @@ -94,7 +94,6 @@ class TestCheckCondition(t_help.TestCase): self.assertEqual(mocklog.warn.call_count, 1) - def check_lps_ret(psc_return, mode=None): if len(psc_return) != 3: raise TypeError("length returned = %d" % len(psc_return)) diff --git a/tests/unittests/test_handler/test_handler_seed_random.py b/tests/unittests/test_handler/test_handler_seed_random.py index 0bcdcb31..34d11f21 100644 --- a/tests/unittests/test_handler/test_handler_seed_random.py +++ b/tests/unittests/test_handler/test_handler_seed_random.py @@ -190,7 +190,8 @@ class TestRandomSeed(t_help.TestCase): c = self._get_cloud('ubuntu', {}) self.whichdata = {} self.assertRaises(ValueError, cc_seed_random.handle, - 'test', {'random_seed': {'command_required': True}}, c, LOG, []) + 'test', {'random_seed': {'command_required': True}}, + c, LOG, []) def test_seed_command_and_required(self): c = self._get_cloud('ubuntu', {}) diff --git a/tests/unittests/test_handler/test_handler_snappy.py b/tests/unittests/test_handler/test_handler_snappy.py index eceb14d9..8aeff53c 100644 --- a/tests/unittests/test_handler/test_handler_snappy.py +++ b/tests/unittests/test_handler/test_handler_snappy.py @@ -125,8 +125,7 @@ class TestInstallPackages(t_help.TestCase): "pkg1.smoser.config": "pkg1.smoser.config-data", "pkg1.config": "pkg1.config-data", "pkg2.smoser_0.0_amd64.snap": "pkg2-snapdata", - "pkg2.smoser_0.0_amd64.config": "pkg2.config", - }) + "pkg2.smoser_0.0_amd64.config": "pkg2.config"}) ret = get_package_ops( packages=[], configs={}, installed=[], fspath=self.tmp) diff --git a/tests/unittests/test_sshutil.py b/tests/unittests/test_sshutil.py index 3b317121..9aeb1cde 100644 --- a/tests/unittests/test_sshutil.py +++ b/tests/unittests/test_sshutil.py @@ -32,7 +32,8 @@ VALID_CONTENT = { ), } -TEST_OPTIONS = ("no-port-forwarding,no-agent-forwarding,no-X11-forwarding," +TEST_OPTIONS = ( + "no-port-forwarding,no-agent-forwarding,no-X11-forwarding," 'command="echo \'Please login as the user \"ubuntu\" rather than the' 'user \"root\".\';echo;sleep 10"') diff --git a/tests/unittests/test_templating.py b/tests/unittests/test_templating.py index 0c19a2c2..b9863650 100644 --- a/tests/unittests/test_templating.py +++ b/tests/unittests/test_templating.py @@ -114,5 +114,6 @@ $a,$b''' codename) out_data = templater.basic_render(in_data, - {'mirror': mirror, 'codename': codename}) + {'mirror': mirror, + 'codename': codename}) self.assertEqual(ex_data, out_data) diff --git a/tools/hacking.py b/tools/hacking.py index 3175df38..1a0631c2 100755 --- a/tools/hacking.py +++ b/tools/hacking.py @@ -47,10 +47,10 @@ def import_normalize(line): # handle "from x import y as z" to "import x.y as z" split_line = line.split() if (line.startswith("from ") and "," not in line and - split_line[2] == "import" and split_line[3] != "*" and - split_line[1] != "__future__" and - (len(split_line) == 4 or - (len(split_line) == 6 and split_line[4] == "as"))): + split_line[2] == "import" and split_line[3] != "*" and + split_line[1] != "__future__" and + (len(split_line) == 4 or + (len(split_line) == 6 and split_line[4] == "as"))): return "import %s.%s" % (split_line[1], split_line[3]) else: return line @@ -74,7 +74,7 @@ def cloud_import_alphabetical(physical_line, line_number, lines): split_line[0] == "import" and split_previous[0] == "import"): if split_line[1] < split_previous[1]: return (0, "N306: imports not in alphabetical order (%s, %s)" - % (split_previous[1], split_line[1])) + % (split_previous[1], split_line[1])) def cloud_docstring_start_space(physical_line): @@ -87,8 +87,8 @@ def cloud_docstring_start_space(physical_line): pos = max([physical_line.find(i) for i in DOCSTRING_TRIPLE]) # start if (pos != -1 and len(physical_line) > pos + 1): if (physical_line[pos + 3] == ' '): - return (pos, "N401: one line docstring should not start with" - " a space") + return (pos, + "N401: one line docstring should not start with a space") def cloud_todo_format(physical_line): @@ -167,4 +167,4 @@ if __name__ == "__main__": finally: if len(_missingImport) > 0: print >> sys.stderr, ("%i imports missing in this test environment" - % len(_missingImport)) + % len(_missingImport)) diff --git a/tools/mock-meta.py b/tools/mock-meta.py index dfbc2a71..1c746f17 100755 --- a/tools/mock-meta.py +++ b/tools/mock-meta.py @@ -126,11 +126,11 @@ class WebException(Exception): def yamlify(data): formatted = yaml.dump(data, - line_break="\n", - indent=4, - explicit_start=True, - explicit_end=True, - default_flow_style=False) + line_break="\n", + indent=4, + explicit_start=True, + explicit_end=True, + default_flow_style=False) return formatted @@ -282,7 +282,7 @@ class MetaDataHandler(object): else: log.warn(("Did not implement action %s, " "returning empty response: %r"), - action, NOT_IMPL_RESPONSE) + action, NOT_IMPL_RESPONSE) return NOT_IMPL_RESPONSE @@ -404,14 +404,17 @@ def setup_logging(log_level, fmt='%(levelname)s: @%(name)s : %(message)s'): def extract_opts(): parser = OptionParser() parser.add_option("-p", "--port", dest="port", action="store", type=int, - default=80, metavar="PORT", - help="port from which to serve traffic (default: %default)") + default=80, metavar="PORT", + help=("port from which to serve traffic" + " (default: %default)")) parser.add_option("-a", "--addr", dest="address", action="store", type=str, - default='0.0.0.0', metavar="ADDRESS", - help="address from which to serve traffic (default: %default)") + default='0.0.0.0', metavar="ADDRESS", + help=("address from which to serve traffic" + " (default: %default)")) parser.add_option("-f", '--user-data-file', dest='user_data_file', - action='store', metavar='FILE', - help="user data filename to serve back to incoming requests") + action='store', metavar='FILE', + help=("user data filename to serve back to" + "incoming requests")) (options, args) = parser.parse_args() out = dict() out['extra'] = args diff --git a/tools/run-pep8 b/tools/run-pep8 index ccd6be5a..086400fc 100755 --- a/tools/run-pep8 +++ b/tools/run-pep8 @@ -1,39 +1,22 @@ #!/bin/bash -if [ $# -eq 0 ]; then - files=( bin/cloud-init $(find * -name "*.py" -type f) ) +pycheck_dirs=( "cloudinit/" "bin/" "tests/" "tools/" ) +# FIXME: cloud-init modifies sys module path, pep8 does not like +# bin_files=( "bin/cloud-init" ) +CR=" +" +[ "$1" = "-v" ] && { verbose="$1"; shift; } || verbose="" + +set -f +if [ $# -eq 0 ]; then unset IFS + IFS="$CR" + files=( "${bin_files[@]}" "${pycheck_dirs[@]}" ) + unset IFS else - files=( "$@" ); + files=( "$@" ) fi -if [ -f 'hacking.py' ] -then - base=`pwd` -else - base=`pwd`/tools/ -fi - -IGNORE="" - -# King Arthur: Be quiet! ... Be Quiet! I Order You to Be Quiet. -IGNORE="$IGNORE,E121" # Continuation line indentation is not a multiple of four -IGNORE="$IGNORE,E123" # Closing bracket does not match indentation of opening bracket's line -IGNORE="$IGNORE,E124" # Closing bracket missing visual indentation -IGNORE="$IGNORE,E125" # Continuation line does not distinguish itself from next logical line -IGNORE="$IGNORE,E126" # Continuation line over-indented for hanging indent -IGNORE="$IGNORE,E127" # Continuation line over-indented for visual indent -IGNORE="$IGNORE,E128" # Continuation line under-indented for visual indent -IGNORE="$IGNORE,E502" # The backslash is redundant between brackets -IGNORE="${IGNORE#,}" # remove the leading ',' added above - -cmd=( - ${base}/hacking.py - - --ignore="$IGNORE" - - "${files[@]}" -) - -echo -e "\nRunning 'cloudinit' pep8:" -echo "${cmd[@]}" -"${cmd[@]}" +myname=${0##*/} +cmd=( "${myname#run-}" $verbose "${files[@]}" ) +echo "Running: " "${cmd[@]}" 1>&2 +exec "${cmd[@]}" -- cgit v1.2.3 From 72c71196d701a1398963b52a30e9ab34a2579a49 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 3 Mar 2016 17:23:13 -0500 Subject: do not duplicate '-generator' in log ame --- systemd/cloud-init-generator | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/systemd/cloud-init-generator b/systemd/cloud-init-generator index b09924ac..3bd2d6b3 100755 --- a/systemd/cloud-init-generator +++ b/systemd/cloud-init-generator @@ -16,7 +16,7 @@ debug() { shift [ "$lvl" -gt "$DEBUG_LEVEL" ] && return if [ -z "$LOG" ]; then - local log="$LOG_D/${0##*/}-generator.log" + local log="$LOG_D/${0##*/}.log" { : > "$log"; } >/dev/null 2>&1 && LOG="$log" || LOG="/dev/kmsg" fi -- cgit v1.2.3 From 3d9153d16b194e7a3139c290e723ef17518e617d Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Thu, 3 Mar 2016 16:32:32 -0600 Subject: Fix pyflake/pyflake3 errors Now we can run make check to assess pep8, pyflakes for python2 or 3 And execute unittests via nosetests (2 and 3). --- Makefile | 1 - cloudinit/util.py | 2 +- tests/unittests/test_datasource/test_azure_helper.py | 2 -- tests/unittests/test_datasource/test_smartos.py | 1 - .../unittests/test_handler/test_handler_power_state.py | 2 +- tools/run-pyflakes | 18 ++++++++++++++++++ tools/run-pyflakes3 | 2 ++ 7 files changed, 22 insertions(+), 6 deletions(-) create mode 100755 tools/run-pyflakes create mode 100755 tools/run-pyflakes3 diff --git a/Makefile b/Makefile index fb65b70b..fc91f829 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,6 @@ pyflakes: pyflakes3: @$(CWD)/tools/run-pyflakes3 - unittest: nosetests $(noseopts) tests/unittests nosetests3 $(noseopts) tests/unittests diff --git a/cloudinit/util.py b/cloudinit/util.py index de37b0f5..e7407ea4 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2148,7 +2148,7 @@ def _read_dmi_syspath(key): LOG.debug("dmi data %s returned %s", dmi_key_path, key_data) return key_data.strip() - except Exception as e: + except Exception: logexc(LOG, "failed read of %s", dmi_key_path) return None diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 8dbdfb0b..1134199b 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -1,6 +1,4 @@ import os -import struct -import unittest from cloudinit.sources.helpers import azure as azure_helper from ..helpers import TestCase diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 5e617b83..616e9f0e 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -31,7 +31,6 @@ import shutil import stat import tempfile import uuid -import unittest from binascii import crc32 import serial diff --git a/tests/unittests/test_handler/test_handler_power_state.py b/tests/unittests/test_handler/test_handler_power_state.py index f9660ff6..04ce5687 100644 --- a/tests/unittests/test_handler/test_handler_power_state.py +++ b/tests/unittests/test_handler/test_handler_power_state.py @@ -106,7 +106,7 @@ def check_lps_ret(psc_return, mode=None): if 'shutdown' not in psc_return[0][0]: errs.append("string 'shutdown' not in cmd") - if 'condition' is None: + if condition is None: errs.append("condition was not returned") if mode is not None: diff --git a/tools/run-pyflakes b/tools/run-pyflakes new file mode 100755 index 00000000..4bea17f4 --- /dev/null +++ b/tools/run-pyflakes @@ -0,0 +1,18 @@ +#!/bin/bash + +PYTHON_VERSION=${PYTHON_VERSION:-2} +CR=" +" +pycheck_dirs=( "cloudinit/" "bin/" "tests/" "tools/" ) + +set -f +if [ $# -eq 0 ]; then + files=( "${pycheck_dirs[@]}" ) +else + files=( "$@" ) +fi + +cmd=( "python${PYTHON_VERSION}" -m "pyflakes" "${files[@]}" ) + +echo "Running: " "${cmd[@]}" 1>&2 +exec "${cmd[@]}" diff --git a/tools/run-pyflakes3 b/tools/run-pyflakes3 new file mode 100755 index 00000000..e9f0863d --- /dev/null +++ b/tools/run-pyflakes3 @@ -0,0 +1,2 @@ +#!/bin/sh +PYTHON_VERSION=3 exec "${0%/*}/run-pyflakes" "$@" -- cgit v1.2.3 From fa5c11836cec515dd23efcfabe9b736edd47c22a Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 3 Mar 2016 17:49:01 -0500 Subject: generator: be more clear on where kernel cmdline came from --- systemd/cloud-init-generator | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/systemd/cloud-init-generator b/systemd/cloud-init-generator index 3bd2d6b3..eb0e1a77 100755 --- a/systemd/cloud-init-generator +++ b/systemd/cloud-init-generator @@ -33,15 +33,18 @@ etc_file() { read_proc_cmdline() { if [ "$CONTAINER" = "lxc" ]; then + _RET_MSG="ignored: \$container=$CONTAINER" _RET="" return 0 fi if systemd-detect-virt --container --quiet; then + _RET_MSG="ignored: detect-virt is container" _RET="" return 0 fi + _RET_MSG="/proc/cmdline" read _RET < /proc/cmdline } @@ -53,7 +56,7 @@ kernel_cmdline() { debug 1 "kernel command line from env KERNEL_CMDLINE: $cmdline" elif read_proc_cmdline; then read_proc_cmdline && cmdline="$_RET" - debug 1 "kernel command line from /proc/cmdline: $cmdline" + debug 1 "kernel command line ($_RET_MSG): $cmdline" fi _RET="unset" cmdline=" $cmdline " -- cgit v1.2.3 From 77a9dbb4aa7e555d4c033c54a60b7084b2ab3373 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 3 Mar 2016 17:49:10 -0500 Subject: cloud-init.target should not do anything no wants or after or before. --- systemd/cloud-init.target | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/systemd/cloud-init.target b/systemd/cloud-init.target index 03f61002..a63babb0 100644 --- a/systemd/cloud-init.target +++ b/systemd/cloud-init.target @@ -2,15 +2,5 @@ # To disable it you can either: # a.) boot with kernel cmdline of 'cloudinit=disabled' # b.) touch a file /etc/cloud/cloud-init.disabled -# cloud-init normally emits a "cloud-config" upstart event to inform third -# parties that cloud-config is available, which does us no good when we're -# using systemd. cloud-config.target serves as this synchronization point -# instead. Services that would "start on cloud-config" with upstart can -# instead use "After=cloud-config.target" and "Wants=cloud-config.target" -# as appropriate. - [Unit] Description=Cloud-init target -Wants=cloud-init-local.service cloud-init.service -After=cloud-init-local.service cloud-init.service - -- cgit v1.2.3 From bbf105baafbe788f7babbda188b513180424e256 Mon Sep 17 00:00:00 2001 From: Sankar Tanguturi Date: Thu, 3 Mar 2016 16:01:39 -0800 Subject: Resolved all the pep8 errors. Executed ./tools/run-pep8 cloudinit/sources/DataSourceOVF.py and no errors were reported. --- cloudinit/sources/DataSourceOVF.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index d92c128c..d07f6219 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -66,13 +66,14 @@ class DataSourceOVF(sources.DataSource): system_type = util.read_dmi_data("system-product-name") if system_type is None: - LOG.debug("No system-product-name found") + LOG.debug("No system-product-name found") elif 'vmware' in system_type.lower(): LOG.debug("VMware Virtualization Platform found") if not util.get_cfg_option_bool(self.sys_cfg, "disable_vmware_customization", True): - deployPkgPluginPath = search_file("/usr/lib/vmware-tools", "libdeployPkgPlugin.so") + deployPkgPluginPath = search_file("/usr/lib/vmware-tools", + "libdeployPkgPlugin.so") if deployPkgPluginPath: vmwareImcConfigFilePath = util.log_time(logfunc=LOG.debug, msg="waiting for configuration file", @@ -80,7 +81,8 @@ class DataSourceOVF(sources.DataSource): args=("/tmp", "cust.cfg")) if vmwareImcConfigFilePath: - LOG.debug("Found VMware DeployPkg Config File Path at %s" % vmwareImcConfigFilePath) + LOG.debug("Found VMware DeployPkg Config File at %s" % + vmwareImcConfigFilePath) else: LOG.debug("Did not find VMware DeployPkg Config File Path") else: @@ -151,7 +153,7 @@ class DataSourceOVF(sources.DataSource): def get_public_ssh_keys(self): if 'public-keys' not in self.metadata: - return [] + return [] pks = self.metadata['public-keys'] if isinstance(pks, (list)): return pks @@ -174,7 +176,7 @@ class DataSourceOVFNet(DataSourceOVF): def wait_for_imc_cfg_file(dirpath, filename, maxwait=180, naplen=5): waited = 0 - + while waited < maxwait: fileFullPath = search_file(dirpath, filename) if fileFullPath: @@ -183,6 +185,7 @@ def wait_for_imc_cfg_file(dirpath, filename, maxwait=180, naplen=5): waited += naplen return None + # This will return a dict with some content # meta-data, user-data, some config def read_vmware_imc(config): @@ -190,13 +193,14 @@ def read_vmware_imc(config): cfg = {} ud = "" if config.host_name: - if config.domain_name: - md['local-hostname'] = config.host_name + "." + config.domain_name - else: - md['local-hostname'] = config.host_name + if config.domain_name: + md['local-hostname'] = config.host_name + "." + config.domain_name + else: + md['local-hostname'] = config.host_name return (md, ud, cfg) + # This will return a dict with some content # meta-data, user-data, some config def read_ovf_environment(contents): @@ -351,7 +355,7 @@ def get_properties(contents): def search_file(dirpath, filename): if not dirpath or not filename: - return None + return None for root, dirs, files in os.walk(dirpath): if filename in files: @@ -359,6 +363,7 @@ def search_file(dirpath, filename): return None + class XmlError(Exception): pass -- cgit v1.2.3 From 2231c45ac3712c5cb7c1b810c838d3f91f424bf2 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 3 Mar 2016 19:10:18 -0500 Subject: packages/debian: make trunk packaging closer to ubuntu The big difference is using: ${python3:Depends} or ${python:Depends} rather than explicitly listing the dependencies (via template ${requires}). which means we get paths of /usr/lib/python3/dist-packages/.. rather than /usr/lib/python3.5/dist-packages/.. when built on xenial. Additionally it seems we no longer need this strange line. # Because setup tools didn't copy data... --- packages/debian/control.in | 2 +- packages/debian/rules.in | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/debian/control.in b/packages/debian/control.in index 5fe16e43..1c4f522c 100644 --- a/packages/debian/control.in +++ b/packages/debian/control.in @@ -18,8 +18,8 @@ Package: cloud-init Architecture: all Depends: procps, ${python}, - ${requires}, ${misc:Depends}, + ${${python}:Depends} Recommends: eatmydata, sudo, software-properties-common, gdisk XB-Python-Version: ${python:Versions} Description: Init scripts for cloud instances diff --git a/packages/debian/rules.in b/packages/debian/rules.in index d6cd23ae..66e80946 100755 --- a/packages/debian/rules.in +++ b/packages/debian/rules.in @@ -1,9 +1,8 @@ ## template:basic #!/usr/bin/make -f - INIT_SYSTEM ?= upstart,systemd -PYVER ?= python${pyver} export PYBUILD_INSTALL_ARGS=--init-system=$(INIT_SYSTEM) +PYVER ?= python${pyver} %: dh $@ --with $(PYVER),systemd --buildsystem pybuild @@ -14,6 +13,7 @@ override_dh_install: cp tools/21-cloudinit.conf debian/cloud-init/etc/rsyslog.d/21-cloudinit.conf override_dh_auto_test: - # Because setup tools didn't copy data... - [ ! -d .pybuild/pythonX.Y_?.?/build/tests ] || cp -r tests/data .pybuild/pythonX.Y_?.?/build/tests http_proxy= make check + +override_dh_systemd_start: + dh_systemd_start --no-restart-on-upgrade --no-start -- cgit v1.2.3 From cc92c32ea6e70de44ad0bd1e7c64f17e7fb23304 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 3 Mar 2016 22:00:14 -0500 Subject: support nocheck in building. --- packages/debian/rules.in | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/debian/rules.in b/packages/debian/rules.in index 66e80946..16f9b817 100755 --- a/packages/debian/rules.in +++ b/packages/debian/rules.in @@ -12,8 +12,13 @@ override_dh_install: install -d debian/cloud-init/etc/rsyslog.d cp tools/21-cloudinit.conf debian/cloud-init/etc/rsyslog.d/21-cloudinit.conf +ifeq (,$(findstring nocheck,$(DEB_BUILD_OPTIONS))) override_dh_auto_test: - http_proxy= make check + http_proxy= make check +endif + +override_dh_systemd_enable: + dh_systemd_enable --no-enable override_dh_systemd_start: dh_systemd_start --no-restart-on-upgrade --no-start -- cgit v1.2.3 From d1c7de36c5d5042919ea2dec6bb5bd00688f9534 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 3 Mar 2016 22:03:34 -0500 Subject: packages/bddeb: copy all files in packages/debian/ just copy all the files that are there. makes adding files easier. --- packages/bddeb | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/bddeb b/packages/bddeb index c4efe264..c141b1ab 100755 --- a/packages/bddeb +++ b/packages/bddeb @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import glob import os import shutil import sys @@ -105,11 +106,11 @@ def write_debian_folder(root, version, revno, pkgmap, util.abs_join(deb_dir, 'rules'), params={'python': python, 'pyver': pyver}) - # Just copy the following directly - for base_fn in ['dirs', 'copyright', 'compat']: - shutil.copy(util.abs_join(find_root(), - 'packages', 'debian', base_fn), - util.abs_join(deb_dir, base_fn)) + # Just copy any other files directly (including .in) + pdeb_d = util.abs_join(find_root(), 'packages', 'debian') + for f in [os.path.join(pdeb_d, f) for f in os.listdir(pdeb_d)]: + if os.path.isfile(f): + shutil.copy(f, util.abs_join(deb_dir, os.path.basename(f))) def main(): -- cgit v1.2.3 From d02a257063db96f487510b9841a9a396ef2675af Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 3 Mar 2016 22:46:37 -0500 Subject: mention link path in generator --- systemd/cloud-init-generator | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/systemd/cloud-init-generator b/systemd/cloud-init-generator index eb0e1a77..9d1e22f0 100755 --- a/systemd/cloud-init-generator +++ b/systemd/cloud-init-generator @@ -114,7 +114,7 @@ main() { debug 0 "[$ret] disable failed, remove $link_path" fi else - debug 1 "already disabled: no change needed" + debug 1 "already disabled: no change needed [no $link_path]" fi else debug 0 "unexpected result '$result'" -- cgit v1.2.3 From 964aeaa9c1eac3fafa73ef9abf344613a093cb06 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 4 Mar 2016 00:01:08 -0500 Subject: postinst/preinst: cleanup old multi-user.target enabled files also, actually enable the services. now this will have them enabled in the cloud-init.target. --- packages/debian/cloud-init.postinst | 17 +++++++++++++++++ packages/debian/cloud-init.preinst | 22 ++++++++++++++++++++++ packages/debian/rules.in | 3 --- 3 files changed, 39 insertions(+), 3 deletions(-) create mode 100644 packages/debian/cloud-init.postinst create mode 100644 packages/debian/cloud-init.preinst diff --git a/packages/debian/cloud-init.postinst b/packages/debian/cloud-init.postinst new file mode 100644 index 00000000..f1222b42 --- /dev/null +++ b/packages/debian/cloud-init.postinst @@ -0,0 +1,17 @@ +#!/bin/sh +set -x +cleanup_lp1552999() { + local oldver="$1" last_bad_ver="0.7.7~bzr1178" + dpkg --compare-versions "$oldver" le "$last_bad_ver" || return 0 + local edir="/etc/systemd/system/multi-user.target.wants" + rm -f "$edir/cloud-config.service" "$edir/cloud-final.service" \ + "$edir/cloud-init-local.service" "$edir/cloud-init.service" +} + + +#DEBHELPER# + +if [ "$1" = "configure" ]; then + oldver="$2" + cleanup_lp1552999 "$oldver" +fi diff --git a/packages/debian/cloud-init.preinst b/packages/debian/cloud-init.preinst new file mode 100644 index 00000000..babaab6c --- /dev/null +++ b/packages/debian/cloud-init.preinst @@ -0,0 +1,22 @@ +#!/bin/sh +# vi: ts=4 expandtab + +set -x +cleanup_lp1552999() { + local oldver="$1" last_bad_ver="0.7.7~bzr1178" + dpkg --compare-versions "$oldver" le "$last_bad_ver" || return 0 + local hdir="/var/lib/systemd/deb-systemd-helper-enabled" + hdir="$hdir/multi-user.target.wants" + local edir="/etc/systemd/system/multi-user.target.wants" + rm -f "$hdir/cloud-config.service" "$hdir/cloud-final.service" \ + "$hdir/cloud-init-local.service" "$hdir/cloud-init.service" +} + + +if [ "$1" = "upgrade" ]; then + oldver="$2" + cleanup_lp1552999 "$oldver" +fi +set +x + +#DEBHELPER# diff --git a/packages/debian/rules.in b/packages/debian/rules.in index 16f9b817..a24720f4 100755 --- a/packages/debian/rules.in +++ b/packages/debian/rules.in @@ -17,8 +17,5 @@ override_dh_auto_test: http_proxy= make check endif -override_dh_systemd_enable: - dh_systemd_enable --no-enable - override_dh_systemd_start: dh_systemd_start --no-restart-on-upgrade --no-start -- cgit v1.2.3 From a782829a083c7170df62dda94d6b7441963dcb94 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 4 Mar 2016 00:10:36 -0500 Subject: remove debug set -x --- packages/debian/cloud-init.postinst | 1 - packages/debian/cloud-init.preinst | 2 -- 2 files changed, 3 deletions(-) diff --git a/packages/debian/cloud-init.postinst b/packages/debian/cloud-init.postinst index f1222b42..cdd0466d 100644 --- a/packages/debian/cloud-init.postinst +++ b/packages/debian/cloud-init.postinst @@ -1,5 +1,4 @@ #!/bin/sh -set -x cleanup_lp1552999() { local oldver="$1" last_bad_ver="0.7.7~bzr1178" dpkg --compare-versions "$oldver" le "$last_bad_ver" || return 0 diff --git a/packages/debian/cloud-init.preinst b/packages/debian/cloud-init.preinst index babaab6c..3c2af06d 100644 --- a/packages/debian/cloud-init.preinst +++ b/packages/debian/cloud-init.preinst @@ -1,7 +1,6 @@ #!/bin/sh # vi: ts=4 expandtab -set -x cleanup_lp1552999() { local oldver="$1" last_bad_ver="0.7.7~bzr1178" dpkg --compare-versions "$oldver" le "$last_bad_ver" || return 0 @@ -17,6 +16,5 @@ if [ "$1" = "upgrade" ]; then oldver="$2" cleanup_lp1552999 "$oldver" fi -set +x #DEBHELPER# -- cgit v1.2.3 From ea5ffb88f2d3b5f55699955520012596de1d8af5 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 4 Mar 2016 00:18:51 -0500 Subject: fix tab in rules --- packages/debian/rules.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/debian/rules.in b/packages/debian/rules.in index a24720f4..988b5bc7 100755 --- a/packages/debian/rules.in +++ b/packages/debian/rules.in @@ -14,7 +14,7 @@ override_dh_install: ifeq (,$(findstring nocheck,$(DEB_BUILD_OPTIONS))) override_dh_auto_test: - http_proxy= make check + http_proxy= make check endif override_dh_systemd_start: -- cgit v1.2.3 From 9b0cbf54a90d2434e2a93e34664646ee8638fc97 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 4 Mar 2016 00:34:37 -0500 Subject: fix packages/debian/rules.in --- packages/debian/rules.in | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/debian/rules.in b/packages/debian/rules.in index 988b5bc7..5420949c 100755 --- a/packages/debian/rules.in +++ b/packages/debian/rules.in @@ -12,9 +12,11 @@ override_dh_install: install -d debian/cloud-init/etc/rsyslog.d cp tools/21-cloudinit.conf debian/cloud-init/etc/rsyslog.d/21-cloudinit.conf -ifeq (,$(findstring nocheck,$(DEB_BUILD_OPTIONS))) override_dh_auto_test: +ifeq (,$(findstring nocheck,$(DEB_BUILD_OPTIONS))) http_proxy= make check +else + @echo check disabled by DEB_BUILD_OPTIONS=$(DEB_BUILD_OPTIONS) endif override_dh_systemd_start: -- cgit v1.2.3 From 70acc910c3368980d7cb8971391a2c9dfaf3fda8 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Fri, 4 Mar 2016 09:51:05 -0600 Subject: pep8: update formatting to pass pep8 1.4.6 (trusty) and 1.6.2 (xenial) make check fails in a trusty sbuild due to different rules on older pep8. Fix formatting to pass in older and newer pep8. --- cloudinit/config/cc_rh_subscription.py | 4 +--- tests/unittests/test_datasource/test_azure.py | 2 +- tools/hacking.py | 4 ++-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/cloudinit/config/cc_rh_subscription.py b/cloudinit/config/cc_rh_subscription.py index 6f474aed..6087c45c 100644 --- a/cloudinit/config/cc_rh_subscription.py +++ b/cloudinit/config/cc_rh_subscription.py @@ -126,10 +126,8 @@ class SubscriptionManager(object): "(True/False " return False, not_bool - if (self.servicelevel is not None) and \ - ((not self.auto_attach) or + if (self.servicelevel is not None) and ((not self.auto_attach) or (util.is_false(str(self.auto_attach)))): - no_auto = ("The service-level key must be used in conjunction " "with the auto-attach key. Please re-run with " "auto-attach: True") diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 4c9c7d8b..444e2799 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -268,7 +268,7 @@ class TestAzureDataSource(TestCase): pos = defuser['passwd'].rfind("$") + 1 self.assertEqual(defuser['passwd'], crypt.crypt(odata['UserPassword'], - defuser['passwd'][0:pos])) + defuser['passwd'][0:pos])) def test_userdata_plain(self): mydata = "FOOBAR" diff --git a/tools/hacking.py b/tools/hacking.py index 1a0631c2..716c1154 100755 --- a/tools/hacking.py +++ b/tools/hacking.py @@ -49,8 +49,8 @@ def import_normalize(line): if (line.startswith("from ") and "," not in line and split_line[2] == "import" and split_line[3] != "*" and split_line[1] != "__future__" and - (len(split_line) == 4 or - (len(split_line) == 6 and split_line[4] == "as"))): + (len(split_line) == 4 or (len(split_line) == 6 and + split_line[4] == "as"))): return "import %s.%s" % (split_line[1], split_line[3]) else: return line -- cgit v1.2.3