From 86715c88aab8561e1ddadac95671f6095d16f9e7 Mon Sep 17 00:00:00 2001 From: Brent Baude Date: Mon, 20 Mar 2017 12:31:15 -0500 Subject: Bounce network interface for Azure when using the built-in path. When deploying on Azure and using only cloud-init, you must "bounce" the network interface to trigger a DDNS update. This allows dhclient to register the hostname with Azure so that DNS works correctly on their private networks (i.e. between vm and vm). The agent path was already doing the bounce so this creates parity between the built-in path and the agent. LP: #1674685 --- cloudinit/sources/DataSourceAzure.py | 78 +++++++++++++++++++++--------------- cloudinit/sources/__init__.py | 2 +- 2 files changed, 47 insertions(+), 33 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index c5af8b84..48a3e1df 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -111,50 +111,62 @@ class DataSourceAzureNet(sources.DataSource): root = sources.DataSource.__str__(self) return "%s [seed=%s]" % (root, self.seed) - def get_metadata_from_agent(self): - temp_hostname = self.metadata.get('local-hostname') + def bounce_network_with_azure_hostname(self): + # When using cloud-init to provision, we have to set the hostname from + # the metadata and "bounce" the network to force DDNS to update via + # dhclient + azure_hostname = self.metadata.get('local-hostname') + LOG.debug("Hostname in metadata is {}".format(azure_hostname)) hostname_command = self.ds_cfg['hostname_bounce']['hostname_command'] - agent_cmd = self.ds_cfg['agent_command'] - LOG.debug("Getting metadata via agent. hostname=%s cmd=%s", - temp_hostname, agent_cmd) - with temporary_hostname(temp_hostname, self.ds_cfg, + + with temporary_hostname(azure_hostname, self.ds_cfg, hostname_command=hostname_command) \ as previous_hostname: if (previous_hostname is not None and - util.is_true(self.ds_cfg.get('set_hostname'))): + util.is_true(self.ds_cfg.get('set_hostname'))): cfg = self.ds_cfg['hostname_bounce'] + + # "Bouncing" the network try: - perform_hostname_bounce(hostname=temp_hostname, + perform_hostname_bounce(hostname=azure_hostname, cfg=cfg, prev_hostname=previous_hostname) except Exception as e: LOG.warn("Failed publishing hostname: %s", e) util.logexc(LOG, "handling set_hostname failed") - try: - invoke_agent(agent_cmd) - except util.ProcessExecutionError: - # claim the datasource even if the command failed - util.logexc(LOG, "agent command '%s' failed.", - self.ds_cfg['agent_command']) - - ddir = self.ds_cfg['data_dir'] - - fp_files = [] - key_value = None - for pk in self.cfg.get('_pubkeys', []): - if pk.get('value', None): - key_value = pk['value'] - LOG.debug("ssh authentication: using value from fabric") - else: - bname = str(pk['fingerprint'] + ".crt") - fp_files += [os.path.join(ddir, bname)] - LOG.debug("ssh authentication: " - "using fingerprint from fabirc") - - missing = util.log_time(logfunc=LOG.debug, msg="waiting for files", - func=wait_for_files, - args=(fp_files,)) + def get_metadata_from_agent(self): + temp_hostname = self.metadata.get('local-hostname') + agent_cmd = self.ds_cfg['agent_command'] + LOG.debug("Getting metadata via agent. hostname=%s cmd=%s", + temp_hostname, agent_cmd) + + self.bounce_network_with_azure_hostname() + + try: + invoke_agent(agent_cmd) + except util.ProcessExecutionError: + # claim the datasource even if the command failed + util.logexc(LOG, "agent command '%s' failed.", + self.ds_cfg['agent_command']) + + ddir = self.ds_cfg['data_dir'] + + fp_files = [] + key_value = None + for pk in self.cfg.get('_pubkeys', []): + if pk.get('value', None): + key_value = pk['value'] + LOG.debug("ssh authentication: using value from fabric") + else: + bname = str(pk['fingerprint'] + ".crt") + fp_files += [os.path.join(ddir, bname)] + LOG.debug("ssh authentication: " + "using fingerprint from fabirc") + + missing = util.log_time(logfunc=LOG.debug, msg="waiting for files", + func=wait_for_files, + args=(fp_files,)) if len(missing): LOG.warn("Did not find files, but going on: %s", missing) @@ -220,6 +232,8 @@ class DataSourceAzureNet(sources.DataSource): write_files(ddir, files, dirmode=0o700) if self.ds_cfg['agent_command'] == AGENT_START_BUILTIN: + self.bounce_network_with_azure_hostname() + metadata_func = partial(get_metadata_from_fabric, fallback_lease_file=self. dhclient_lease_file) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 3d01072f..18294505 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -210,7 +210,7 @@ class DataSource(object): else: hostname = toks[0] - if fqdn: + if fqdn and domain != defdomain: return "%s.%s" % (hostname, domain) else: return hostname -- cgit v1.2.3 From e1a1854e3f30bed18467e6c0feb896338ad4419a Mon Sep 17 00:00:00 2001 From: root Date: Tue, 21 Mar 2017 15:34:59 +0000 Subject: Bigstep: fix bug when executing in python3. Python 2's json.loads would accept bytes, so this bug was only exposed in python3. --- cloudinit/sources/DataSourceBigstep.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceBigstep.py b/cloudinit/sources/DataSourceBigstep.py index 5ffdcb25..d7fcd45a 100644 --- a/cloudinit/sources/DataSourceBigstep.py +++ b/cloudinit/sources/DataSourceBigstep.py @@ -27,7 +27,7 @@ class DataSourceBigstep(sources.DataSource): if url is None: return False response = url_helper.readurl(url) - decoded = json.loads(response.contents) + decoded = json.loads(response.contents.decode()) self.metadata = decoded["metadata"] self.vendordata_raw = decoded["vendordata_raw"] self.userdata_raw = decoded["userdata_raw"] -- cgit v1.2.3 From 35cf3415f9748c880db4d3c004f3410c3aa2cab2 Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Tue, 21 Mar 2017 14:18:46 -0600 Subject: test: add running of pylint Now tox will run pylint. The .pylintrc file sets pylint to only produce errors, and will ignore certain classes that are known problematic (six). --- .pylintrc | 39 +++++++++++++++++++++++++++++++ cloudinit/net/network_state.py | 5 ++-- cloudinit/net/renderer.py | 5 ++++ cloudinit/sources/DataSourceAltCloud.py | 3 +-- cloudinit/sources/DataSourceOpenNebula.py | 4 ++-- cloudinit/sources/__init__.py | 2 +- cloudinit/url_helper.py | 2 +- tox.ini | 10 +++++++- 8 files changed, 61 insertions(+), 9 deletions(-) create mode 100644 .pylintrc (limited to 'cloudinit/sources') diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 00000000..b8cda03c --- /dev/null +++ b/.pylintrc @@ -0,0 +1,39 @@ +[MASTER] + +# --go-faster, use multiple processes to speed up Pylint +jobs=4 + + +[MESSAGES CONTROL] + +# Errors only +disable=C, F, I, R, W + + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +output-format=colorized + +# Just the errors please, no full report +reports=no + + +[TYPECHECK] + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules=six.moves,pkg_resources + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members=types,http.client,command_handlers + diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 701aaa4e..692b6007 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -214,7 +214,7 @@ class NetworkStateInterpreter(object): return util.yaml_dumps(self._network_state) def as_dict(self): - return {'version': self.version, 'config': self.config} + return {'version': self._version, 'config': self._config} def get_network_state(self): ns = self.network_state @@ -611,7 +611,8 @@ class NetworkStateInterpreter(object): self.handle_vlan(vlan_cmd) def handle_wifis(self, command): - raise NotImplemented('NetworkState V2: Skipping wifi configuration') + raise NotImplementedError("NetworkState V2: " + "Skipping wifi configuration") def _v2_common(self, cfg): LOG.debug('v2_common: handling config:\n%s', cfg) diff --git a/cloudinit/net/renderer.py b/cloudinit/net/renderer.py index a5b2b573..c68658dc 100644 --- a/cloudinit/net/renderer.py +++ b/cloudinit/net/renderer.py @@ -5,6 +5,7 @@ # # This file is part of cloud-init. See LICENSE file for license information. +import abc import six from .network_state import parse_net_config_data @@ -37,6 +38,10 @@ class Renderer(object): iface['mac_address'])) return content.getvalue() + @abc.abstractmethod + def render_network_state(self, network_state, target=None): + """Render network state.""" + def render_network_config(self, network_config, target=None): return self.render_network_state( network_state=parse_net_config_data(network_config), target=target) diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py index c2b0eac2..8528fa10 100644 --- a/cloudinit/sources/DataSourceAltCloud.py +++ b/cloudinit/sources/DataSourceAltCloud.py @@ -201,8 +201,7 @@ class DataSourceAltCloud(sources.DataSource): util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err) return False except OSError as _err: - util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), - _err.message) + util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err) return False try: diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 1f1baf46..cd75e6ea 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -286,12 +286,12 @@ def parse_shell_config(content, keylist=None, bash=None, asuser=None, output = output[0:-1] # remove trailing null # go through output. First _start_ is for 'preset', second for 'target'. - # Add to target only things were changed and not in volitile + # Add to ret only things were changed and not in excluded. for line in output.split("\x00"): try: (key, val) = line.split("=", 1) if target is preset: - target[key] = val + preset[key] = val elif (key not in excluded and (key in keylist_in or preset.get(key) != val)): ret[key] = val diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 18294505..5c99437e 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -50,7 +50,7 @@ class DataSource(object): self.distro = distro self.paths = paths self.userdata = None - self.metadata = None + self.metadata = {} self.userdata_raw = None self.vendordata = None self.vendordata_raw = None diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 312b0460..2f6a158e 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -45,7 +45,7 @@ try: from distutils.version import LooseVersion import pkg_resources _REQ = pkg_resources.get_distribution('requests') - _REQ_VER = LooseVersion(_REQ.version) + _REQ_VER = LooseVersion(_REQ.version) # pylint: disable=no-member if _REQ_VER >= LooseVersion('0.8.8'): SSL_ENABLED = True if _REQ_VER >= LooseVersion('0.7.0') and _REQ_VER < LooseVersion('1.0.0'): diff --git a/tox.ini b/tox.ini index f016f206..bf9046af 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27, py3, flake8, xenial +envlist = py27, py3, flake8, xenial, pylint recreate = True [testenv] @@ -17,6 +17,10 @@ commands = {envpython} -m flake8 {posargs:cloudinit/ tests/ tools/} setenv = LC_ALL = en_US.utf-8 +[testenv:pylint] +deps = pylint==1.6.5 +commands = {envpython} -m pylint {posargs:cloudinit} + [testenv:py3] basepython = python3 commands = {envpython} -m nose {posargs:--with-coverage \ @@ -88,6 +92,10 @@ deps = pycodestyle commands = {envpython} -m pyflakes {posargs:cloudinit/ tests/ tools/} deps = pyflakes +[testenv:tip-pylint] +commands = {envpython} -m pylint {posargs:cloudinit} +deps = pylint + [testenv:citest] basepython = python3 commands = {envpython} -m tests.cloud_tests {posargs} -- cgit v1.2.3 From 443095f4d4b6feba30c7011b7ab48adb2a40fcf5 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 16 Mar 2017 11:20:02 -0400 Subject: ConfigDrive: support reading config drive data from /config-drive. This is thie cloud-init part of a fix to allow nova-lxd to provide config drive data. The other part will be done in nova-lxd. The agreement here is that nova-lxd will copy the contents of the config drive to /config-drive in the container. LP: #1673411 --- cloudinit/sources/DataSourceConfigDrive.py | 13 ++++++++----- tools/ds-identify | 13 +++++++++++++ 2 files changed, 21 insertions(+), 5 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 8a448dc9..46dd89e0 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -54,13 +54,16 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): found = None md = {} results = {} - if os.path.isdir(self.seed_dir): + for sdir in (self.seed_dir, "/config-drive"): + if not os.path.isdir(sdir): + continue try: - results = read_config_drive(self.seed_dir) - found = self.seed_dir + results = read_config_drive(sdir) + found = sdir + break except openstack.NonReadable: - util.logexc(LOG, "Failed reading config drive from %s", - self.seed_dir) + util.logexc(LOG, "Failed reading config drive from %s", sdir) + if not found: for dev in find_candidate_devs(): try: diff --git a/tools/ds-identify b/tools/ds-identify index b3220c1f..bf09a3ad 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -538,6 +538,19 @@ check_configdrive_v2() { if has_fs_with_label "config-2"; then return ${DS_FOUND} fi + # look in /config-drive /seed/config_drive for a directory + # openstack/YYYY-MM-DD format with a file meta_data.json + local d="" + for d in /config-drive "${PATH_VAR_LIB_CLOUD}/seed/config_drive"; do + set +f; set -- "$d/openstack/"2???-??-??/meta_data.json; set -f; + [ -f "$1" ] && return ${DS_FOUND} + done + # at least one cloud (softlayer) seeds config drive with only 'latest'. + local lpath="openstack/latest/meta_data.json" + if [ -e "${PATH_VAR_LIB_CLOUD}/$lpath" ]; then + debug 1 "config drive seeded directory had only 'latest'" + return ${DS_FOUND} + fi return ${DS_NOT_FOUND} } -- cgit v1.2.3 From 328fe5ab399b1f5b48d1985f41fc2ef66e368922 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 27 Mar 2017 12:43:15 -0400 Subject: GCE: Search GCE in ds-identify, consider serial number in check. While documentation indicates that the smbios product name should contain 'Google Compute Engine', experimentation and bug reports indicate that is not always the case. The change here is to change the check for GCE to also consider a serial number that starts with 'GoogleCompute-'. Also, ds-identify was not currently searching for GCE if no config of datasource_list was found. Most images have a datasource_list defined. So update the list to include GCE. LP: #1674861 --- cloudinit/sources/DataSourceGCE.py | 18 ++++++++++++++++++ tests/unittests/test_datasource/test_gce.py | 14 +++++++++++++- tools/ds-identify | 14 +++++++++++++- 3 files changed, 44 insertions(+), 2 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index b1a1c8f2..637c9505 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -62,6 +62,9 @@ class DataSourceGCE(sources.DataSource): return public_key def get_data(self): + if not platform_reports_gce(): + return False + # url_map: (our-key, path, required, is_text) url_map = [ ('instance-id', ('instance/id',), True, True), @@ -144,6 +147,21 @@ class DataSourceGCE(sources.DataSource): return self.availability_zone.rsplit('-', 1)[0] +def platform_reports_gce(): + pname = util.read_dmi_data('system-product-name') or "N/A" + if pname == "Google Compute Engine": + return True + + # system-product-name is not always guaranteed (LP: #1674861) + serial = util.read_dmi_data('system-serial-number') or "N/A" + if serial.startswith("GoogleCloud-"): + return True + + LOG.debug("Not running on google cloud. product-name=%s serial=%s", + pname, serial) + return False + + # Used to match classes to dependencies datasources = [ (DataSourceGCE, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py index 4f83454e..3eaa58e3 100644 --- a/tests/unittests/test_datasource/test_gce.py +++ b/tests/unittests/test_datasource/test_gce.py @@ -5,6 +5,7 @@ # This file is part of cloud-init. See LICENSE file for license information. import httpretty +import mock import re from base64 import b64encode, b64decode @@ -71,6 +72,11 @@ class TestDataSourceGCE(test_helpers.HttprettyTestCase): self.ds = DataSourceGCE.DataSourceGCE( settings.CFG_BUILTIN, None, helpers.Paths({})) + self.m_platform_reports_gce = mock.patch( + 'cloudinit.sources.DataSourceGCE.platform_reports_gce', + return_value=True) + self.m_platform_reports_gce.start() + self.addCleanup(self.m_platform_reports_gce.stop) super(TestDataSourceGCE, self).setUp() def test_connection(self): @@ -153,7 +159,13 @@ class TestDataSourceGCE(test_helpers.HttprettyTestCase): def test_only_last_part_of_zone_used_for_availability_zone(self): _set_mock_metadata() - self.ds.get_data() + r = self.ds.get_data() + self.assertEqual(True, r) self.assertEqual('bar', self.ds.availability_zone) + def test_get_data_returns_false_if_not_on_gce(self): + self.m_platform_reports_gce.return_value = False + self.assertEqual(False, self.ds.get_data()) + + # vi: ts=4 expandtab diff --git a/tools/ds-identify b/tools/ds-identify index bf09a3ad..15d6600e 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -108,7 +108,7 @@ DI_DSNAME="" # this has to match the builtin list in cloud-init, it is what will # be searched if there is no setting found in config. DI_DSLIST_DEFAULT="MAAS ConfigDrive NoCloud AltCloud Azure Bigstep \ -CloudSigma CloudStack DigitalOcean Ec2 OpenNebula OpenStack OVF SmartOS" +CloudSigma CloudStack DigitalOcean Ec2 GCE OpenNebula OpenStack OVF SmartOS" DI_DSLIST="" DI_MODE="" DI_ON_FOUND="" @@ -383,6 +383,14 @@ dmi_product_name_matches() { return 1 } +dmi_product_serial_matches() { + is_container && return 1 + case "${DI_DMI_PRODUCT_SERIAL}" in + $1) return 0;; + esac + return 1 +} + dmi_product_name_is() { is_container && return 1 [ "${DI_DMI_PRODUCT_NAME}" = "$1" ] @@ -770,6 +778,10 @@ dscheck_GCE() { if dmi_product_name_is "Google Compute Engine"; then return ${DS_FOUND} fi + # product name is not guaranteed (LP: #1674861) + if dmi_product_serial_matches "GoogleCloud-*"; then + return ${DS_FOUND} + fi return ${DS_NOT_FOUND} } -- cgit v1.2.3 From 61eb03fef92f435434d974fb46439189ef0b5f97 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 3 Apr 2017 10:04:43 -0400 Subject: OpenStack: add 'dvs' to the list of physical link types. Links presented in network_data.json to the guest running on ESXi are of type 'dvs'. LP: #1674946 --- cloudinit/sources/helpers/openstack.py | 1 + 1 file changed, 1 insertion(+) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 096062d5..61cd36bd 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -52,6 +52,7 @@ OS_VERSIONS = ( PHYSICAL_TYPES = ( None, 'bridge', + 'dvs', 'ethernet', 'hw_veb', 'hyperv', -- cgit v1.2.3