From 648dbbf6b090c81e989f1ab70bf99f4de16a6a70 Mon Sep 17 00:00:00 2001 From: Brent Baude Date: Wed, 10 Aug 2016 16:36:49 -0600 Subject: Get Azure endpoint server from DHCP client It is more efficient and cross-distribution safe to use the hooks function from dhclient to obtain the Azure endpoint server (DHCP option 245). This is done by providing shell scritps that are called by the hooks infrastructure of both dhclient and NetworkManager. The hooks then invoke 'cloud-init dhclient-hook' that maintains json data with the dhclient options in /run/cloud-init/dhclient.hooks/.json . The azure helper then pulls the value from /run/cloud-init/dhclient.hooks/.json file(s). If that file does not exist or the value is not present, it will then fall back to the original method of scraping the dhcp client lease file. --- cloudinit/sources/helpers/azure.py | 99 +++++++++++++++++++++++++++++++++----- 1 file changed, 87 insertions(+), 12 deletions(-) (limited to 'cloudinit/sources/helpers/azure.py') diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 63ccf10e..6e43440f 100644 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -1,3 +1,4 @@ +import json import logging import os import re @@ -6,6 +7,7 @@ import struct import tempfile import time +from cloudinit import stages from contextlib import contextmanager from xml.etree import ElementTree @@ -187,19 +189,32 @@ class WALinuxAgentShim(object): ' ', '']) - def __init__(self): + def __init__(self, fallback_lease_file=None): LOG.debug('WALinuxAgentShim instantiated...') - self.endpoint = self.find_endpoint() + self.dhcpoptions = None + self._endpoint = None self.openssl_manager = None self.values = {} + self.lease_file = fallback_lease_file def clean_up(self): if self.openssl_manager is not None: self.openssl_manager.clean_up() @staticmethod - def get_ip_from_lease_value(lease_value): - unescaped_value = lease_value.replace('\\', '') + def _get_hooks_dir(): + _paths = stages.Init() + return os.path.join(_paths.paths.get_runpath(), "dhclient.hooks") + + @property + def endpoint(self): + if self._endpoint is None: + self._endpoint = self.find_endpoint(self.lease_file) + return self._endpoint + + @staticmethod + def get_ip_from_lease_value(fallback_lease_value): + unescaped_value = fallback_lease_value.replace('\\', '') if len(unescaped_value) > 4: hex_string = '' for hex_pair in unescaped_value.split(':'): @@ -213,15 +228,75 @@ class WALinuxAgentShim(object): return socket.inet_ntoa(packed_bytes) @staticmethod - def find_endpoint(): - LOG.debug('Finding Azure endpoint...') - content = util.load_file('/var/lib/dhcp/dhclient.eth0.leases') - value = None + def _get_value_from_leases_file(fallback_lease_file): + leases = [] + content = util.load_file(fallback_lease_file) + LOG.debug("content is {}".format(content)) for line in content.splitlines(): if 'unknown-245' in line: - value = line.strip(' ').split(' ', 2)[-1].strip(';\n"') + # Example line from Ubuntu + # option unknown-245 a8:3f:81:10; + leases.append(line.strip(' ').split(' ', 2)[-1].strip(';\n"')) + # Return the "most recent" one in the list + if len(leases) < 1: + return None + else: + return leases[-1] + + @staticmethod + def _load_dhclient_json(): + dhcp_options = {} + hooks_dir = WALinuxAgentShim._get_hooks_dir() + if not os.path.exists(hooks_dir): + LOG.debug("%s not found.", hooks_dir) + return None + hook_files = [os.path.join(hooks_dir, x) + for x in os.listdir(hooks_dir)] + for hook_file in hook_files: + try: + name = os.path.basename(hook_file).replace('.json', '') + dhcp_options[name] = json.loads(util.load_file((hook_file))) + except ValueError: + raise ValueError("%s is not valid JSON data", hook_file) + return dhcp_options + + @staticmethod + def _get_value_from_dhcpoptions(dhcp_options): + if dhcp_options is None: + return None + # the MS endpoint server is given to us as DHPC option 245 + _value = None + for interface in dhcp_options: + _value = dhcp_options[interface].get('unknown_245', None) + if _value is not None: + LOG.debug("Endpoint server found in dhclient options") + break + return _value + + @staticmethod + def find_endpoint(fallback_lease_file=None): + LOG.debug('Finding Azure endpoint...') + value = None + # Option-245 stored in /run/cloud-init/dhclient.hooks/.json + # a dhclient exit hook that calls cloud-init-dhclient-hook + dhcp_options = WALinuxAgentShim._load_dhclient_json() + value = WALinuxAgentShim._get_value_from_dhcpoptions(dhcp_options) if value is None: - raise ValueError('No endpoint found in DHCP config.') + # Fallback and check the leases file if unsuccessful + LOG.debug("Unable to find endpoint in dhclient logs. " + " Falling back to check lease files") + if fallback_lease_file is None: + LOG.warn("No fallback lease file was specified.") + value = None + else: + LOG.debug("Looking for endpoint in lease file %s", + fallback_lease_file) + value = WALinuxAgentShim._get_value_from_leases_file( + fallback_lease_file) + + if value is None: + raise ValueError('No endpoint found.') + endpoint_ip_address = WALinuxAgentShim.get_ip_from_lease_value(value) LOG.debug('Azure endpoint found at %s', endpoint_ip_address) return endpoint_ip_address @@ -271,8 +346,8 @@ class WALinuxAgentShim(object): LOG.info('Reported ready to Azure fabric.') -def get_metadata_from_fabric(): - shim = WALinuxAgentShim() +def get_metadata_from_fabric(fallback_lease_file=None): + shim = WALinuxAgentShim(fallback_lease_file=fallback_lease_file) try: return shim.register_with_azure_and_fetch_data() finally: -- cgit v1.2.3 From 64522efe710faf6fa1615dbb60a2fc4cc8a7c278 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 18 Aug 2016 12:25:29 -0400 Subject: azure dhclient-hook cleanups This adds some function to the generator to maintain the presense of a flag file '/run/cloud-init/enabled' indicating that cloud-init is enabled. Then, only run the dhclient hooks if on Azure and cloud-init is enabled. The test for is_azure currently only checks to see that the board vendor is Microsoft, not actually that we are on azure. Running should not be harmful anywhere, other than slowing down dhclient. The value of this additional code is that then dhclient having run does not task the system with the load of cloud-init. Additionally, some changes to config are done here. * rename 'dhclient_leases' to 'dhclient_lease_file' * move that to the datasource config (datasource/Azure/dhclient_lease_file) Also, it removes the config in config/cloud.cfg that set agent_command to __builtin__. This means that by default cloud-init still needs the agent installed. The suggested follow-on improvement is to use __builtin__ if there is no walinux-agent installed. --- cloudinit/sources/DataSourceAzure.py | 13 +++++++------ cloudinit/sources/helpers/azure.py | 3 ++- config/cloud.cfg | 6 ------ doc/sources/azure/README.rst | 9 +++------ systemd/cloud-init-generator | 5 +++++ tools/hook-dhclient | 25 ++++++++++++++++++++----- tools/hook-network-manager | 23 +++++++++++++++++++---- tools/hook-rhel.sh | 15 +++++++++++++++ 8 files changed, 71 insertions(+), 28 deletions(-) (limited to 'cloudinit/sources/helpers/azure.py') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index a251fe01..dbc2bb68 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -54,6 +54,7 @@ BUILTIN_DS_CONFIG = { 'hostname_command': 'hostname', }, 'disk_aliases': {'ephemeral0': '/dev/sdb'}, + 'dhclient_lease_file': '/var/lib/dhcp/dhclient.eth0.leases', } BUILTIN_CLOUD_CONFIG = { @@ -106,8 +107,6 @@ def temporary_hostname(temp_hostname, cfg, hostname_command='hostname'): class DataSourceAzureNet(sources.DataSource): - FALLBACK_LEASE = '/var/lib/dhcp/dhclient.eth0.leases' - def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) self.seed_dir = os.path.join(paths.seed_dir, 'azure') @@ -116,8 +115,7 @@ class DataSourceAzureNet(sources.DataSource): self.ds_cfg = util.mergemanydict([ util.get_cfg_by_path(sys_cfg, DS_CFG_PATH, {}), BUILTIN_DS_CONFIG]) - self.dhclient_lease_file = self.paths.cfgs.get('dhclient_lease', - self.FALLBACK_LEASE) + self.dhclient_lease_file = self.ds_cfg.get('dhclient_lease_file') def __str__(self): root = sources.DataSource.__str__(self) @@ -126,6 +124,9 @@ class DataSourceAzureNet(sources.DataSource): def get_metadata_from_agent(self): temp_hostname = self.metadata.get('local-hostname') hostname_command = self.ds_cfg['hostname_bounce']['hostname_command'] + agent_cmd = self.ds_cfg['agent_command'] + LOG.debug("Getting metadata via agent. hostname=%s cmd=%s", + temp_hostname, agent_cmd) with temporary_hostname(temp_hostname, self.ds_cfg, hostname_command=hostname_command) \ as previous_hostname: @@ -141,7 +142,7 @@ class DataSourceAzureNet(sources.DataSource): util.logexc(LOG, "handling set_hostname failed") try: - invoke_agent(self.ds_cfg['agent_command']) + invoke_agent(agent_cmd) except util.ProcessExecutionError: # claim the datasource even if the command failed util.logexc(LOG, "agent command '%s' failed.", @@ -234,13 +235,13 @@ class DataSourceAzureNet(sources.DataSource): dhclient_lease_file) else: metadata_func = self.get_metadata_from_agent + try: fabric_data = metadata_func() except Exception as exc: LOG.info("Error communicating with Azure fabric; assume we aren't" " on Azure.", exc_info=True) return False - self.metadata['instance-id'] = util.read_dmi_data('system-uuid') self.metadata.update(fabric_data) diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 6e43440f..689ed4cc 100644 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -190,7 +190,8 @@ class WALinuxAgentShim(object): '']) def __init__(self, fallback_lease_file=None): - LOG.debug('WALinuxAgentShim instantiated...') + LOG.debug('WALinuxAgentShim instantiated, fallback_lease_file=%s', + fallback_lease_file) self.dhcpoptions = None self._endpoint = None self.openssl_manager = None diff --git a/config/cloud.cfg b/config/cloud.cfg index 93ef3423..2d7fb473 100644 --- a/config/cloud.cfg +++ b/config/cloud.cfg @@ -98,7 +98,6 @@ system_info: cloud_dir: /var/lib/cloud/ templates_dir: /etc/cloud/templates/ upstart_dir: /etc/init/ - dhclient_lease: package_mirrors: - arches: [i386, amd64] failsafe: @@ -115,8 +114,3 @@ system_info: primary: http://ports.ubuntu.com/ubuntu-ports security: http://ports.ubuntu.com/ubuntu-ports ssh_svcname: ssh -datasource: - Azure: - set_hostname: False - agent_command: __builtin__ - diff --git a/doc/sources/azure/README.rst b/doc/sources/azure/README.rst index 48f3cc7a..ec7d9e84 100644 --- a/doc/sources/azure/README.rst +++ b/doc/sources/azure/README.rst @@ -30,13 +30,10 @@ datasource: If those files are not available, the fallback is to check the leases file for the endpoint server (again option 245). -You can define the path to the lease file with the 'dhclient_lease' configuration -value under system_info: and paths:. For example: +You can define the path to the lease file with the 'dhclient_lease_file' +configuration. The default value is /var/lib/dhcp/dhclient.eth0.leases. - dhclient_lease: /var/lib/dhcp/dhclient.eth0.leases - -If no configuration value is provided, the dhclient_lease value will fallback to -/var/lib/dhcp/dhclient.eth0.leases. + dhclient_lease_file: /var/lib/dhcp/dhclient.eth0.leases walinuxagent ------------ diff --git a/systemd/cloud-init-generator b/systemd/cloud-init-generator index 2d319695..fedb6309 100755 --- a/systemd/cloud-init-generator +++ b/systemd/cloud-init-generator @@ -6,6 +6,7 @@ DEBUG_LEVEL=1 LOG_D="/run/cloud-init" ENABLE="enabled" DISABLE="disabled" +RUN_ENABLED_FILE="$LOG_D/$ENABLE" CLOUD_SYSTEM_TARGET="/lib/systemd/system/cloud-init.target" CLOUD_TARGET_NAME="cloud-init.target" # lxc sets 'container', but lets make that explicitly a global @@ -107,6 +108,7 @@ main() { "ln $CLOUD_SYSTEM_TARGET $link_path" fi fi + : > "$RUN_ENABLED_FILE" elif [ "$result" = "$DISABLE" ]; then if [ -f "$link_path" ]; then if rm -f "$link_path"; then @@ -118,6 +120,9 @@ main() { else debug 1 "already disabled: no change needed [no $link_path]" fi + if [ -e "$RUN_ENABLED_FILE" ]; then + rm -f "$RUN_ENABLED_FILE" + fi else debug 0 "unexpected result '$result'" ret=3 diff --git a/tools/hook-dhclient b/tools/hook-dhclient index d099979a..6a4626c6 100755 --- a/tools/hook-dhclient +++ b/tools/hook-dhclient @@ -1,9 +1,24 @@ #!/bin/sh # This script writes DHCP lease information into the cloud-init run directory # It is sourced, not executed. For more information see dhclient-script(8). +is_azure() { + local dmi_path="/sys/class/dmi/id/board_vendor" vendor="" + if [ -e "$dmi_path" ] && read vendor < "$dmi_path"; then + [ "$vendor" = "Microsoft Corporation" ] && return 0 + fi + return 1 +} -case "$reason" in - BOUND) cloud-init dhclient-hook up "$interface";; - DOWN|RELEASE|REBOOT|STOP|EXPIRE) - cloud-init dhclient-hook down "$interface";; -esac +is_enabled() { + # only execute hooks if cloud-init is enabled and on azure + [ -e /run/cloud-init/enabled ] || return 1 + is_azure +} + +if is_enabled; then + case "$reason" in + BOUND) cloud-init dhclient-hook up "$interface";; + DOWN|RELEASE|REBOOT|STOP|EXPIRE) + cloud-init dhclient-hook down "$interface";; + esac +fi diff --git a/tools/hook-network-manager b/tools/hook-network-manager index 447b134e..98a36c8a 100755 --- a/tools/hook-network-manager +++ b/tools/hook-network-manager @@ -2,8 +2,23 @@ # This script hooks into NetworkManager(8) via its scripts # arguments are 'interface-name' and 'action' # +is_azure() { + local dmi_path="/sys/class/dmi/id/board_vendor" vendor="" + if [ -e "$dmi_path" ] && read vendor < "$dmi_path"; then + [ "$vendor" = "Microsoft Corporation" ] && return 0 + fi + return 1 +} -case "$1:$2" in - *:up) exec cloud-init dhclient-hook up "$1";; - *:down) exec cloud-init dhclient-hook down "$1";; -esac +is_enabled() { + # only execute hooks if cloud-init is enabled and on azure + [ -e /run/cloud-init/enabled ] || return 1 + is_azure +} + +if is_enabled; then + case "$1:$2" in + *:up) exec cloud-init dhclient-hook up "$1";; + *:down) exec cloud-init dhclient-hook down "$1";; + esac +fi diff --git a/tools/hook-rhel.sh b/tools/hook-rhel.sh index 5e963a89..8232414c 100755 --- a/tools/hook-rhel.sh +++ b/tools/hook-rhel.sh @@ -2,11 +2,26 @@ # Current versions of RHEL and CentOS do not honor the directory # /etc/dhcp/dhclient-exit-hooks.d so this file can be placed in # /etc/dhcp/dhclient.d instead +is_azure() { + local dmi_path="/sys/class/dmi/id/board_vendor" vendor="" + if [ -e "$dmi_path" ] && read vendor < "$dmi_path"; then + [ "$vendor" = "Microsoft Corporation" ] && return 0 + fi + return 1 +} + +is_enabled() { + # only execute hooks if cloud-init is enabled and on azure + [ -e /run/cloud-init/enabled ] || return 1 + is_azure +} hook-rhel_config(){ + is_enabled || return 0 cloud-init dhclient-hook up "$interface" } hook-rhel_restore(){ + is_enabled || return 0 cloud-init dhclient-hook down "$interface" } -- cgit v1.2.3