From b455902450e3f9ccb0cb876b460bdc7d5f6e24db Mon Sep 17 00:00:00 2001
From: Ryan Harper <ryan.harper@canonical.com>
Date: Wed, 10 Aug 2016 14:49:30 -0600
Subject: add ntp config module

Add support for installing and configuring ntp service, exposing the
minimum config of servers or pools to be added. If none are defined
then fallback on generating a list of pools by distro hosted at
pool.ntp.org (which matches what's found in the default ntp.conf
shipped in the respective distro).
---
 config/cloud.cfg | 1 +
 1 file changed, 1 insertion(+)

(limited to 'config')

diff --git a/config/cloud.cfg b/config/cloud.cfg
index a6afcc83..2d7fb473 100644
--- a/config/cloud.cfg
+++ b/config/cloud.cfg
@@ -45,6 +45,7 @@ cloud_config_modules:
  - emit_upstart
  - disk_setup
  - mounts
+ - ntp
  - ssh-import-id
  - locale
  - set-passwords
-- 
cgit v1.2.3


From 648dbbf6b090c81e989f1ab70bf99f4de16a6a70 Mon Sep 17 00:00:00 2001
From: Brent Baude <bbaude@redhat.com>
Date: Wed, 10 Aug 2016 16:36:49 -0600
Subject: Get Azure endpoint server from DHCP client

It is more efficient and cross-distribution safe to use the hooks function
from dhclient to obtain the Azure endpoint server (DHCP option 245).

This is done by providing shell scritps that are called by the hooks
infrastructure of both dhclient and NetworkManager.  The hooks then
invoke 'cloud-init dhclient-hook' that maintains json data
with the dhclient options in
/run/cloud-init/dhclient.hooks/<interface>.json .

The azure helper then pulls the value from
/run/cloud-init/dhclient.hooks/<interface>.json file(s). If that file does
not exist or the value is not present, it will then fall back to the
original method of scraping the dhcp client lease file.
---
 cloudinit/atomic_helper.py                         | 25 ++++++
 cloudinit/cmd/main.py                              | 45 ++++++----
 cloudinit/dhclient_hook.py                         | 50 +++++++++++
 cloudinit/sources/DataSourceAzure.py               | 15 ++--
 cloudinit/sources/helpers/azure.py                 | 99 +++++++++++++++++++---
 config/cloud.cfg                                   |  6 ++
 doc/sources/azure/README.rst                       | 32 ++++++-
 setup.py                                           |  2 +
 .../unittests/test_datasource/test_azure_helper.py | 15 +++-
 tools/hook-dhclient                                |  9 ++
 tools/hook-network-manager                         |  9 ++
 tools/hook-rhel.sh                                 | 12 +++
 12 files changed, 277 insertions(+), 42 deletions(-)
 create mode 100644 cloudinit/atomic_helper.py
 create mode 100644 cloudinit/dhclient_hook.py
 create mode 100755 tools/hook-dhclient
 create mode 100755 tools/hook-network-manager
 create mode 100755 tools/hook-rhel.sh

(limited to 'config')

diff --git a/cloudinit/atomic_helper.py b/cloudinit/atomic_helper.py
new file mode 100644
index 00000000..15319f71
--- /dev/null
+++ b/cloudinit/atomic_helper.py
@@ -0,0 +1,25 @@
+#!/usr/bin/python
+# vi: ts=4 expandtab
+
+import json
+import os
+import tempfile
+
+
+def atomic_write_file(path, content, mode='w'):
+    tf = None
+    try:
+        tf = tempfile.NamedTemporaryFile(dir=os.path.dirname(path),
+                                         delete=False, mode=mode)
+        tf.write(content)
+        tf.close()
+        os.rename(tf.name, path)
+    except Exception as e:
+        if tf is not None:
+            os.unlink(tf.name)
+        raise e
+
+
+def atomic_write_json(path, data):
+    return atomic_write_file(path, json.dumps(data, indent=1,
+                                              sort_keys=True) + "\n")
diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py
index 63621c1d..ba22b168 100644
--- a/cloudinit/cmd/main.py
+++ b/cloudinit/cmd/main.py
@@ -25,7 +25,6 @@ import argparse
 import json
 import os
 import sys
-import tempfile
 import time
 import traceback
 
@@ -47,6 +46,10 @@ from cloudinit.reporting import events
 from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE,
                                 CLOUD_CONFIG)
 
+from cloudinit.atomic_helper import atomic_write_json
+
+from cloudinit.dhclient_hook import LogDhclient
+
 
 # Pretty little cheetah formatted welcome message template
 WELCOME_MSG_TPL = ("Cloud-init v. ${version} running '${action}' at "
@@ -452,22 +455,10 @@ def main_single(name, args):
         return 0
 
 
-def atomic_write_file(path, content, mode='w'):
-    tf = None
-    try:
-        tf = tempfile.NamedTemporaryFile(dir=os.path.dirname(path),
-                                         delete=False, mode=mode)
-        tf.write(content)
-        tf.close()
-        os.rename(tf.name, path)
-    except Exception as e:
-        if tf is not None:
-            os.unlink(tf.name)
-        raise e
-
-
-def atomic_write_json(path, data):
-    return atomic_write_file(path, json.dumps(data, indent=1) + "\n")
+def dhclient_hook(name, args):
+    record = LogDhclient(args)
+    record.check_hooks_dir()
+    record.record()
 
 
 def status_wrapper(name, args, data_d=None, link_d=None):
@@ -627,7 +618,6 @@ def main(sysv_args=None):
     # This subcommand allows you to run a single module
     parser_single = subparsers.add_parser('single',
                                           help=('run a single module '))
-    parser_single.set_defaults(action=('single', main_single))
     parser_single.add_argument("--name", '-n', action="store",
                                help="module name to run",
                                required=True)
@@ -644,6 +634,16 @@ def main(sysv_args=None):
                                      ' pass to this module'))
     parser_single.set_defaults(action=('single', main_single))
 
+    parser_dhclient = subparsers.add_parser('dhclient-hook',
+                                            help=('run the dhclient hook'
+                                                  'to record network info'))
+    parser_dhclient.add_argument("net_action",
+                                 help=('action taken on the interface'))
+    parser_dhclient.add_argument("net_interface",
+                                 help=('the network interface being acted'
+                                       ' upon'))
+    parser_dhclient.set_defaults(action=('dhclient_hook', dhclient_hook))
+
     args = parser.parse_args(args=sysv_args)
 
     try:
@@ -677,9 +677,18 @@ def main(sysv_args=None):
                         "running single module %s" % args.name)
         report_on = args.report
 
+    elif name == 'dhclient_hook':
+        rname, rdesc = ("dhclient-hook",
+                        "running dhclient-hook module")
+
     args.reporter = events.ReportEventStack(
         rname, rdesc, reporting_enabled=report_on)
+
     with args.reporter:
         return util.log_time(
             logfunc=LOG.debug, msg="cloud-init mode '%s'" % name,
             get_uptime=True, func=functor, args=(name, args))
+
+
+if __name__ == '__main__':
+    main(sys.argv)
diff --git a/cloudinit/dhclient_hook.py b/cloudinit/dhclient_hook.py
new file mode 100644
index 00000000..9dcbe39c
--- /dev/null
+++ b/cloudinit/dhclient_hook.py
@@ -0,0 +1,50 @@
+#!/usr/bin/python
+# vi: ts=4 expandtab
+
+import os
+
+from cloudinit.atomic_helper import atomic_write_json
+from cloudinit import log as logging
+from cloudinit import stages
+
+LOG = logging.getLogger(__name__)
+
+
+class LogDhclient(object):
+
+    def __init__(self, cli_args):
+        self.hooks_dir = self._get_hooks_dir()
+        self.net_interface = cli_args.net_interface
+        self.net_action = cli_args.net_action
+        self.hook_file = os.path.join(self.hooks_dir,
+                                      self.net_interface + ".json")
+
+    @staticmethod
+    def _get_hooks_dir():
+        i = stages.Init()
+        return os.path.join(i.paths.get_runpath(), 'dhclient.hooks')
+
+    def check_hooks_dir(self):
+        if not os.path.exists(self.hooks_dir):
+            os.makedirs(self.hooks_dir)
+        else:
+            # If the action is down and the json file exists, we need to
+            # delete the file
+            if self.net_action is 'down' and os.path.exists(self.hook_file):
+                os.remove(self.hook_file)
+
+    @staticmethod
+    def get_vals(info):
+        new_info = {}
+        for k, v in info.items():
+            if k.startswith("DHCP4_") or k.startswith("new_"):
+                key = (k.replace('DHCP4_', '').replace('new_', '')).lower()
+                new_info[key] = v
+        return new_info
+
+    def record(self):
+        envs = os.environ
+        if self.hook_file is None:
+            return
+        atomic_write_json(self.hook_file, self.get_vals(envs))
+        LOG.debug("Wrote dhclient options in %s", self.hook_file)
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index 8c7e8673..a251fe01 100644
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -20,18 +20,17 @@ import base64
 import contextlib
 import crypt
 import fnmatch
+from functools import partial
 import os
 import os.path
 import time
-import xml.etree.ElementTree as ET
-
 from xml.dom import minidom
-
-from cloudinit.sources.helpers.azure import get_metadata_from_fabric
+import xml.etree.ElementTree as ET
 
 from cloudinit import log as logging
 from cloudinit.settings import PER_ALWAYS
 from cloudinit import sources
+from cloudinit.sources.helpers.azure import get_metadata_from_fabric
 from cloudinit import util
 
 LOG = logging.getLogger(__name__)
@@ -107,6 +106,8 @@ def temporary_hostname(temp_hostname, cfg, hostname_command='hostname'):
 
 
 class DataSourceAzureNet(sources.DataSource):
+    FALLBACK_LEASE = '/var/lib/dhcp/dhclient.eth0.leases'
+
     def __init__(self, sys_cfg, distro, paths):
         sources.DataSource.__init__(self, sys_cfg, distro, paths)
         self.seed_dir = os.path.join(paths.seed_dir, 'azure')
@@ -115,6 +116,8 @@ class DataSourceAzureNet(sources.DataSource):
         self.ds_cfg = util.mergemanydict([
             util.get_cfg_by_path(sys_cfg, DS_CFG_PATH, {}),
             BUILTIN_DS_CONFIG])
+        self.dhclient_lease_file = self.paths.cfgs.get('dhclient_lease',
+                                                       self.FALLBACK_LEASE)
 
     def __str__(self):
         root = sources.DataSource.__str__(self)
@@ -226,7 +229,9 @@ class DataSourceAzureNet(sources.DataSource):
         write_files(ddir, files, dirmode=0o700)
 
         if self.ds_cfg['agent_command'] == '__builtin__':
-            metadata_func = get_metadata_from_fabric
+            metadata_func = partial(get_metadata_from_fabric,
+                                    fallback_lease_file=self.
+                                    dhclient_lease_file)
         else:
             metadata_func = self.get_metadata_from_agent
         try:
diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py
index 63ccf10e..6e43440f 100644
--- a/cloudinit/sources/helpers/azure.py
+++ b/cloudinit/sources/helpers/azure.py
@@ -1,3 +1,4 @@
+import json
 import logging
 import os
 import re
@@ -6,6 +7,7 @@ import struct
 import tempfile
 import time
 
+from cloudinit import stages
 from contextlib import contextmanager
 from xml.etree import ElementTree
 
@@ -187,19 +189,32 @@ class WALinuxAgentShim(object):
         '  </Container>',
         '</Health>'])
 
-    def __init__(self):
+    def __init__(self, fallback_lease_file=None):
         LOG.debug('WALinuxAgentShim instantiated...')
-        self.endpoint = self.find_endpoint()
+        self.dhcpoptions = None
+        self._endpoint = None
         self.openssl_manager = None
         self.values = {}
+        self.lease_file = fallback_lease_file
 
     def clean_up(self):
         if self.openssl_manager is not None:
             self.openssl_manager.clean_up()
 
     @staticmethod
-    def get_ip_from_lease_value(lease_value):
-        unescaped_value = lease_value.replace('\\', '')
+    def _get_hooks_dir():
+        _paths = stages.Init()
+        return os.path.join(_paths.paths.get_runpath(), "dhclient.hooks")
+
+    @property
+    def endpoint(self):
+        if self._endpoint is None:
+            self._endpoint = self.find_endpoint(self.lease_file)
+        return self._endpoint
+
+    @staticmethod
+    def get_ip_from_lease_value(fallback_lease_value):
+        unescaped_value = fallback_lease_value.replace('\\', '')
         if len(unescaped_value) > 4:
             hex_string = ''
             for hex_pair in unescaped_value.split(':'):
@@ -213,15 +228,75 @@ class WALinuxAgentShim(object):
         return socket.inet_ntoa(packed_bytes)
 
     @staticmethod
-    def find_endpoint():
-        LOG.debug('Finding Azure endpoint...')
-        content = util.load_file('/var/lib/dhcp/dhclient.eth0.leases')
-        value = None
+    def _get_value_from_leases_file(fallback_lease_file):
+        leases = []
+        content = util.load_file(fallback_lease_file)
+        LOG.debug("content is {}".format(content))
         for line in content.splitlines():
             if 'unknown-245' in line:
-                value = line.strip(' ').split(' ', 2)[-1].strip(';\n"')
+                # Example line from Ubuntu
+                # option unknown-245 a8:3f:81:10;
+                leases.append(line.strip(' ').split(' ', 2)[-1].strip(';\n"'))
+        # Return the "most recent" one in the list
+        if len(leases) < 1:
+            return None
+        else:
+            return leases[-1]
+
+    @staticmethod
+    def _load_dhclient_json():
+        dhcp_options = {}
+        hooks_dir = WALinuxAgentShim._get_hooks_dir()
+        if not os.path.exists(hooks_dir):
+            LOG.debug("%s not found.", hooks_dir)
+            return None
+        hook_files = [os.path.join(hooks_dir, x)
+                      for x in os.listdir(hooks_dir)]
+        for hook_file in hook_files:
+            try:
+                name = os.path.basename(hook_file).replace('.json', '')
+                dhcp_options[name] = json.loads(util.load_file((hook_file)))
+            except ValueError:
+                raise ValueError("%s is not valid JSON data", hook_file)
+        return dhcp_options
+
+    @staticmethod
+    def _get_value_from_dhcpoptions(dhcp_options):
+        if dhcp_options is None:
+            return None
+        # the MS endpoint server is given to us as DHPC option 245
+        _value = None
+        for interface in dhcp_options:
+            _value = dhcp_options[interface].get('unknown_245', None)
+            if _value is not None:
+                LOG.debug("Endpoint server found in dhclient options")
+                break
+        return _value
+
+    @staticmethod
+    def find_endpoint(fallback_lease_file=None):
+        LOG.debug('Finding Azure endpoint...')
+        value = None
+        # Option-245 stored in /run/cloud-init/dhclient.hooks/<ifc>.json
+        # a dhclient exit hook that calls cloud-init-dhclient-hook
+        dhcp_options = WALinuxAgentShim._load_dhclient_json()
+        value = WALinuxAgentShim._get_value_from_dhcpoptions(dhcp_options)
         if value is None:
-            raise ValueError('No endpoint found in DHCP config.')
+            # Fallback and check the leases file if unsuccessful
+            LOG.debug("Unable to find endpoint in dhclient logs. "
+                      " Falling back to check lease files")
+            if fallback_lease_file is None:
+                LOG.warn("No fallback lease file was specified.")
+                value = None
+            else:
+                LOG.debug("Looking for endpoint in lease file %s",
+                          fallback_lease_file)
+                value = WALinuxAgentShim._get_value_from_leases_file(
+                    fallback_lease_file)
+
+        if value is None:
+            raise ValueError('No endpoint found.')
+
         endpoint_ip_address = WALinuxAgentShim.get_ip_from_lease_value(value)
         LOG.debug('Azure endpoint found at %s', endpoint_ip_address)
         return endpoint_ip_address
@@ -271,8 +346,8 @@ class WALinuxAgentShim(object):
         LOG.info('Reported ready to Azure fabric.')
 
 
-def get_metadata_from_fabric():
-    shim = WALinuxAgentShim()
+def get_metadata_from_fabric(fallback_lease_file=None):
+    shim = WALinuxAgentShim(fallback_lease_file=fallback_lease_file)
     try:
         return shim.register_with_azure_and_fetch_data()
     finally:
diff --git a/config/cloud.cfg b/config/cloud.cfg
index 2d7fb473..93ef3423 100644
--- a/config/cloud.cfg
+++ b/config/cloud.cfg
@@ -98,6 +98,7 @@ system_info:
       cloud_dir: /var/lib/cloud/
       templates_dir: /etc/cloud/templates/
       upstart_dir: /etc/init/
+      dhclient_lease: 
    package_mirrors:
      - arches: [i386, amd64]
        failsafe:
@@ -114,3 +115,8 @@ system_info:
          primary: http://ports.ubuntu.com/ubuntu-ports
          security: http://ports.ubuntu.com/ubuntu-ports
    ssh_svcname: ssh
+datasource:
+  Azure:
+    set_hostname: False
+    agent_command: __builtin__
+
diff --git a/doc/sources/azure/README.rst b/doc/sources/azure/README.rst
index 8239d1fa..48f3cc7a 100644
--- a/doc/sources/azure/README.rst
+++ b/doc/sources/azure/README.rst
@@ -9,10 +9,34 @@ Azure Platform
 The azure cloud-platform provides initial data to an instance via an attached
 CD formated in UDF.  That CD contains a 'ovf-env.xml' file that provides some
 information.  Additional information is obtained via interaction with the
-"endpoint".  The ip address of the endpoint is advertised to the instance
-inside of dhcp option 245.  On ubuntu, that can be seen in
-/var/lib/dhcp/dhclient.eth0.leases as a colon delimited hex value (example:
-``option unknown-245 64:41:60:82;`` is 100.65.96.130)
+"endpoint".
+
+To find the endpoint, we now leverage the dhcp client's ability to log its
+known values on exit.  The endpoint server is special DHCP option 245.
+Depending on your networking stack, this can be done
+by calling a script in /etc/dhcp/dhclient-exit-hooks or a file in
+/etc/NetworkManager/dispatcher.d.  Both of these call a sub-command
+'dhclient_hook' of cloud-init itself. This sub-command will write the client
+information in json format to /run/cloud-init/dhclient.hook/<interface>.json.
+
+In order for cloud-init to leverage this method to find the endpoint, the
+cloud.cfg file must contain:
+
+datasource:
+  Azure:
+    set_hostname: False
+    agent_command: __builtin__
+
+If those files are not available, the fallback is to check the leases file
+for the endpoint server (again option 245).
+
+You can define the path to the lease file with the 'dhclient_lease' configuration
+value under system_info: and paths:.  For example:
+
+      dhclient_lease: /var/lib/dhcp/dhclient.eth0.leases
+
+If no configuration value is provided, the dhclient_lease value will fallback to
+/var/lib/dhcp/dhclient.eth0.leases.
 
 walinuxagent
 ------------
diff --git a/setup.py b/setup.py
index 4abbb67e..bbadd7bf 100755
--- a/setup.py
+++ b/setup.py
@@ -176,6 +176,8 @@ else:
         (ETC + '/cloud', glob('config/*.cfg')),
         (ETC + '/cloud/cloud.cfg.d', glob('config/cloud.cfg.d/*')),
         (ETC + '/cloud/templates', glob('templates/*')),
+        (ETC + '/NetworkManager/dispatcher.d/', ['tools/hook-network-manager']),
+        (ETC + '/dhcp/dhclient-exit-hooks.d/', ['tools/hook-dhclient']),
         (USR_LIB_EXEC + '/cloud-init', ['tools/uncloud-init',
                                         'tools/write-ssh-key-fingerprints']),
         (USR + '/share/doc/cloud-init', [f for f in glob('doc/*') if is_f(f)]),
diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py
index 65202ff0..64523e16 100644
--- a/tests/unittests/test_datasource/test_azure_helper.py
+++ b/tests/unittests/test_datasource/test_azure_helper.py
@@ -54,13 +54,17 @@ class TestFindEndpoint(TestCase):
         self.load_file = patches.enter_context(
             mock.patch.object(azure_helper.util, 'load_file'))
 
+        self.dhcp_options = patches.enter_context(
+            mock.patch.object(azure_helper.WALinuxAgentShim,
+                              '_load_dhclient_json'))
+
     def test_missing_file(self):
-        self.load_file.side_effect = IOError
-        self.assertRaises(IOError,
+        self.assertRaises(ValueError,
                           azure_helper.WALinuxAgentShim.find_endpoint)
 
     def test_missing_special_azure_line(self):
         self.load_file.return_value = ''
+        self.dhcp_options.return_value = {'eth0': {'key': 'value'}}
         self.assertRaises(ValueError,
                           azure_helper.WALinuxAgentShim.find_endpoint)
 
@@ -72,13 +76,18 @@ class TestFindEndpoint(TestCase):
             ' option unknown-245 {0};'.format(encoded_address),
             '}'])
 
+    def test_from_dhcp_client(self):
+        self.dhcp_options.return_value = {"eth0": {"unknown_245": "5:4:3:2"}}
+        self.assertEqual('5.4.3.2',
+                         azure_helper.WALinuxAgentShim.find_endpoint(None))
+
     def test_latest_lease_used(self):
         encoded_addresses = ['5:4:3:2', '4:3:2:1']
         file_content = '\n'.join([self._build_lease_content(encoded_address)
                                   for encoded_address in encoded_addresses])
         self.load_file.return_value = file_content
         self.assertEqual(encoded_addresses[-1].replace(':', '.'),
-                         azure_helper.WALinuxAgentShim.find_endpoint())
+                         azure_helper.WALinuxAgentShim.find_endpoint("foobar"))
 
 
 class TestExtractIpAddressFromLeaseValue(TestCase):
diff --git a/tools/hook-dhclient b/tools/hook-dhclient
new file mode 100755
index 00000000..d099979a
--- /dev/null
+++ b/tools/hook-dhclient
@@ -0,0 +1,9 @@
+#!/bin/sh
+# This script writes DHCP lease information into the cloud-init run directory
+# It is sourced, not executed.  For more information see dhclient-script(8).
+
+case "$reason" in
+   BOUND) cloud-init dhclient-hook up "$interface";;
+   DOWN|RELEASE|REBOOT|STOP|EXPIRE)
+      cloud-init dhclient-hook down "$interface";;
+esac
diff --git a/tools/hook-network-manager b/tools/hook-network-manager
new file mode 100755
index 00000000..447b134e
--- /dev/null
+++ b/tools/hook-network-manager
@@ -0,0 +1,9 @@
+#!/bin/sh
+# This script hooks into NetworkManager(8) via its scripts
+# arguments are 'interface-name' and 'action'
+#
+
+case "$1:$2" in
+   *:up) exec cloud-init dhclient-hook up "$1";;
+   *:down) exec cloud-init dhclient-hook down "$1";;
+esac
diff --git a/tools/hook-rhel.sh b/tools/hook-rhel.sh
new file mode 100755
index 00000000..5e963a89
--- /dev/null
+++ b/tools/hook-rhel.sh
@@ -0,0 +1,12 @@
+#!/bin/sh
+# Current versions of RHEL and CentOS do not honor the directory
+# /etc/dhcp/dhclient-exit-hooks.d so this file can be placed in
+# /etc/dhcp/dhclient.d instead
+
+hook-rhel_config(){
+    cloud-init dhclient-hook up "$interface"
+}
+
+hook-rhel_restore(){
+    cloud-init dhclient-hook down "$interface"
+}
-- 
cgit v1.2.3


From 64522efe710faf6fa1615dbb60a2fc4cc8a7c278 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@brickies.net>
Date: Thu, 18 Aug 2016 12:25:29 -0400
Subject: azure dhclient-hook cleanups

This adds some function to the generator to maintain the presense of a
flag file '/run/cloud-init/enabled' indicating that cloud-init is enabled.

Then, only run the dhclient hooks if on Azure and cloud-init is enabled.
The test for is_azure currently only checks to see that the board vendor
is Microsoft, not actually that we are on azure.  Running should not be
harmful anywhere, other than slowing down dhclient.

The value of this additional code is that then dhclient having run
does not task the system with the load of cloud-init.

Additionally, some changes to config are done here.
 * rename 'dhclient_leases' to 'dhclient_lease_file'
 * move that to the datasource config (datasource/Azure/dhclient_lease_file)

Also, it removes the config in config/cloud.cfg that set agent_command
to __builtin__.  This means that by default cloud-init still needs
the agent installed.  The suggested follow-on improvement is to
use __builtin__ if there is no walinux-agent installed.
---
 cloudinit/sources/DataSourceAzure.py | 13 +++++++------
 cloudinit/sources/helpers/azure.py   |  3 ++-
 config/cloud.cfg                     |  6 ------
 doc/sources/azure/README.rst         |  9 +++------
 systemd/cloud-init-generator         |  5 +++++
 tools/hook-dhclient                  | 25 ++++++++++++++++++++-----
 tools/hook-network-manager           | 23 +++++++++++++++++++----
 tools/hook-rhel.sh                   | 15 +++++++++++++++
 8 files changed, 71 insertions(+), 28 deletions(-)

(limited to 'config')

diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index a251fe01..dbc2bb68 100644
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -54,6 +54,7 @@ BUILTIN_DS_CONFIG = {
         'hostname_command': 'hostname',
     },
     'disk_aliases': {'ephemeral0': '/dev/sdb'},
+    'dhclient_lease_file': '/var/lib/dhcp/dhclient.eth0.leases',
 }
 
 BUILTIN_CLOUD_CONFIG = {
@@ -106,8 +107,6 @@ def temporary_hostname(temp_hostname, cfg, hostname_command='hostname'):
 
 
 class DataSourceAzureNet(sources.DataSource):
-    FALLBACK_LEASE = '/var/lib/dhcp/dhclient.eth0.leases'
-
     def __init__(self, sys_cfg, distro, paths):
         sources.DataSource.__init__(self, sys_cfg, distro, paths)
         self.seed_dir = os.path.join(paths.seed_dir, 'azure')
@@ -116,8 +115,7 @@ class DataSourceAzureNet(sources.DataSource):
         self.ds_cfg = util.mergemanydict([
             util.get_cfg_by_path(sys_cfg, DS_CFG_PATH, {}),
             BUILTIN_DS_CONFIG])
-        self.dhclient_lease_file = self.paths.cfgs.get('dhclient_lease',
-                                                       self.FALLBACK_LEASE)
+        self.dhclient_lease_file = self.ds_cfg.get('dhclient_lease_file')
 
     def __str__(self):
         root = sources.DataSource.__str__(self)
@@ -126,6 +124,9 @@ class DataSourceAzureNet(sources.DataSource):
     def get_metadata_from_agent(self):
         temp_hostname = self.metadata.get('local-hostname')
         hostname_command = self.ds_cfg['hostname_bounce']['hostname_command']
+        agent_cmd = self.ds_cfg['agent_command']
+        LOG.debug("Getting metadata via agent.  hostname=%s cmd=%s",
+                  temp_hostname, agent_cmd)
         with temporary_hostname(temp_hostname, self.ds_cfg,
                                 hostname_command=hostname_command) \
                 as previous_hostname:
@@ -141,7 +142,7 @@ class DataSourceAzureNet(sources.DataSource):
                     util.logexc(LOG, "handling set_hostname failed")
 
             try:
-                invoke_agent(self.ds_cfg['agent_command'])
+                invoke_agent(agent_cmd)
             except util.ProcessExecutionError:
                 # claim the datasource even if the command failed
                 util.logexc(LOG, "agent command '%s' failed.",
@@ -234,13 +235,13 @@ class DataSourceAzureNet(sources.DataSource):
                                     dhclient_lease_file)
         else:
             metadata_func = self.get_metadata_from_agent
+
         try:
             fabric_data = metadata_func()
         except Exception as exc:
             LOG.info("Error communicating with Azure fabric; assume we aren't"
                      " on Azure.", exc_info=True)
             return False
-
         self.metadata['instance-id'] = util.read_dmi_data('system-uuid')
         self.metadata.update(fabric_data)
 
diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py
index 6e43440f..689ed4cc 100644
--- a/cloudinit/sources/helpers/azure.py
+++ b/cloudinit/sources/helpers/azure.py
@@ -190,7 +190,8 @@ class WALinuxAgentShim(object):
         '</Health>'])
 
     def __init__(self, fallback_lease_file=None):
-        LOG.debug('WALinuxAgentShim instantiated...')
+        LOG.debug('WALinuxAgentShim instantiated, fallback_lease_file=%s',
+                  fallback_lease_file)
         self.dhcpoptions = None
         self._endpoint = None
         self.openssl_manager = None
diff --git a/config/cloud.cfg b/config/cloud.cfg
index 93ef3423..2d7fb473 100644
--- a/config/cloud.cfg
+++ b/config/cloud.cfg
@@ -98,7 +98,6 @@ system_info:
       cloud_dir: /var/lib/cloud/
       templates_dir: /etc/cloud/templates/
       upstart_dir: /etc/init/
-      dhclient_lease: 
    package_mirrors:
      - arches: [i386, amd64]
        failsafe:
@@ -115,8 +114,3 @@ system_info:
          primary: http://ports.ubuntu.com/ubuntu-ports
          security: http://ports.ubuntu.com/ubuntu-ports
    ssh_svcname: ssh
-datasource:
-  Azure:
-    set_hostname: False
-    agent_command: __builtin__
-
diff --git a/doc/sources/azure/README.rst b/doc/sources/azure/README.rst
index 48f3cc7a..ec7d9e84 100644
--- a/doc/sources/azure/README.rst
+++ b/doc/sources/azure/README.rst
@@ -30,13 +30,10 @@ datasource:
 If those files are not available, the fallback is to check the leases file
 for the endpoint server (again option 245).
 
-You can define the path to the lease file with the 'dhclient_lease' configuration
-value under system_info: and paths:.  For example:
+You can define the path to the lease file with the 'dhclient_lease_file'
+configuration.  The default value is /var/lib/dhcp/dhclient.eth0.leases.
 
-      dhclient_lease: /var/lib/dhcp/dhclient.eth0.leases
-
-If no configuration value is provided, the dhclient_lease value will fallback to
-/var/lib/dhcp/dhclient.eth0.leases.
+    dhclient_lease_file: /var/lib/dhcp/dhclient.eth0.leases
 
 walinuxagent
 ------------
diff --git a/systemd/cloud-init-generator b/systemd/cloud-init-generator
index 2d319695..fedb6309 100755
--- a/systemd/cloud-init-generator
+++ b/systemd/cloud-init-generator
@@ -6,6 +6,7 @@ DEBUG_LEVEL=1
 LOG_D="/run/cloud-init"
 ENABLE="enabled"
 DISABLE="disabled"
+RUN_ENABLED_FILE="$LOG_D/$ENABLE"
 CLOUD_SYSTEM_TARGET="/lib/systemd/system/cloud-init.target"
 CLOUD_TARGET_NAME="cloud-init.target"
 # lxc sets 'container', but lets make that explicitly a global
@@ -107,6 +108,7 @@ main() {
                     "ln $CLOUD_SYSTEM_TARGET $link_path"
             fi
         fi
+        : > "$RUN_ENABLED_FILE"
     elif [ "$result" = "$DISABLE" ]; then
         if [ -f "$link_path" ]; then
             if rm -f "$link_path"; then
@@ -118,6 +120,9 @@ main() {
         else
             debug 1 "already disabled: no change needed [no $link_path]"
         fi
+        if [ -e "$RUN_ENABLED_FILE" ]; then
+            rm -f "$RUN_ENABLED_FILE"
+        fi
     else
         debug 0 "unexpected result '$result'"
         ret=3
diff --git a/tools/hook-dhclient b/tools/hook-dhclient
index d099979a..6a4626c6 100755
--- a/tools/hook-dhclient
+++ b/tools/hook-dhclient
@@ -1,9 +1,24 @@
 #!/bin/sh
 # This script writes DHCP lease information into the cloud-init run directory
 # It is sourced, not executed.  For more information see dhclient-script(8).
+is_azure() {
+    local dmi_path="/sys/class/dmi/id/board_vendor" vendor=""
+    if [ -e "$dmi_path" ] && read vendor < "$dmi_path"; then
+        [ "$vendor" = "Microsoft Corporation" ] && return 0
+    fi
+    return 1
+}
 
-case "$reason" in
-   BOUND) cloud-init dhclient-hook up "$interface";;
-   DOWN|RELEASE|REBOOT|STOP|EXPIRE)
-      cloud-init dhclient-hook down "$interface";;
-esac
+is_enabled() {
+    # only execute hooks if cloud-init is enabled and on azure
+    [ -e /run/cloud-init/enabled ] || return 1
+    is_azure
+}
+
+if is_enabled; then
+   case "$reason" in
+      BOUND) cloud-init dhclient-hook up "$interface";;
+      DOWN|RELEASE|REBOOT|STOP|EXPIRE)
+         cloud-init dhclient-hook down "$interface";;
+   esac
+fi
diff --git a/tools/hook-network-manager b/tools/hook-network-manager
index 447b134e..98a36c8a 100755
--- a/tools/hook-network-manager
+++ b/tools/hook-network-manager
@@ -2,8 +2,23 @@
 # This script hooks into NetworkManager(8) via its scripts
 # arguments are 'interface-name' and 'action'
 #
+is_azure() {
+    local dmi_path="/sys/class/dmi/id/board_vendor" vendor=""
+    if [ -e "$dmi_path" ] && read vendor < "$dmi_path"; then
+        [ "$vendor" = "Microsoft Corporation" ] && return 0
+    fi
+    return 1
+}
 
-case "$1:$2" in
-   *:up) exec cloud-init dhclient-hook up "$1";;
-   *:down) exec cloud-init dhclient-hook down "$1";;
-esac
+is_enabled() {
+    # only execute hooks if cloud-init is enabled and on azure
+    [ -e /run/cloud-init/enabled ] || return 1
+    is_azure
+}
+
+if is_enabled; then
+    case "$1:$2" in
+        *:up) exec cloud-init dhclient-hook up "$1";;
+        *:down) exec cloud-init dhclient-hook down "$1";;
+    esac
+fi
diff --git a/tools/hook-rhel.sh b/tools/hook-rhel.sh
index 5e963a89..8232414c 100755
--- a/tools/hook-rhel.sh
+++ b/tools/hook-rhel.sh
@@ -2,11 +2,26 @@
 # Current versions of RHEL and CentOS do not honor the directory
 # /etc/dhcp/dhclient-exit-hooks.d so this file can be placed in
 # /etc/dhcp/dhclient.d instead
+is_azure() {
+    local dmi_path="/sys/class/dmi/id/board_vendor" vendor=""
+    if [ -e "$dmi_path" ] && read vendor < "$dmi_path"; then
+        [ "$vendor" = "Microsoft Corporation" ] && return 0
+    fi
+    return 1
+}
+
+is_enabled() {
+    # only execute hooks if cloud-init is enabled and on azure
+    [ -e /run/cloud-init/enabled ] || return 1
+    is_azure
+}
 
 hook-rhel_config(){
+    is_enabled || return 0
     cloud-init dhclient-hook up "$interface"
 }
 
 hook-rhel_restore(){
+    is_enabled || return 0
     cloud-init dhclient-hook down "$interface"
 }
-- 
cgit v1.2.3


From 34a26f7f59f2963691e36ca0476bec9fc9ccef63 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@brickies.net>
Date: Thu, 8 Sep 2016 13:17:37 -0400
Subject: systemd: Better support package and upgrade.

In systemd, package installation before the system is fully booted
(systemctl is-system-running ==  starting) may result in the package not
being started.  Upgrade (package_upgrade: true) can also cause failure if
that is done during systemd boot.

The solution here is:
 a.) move config modules that do or may do package installation to
     'final_modules'. That list is:
     - snappy
     - package-update-upgrade-install
     - fan
     - landscape
     - lxd
     - puppet
     - chef
     - salt-minion
     - mcollective
 b.) move cloud-final.service to run as 'Type=idle'

LP: #1576692, #1621336
---
 config/cloud.cfg            | 16 ++++++++--------
 systemd/cloud-final.service |  2 +-
 2 files changed, 9 insertions(+), 9 deletions(-)

(limited to 'config')

diff --git a/config/cloud.cfg b/config/cloud.cfg
index 2d7fb473..7c94ec5c 100644
--- a/config/cloud.cfg
+++ b/config/cloud.cfg
@@ -49,25 +49,25 @@ cloud_config_modules:
  - ssh-import-id
  - locale
  - set-passwords
- - snappy
  - grub-dpkg
  - apt-pipelining
  - apt-configure
+ - timezone
+ - disable-ec2-metadata
+ - runcmd
+ - byobu
+
+# The modules that run in the 'final' stage
+cloud_final_modules:
+ - snappy
  - package-update-upgrade-install
  - fan
  - landscape
- - timezone
  - lxd
  - puppet
  - chef
  - salt-minion
  - mcollective
- - disable-ec2-metadata
- - runcmd
- - byobu
-
-# The modules that run in the 'final' stage
-cloud_final_modules:
  - rightscale_userdata
  - scripts-vendor
  - scripts-per-once
diff --git a/systemd/cloud-final.service b/systemd/cloud-final.service
index 3927710f..b8f69b78 100644
--- a/systemd/cloud-final.service
+++ b/systemd/cloud-final.service
@@ -1,6 +1,6 @@
 [Unit]
 Description=Execute cloud user/final scripts
-After=network-online.target cloud-config.service rc-local.service
+After=network-online.target cloud-config.service rc-local.service multi-user.target
 Wants=network-online.target cloud-config.service
 
 [Service]
-- 
cgit v1.2.3