Complete initial cleanup for refactoring/rework.

Some of the cleanups were the following

1. Using standard (logged) utility functions for sub process work, writing, reading files, and other file system/operating system options
2. Having distrobutions impelement there own subclasses to handle system specifics (if applicable)
3. Having a cloud wrapper that provides just the functionality we want to expose (cloud.py)
4. Using a path class instead of globals for all cloud init paths (it is configured via config)
5. Removal of as much shared global state as possible (there should be none, minus a set of constants)
6. Other various cleanups that remove transforms/handlers/modules from reading/writing/chmoding there own files. 
 a. They should be using util functions to take advantage of the logging that is now enabled in those util functions (very useful for debugging)
7. Urls being read and checked from a single module that serves this and only this purpose (+1 for code organization)
8. Updates to log whenever a transform decides not to run
9. Ensure whenever a exception is thrown (and possibly captured) that the util.logexc function is called 
 a. For debugging, tracing this is important to not just drop them on the floor.
10. Code shuffling into utils.py where it makes sense (and where it could serve a benefit for other code now or in the future)

1 files changed, 49 insertions, 52 deletions

diff --git a/cloudinit/transforms/cc_chef.py b/cloudinit/transforms/cc_chef.py
index 941e04fe..12c2f539 100644
--- a/cloudinit/transforms/cc_chef.py
+++ b/cloudinit/transforms/cc_chef.py
@@ -18,53 +18,59 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import os
-import subprocess
 import json
-import cloudinit.CloudConfig as cc
-import cloudinit.util as util
+import os
+
+from cloudinit import templater
+from cloudinit import util
 
 ruby_version_default = "1.8"
 
 
-def handle(_name, cfg, cloud, log, _args):
+def handle(name, cfg, cloud, log, _args):
+
     # If there isn't a chef key in the configuration don't do anything
     if 'chef' not in cfg:
+        log.debug("Skipping module named %s, no 'chef' key in configuration", name)
         return
     chef_cfg = cfg['chef']
 
     # ensure the chef directories we use exist
-    mkdirs(['/etc/chef', '/var/log/chef', '/var/lib/chef',
-            '/var/cache/chef', '/var/backups/chef', '/var/run/chef'])
+    util.ensure_dirs(['/etc/chef', '/var/log/chef', '/var/lib/chef',
+                     '/var/cache/chef', '/var/backups/chef', '/var/run/chef'])
 
     # set the validation key based on the presence of either 'validation_key'
     # or 'validation_cert'. In the case where both exist, 'validation_key'
     # takes precedence
     for key in ('validation_key', 'validation_cert'):
         if key in chef_cfg and chef_cfg[key]:
-            with open('/etc/chef/validation.pem', 'w') as validation_key_fh:
-                validation_key_fh.write(chef_cfg[key])
+            util.write_file('/etc/chef/validation.pem', chef_cfg[key])
             break
 
     # create the chef config from template
-    util.render_to_file('chef_client.rb', '/etc/chef/client.rb',
-        {'server_url': chef_cfg['server_url'],
-         'node_name': util.get_cfg_option_str(chef_cfg, 'node_name',
-                                          cloud.datasource.get_instance_id()),
-         'environment': util.get_cfg_option_str(chef_cfg, 'environment',
-                                                '_default'),
-         'validation_name': chef_cfg['validation_name']})
+    template_fn = cloud.get_template_filename('chef_client.rb')
+    if template_fn:
+        params = {
+            'server_url': chef_cfg['server_url'],
+            'node_name': util.get_cfg_option_str(chef_cfg, 'node_name',
+                                    cloud.datasource.get_instance_id()),
+            'environment': util.get_cfg_option_str(chef_cfg, 'environment',
+                                                   '_default'),
+            'validation_name': chef_cfg['validation_name']
+        }
+        templater.render_to_file(template_fn, '/etc/chef/client.rb', params)
+    else:
+        log.warn("No template found, not rendering to /etc/chef/client.rb")
 
     # set the firstboot json
-    with open('/etc/chef/firstboot.json', 'w') as firstboot_json_fh:
-        initial_json = {}
-        if 'run_list' in chef_cfg:
-            initial_json['run_list'] = chef_cfg['run_list']
-        if 'initial_attributes' in chef_cfg:
-            initial_attributes = chef_cfg['initial_attributes']
-            for k in initial_attributes.keys():
-                initial_json[k] = initial_attributes[k]
-        firstboot_json_fh.write(json.dumps(initial_json))
+    initial_json = {}
+    if 'run_list' in chef_cfg:
+        initial_json['run_list'] = chef_cfg['run_list']
+    if 'initial_attributes' in chef_cfg:
+        initial_attributes = chef_cfg['initial_attributes']
+        for k in initial_attributes.keys():
+            initial_json[k] = initial_attributes[k]
+    util.write_file('/etc/chef/firstboot.json', json.dumps(initial_json))
 
     # If chef is not installed, we install chef based on 'install_type'
     if not os.path.isfile('/usr/bin/chef-client'):
@@ -75,14 +81,15 @@ def handle(_name, cfg, cloud, log, _args):
             chef_version = util.get_cfg_option_str(chef_cfg, 'version', None)
             ruby_version = util.get_cfg_option_str(chef_cfg, 'ruby_version',
                                                    ruby_version_default)
-            install_chef_from_gems(ruby_version, chef_version)
+            install_chef_from_gems(cloud.distro, ruby_version, chef_version)
             # and finally, run chef-client
-            log.debug('running chef-client')
-            subprocess.check_call(['/usr/bin/chef-client', '-d', '-i', '1800',
-                                   '-s', '20'])
-        else:
+            log.debug('Running chef-client')
+            util.subp(['/usr/bin/chef-client', '-d', '-i', '1800', '-s', '20'])
+        elif install_type == 'packages':
             # this will install and run the chef-client from packages
-            cc.install_packages(('chef',))
+            cloud.distro.install_packages(('chef',))
+        else:
+            log.warn("Unknown chef install type %s", install_type)
 
 
 def get_ruby_packages(version):
@@ -90,30 +97,20 @@ def get_ruby_packages(version):
     pkgs = ['ruby%s' % version, 'ruby%s-dev' % version]
     if version == "1.8":
         pkgs.extend(('libopenssl-ruby1.8', 'rubygems1.8'))
-    return(pkgs)
+    return pkgs
 
 
-def install_chef_from_gems(ruby_version, chef_version=None):
-    cc.install_packages(get_ruby_packages(ruby_version))
+def install_chef_from_gems(ruby_version, chef_version, distro):
+    distro.install_packages(get_ruby_packages(ruby_version))
     if not os.path.exists('/usr/bin/gem'):
-        os.symlink('/usr/bin/gem%s' % ruby_version, '/usr/bin/gem')
+        util.sym_link('/usr/bin/gem%s' % ruby_version, '/usr/bin/gem')
     if not os.path.exists('/usr/bin/ruby'):
-        os.symlink('/usr/bin/ruby%s' % ruby_version, '/usr/bin/ruby')
+        util.sym_link('/usr/bin/ruby%s' % ruby_version, '/usr/bin/ruby')
     if chef_version:
-        subprocess.check_call(['/usr/bin/gem', 'install', 'chef',
-                               '-v %s' % chef_version, '--no-ri',
-                               '--no-rdoc', '--bindir', '/usr/bin', '-q'])
+        util.subp(['/usr/bin/gem', 'install', 'chef',
+                  '-v %s' % chef_version, '--no-ri',
+                  '--no-rdoc', '--bindir', '/usr/bin', '-q'])
     else:
-        subprocess.check_call(['/usr/bin/gem', 'install', 'chef',
-                               '--no-ri', '--no-rdoc', '--bindir',
-                               '/usr/bin', '-q'])
-
-
-def ensure_dir(d):
-    if not os.path.exists(d):
-        os.makedirs(d)
-
-
-def mkdirs(dirs):
-    for d in dirs:
-        ensure_dir(d)
+        util.subp(['/usr/bin/gem', 'install', 'chef',
+                  '--no-ri', '--no-rdoc', '--bindir',
+                  '/usr/bin', '-q'])