Complete initial cleanup for refactoring/rework.

Some of the cleanups were the following

1. Using standard (logged) utility functions for sub process work, writing, reading files, and other file system/operating system options
2. Having distrobutions impelement there own subclasses to handle system specifics (if applicable)
3. Having a cloud wrapper that provides just the functionality we want to expose (cloud.py)
4. Using a path class instead of globals for all cloud init paths (it is configured via config)
5. Removal of as much shared global state as possible (there should be none, minus a set of constants)
6. Other various cleanups that remove transforms/handlers/modules from reading/writing/chmoding there own files. 
 a. They should be using util functions to take advantage of the logging that is now enabled in those util functions (very useful for debugging)
7. Urls being read and checked from a single module that serves this and only this purpose (+1 for code organization)
8. Updates to log whenever a transform decides not to run
9. Ensure whenever a exception is thrown (and possibly captured) that the util.logexc function is called 
 a. For debugging, tracing this is important to not just drop them on the floor.
10. Code shuffling into utils.py where it makes sense (and where it could serve a benefit for other code now or in the future)

1 files changed, 66 insertions, 160 deletions

diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py
index 61a0038f..27196265 100644
--- a/cloudinit/sources/DataSourceMAAS.py
+++ b/cloudinit/sources/DataSourceMAAS.py
@@ -1,8 +1,10 @@
 # vi: ts=4 expandtab
 #
 #    Copyright (C) 2012 Canonical Ltd.
+#    Copyright (C) 2012 Yahoo! Inc.
 #
 #    Author: Scott Moser <scott.moser@canonical.com>
+#    Author: Joshua Harlow <harlowja@yahoo-inc.com>
 #
 #    This program is free software: you can redistribute it and/or modify
 #    it under the terms of the GNU General Public License version 3, as
@@ -16,22 +18,22 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import cloudinit.DataSource as DataSource
-
-from cloudinit import seeddir as base_seeddir
-from cloudinit import log
-import cloudinit.util as util
+import os
 import errno
 import oauth.oauth as oauth
-import os.path
-import urllib2
 import time
+import urllib2
 
+from cloudinit import log as logging
+from cloudinit import sources
+from cloudinit import url_helper as uhelp
+from cloudinit import util
 
+LOG = logging.getLogger(__name__)
 MD_VERSION = "2012-03-01"
 
 
-class DataSourceMAAS(DataSource.DataSource):
+class DataSourceMAAS(sources.DataSource):
     """
     DataSourceMAAS reads instance information from MAAS.
     Given a config metadata_url, and oauth tokens, it expects to find
@@ -40,61 +42,64 @@ class DataSourceMAAS(DataSource.DataSource):
       user-data
       hostname
     """
-    seeddir = base_seeddir + '/maas'
-    baseurl = None
+    def __init__(self, sys_cfg, distro, paths):
+        sources.DataSource.__init__(self, sys_cfg, distro, paths)
+        self.base_url = None
+        self.seed_dir = os.path.join(paths.seed_dir, 'maas')
 
     def __str__(self):
-        return("DataSourceMAAS[%s]" % self.baseurl)
+        return "%s[%s]" % (util.obj_name(self), self.base_url)
 
     def get_data(self):
         mcfg = self.ds_cfg
 
         try:
-            (userdata, metadata) = read_maas_seed_dir(self.seeddir)
+            (userdata, metadata) = read_maas_seed_dir(self.seed_dir)
             self.userdata_raw = userdata
             self.metadata = metadata
-            self.baseurl = self.seeddir
+            self.base_url = self.seed_dir
             return True
         except MAASSeedDirNone:
             pass
         except MAASSeedDirMalformed as exc:
-            log.warn("%s was malformed: %s\n" % (self.seeddir, exc))
+            LOG.warn("%s was malformed: %s" % (self.seed_dir, exc))
             raise
 
-        try:
-            # if there is no metadata_url, then we're not configured
-            url = mcfg.get('metadata_url', None)
-            if url == None:
-                return False
+        # If there is no metadata_url, then we're not configured
+        url = mcfg.get('metadata_url', None)
+        if not url:
+            return False
 
+        try:
             if not self.wait_for_metadata_service(url):
                 return False
 
-            self.baseurl = url
+            self.base_url = url
 
-            (userdata, metadata) = read_maas_seed_url(self.baseurl,
-                self.md_headers)
+            (userdata, metadata) = read_maas_seed_url(self.base_url,
+                                                      self.md_headers)
             self.userdata_raw = userdata
             self.metadata = metadata
             return True
         except Exception:
-            util.logexc(log)
+            util.logexc(LOG, "Failed fetching metadata from url %s", url)
             return False
 
     def md_headers(self, url):
         mcfg = self.ds_cfg
 
-        # if we are missing token_key, token_secret or consumer_key
+        # If we are missing token_key, token_secret or consumer_key
         # then just do non-authed requests
         for required in ('token_key', 'token_secret', 'consumer_key'):
             if required not in mcfg:
-                return({})
+                return {}
 
         consumer_secret = mcfg.get('consumer_secret', "")
-
-        return(oauth_headers(url=url, consumer_key=mcfg['consumer_key'],
-            token_key=mcfg['token_key'], token_secret=mcfg['token_secret'],
-            consumer_secret=consumer_secret))
+        return oauth_headers(url=url,
+                             consumer_key=mcfg['consumer_key'],
+                             token_key=mcfg['token_key'],
+                             token_secret=mcfg['token_secret'],
+                             consumer_secret=consumer_secret)
 
     def wait_for_metadata_service(self, url):
         mcfg = self.ds_cfg
@@ -103,32 +108,31 @@ class DataSourceMAAS(DataSource.DataSource):
         try:
             max_wait = int(mcfg.get("max_wait", max_wait))
         except Exception:
-            util.logexc(log)
-            log.warn("Failed to get max wait. using %s" % max_wait)
+            util.logexc(LOG, "Failed to get max wait. using %s", max_wait)
 
         if max_wait == 0:
             return False
 
         timeout = 50
         try:
-            timeout = int(mcfg.get("timeout", timeout))
+            if timeout in mcfg:
+                timeout = int(mcfg.get("timeout", timeout))
         except Exception:
-            util.logexc(log)
-            log.warn("Failed to get timeout, using %s" % timeout)
+            LOG.warn("Failed to get timeout, using %s" % timeout)
 
         starttime = time.time()
         check_url = "%s/%s/meta-data/instance-id" % (url, MD_VERSION)
         url = util.wait_for_url(urls=[check_url], max_wait=max_wait,
-            timeout=timeout, status_cb=log.warn,
-            headers_cb=self.md_headers)
+                                timeout=timeout, status_cb=LOG.warn,
+                                headers_cb=self.md_headers)
 
         if url:
-            log.debug("Using metadata source: '%s'" % url)
+            LOG.info("Using metadata source: '%s'", url)
         else:
-            log.critical("giving up on md after %i seconds\n" %
-                         int(time.time() - starttime))
+            LOG.critical("Giving up on md from %s after %i seconds",
+                            urls, int(time.time() - starttime))
 
-        return (bool(url))
+        return bool(url)
 
 
 def read_maas_seed_dir(seed_d):
@@ -139,22 +143,19 @@ def read_maas_seed_dir(seed_d):
       * local-hostname
       * user-data
     """
-    files = ('local-hostname', 'instance-id', 'user-data', 'public-keys')
-    md = {}
-
     if not os.path.isdir(seed_d):
         raise MAASSeedDirNone("%s: not a directory")
 
+    files = ('local-hostname', 'instance-id', 'user-data', 'public-keys')
+    md = {}
     for fname in files:
         try:
-            with open(os.path.join(seed_d, fname)) as fp:
-                md[fname] = fp.read()
-                fp.close()
+            md[fname] = util.load_file(os.path.join(seed_d, fname))
         except IOError as e:
             if e.errno != errno.ENOENT:
                 raise
 
-    return(check_seed_contents(md, seed_d))
+    return check_seed_contents(md, seed_d)
 
 
 def read_maas_seed_url(seed_url, header_cb=None, timeout=None,
@@ -169,29 +170,26 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None,
       * <seed_url>/<version>/meta-data/local-hostname
       * <seed_url>/<version>/user-data
     """
-    files = ('meta-data/local-hostname',
-             'meta-data/instance-id',
-             'meta-data/public-keys',
-             'user-data')
-
     base_url = "%s/%s" % (seed_url, version)
+    files = {
+        'local-hostname': "%s/%s" % (base_url, 'meta-data/local-hostname'),
+        'instance-id': "%s/%s" % (base_url, 'meta-data/instance-id'),
+        'public-keys': "%s/%s" % (base_url, 'meta-data/public-keys'),
+        'user-data': "%s/%s" % (base_url, 'user-data'),
+    }
     md = {}
-    for fname in files:
-        url = "%s/%s" % (base_url, fname)
+    for (name, url) in files:
         if header_cb:
             headers = header_cb(url)
         else:
             headers = {}
-
         try:
-            req = urllib2.Request(url, data=None, headers=headers)
-            resp = urllib2.urlopen(req, timeout=timeout)
-            md[os.path.basename(fname)] = resp.read()
+            (resp, sc) = uhelp.readurl(url, headers=headers, timeout=timeout)
+            md[name] = resp
         except urllib2.HTTPError as e:
             if e.code != 404:
                 raise
-
-    return(check_seed_contents(md, seed_url))
+    return check_seed_contents(md, seed_url)
 
 
 def check_seed_contents(content, seed):
@@ -201,11 +199,10 @@ def check_seed_contents(content, seed):
        Raise MAASSeedDirMalformed or MAASSeedDirNone
     """
     md_required = ('instance-id', 'local-hostname')
-    found = content.keys()
-
     if len(content) == 0:
         raise MAASSeedDirNone("%s: no data files found" % seed)
 
+    found = content.keys()
     missing = [k for k in md_required if k not in found]
     if len(missing):
         raise MAASSeedDirMalformed("%s: missing files %s" % (seed, missing))
@@ -217,7 +214,7 @@ def check_seed_contents(content, seed):
             continue
         md[key] = val
 
-    return(userdata, md)
+    return (userdata, md)
 
 
 def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret):
@@ -232,8 +229,8 @@ def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret):
     }
     req = oauth.OAuthRequest(http_url=url, parameters=params)
     req.sign_request(oauth.OAuthSignatureMethod_PLAINTEXT(),
-        consumer, token)
-    return(req.to_header())
+                     consumer, token)
+    return req.to_header()
 
 
 class MAASSeedDirNone(Exception):
@@ -244,102 +241,11 @@ class MAASSeedDirMalformed(Exception):
     pass
 
 
+# Used to match classes to dependencies
 datasources = [
-  (DataSourceMAAS, (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)),
+  (DataSourceMAAS, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
 ]
 
-
-# return a list of data sources that match this set of dependencies
+# Return a list of data sources that match this set of dependencies
 def get_datasource_list(depends):
-    return(DataSource.list_from_depends(depends, datasources))
-
-
-if __name__ == "__main__":
-    def main():
-        """
-        Call with single argument of directory or http or https url.
-        If url is given additional arguments are allowed, which will be
-        interpreted as consumer_key, token_key, token_secret, consumer_secret
-        """
-        import argparse
-        import pprint
-
-        parser = argparse.ArgumentParser(description='Interact with MAAS DS')
-        parser.add_argument("--config", metavar="file",
-            help="specify DS config file", default=None)
-        parser.add_argument("--ckey", metavar="key",
-            help="the consumer key to auth with", default=None)
-        parser.add_argument("--tkey", metavar="key",
-            help="the token key to auth with", default=None)
-        parser.add_argument("--csec", metavar="secret",
-            help="the consumer secret (likely '')", default="")
-        parser.add_argument("--tsec", metavar="secret",
-            help="the token secret to auth with", default=None)
-        parser.add_argument("--apiver", metavar="version",
-            help="the apiver to use ("" can be used)", default=MD_VERSION)
-
-        subcmds = parser.add_subparsers(title="subcommands", dest="subcmd")
-        subcmds.add_parser('crawl', help="crawl the datasource")
-        subcmds.add_parser('get', help="do a single GET of provided url")
-        subcmds.add_parser('check-seed', help="read andn verify seed at url")
-
-        parser.add_argument("url", help="the data source to query")
-
-        args = parser.parse_args()
-
-        creds = {'consumer_key': args.ckey, 'token_key': args.tkey,
-            'token_secret': args.tsec, 'consumer_secret': args.csec}
-
-        if args.config:
-            import yaml
-            with open(args.config) as fp:
-                cfg = yaml.load(fp)
-            if 'datasource' in cfg:
-                cfg = cfg['datasource']['MAAS']
-            for key in creds.keys():
-                if key in cfg and creds[key] == None:
-                    creds[key] = cfg[key]
-
-        def geturl(url, headers_cb):
-            req = urllib2.Request(url, data=None, headers=headers_cb(url))
-            return(urllib2.urlopen(req).read())
-
-        def printurl(url, headers_cb):
-            print "== %s ==\n%s\n" % (url, geturl(url, headers_cb))
-
-        def crawl(url, headers_cb=None):
-            if url.endswith("/"):
-                for line in geturl(url, headers_cb).splitlines():
-                    if line.endswith("/"):
-                        crawl("%s%s" % (url, line), headers_cb)
-                    else:
-                        printurl("%s%s" % (url, line), headers_cb)
-            else:
-                printurl(url, headers_cb)
-
-        def my_headers(url):
-            headers = {}
-            if creds.get('consumer_key', None) != None:
-                headers = oauth_headers(url, **creds)
-            return headers
-
-        if args.subcmd == "check-seed":
-            if args.url.startswith("http"):
-                (userdata, metadata) = read_maas_seed_url(args.url,
-                    header_cb=my_headers, version=args.apiver)
-            else:
-                (userdata, metadata) = read_maas_seed_url(args.url)
-            print "=== userdata ==="
-            print userdata
-            print "=== metadata ==="
-            pprint.pprint(metadata)
-
-        elif args.subcmd == "get":
-            printurl(args.url, my_headers)
-
-        elif args.subcmd == "crawl":
-            if not args.url.endswith("/"):
-                args.url = "%s/" % args.url
-            crawl(args.url, my_headers)
-
-    main()
+    return sources.list_from_depends(depends, datasources)