MAAS: add vendor-data support

Add vendor-data support to maas which will behave like the openstack
vendor-data does.  Data returned from maas must be yaml loadable.

Also update the main in DataSourceMAAS to "just work" on a maas
deployed system.

LP: #1612313

5 files changed, 141 insertions, 114 deletions

diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py
index 91d6ff13..5c9edabe 100644
--- a/cloudinit/sources/DataSourceConfigDrive.py
+++ b/cloudinit/sources/DataSourceConfigDrive.py
@@ -134,7 +134,7 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource):
         vd = results.get('vendordata')
         self.vendordata_pure = vd
         try:
-            self.vendordata_raw = openstack.convert_vendordata_json(vd)
+            self.vendordata_raw = sources.convert_vendordata(vd)
         except ValueError as e:
             LOG.warn("Invalid content in vendor-data: %s", e)
             self.vendordata_raw = None
diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py
index d828f078..ab93c0a2 100644
--- a/cloudinit/sources/DataSourceMAAS.py
+++ b/cloudinit/sources/DataSourceMAAS.py
@@ -20,7 +20,6 @@
 
 from __future__ import print_function
 
-import errno
 import os
 import time
 
@@ -32,7 +31,14 @@ from cloudinit import util
 LOG = logging.getLogger(__name__)
 MD_VERSION = "2012-03-01"
 
-BINARY_FIELDS = ('user-data',)
+DS_FIELDS = [
+    # remote path, location in dictionary, binary data?, optional?
+    ("meta-data/instance-id", 'meta-data/instance-id', False, False),
+    ("meta-data/local-hostname", 'meta-data/local-hostname', False, False),
+    ("meta-data/public-keys", 'meta-data/public-keys', False, True),
+    ('meta-data/vendor-data', 'vendor-data', True, True),
+    ('user-data', 'user-data', True, True),
+]
 
 
 class DataSourceMAAS(sources.DataSource):
@@ -43,6 +49,7 @@ class DataSourceMAAS(sources.DataSource):
       instance-id
       user-data
       hostname
+      vendor-data
     """
     def __init__(self, sys_cfg, distro, paths):
         sources.DataSource.__init__(self, sys_cfg, distro, paths)
@@ -71,10 +78,7 @@ class DataSourceMAAS(sources.DataSource):
         mcfg = self.ds_cfg
 
         try:
-            (userdata, metadata) = read_maas_seed_dir(self.seed_dir)
-            self.userdata_raw = userdata
-            self.metadata = metadata
-            self.base_url = self.seed_dir
+            self._set_data(self.seed_dir, read_maas_seed_dir(self.seed_dir))
             return True
         except MAASSeedDirNone:
             pass
@@ -95,18 +99,29 @@ class DataSourceMAAS(sources.DataSource):
             if not self.wait_for_metadata_service(url):
                 return False
 
-            self.base_url = url
-
-            (userdata, metadata) = read_maas_seed_url(
-                self.base_url, read_file_or_url=self.oauth_helper.readurl,
-                paths=self.paths, retries=1)
-            self.userdata_raw = userdata
-            self.metadata = metadata
+            self._set_data(
+                url, read_maas_seed_url(
+                    url, read_file_or_url=self.oauth_helper.readurl,
+                    paths=self.paths, retries=1))
             return True
         except Exception:
             util.logexc(LOG, "Failed fetching metadata from url %s", url)
             return False
 
+    def _set_data(self, url, data):
+        # takes a url for base_url and a tuple of userdata, metadata, vd.
+        self.base_url = url
+        ud, md, vd = data
+        self.userdata_raw = ud
+        self.metadata = md
+        self.vendordata_pure = vd
+        if vd:
+            try:
+                self.vendordata_raw = sources.convert_vendordata(vd)
+            except ValueError as e:
+                LOG.warn("Invalid content in vendor-data: %s", e)
+                self.vendordata_raw = None
+
     def wait_for_metadata_service(self, url):
         mcfg = self.ds_cfg
         max_wait = 120
@@ -126,6 +141,8 @@ class DataSourceMAAS(sources.DataSource):
             LOG.warn("Failed to get timeout, using %s" % timeout)
 
         starttime = time.time()
+        if url.endswith("/"):
+            url = url[:-1]
         check_url = "%s/%s/meta-data/instance-id" % (url, MD_VERSION)
         urls = [check_url]
         url = self.oauth_helper.wait_for_url(
@@ -141,27 +158,13 @@ class DataSourceMAAS(sources.DataSource):
 
 
 def read_maas_seed_dir(seed_d):
-    """
-    Return user-data and metadata for a maas seed dir in seed_d.
-    Expected format of seed_d are the following files:
-      * instance-id
-      * local-hostname
-      * user-data
-    """
-    if not os.path.isdir(seed_d):
+    if seed_d.startswith("file://"):
+        seed_d = seed_d[7:]
+    if not os.path.isdir(seed_d) or len(os.listdir(seed_d)) == 0:
         raise MAASSeedDirNone("%s: not a directory")
 
-    files = ('local-hostname', 'instance-id', 'user-data', 'public-keys')
-    md = {}
-    for fname in files:
-        try:
-            md[fname] = util.load_file(os.path.join(seed_d, fname),
-                                       decode=fname not in BINARY_FIELDS)
-        except IOError as e:
-            if e.errno != errno.ENOENT:
-                raise
-
-    return check_seed_contents(md, seed_d)
+    # seed_dir looks in seed_dir, not seed_dir/VERSION
+    return read_maas_seed_url("file://%s" % seed_d, version=None)
 
 
 def read_maas_seed_url(seed_url, read_file_or_url=None, timeout=None,
@@ -175,73 +178,78 @@ def read_maas_seed_url(seed_url, read_file_or_url=None, timeout=None,
       * <seed_url>/<version>/meta-data/instance-id
       * <seed_url>/<version>/meta-data/local-hostname
       * <seed_url>/<version>/user-data
+    If version is None, then <version>/ will not be used.
     """
-    base_url = "%s/%s" % (seed_url, version)
-    file_order = [
-        'local-hostname',
-        'instance-id',
-        'public-keys',
-        'user-data',
-    ]
-    files = {
-        'local-hostname': "%s/%s" % (base_url, 'meta-data/local-hostname'),
-        'instance-id': "%s/%s" % (base_url, 'meta-data/instance-id'),
-        'public-keys': "%s/%s" % (base_url, 'meta-data/public-keys'),
-        'user-data': "%s/%s" % (base_url, 'user-data'),
-    }
-
     if read_file_or_url is None:
         read_file_or_url = util.read_file_or_url
 
+    if seed_url.endswith("/"):
+        seed_url = seed_url[:-1]
+
     md = {}
-    for name in file_order:
-        url = files.get(name)
-        if name == 'user-data':
-            item_retries = 0
+    for path, dictname, binary, optional in DS_FIELDS:
+        if version is None:
+            url = "%s/%s" % (seed_url, path)
         else:
-            item_retries = retries
-
+            url = "%s/%s/%s" % (seed_url, version, path)
         try:
             ssl_details = util.fetch_ssl_details(paths)
-            resp = read_file_or_url(url, retries=item_retries,
-                                    timeout=timeout, ssl_details=ssl_details)
+            resp = read_file_or_url(url, retries=retries, timeout=timeout,
+                                    ssl_details=ssl_details)
             if resp.ok():
-                if name in BINARY_FIELDS:
-                    md[name] = resp.contents
+                if binary:
+                    md[path] = resp.contents
                 else:
-                    md[name] = util.decode_binary(resp.contents)
+                    md[path] = util.decode_binary(resp.contents)
             else:
                 LOG.warn(("Fetching from %s resulted in"
                           " an invalid http code %s"), url, resp.code)
         except url_helper.UrlError as e:
-            if e.code != 404:
-                raise
+            if e.code == 404 and not optional:
+                raise MAASSeedDirMalformed(
+                    "Missing required %s: %s" % (path, e))
+            elif e.code != 404:
+                raise e
+
     return check_seed_contents(md, seed_url)
 
 
 def check_seed_contents(content, seed):
-    """Validate if content is Is the content a dict that is valid as a
-       return for a datasource.
-       Either return a (userdata, metadata) tuple or
+    """Validate if dictionary content valid as a return for a datasource.
+       Either return a (userdata, metadata, vendordata) tuple or
        Raise MAASSeedDirMalformed or MAASSeedDirNone
     """
-    md_required = ('instance-id', 'local-hostname')
-    if len(content) == 0:
+    ret = {}
+    missing = []
+    for spath, dpath, _binary, optional in DS_FIELDS:
+        if spath not in content:
+            if not optional:
+                missing.append(spath)
+            continue
+
+        if "/" in dpath:
+            top, _, p = dpath.partition("/")
+            if top not in ret:
+                ret[top] = {}
+            ret[top][p] = content[spath]
+        else:
+            ret[dpath] = content[spath]
+
+    if len(ret) == 0:
         raise MAASSeedDirNone("%s: no data files found" % seed)
 
-    found = list(content.keys())
-    missing = [k for k in md_required if k not in found]
-    if len(missing):
+    if missing:
         raise MAASSeedDirMalformed("%s: missing files %s" % (seed, missing))
 
-    userdata = content.get('user-data', b"")
-    md = {}
-    for (key, val) in content.items():
-        if key == 'user-data':
-            continue
-        md[key] = val
+    vd_data = None
+    if ret.get('vendor-data'):
+        err = object()
+        vd_data = util.load_yaml(ret.get('vendor-data'), default=err,
+                                 allowed=(object))
+        if vd_data is err:
+            raise MAASSeedDirMalformed("vendor-data was not loadable as yaml.")
 
-    return (userdata, md)
+    return ret.get('user-data'), ret.get('meta-data'), vd_data
 
 
 class MAASSeedDirNone(Exception):
@@ -272,6 +280,7 @@ if __name__ == "__main__":
         """
         import argparse
         import pprint
+        import sys
 
         parser = argparse.ArgumentParser(description='Interact with MAAS DS')
         parser.add_argument("--config", metavar="file",
@@ -289,17 +298,25 @@ if __name__ == "__main__":
                             default=MD_VERSION)
 
         subcmds = parser.add_subparsers(title="subcommands", dest="subcmd")
-        subcmds.add_parser('crawl', help="crawl the datasource")
-        subcmds.add_parser('get', help="do a single GET of provided url")
-        subcmds.add_parser('check-seed', help="read andn verify seed at url")
-
-        parser.add_argument("url", help="the data source to query")
+        for (name, help) in (('crawl', 'crawl the datasource'),
+                             ('get', 'do a single GET of provided url'),
+                             ('check-seed', 'read and verify seed at url')):
+            p = subcmds.add_parser(name, help=help)
+            p.add_argument("url", help="the datasource url", nargs='?',
+                           default=None)
 
         args = parser.parse_args()
 
         creds = {'consumer_key': args.ckey, 'token_key': args.tkey,
                  'token_secret': args.tsec, 'consumer_secret': args.csec}
 
+        maaspkg_cfg = "/etc/cloud/cloud.cfg.d/90_dpkg_maas.cfg"
+        if (args.config is None and args.url is None and
+                os.path.exists(maaspkg_cfg) and
+                os.access(maaspkg_cfg, os.R_OK)):
+            sys.stderr.write("Used config in %s.\n" % maaspkg_cfg)
+            args.config = maaspkg_cfg
+
         if args.config:
             cfg = util.read_conf(args.config)
             if 'datasource' in cfg:
@@ -307,6 +324,12 @@ if __name__ == "__main__":
             for key in creds.keys():
                 if key in cfg and creds[key] is None:
                     creds[key] = cfg[key]
+            if args.url is None and 'metadata_url' in cfg:
+                args.url = cfg['metadata_url']
+
+        if args.url is None:
+            sys.stderr.write("Must provide a url or a config with url.\n")
+            sys.exit(1)
 
         oauth_helper = url_helper.OauthUrlHelper(**creds)
 
@@ -331,16 +354,20 @@ if __name__ == "__main__":
                 printurl(url)
 
         if args.subcmd == "check-seed":
+            sys.stderr.write("Checking seed at %s\n" % args.url)
             readurl = oauth_helper.readurl
             if args.url[0] == "/" or args.url.startswith("file://"):
-                readurl = None
-            (userdata, metadata) = read_maas_seed_url(
-                args.url, version=args.apiver, read_file_or_url=readurl,
-                retries=2)
-            print("=== userdata ===")
-            print(userdata.decode())
-            print("=== metadata ===")
+                (userdata, metadata, vd) = read_maas_seed_dir(args.url)
+            else:
+                (userdata, metadata, vd) = read_maas_seed_url(
+                    args.url, version=args.apiver, read_file_or_url=readurl,
+                    retries=2)
+            print("=== user-data ===")
+            print("N/A" if userdata is None else userdata.decode())
+            print("=== meta-data ===")
             pprint.pprint(metadata)
+            print("=== vendor-data ===")
+            pprint.pprint("N/A" if vd is None else vd)
 
         elif args.subcmd == "get":
             printurl(args.url)
diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py
index c06d17f3..82558214 100644
--- a/cloudinit/sources/DataSourceOpenStack.py
+++ b/cloudinit/sources/DataSourceOpenStack.py
@@ -138,7 +138,7 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
         vd = results.get('vendordata')
         self.vendordata_pure = vd
         try:
-            self.vendordata_raw = openstack.convert_vendordata_json(vd)
+            self.vendordata_raw = sources.convert_vendordata(vd)
         except ValueError as e:
             LOG.warn("Invalid content in vendor-data: %s", e)
             self.vendordata_raw = None
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index 87b8e524..d1395270 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -21,8 +21,8 @@
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import abc
+import copy
 import os
-
 import six
 
 from cloudinit import importer
@@ -355,6 +355,31 @@ def instance_id_matches_system_uuid(instance_id, field='system-uuid'):
     return instance_id.lower() == dmi_value.lower()
 
 
+def convert_vendordata(data, recurse=True):
+    """data: a loaded object (strings, arrays, dicts).
+    return something suitable for cloudinit vendordata_raw.
+
+    if data is:
+       None: return None
+       string: return string
+       list: return data
+             the list is then processed in UserDataProcessor
+       dict: return convert_vendordata(data.get('cloud-init'))
+    """
+    if not data:
+        return None
+    if isinstance(data, six.string_types):
+        return data
+    if isinstance(data, list):
+        return copy.deepcopy(data)
+    if isinstance(data, dict):
+        if recurse is True:
+            return convert_vendordata(data.get('cloud-init'),
+                                      recurse=False)
+        raise ValueError("vendordata['cloud-init'] cannot be dict")
+    raise ValueError("Unknown data type for vendordata: %s" % type(data))
+
+
 # 'depends' is a list of dependencies (DEP_FILESYSTEM)
 # ds_list is a list of 2 item lists
 # ds_list = [
diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index 461fbd0d..84322e0e 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -621,28 +621,3 @@ def convert_net_json(network_json=None, known_macs=None):
         config.append(cfg)
 
     return {'version': 1, 'config': config}
-
-
-def convert_vendordata_json(data, recurse=True):
-    """data: a loaded json *object* (strings, arrays, dicts).
-    return something suitable for cloudinit vendordata_raw.
-
-    if data is:
-       None: return None
-       string: return string
-       list: return data
-             the list is then processed in UserDataProcessor
-       dict: return convert_vendordata_json(data.get('cloud-init'))
-    """
-    if not data:
-        return None
-    if isinstance(data, six.string_types):
-        return data
-    if isinstance(data, list):
-        return copy.deepcopy(data)
-    if isinstance(data, dict):
-        if recurse is True:
-            return convert_vendordata_json(data.get('cloud-init'),
-                                           recurse=False)
-        raise ValueError("vendordata['cloud-init'] cannot be dict")
-    raise ValueError("Unknown data type for vendordata: %s" % type(data))