diff options
Diffstat (limited to 'cloudinit/sources/DataSourceMAAS.py')
| -rw-r--r-- | cloudinit/sources/DataSourceMAAS.py | 199 |
1 files changed, 113 insertions, 86 deletions
diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index d828f078..ab93c0a2 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -20,7 +20,6 @@ from __future__ import print_function -import errno import os import time @@ -32,7 +31,14 @@ from cloudinit import util LOG = logging.getLogger(__name__) MD_VERSION = "2012-03-01" -BINARY_FIELDS = ('user-data',) +DS_FIELDS = [ + # remote path, location in dictionary, binary data?, optional? + ("meta-data/instance-id", 'meta-data/instance-id', False, False), + ("meta-data/local-hostname", 'meta-data/local-hostname', False, False), + ("meta-data/public-keys", 'meta-data/public-keys', False, True), + ('meta-data/vendor-data', 'vendor-data', True, True), + ('user-data', 'user-data', True, True), +] class DataSourceMAAS(sources.DataSource): @@ -43,6 +49,7 @@ class DataSourceMAAS(sources.DataSource): instance-id user-data hostname + vendor-data """ def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) @@ -71,10 +78,7 @@ class DataSourceMAAS(sources.DataSource): mcfg = self.ds_cfg try: - (userdata, metadata) = read_maas_seed_dir(self.seed_dir) - self.userdata_raw = userdata - self.metadata = metadata - self.base_url = self.seed_dir + self._set_data(self.seed_dir, read_maas_seed_dir(self.seed_dir)) return True except MAASSeedDirNone: pass @@ -95,18 +99,29 @@ class DataSourceMAAS(sources.DataSource): if not self.wait_for_metadata_service(url): return False - self.base_url = url - - (userdata, metadata) = read_maas_seed_url( - self.base_url, read_file_or_url=self.oauth_helper.readurl, - paths=self.paths, retries=1) - self.userdata_raw = userdata - self.metadata = metadata + self._set_data( + url, read_maas_seed_url( + url, read_file_or_url=self.oauth_helper.readurl, + paths=self.paths, retries=1)) return True except Exception: util.logexc(LOG, "Failed fetching metadata from url %s", url) return False + def _set_data(self, url, data): + # takes a url for base_url and a tuple of userdata, metadata, vd. + self.base_url = url + ud, md, vd = data + self.userdata_raw = ud + self.metadata = md + self.vendordata_pure = vd + if vd: + try: + self.vendordata_raw = sources.convert_vendordata(vd) + except ValueError as e: + LOG.warn("Invalid content in vendor-data: %s", e) + self.vendordata_raw = None + def wait_for_metadata_service(self, url): mcfg = self.ds_cfg max_wait = 120 @@ -126,6 +141,8 @@ class DataSourceMAAS(sources.DataSource): LOG.warn("Failed to get timeout, using %s" % timeout) starttime = time.time() + if url.endswith("/"): + url = url[:-1] check_url = "%s/%s/meta-data/instance-id" % (url, MD_VERSION) urls = [check_url] url = self.oauth_helper.wait_for_url( @@ -141,27 +158,13 @@ class DataSourceMAAS(sources.DataSource): def read_maas_seed_dir(seed_d): - """ - Return user-data and metadata for a maas seed dir in seed_d. - Expected format of seed_d are the following files: - * instance-id - * local-hostname - * user-data - """ - if not os.path.isdir(seed_d): + if seed_d.startswith("file://"): + seed_d = seed_d[7:] + if not os.path.isdir(seed_d) or len(os.listdir(seed_d)) == 0: raise MAASSeedDirNone("%s: not a directory") - files = ('local-hostname', 'instance-id', 'user-data', 'public-keys') - md = {} - for fname in files: - try: - md[fname] = util.load_file(os.path.join(seed_d, fname), - decode=fname not in BINARY_FIELDS) - except IOError as e: - if e.errno != errno.ENOENT: - raise - - return check_seed_contents(md, seed_d) + # seed_dir looks in seed_dir, not seed_dir/VERSION + return read_maas_seed_url("file://%s" % seed_d, version=None) def read_maas_seed_url(seed_url, read_file_or_url=None, timeout=None, @@ -175,73 +178,78 @@ def read_maas_seed_url(seed_url, read_file_or_url=None, timeout=None, * <seed_url>/<version>/meta-data/instance-id * <seed_url>/<version>/meta-data/local-hostname * <seed_url>/<version>/user-data + If version is None, then <version>/ will not be used. """ - base_url = "%s/%s" % (seed_url, version) - file_order = [ - 'local-hostname', - 'instance-id', - 'public-keys', - 'user-data', - ] - files = { - 'local-hostname': "%s/%s" % (base_url, 'meta-data/local-hostname'), - 'instance-id': "%s/%s" % (base_url, 'meta-data/instance-id'), - 'public-keys': "%s/%s" % (base_url, 'meta-data/public-keys'), - 'user-data': "%s/%s" % (base_url, 'user-data'), - } - if read_file_or_url is None: read_file_or_url = util.read_file_or_url + if seed_url.endswith("/"): + seed_url = seed_url[:-1] + md = {} - for name in file_order: - url = files.get(name) - if name == 'user-data': - item_retries = 0 + for path, dictname, binary, optional in DS_FIELDS: + if version is None: + url = "%s/%s" % (seed_url, path) else: - item_retries = retries - + url = "%s/%s/%s" % (seed_url, version, path) try: ssl_details = util.fetch_ssl_details(paths) - resp = read_file_or_url(url, retries=item_retries, - timeout=timeout, ssl_details=ssl_details) + resp = read_file_or_url(url, retries=retries, timeout=timeout, + ssl_details=ssl_details) if resp.ok(): - if name in BINARY_FIELDS: - md[name] = resp.contents + if binary: + md[path] = resp.contents else: - md[name] = util.decode_binary(resp.contents) + md[path] = util.decode_binary(resp.contents) else: LOG.warn(("Fetching from %s resulted in" " an invalid http code %s"), url, resp.code) except url_helper.UrlError as e: - if e.code != 404: - raise + if e.code == 404 and not optional: + raise MAASSeedDirMalformed( + "Missing required %s: %s" % (path, e)) + elif e.code != 404: + raise e + return check_seed_contents(md, seed_url) def check_seed_contents(content, seed): - """Validate if content is Is the content a dict that is valid as a - return for a datasource. - Either return a (userdata, metadata) tuple or + """Validate if dictionary content valid as a return for a datasource. + Either return a (userdata, metadata, vendordata) tuple or Raise MAASSeedDirMalformed or MAASSeedDirNone """ - md_required = ('instance-id', 'local-hostname') - if len(content) == 0: + ret = {} + missing = [] + for spath, dpath, _binary, optional in DS_FIELDS: + if spath not in content: + if not optional: + missing.append(spath) + continue + + if "/" in dpath: + top, _, p = dpath.partition("/") + if top not in ret: + ret[top] = {} + ret[top][p] = content[spath] + else: + ret[dpath] = content[spath] + + if len(ret) == 0: raise MAASSeedDirNone("%s: no data files found" % seed) - found = list(content.keys()) - missing = [k for k in md_required if k not in found] - if len(missing): + if missing: raise MAASSeedDirMalformed("%s: missing files %s" % (seed, missing)) - userdata = content.get('user-data', b"") - md = {} - for (key, val) in content.items(): - if key == 'user-data': - continue - md[key] = val + vd_data = None + if ret.get('vendor-data'): + err = object() + vd_data = util.load_yaml(ret.get('vendor-data'), default=err, + allowed=(object)) + if vd_data is err: + raise MAASSeedDirMalformed("vendor-data was not loadable as yaml.") - return (userdata, md) + return ret.get('user-data'), ret.get('meta-data'), vd_data class MAASSeedDirNone(Exception): @@ -272,6 +280,7 @@ if __name__ == "__main__": """ import argparse import pprint + import sys parser = argparse.ArgumentParser(description='Interact with MAAS DS') parser.add_argument("--config", metavar="file", @@ -289,17 +298,25 @@ if __name__ == "__main__": default=MD_VERSION) subcmds = parser.add_subparsers(title="subcommands", dest="subcmd") - subcmds.add_parser('crawl', help="crawl the datasource") - subcmds.add_parser('get', help="do a single GET of provided url") - subcmds.add_parser('check-seed', help="read andn verify seed at url") - - parser.add_argument("url", help="the data source to query") + for (name, help) in (('crawl', 'crawl the datasource'), + ('get', 'do a single GET of provided url'), + ('check-seed', 'read and verify seed at url')): + p = subcmds.add_parser(name, help=help) + p.add_argument("url", help="the datasource url", nargs='?', + default=None) args = parser.parse_args() creds = {'consumer_key': args.ckey, 'token_key': args.tkey, 'token_secret': args.tsec, 'consumer_secret': args.csec} + maaspkg_cfg = "/etc/cloud/cloud.cfg.d/90_dpkg_maas.cfg" + if (args.config is None and args.url is None and + os.path.exists(maaspkg_cfg) and + os.access(maaspkg_cfg, os.R_OK)): + sys.stderr.write("Used config in %s.\n" % maaspkg_cfg) + args.config = maaspkg_cfg + if args.config: cfg = util.read_conf(args.config) if 'datasource' in cfg: @@ -307,6 +324,12 @@ if __name__ == "__main__": for key in creds.keys(): if key in cfg and creds[key] is None: creds[key] = cfg[key] + if args.url is None and 'metadata_url' in cfg: + args.url = cfg['metadata_url'] + + if args.url is None: + sys.stderr.write("Must provide a url or a config with url.\n") + sys.exit(1) oauth_helper = url_helper.OauthUrlHelper(**creds) @@ -331,16 +354,20 @@ if __name__ == "__main__": printurl(url) if args.subcmd == "check-seed": + sys.stderr.write("Checking seed at %s\n" % args.url) readurl = oauth_helper.readurl if args.url[0] == "/" or args.url.startswith("file://"): - readurl = None - (userdata, metadata) = read_maas_seed_url( - args.url, version=args.apiver, read_file_or_url=readurl, - retries=2) - print("=== userdata ===") - print(userdata.decode()) - print("=== metadata ===") + (userdata, metadata, vd) = read_maas_seed_dir(args.url) + else: + (userdata, metadata, vd) = read_maas_seed_url( + args.url, version=args.apiver, read_file_or_url=readurl, + retries=2) + print("=== user-data ===") + print("N/A" if userdata is None else userdata.decode()) + print("=== meta-data ===") pprint.pprint(metadata) + print("=== vendor-data ===") + pprint.pprint("N/A" if vd is None else vd) elif args.subcmd == "get": printurl(args.url) |