path: root/cloudinit/cloud.py

from time import time

import cPickle as pickle
import contextlib
import os
import sys
import weakref


from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS,
                                OLD_CLOUD_CONFIG, CLOUD_CONFIG,
                                CFG_BUILTIN, CUR_INSTANCE_LINK)
from cloudinit import (get_builtin_cfg, get_base_cfg)
from cloudinit import log as logging
from cloudinit import parts
from cloudinit import sources
from cloudinit import util
from cloudinit import user_data
from cloudinit import handlers

LOG = logging.getLogger(__name__)


class CloudSemaphores(object):
    def __init__(self, paths):
        self.paths = paths

    # acquire lock on 'name' for given 'freq' and run function 'func'
    # if 'clear_on_fail' is True and 'func' throws an exception
    # then remove the lock (so it would run again)
    def run_functor(self, name, freq, functor, args=None, clear_on_fail=False):
        if not args:
            args = []
        if self.has_run(name, freq):
            LOG.debug("%s already ran %s", name, freq)
            return False
        with self.lock(name, freq, clear_on_fail) as lock:
            if not lock:
                raise RuntimeError("Failed to acquire lock on %s" % name)
            else:
                LOG.debug("Running %s with args %s using lock %s", func, args, lock)
                func(*args)
        return True

    @contextlib.contextmanager
    def lock(self, name, freq, clear_on_fail=False):
        try:
            yield self._acquire(name, freq)
        except:
            if clear_on_fail:
                self.clear(name, freq)
            raise

    def clear(self, name, freq):
        sem_file = self._getpath(name, freq)
        try:
            util.del_file(sem_file)
        except IOError:
            return False
        return True

    def _acquire(self, name, freq):
        if self.has_run(name, freq):
            return None
        # This is a race condition since nothing atomic is happening
        # here, but this should be ok due to the nature of when
        # and where cloud-init runs... (file writing is not a lock..)
        sem_file = self._getpath(name, freq)
        contents = "%s\n" % str(time())
        try:
            util.write_file(sem_file, contents)
        except (IOError, OSError):
            return None
        return sem_file

    def has_run(self, name, freq):
        if freq == PER_ALWAYS:
            return False
        sem_file = self._get_path(name, freq)
        if os.path.exists(sem_file):
            return True
        return False

    def _get_path(self, name, freq):
        sem_path = self.init.get_ipath("sem")
        if freq == PER_INSTANCE:
            return os.path.join(sem_path, name)
        return os.path.join(sem_path, "%s.%s" % (name, freq))


class CloudPaths(object):
    def __init__(self, init):
        self.config = CLOUD_CONFIG
        self.old_config = OLD_CLOUD_CONFIG
        self.var_dir = VAR_LIB_DIR
        self.instance_link = CUR_INSTANCE_LINK
        self.init = weakref.proxy(init)
        self.upstart_conf_d = "/etc/init"

    def _get_path_key(self, name):
        return PATH_MAP.get(name)

    # get_ipath_cur: get the current instance path for an item
    def get_ipath_cur(self, name=None):
        add_on = self._get_path_key(name)
        ipath = os.path.join(self.var_dir, 'instance')
        if add_on:
            ipath = os.path.join(ipath, add_on)
        return ipath

    # get_cpath : get the "clouddir" (/var/lib/cloud/<name>)
    # for a name in dirmap
    def get_cpath(self, name=None):
        cpath = self.var_dir
        add_on = self._get_path_key(name)
        if add_on:
            cpath = os.path.join(cpath, add_on)
        return cpath

    # get_ipath : get the instance path for a name in pathmap
    # (/var/lib/cloud/instances/<instance>/<name>)
    def get_ipath(self, name=None):
        iid = self.init.datasource.get_instance_id()
        ipath = os.path.join(self.var_dir, 'instances', iid)
        add_on = self._get_path_key(name)
        if add_on:
            ipath = os.path.join(ipath, add_on)
        return ipath


class CloudPartData(object):
    def __init__(self, datasource, paths):
        self.datasource = datasource
        self.paths = paths

    def get_userdata(self):
        return self.datasource.get_userdata()

    def get_public_ssh_keys(self):
        return self.datasource.get_public_ssh_keys()

    def get_locale(self):
        return self.datasource.get_locale()

    def get_mirror(self):
        return self.datasource.get_local_mirror()

    def get_hostname(self, fqdn=False):
        return self.datasource.get_hostname(fqdn=fqdn)

    def device_name_to_device(self, name):
        return self.datasource.device_name_to_device(name)

    def get_ipath_cur(self, name=None):
        return self.paths.get_ipath_cur(name)

    def get_cpath(self, name=None):
        return self.paths.get_cpath(name)

    def get_ipath(self, name=None):
        return self.paths.get_ipath(name)


class CloudInit(object):
    def __init__(self, ds_deps=None):
        self.datasource = None
        if ds_deps:
            self.ds_deps = ds_deps
        else:
            self.ds_deps = [sources.DEP_FILESYSTEM, sources.DEP_NETWORK]
        self.paths = CloudPaths(self)
        self.cfg = self._read_cfg()

    def _read_cfg_old(self):
        # support reading the old ConfigObj format file and merging
        # it into the yaml dictionary
        try:
            from configobj import ConfigObj
        except ImportError:
            ConfigObj = None
        if not ConfigObj:
            return {}
        old_cfg = ConfigObj(self.paths.old_config_fn)
        return dict(old_cfg)

    def read_cfg(self):
        if not self.cfg:
            self.cfg = self._read_cfg()
        return self.cfg

    def _read_cfg(self):
        starting_config = get_builtin_cfg()
        try:
            conf = get_base_cfg(self.paths.config, starting_config)
        except Exception:
            conf = starting_config
        old_conf = self._read_cfg_old()
        conf = util.mergedict(conf, old_conf)
        return conf
    
    def restore_from_cache(self):
        pickled_fn = self.paths.get_ipath_cur('obj_pkl')
        try:
            # we try to restore from a current link and static path
            # by using the instance link, if purge_cache was called
            # the file wont exist
            self.datasource = pickle.loads(util.load_file(pickled_fn))
            return True
        except Exception as e:
            LOG.debug("Failed loading pickled datasource from %s due to %s", pickled_fn, e)
            return False
    
    def write_to_cache(self):
        pickled_fn = self.paths.get_ipath_cur("obj_pkl")
        try:
            contents = pickle.dumps(self.datasource)
            util.write_file(pickled_fn, contents, mode=0400)
        except Exception as e:
            LOG.debug("Failed pickling datasource to %s due to %s", pickled_fn, e)
            return False
    
    def get_data_source(self):
        if self.datasource:
            return True
        if self.restore_from_cache():
            LOG.debug("Restored from cache datasource: %s" % self.datasource)
            return True
        (ds, dsname) = sources.find_source(self.cfg, self.ds_deps)
        LOG.debug("Loaded datasource %s:%s", dsname, ds)
        self.datasource = ds
        return True
    
    def set_cur_instance(self):
        # Ensure we are hooked into the right symlink for the current instance
        idir = self.paths.get_ipath()
        util.del_file(self.paths.instance_link)
        util.sym_link(idir, self.paths.instance_link)

        dlist = []
        for d in ["handlers", "scripts", "sem"]:
            dlist.append(os.path.join(idir, d))
        util.ensure_dirs(dlist)

        # Write out information on what is being used for the current instance
        # and what may have been used for a previous instance...
        dp = self.paths.get_cpath('data')
        ds = "%s: %s\n" % (self.datasource.__class__, self.datasource)
        previous_ds = ''
        ds_fn = os.path.join(idir, 'datasource')
        try:
            previous_ds = util.load_file(ds_fn).strip()
        except IOError as e:
            pass
        if not previous_ds:
            # TODO: ?? is this right
            previous_ds = ds
        util.write_file(ds_fn, ds)
        util.write_file(os.path.join(dp, 'previous-datasource'), previous_ds)
        iid = self.datasource.get_instance_id()
        previous_iid = ''
        p_iid_fn = os.path.join(dp, 'previous-instance-id')
        try:
            previous_iid = util.load_file(p_iid_fn).strip()
        except IOError as e:
            pass
        if not previous_iid:
            # TODO: ?? is this right
            previous_iid = iid
        util.write_file(p_iid_fn, "%s\n" % previous_iid)

    def update_cache(self):
        self.write_to_cache()
        self.store_userdata()

    def store_userdata(self):
        raw_ud = "%s" % (self.datasource.get_userdata_raw())
        util.write_file(self.paths.get_ipath('userdata_raw'), raw_ud, 0600)
        ud = "%s" % (self.datasource.get_userdata())
        util.write_file(self.paths.get_ipath('userdata'), ud, 0600)

    def consume_userdata(self, frequency=PER_INSTANCE):
        cdir = self.paths.get_cpath("handlers")
        idir = self.paths.get_ipath("handlers")
    
        # Add the path to the plugins dir to the top of our list for import
        # instance dir should be read before cloud-dir
        sys.path.insert(0, cdir)
        sys.path.insert(0, idir)

        # Data will be a little proxy that modules can use
        data = CloudPartData(self.datasource, self.paths)

        # This keeps track of all the active handlers
        handlers = CloudHandlers(self)

        # Add handlers in cdir
        potential_handlers = parts.find_module_files(cdir)
        for (fname, modname) in potential_handlers.iteritems():
            try:
                mod = parts.fixup_module(importer.import_module(modname))
                types = handlers.register(mod)
                LOG.debug("Added handler for [%s] from %s", types, fname)
            except:
                LOG.exception("Failed to register handler from %s", fname)

        def_handlers = handlers.register_defaults()
        if def_handlers:
            LOG.debug("Registered default handlers for [%s]", def_handlers)

        # Init the handlers first
        # Ensure userdata fetched before activation
        called = []
        for (_mtype, mod) in handlers.iteritems():
            if mod in called:
                continue
            parts.call_begin(mod, data, frequency)
            called.append(mod)

        # Walk the user data
        part_data = {
            'handlers': handlers,
            'handlerdir': idir,
            'data': data, 
            'frequency': frequency,
            'handlercount': 0,
        }
        user_data.walk(data.get_userdata(), parts.walker_callback, data=part_data)

        # Give callbacks opportunity to finalize
        called = []
        for (_mtype, mod) in handlers.iteritems():
            if mod in called:
                continue
            parts.call_end(mod, data, frequency)
            called.append(mod)


class CloudHandlers(object):

    def __init__(self, paths):
        self.paths = paths
        self.registered = {}

    def __contains__(self, item):
        return self.is_registered(item)

    def __getitem__(self, key):
        return self._get_handler(key)

    def is_registered(self, content_type):
        return content_type in self.registered

    def register(self, mod):
        types = set()
        for t in mod.list_types():
            self.registered[t] = handler
            types.add(t)
        return types

    def _get_handler(self, content_type):
        return self.registered[content_type]

    def items(self):
        return self.registered.items()

    def iteritems(self):
        return self.registered.iteritems()

    def _get_default_handlers(self):
        def_handlers = []
        if self.paths.get_ipath("cloud_config"):
            def_handlers.append(parts.CloudConfigPartHandler(self.paths.get_ipath("cloud_config")))
        if self.paths.get_ipath_cur('scripts'):
            def_handlers.append(parts.ShellScriptPartHandler(self.paths.get_ipath_cur('scripts')))
        if self.paths.get_ipath("boothooks"):
            def_handlers.append(parts.BootHookPartHandler(self.paths.get_ipath("boothooks")))
        if self.paths.upstart_conf_d:
            def_handlers.append(parts.UpstartJobPartHandler(self.paths.upstart_conf_d))
        return def_handlers

    def register_defaults(self):
        registered = set()
        for h in self._get_default_handlers():
            for t in h.list_types():
                if not self.is_registered(t)
                    self.register_handler(t, h)
                    registered.add(t)
        return registered


class CloudConfig(object):

    def __init__(self, cfgfile, cloud):
        self.cloud = cloud
        self.cfg = self._get_config(cfgfile)
        self.paths = cloud.paths
        self.sems = CloudSemaphores(self.paths)

    def _get_config(self, cfgfile):

        cfg = None
        try:
            cfg = util.read_conf(cfgfile)
        except:
            LOG.exception(("Failed loading of cloud config '%s'. "
                          "Continuing with empty config."), cfgfile)
        if not cfg:
            cfg = {}

        ds_cfg = None
        try:
            ds_cfg = self.cloud.datasource.get_config_obj()
        except:
            LOG.exception("Failed loading of datasource config.")
        if not ds_cfg:
            ds_cfg = {}

        cfg = util.mergedict(cfg, ds_cfg)
        cloud_cfg = self.cloud.cfg or {}
        return util.mergedict(cfg, cloud_cfg)

    def extract(self, name):
        modname = handlers.form_module_name(name)
        if not modname:
            return None
        return handlers.fixup_module(importer.import_module(modname))

    def handle(self, name, mod, args, freq=None):
        def_freq = mod.frequency 
        if not freq:
            freq = def_freq
        c_name = "config-%s" % (name)
        real_args = [name, self.cfg, self.cloud, LOG, args]
        return self.sems.run_functor(c_name, freq, mod.handle, real_args)