diff options
16 files changed, 2604 insertions, 597 deletions
diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 05ad4b03..609dbb51 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -23,80 +23,182 @@ import os import re from cloudinit import gpg +from cloudinit import log as logging from cloudinit import templater from cloudinit import util -distros = ['ubuntu', 'debian'] - -PROXY_TPL = "Acquire::HTTP::Proxy \"%s\";\n" -APT_CONFIG_FN = "/etc/apt/apt.conf.d/94cloud-init-config" -APT_PROXY_FN = "/etc/apt/apt.conf.d/95cloud-init-proxy" +LOG = logging.getLogger(__name__) # this will match 'XXX:YYY' (ie, 'cloud-archive:foo' or 'ppa:bar') ADD_APT_REPO_MATCH = r"^[\w-]+:\w" +# place where apt stores cached repository data +APT_LISTS = "/var/lib/apt/lists" -def handle(name, cfg, cloud, log, _args): - if util.is_false(cfg.get('apt_configure_enabled', True)): - log.debug("Skipping module named %s, disabled by config.", name) - return - - release = get_release() - mirrors = find_apt_mirror_info(cloud, cfg) - if not mirrors or "primary" not in mirrors: - log.debug(("Skipping module named %s," - " no package 'mirror' located"), name) - return - - # backwards compatibility - mirror = mirrors["primary"] - mirrors["mirror"] = mirror - - log.debug("Mirror info: %s" % mirrors) - - if not util.get_cfg_option_bool(cfg, - 'apt_preserve_sources_list', False): - generate_sources_list(cfg, release, mirrors, cloud, log) - old_mirrors = cfg.get('apt_old_mirrors', - {"primary": "archive.ubuntu.com/ubuntu", - "security": "security.ubuntu.com/ubuntu"}) - rename_apt_lists(old_mirrors, mirrors) +# Files to store proxy information +APT_CONFIG_FN = "/etc/apt/apt.conf.d/94cloud-init-config" +APT_PROXY_FN = "/etc/apt/apt.conf.d/90cloud-init-aptproxy" + +# Default keyserver to use +DEFAULT_KEYSERVER = "keyserver.ubuntu.com" + +# Default archive mirrors +PRIMARY_ARCH_MIRRORS = {"PRIMARY": "http://archive.ubuntu.com/ubuntu/", + "SECURITY": "http://security.ubuntu.com/ubuntu/"} +PORTS_MIRRORS = {"PRIMARY": "http://ports.ubuntu.com/ubuntu-ports", + "SECURITY": "http://ports.ubuntu.com/ubuntu-ports"} +PRIMARY_ARCHES = ['amd64', 'i386'] +PORTS_ARCHES = ['s390x', 'arm64', 'armhf', 'powerpc', 'ppc64el'] + + +def get_default_mirrors(arch=None, target=None): + """returns the default mirrors for the target. These depend on the + architecture, for more see: + https://wiki.ubuntu.com/UbuntuDevelopment/PackageArchive#Ports""" + if arch is None: + arch = util.get_architecture(target) + if arch in PRIMARY_ARCHES: + return PRIMARY_ARCH_MIRRORS.copy() + if arch in PORTS_ARCHES: + return PORTS_MIRRORS.copy() + raise ValueError("No default mirror known for arch %s" % arch) + + +def handle(name, ocfg, cloud, log, _): + """process the config for apt_config. This can be called from + curthooks if a global apt config was provided or via the "apt" + standalone command.""" + # keeping code close to curtin codebase via entry handler + target = None + if log is not None: + global LOG + LOG = log + # feed back converted config, but only work on the subset under 'apt' + ocfg = convert_to_v3_apt_format(ocfg) + cfg = ocfg.get('apt', {}) + + if not isinstance(cfg, dict): + raise ValueError("Expected dictionary for 'apt' config, found %s", + type(cfg)) + + LOG.debug("handling apt (module %s) with apt config '%s'", name, cfg) + + release = util.lsb_release(target=target)['codename'] + arch = util.get_architecture(target) + mirrors = find_apt_mirror_info(cfg, cloud, arch=arch) + LOG.debug("Apt Mirror info: %s", mirrors) + + apply_debconf_selections(cfg, target) + + if util.is_false(cfg.get('preserve_sources_list', False)): + generate_sources_list(cfg, release, mirrors, cloud) + rename_apt_lists(mirrors, target) try: apply_apt_config(cfg, APT_PROXY_FN, APT_CONFIG_FN) - except Exception as e: - log.warn("failed to proxy or apt config info: %s", e) + except (IOError, OSError): + LOG.exception("Failed to apply proxy or apt config info:") - # Process 'apt_sources' - if 'apt_sources' in cfg: + # Process 'apt_source -> sources {dict}' + if 'sources' in cfg: params = mirrors params['RELEASE'] = release - params['MIRROR'] = mirror + params['MIRROR'] = mirrors["MIRROR"] + matcher = None matchcfg = cfg.get('add_apt_repo_match', ADD_APT_REPO_MATCH) if matchcfg: matcher = re.compile(matchcfg).search + + add_apt_sources(cfg['sources'], cloud, target=target, + template_params=params, aa_repo_match=matcher) + + +def debconf_set_selections(selections, target=None): + util.subp(['debconf-set-selections'], data=selections, target=target, + capture=True) + + +def dpkg_reconfigure(packages, target=None): + # For any packages that are already installed, but have preseed data + # we populate the debconf database, but the filesystem configuration + # would be preferred on a subsequent dpkg-reconfigure. + # so, what we have to do is "know" information about certain packages + # to unconfigure them. + unhandled = [] + to_config = [] + for pkg in packages: + if pkg in CONFIG_CLEANERS: + LOG.debug("unconfiguring %s", pkg) + CONFIG_CLEANERS[pkg](target) + to_config.append(pkg) else: - def matcher(x): - return False + unhandled.append(pkg) + + if len(unhandled): + LOG.warn("The following packages were installed and preseeded, " + "but cannot be unconfigured: %s", unhandled) + + if len(to_config): + util.subp(['dpkg-reconfigure', '--frontend=noninteractive'] + + list(to_config), data=None, target=target, capture=True) + + +def apply_debconf_selections(cfg, target=None): + """apply_debconf_selections - push content to debconf""" + # debconf_selections: + # set1: | + # cloud-init cloud-init/datasources multiselect MAAS + # set2: pkg pkg/value string bar + selsets = cfg.get('debconf_selections') + if not selsets: + LOG.debug("debconf_selections was not set in config") + return - errors = add_apt_sources(cfg['apt_sources'], params, - aa_repo_match=matcher) - for e in errors: - log.warn("Add source error: %s", ':'.join(e)) + selections = '\n'.join( + [selsets[key] for key in sorted(selsets.keys())]) + debconf_set_selections(selections.encode() + b"\n", target=target) - dconf_sel = util.get_cfg_option_str(cfg, 'debconf_selections', False) - if dconf_sel: - log.debug("Setting debconf selections per cloud config") - try: - util.subp(('debconf-set-selections', '-'), dconf_sel) - except Exception: - util.logexc(log, "Failed to run debconf-set-selections") + # get a complete list of packages listed in input + pkgs_cfgd = set() + for key, content in selsets.items(): + for line in content.splitlines(): + if line.startswith("#"): + continue + pkg = re.sub(r"[:\s].*", "", line) + pkgs_cfgd.add(pkg) + + pkgs_installed = util.get_installed_packages(target) + + LOG.debug("pkgs_cfgd: %s", pkgs_cfgd) + need_reconfig = pkgs_cfgd.intersection(pkgs_installed) + + if len(need_reconfig) == 0: + LOG.debug("no need for reconfig") + return + + dpkg_reconfigure(need_reconfig, target=target) + + +def clean_cloud_init(target): + """clean out any local cloud-init config""" + flist = glob.glob( + util.target_path(target, "/etc/cloud/cloud.cfg.d/*dpkg*")) + + LOG.debug("cleaning cloud-init config from: %s", flist) + for dpkg_cfg in flist: + os.unlink(dpkg_cfg) def mirrorurl_to_apt_fileprefix(mirror): + """mirrorurl_to_apt_fileprefix + Convert a mirror url to the file prefix used by apt on disk to + store cache information for that mirror. + To do so do: + - take off ???:// + - drop tailing / + - convert in string / to _""" string = mirror - # take off http:// or ftp:// if string.endswith("/"): string = string[0:-1] pos = string.find("://") @@ -106,174 +208,365 @@ def mirrorurl_to_apt_fileprefix(mirror): return string -def rename_apt_lists(old_mirrors, new_mirrors, lists_d="/var/lib/apt/lists"): - for (name, omirror) in old_mirrors.items(): +def rename_apt_lists(new_mirrors, target=None): + """rename_apt_lists - rename apt lists to preserve old cache data""" + default_mirrors = get_default_mirrors(util.get_architecture(target)) + + pre = util.target_path(target, APT_LISTS) + for (name, omirror) in default_mirrors.items(): nmirror = new_mirrors.get(name) if not nmirror: continue - oprefix = os.path.join(lists_d, mirrorurl_to_apt_fileprefix(omirror)) - nprefix = os.path.join(lists_d, mirrorurl_to_apt_fileprefix(nmirror)) + + oprefix = pre + os.path.sep + mirrorurl_to_apt_fileprefix(omirror) + nprefix = pre + os.path.sep + mirrorurl_to_apt_fileprefix(nmirror) if oprefix == nprefix: continue olen = len(oprefix) for filename in glob.glob("%s_*" % oprefix): - util.rename(filename, "%s%s" % (nprefix, filename[olen:])) - - -def get_release(): - (stdout, _stderr) = util.subp(['lsb_release', '-cs']) - return stdout.strip() - - -def generate_sources_list(cfg, codename, mirrors, cloud, log): - params = {'codename': codename} + newname = "%s%s" % (nprefix, filename[olen:]) + LOG.debug("Renaming apt list %s to %s", filename, newname) + try: + os.rename(filename, newname) + except OSError: + # since this is a best effort task, warn with but don't fail + LOG.warn("Failed to rename apt list:", exc_info=True) + + +def mirror_to_placeholder(tmpl, mirror, placeholder): + """mirror_to_placeholder + replace the specified mirror in a template with a placeholder string + Checks for existance of the expected mirror and warns if not found""" + if mirror not in tmpl: + LOG.warn("Expected mirror '%s' not found in: %s", mirror, tmpl) + return tmpl.replace(mirror, placeholder) + + +def map_known_suites(suite): + """there are a few default names which will be auto-extended. + This comes at the inability to use those names literally as suites, + but on the other hand increases readability of the cfg quite a lot""" + mapping = {'updates': '$RELEASE-updates', + 'backports': '$RELEASE-backports', + 'security': '$RELEASE-security', + 'proposed': '$RELEASE-proposed', + 'release': '$RELEASE'} + try: + retsuite = mapping[suite] + except KeyError: + retsuite = suite + return retsuite + + +def disable_suites(disabled, src, release): + """reads the config for suites to be disabled and removes those + from the template""" + if not disabled: + return src + + retsrc = src + for suite in disabled: + suite = map_known_suites(suite) + releasesuite = templater.render_string(suite, {'RELEASE': release}) + LOG.debug("Disabling suite %s as %s", suite, releasesuite) + + newsrc = "" + for line in retsrc.splitlines(True): + if line.startswith("#"): + newsrc += line + continue + + # sources.list allow options in cols[1] which can have spaces + # so the actual suite can be [2] or later. example: + # deb [ arch=amd64,armel k=v ] http://example.com/debian + cols = line.split() + if len(cols) > 1: + pcol = 2 + if cols[1].startswith("["): + for col in cols[1:]: + pcol += 1 + if col.endswith("]"): + break + + if cols[pcol] == releasesuite: + line = '# suite disabled by cloud-init: %s' % line + newsrc += line + retsrc = newsrc + + return retsrc + + +def generate_sources_list(cfg, release, mirrors, cloud): + """generate_sources_list + create a source.list file based on a custom or default template + by replacing mirrors and release in the template""" + aptsrc = "/etc/apt/sources.list" + params = {'RELEASE': release, 'codename': release} for k in mirrors: params[k] = mirrors[k] + params[k.lower()] = mirrors[k] - custtmpl = cfg.get('apt_custom_sources_list', None) - if custtmpl is not None: - templater.render_string_to_file(custtmpl, - '/etc/apt/sources.list', params) - return - - template_fn = cloud.get_template_filename('sources.list.%s' % - (cloud.distro.name)) - if not template_fn: - template_fn = cloud.get_template_filename('sources.list') + tmpl = cfg.get('sources_list', None) + if tmpl is None: + LOG.info("No custom template provided, fall back to builtin") + template_fn = cloud.get_template_filename('sources.list.%s' % + (cloud.distro.name)) if not template_fn: - log.warn("No template found, not rendering /etc/apt/sources.list") + template_fn = cloud.get_template_filename('sources.list') + if not template_fn: + LOG.warn("No template found, not rendering /etc/apt/sources.list") return + tmpl = util.load_file(template_fn) - templater.render_to_file(template_fn, '/etc/apt/sources.list', params) + rendered = templater.render_string(tmpl, params) + disabled = disable_suites(cfg.get('disable_suites'), rendered, release) + util.write_file(aptsrc, disabled, mode=0o644) -def add_apt_key_raw(key): +def add_apt_key_raw(key, target=None): """ actual adding of a key as defined in key argument to the system """ + LOG.debug("Adding key:\n'%s'", key) try: - util.subp(('apt-key', 'add', '-'), key) + util.subp(['apt-key', 'add', '-'], data=key.encode(), target=target) except util.ProcessExecutionError: - raise ValueError('failed to add apt GPG Key to apt keyring') + LOG.exception("failed to add apt GPG Key to apt keyring") + raise -def add_apt_key(ent): +def add_apt_key(ent, target=None): """ - add key to the system as defined in ent (if any) - supports raw keys or keyid's - The latter will as a first step fetch the raw key from a keyserver + Add key to the system as defined in ent (if any). + Supports raw keys or keyid's + The latter will as a first step fetched to get the raw key """ if 'keyid' in ent and 'key' not in ent: - keyserver = "keyserver.ubuntu.com" + keyserver = DEFAULT_KEYSERVER if 'keyserver' in ent: keyserver = ent['keyserver'] - ent['key'] = gpg.get_key_by_id(ent['keyid'], keyserver) + + ent['key'] = gpg.getkeybyid(ent['keyid'], keyserver) if 'key' in ent: - add_apt_key_raw(ent['key']) + add_apt_key_raw(ent['key'], target) -def convert_to_new_format(srclist): - """convert_to_new_format - convert the old list based format to the new dict based one - """ - srcdict = {} - if isinstance(srclist, list): - for srcent in srclist: - if 'filename' not in srcent: - # file collides for multiple !filename cases for compatibility - # yet we need them all processed, so not same dictionary key - srcent['filename'] = "cloud_config_sources.list" - key = util.rand_dict_key(srcdict, "cloud_config_sources.list") - else: - # all with filename use that as key (matching new format) - key = srcent['filename'] - srcdict[key] = srcent - elif isinstance(srclist, dict): - srcdict = srclist - else: - raise ValueError("unknown apt_sources format") - - return srcdict +def update_packages(cloud): + cloud.distro.update_package_sources() -def add_apt_sources(srclist, template_params=None, aa_repo_match=None): +def add_apt_sources(srcdict, cloud, target=None, template_params=None, + aa_repo_match=None): """ add entries in /etc/apt/sources.list.d for each abbreviated - sources.list entry in 'srclist'. When rendering template, also + sources.list entry in 'srcdict'. When rendering template, also include the values in dictionary searchList """ if template_params is None: template_params = {} if aa_repo_match is None: - def _aa_repo_match(x): - return False - aa_repo_match = _aa_repo_match + raise ValueError('did not get a valid repo matcher') - errorlist = [] - srcdict = convert_to_new_format(srclist) + if not isinstance(srcdict, dict): + raise TypeError('unknown apt format: %s' % (srcdict)) for filename in srcdict: ent = srcdict[filename] + LOG.debug("adding source/key '%s'", ent) if 'filename' not in ent: ent['filename'] = filename - # keys can be added without specifying a source - try: - add_apt_key(ent) - except ValueError as detail: - errorlist.append([ent, detail]) + add_apt_key(ent, target) if 'source' not in ent: - errorlist.append(["", "missing source"]) continue source = ent['source'] source = templater.render_string(source, template_params) - if not ent['filename'].startswith(os.path.sep): + if not ent['filename'].startswith("/"): ent['filename'] = os.path.join("/etc/apt/sources.list.d/", ent['filename']) + if not ent['filename'].endswith(".list"): + ent['filename'] += ".list" if aa_repo_match(source): try: - util.subp(["add-apt-repository", source]) - except util.ProcessExecutionError as e: - errorlist.append([source, - ("add-apt-repository failed. " + str(e))]) + util.subp(["add-apt-repository", source], target=target) + except util.ProcessExecutionError: + LOG.exception("add-apt-repository failed.") + raise continue + sourcefn = util.target_path(target, ent['filename']) try: contents = "%s\n" % (source) - util.write_file(ent['filename'], contents, omode="ab") - except Exception: - errorlist.append([source, - "failed write to file %s" % ent['filename']]) + util.write_file(sourcefn, contents, omode="a") + except IOError as detail: + LOG.exception("failed write to file %s: %s", sourcefn, detail) + raise - return errorlist + update_packages(cloud) + return -def find_apt_mirror_info(cloud, cfg): - """find an apt_mirror given the cloud and cfg provided.""" - mirror = None +def convert_v1_to_v2_apt_format(srclist): + """convert v1 apt format to v2 (dict in apt_sources)""" + srcdict = {} + if isinstance(srclist, list): + LOG.debug("apt config: convert V1 to V2 format (source list to dict)") + for srcent in srclist: + if 'filename' not in srcent: + # file collides for multiple !filename cases for compatibility + # yet we need them all processed, so not same dictionary key + srcent['filename'] = "cloud_config_sources.list" + key = util.rand_dict_key(srcdict, "cloud_config_sources.list") + else: + # all with filename use that as key (matching new format) + key = srcent['filename'] + srcdict[key] = srcent + elif isinstance(srclist, dict): + srcdict = srclist + else: + raise ValueError("unknown apt_sources format") + + return srcdict + + +def convert_key(oldcfg, aptcfg, oldkey, newkey): + """convert an old key to the new one if the old one exists + returns true if a key was found and converted""" + if oldcfg.get(oldkey, None) is not None: + aptcfg[newkey] = oldcfg.get(oldkey) + del oldcfg[oldkey] + return True + return False + + +def convert_mirror(oldcfg, aptcfg): + """convert old apt_mirror keys into the new more advanced mirror spec""" + keymap = [('apt_mirror', 'uri'), + ('apt_mirror_search', 'search'), + ('apt_mirror_search_dns', 'search_dns')] + converted = False + newmcfg = {'arches': ['default']} + for oldkey, newkey in keymap: + if convert_key(oldcfg, newmcfg, oldkey, newkey): + converted = True + + # only insert new style config if anything was converted + if converted: + aptcfg['primary'] = [newmcfg] + + +def convert_v2_to_v3_apt_format(oldcfg): + """convert old to new keys and adapt restructured mirror spec""" + oldkeys = ['apt_sources', 'apt_mirror', 'apt_mirror_search', + 'apt_mirror_search_dns', 'apt_proxy', 'apt_http_proxy', + 'apt_ftp_proxy', 'apt_https_proxy', + 'apt_preserve_sources_list', 'apt_custom_sources_list', + 'add_apt_repo_match'] + needtoconvert = [] + for oldkey in oldkeys: + if oldcfg.get(oldkey, None) is not None: + needtoconvert.append(oldkey) + + # no old config, so no new one to be created + if not needtoconvert: + return oldcfg + LOG.debug("apt config: convert V2 to V3 format for keys '%s'", + ", ".join(needtoconvert)) + + if oldcfg.get('apt', None) is not None: + msg = ("Error in apt configuration: " + "old and new format of apt features are mutually exclusive " + "('apt':'%s' vs '%s' key)" % (oldcfg.get('apt', None), + ", ".join(needtoconvert))) + LOG.error(msg) + raise ValueError(msg) + + # create new format from old keys + aptcfg = {} + + # renames / moves under the apt key + convert_key(oldcfg, aptcfg, 'add_apt_repo_match', 'add_apt_repo_match') + convert_key(oldcfg, aptcfg, 'apt_proxy', 'proxy') + convert_key(oldcfg, aptcfg, 'apt_http_proxy', 'http_proxy') + convert_key(oldcfg, aptcfg, 'apt_https_proxy', 'https_proxy') + convert_key(oldcfg, aptcfg, 'apt_ftp_proxy', 'ftp_proxy') + convert_key(oldcfg, aptcfg, 'apt_custom_sources_list', 'sources_list') + convert_key(oldcfg, aptcfg, 'apt_preserve_sources_list', + 'preserve_sources_list') + # dict format not changed since v2, just renamed and moved + convert_key(oldcfg, aptcfg, 'apt_sources', 'sources') + + convert_mirror(oldcfg, aptcfg) + + for oldkey in oldkeys: + if oldcfg.get(oldkey, None) is not None: + raise ValueError("old apt key '%s' left after conversion" % oldkey) + + # insert new format into config and return full cfg with only v3 content + oldcfg['apt'] = aptcfg + return oldcfg + + +def convert_to_v3_apt_format(cfg): + """convert the old list based format to the new dict based one. After that + convert the old dict keys/format to v3 a.k.a 'new apt config'""" + # V1 -> V2, the apt_sources entry from list to dict + apt_sources = cfg.get('apt_sources', None) + if apt_sources is not None: + cfg['apt_sources'] = convert_v1_to_v2_apt_format(apt_sources) + + # V2 -> V3, move all former globals under the "apt" key + # Restructure into new key names and mirror hierarchy + cfg = convert_v2_to_v3_apt_format(cfg) + + return cfg + + +def search_for_mirror(candidates): + """ + Search through a list of mirror urls for one that works + This needs to return quickly. + """ + if candidates is None: + return None + + LOG.debug("search for mirror in candidates: '%s'", candidates) + for cand in candidates: + try: + if util.is_resolvable_url(cand): + LOG.debug("found working mirror: '%s'", cand) + return cand + except Exception: + pass + return None - # this is less preferred way of specifying mirror preferred would be to - # use the distro's search or package_mirror. - mirror = cfg.get("apt_mirror", None) - search = cfg.get("apt_mirror_search", None) - if not mirror and search: - mirror = util.search_for_mirror(search) +def search_for_mirror_dns(configured, mirrortype, cfg, cloud): + """ + Try to resolve a list of predefines DNS names to pick mirrors + """ + mirror = None - if (not mirror and - util.get_cfg_option_bool(cfg, "apt_mirror_search_dns", False)): + if configured: mydom = "" doms = [] + if mirrortype == "primary": + mirrordns = "mirror" + elif mirrortype == "security": + mirrordns = "security-mirror" + else: + raise ValueError("unknown mirror type") + # if we have a fqdn, then search its domain portion first - (_hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) + (_, fqdn) = util.get_hostname_fqdn(cfg, cloud) mydom = ".".join(fqdn.split(".")[1:]) if mydom: doms.append(".%s" % mydom) @@ -282,38 +575,136 @@ def find_apt_mirror_info(cloud, cfg): mirror_list = [] distro = cloud.distro.name - mirrorfmt = "http://%s-mirror%s/%s" % (distro, "%s", distro) + mirrorfmt = "http://%s-%s%s/%s" % (distro, mirrordns, "%s", distro) for post in doms: mirror_list.append(mirrorfmt % (post)) - mirror = util.search_for_mirror(mirror_list) + mirror = search_for_mirror(mirror_list) + + return mirror + +def update_mirror_info(pmirror, smirror, arch, cloud): + """sets security mirror to primary if not defined. + returns defaults if no mirrors are defined""" + if pmirror is not None: + if smirror is None: + smirror = pmirror + return {'PRIMARY': pmirror, + 'SECURITY': smirror} + + # None specified at all, get default mirrors from cloud mirror_info = cloud.datasource.get_package_mirror_info() + if mirror_info: + # get_package_mirror_info() returns a dictionary with + # arbitrary key/value pairs including 'primary' and 'security' keys. + # caller expects dict with PRIMARY and SECURITY. + m = mirror_info.copy() + m['PRIMARY'] = m['primary'] + m['SECURITY'] = m['security'] + + return m + + # if neither apt nor cloud configured mirrors fall back to + return get_default_mirrors(arch) + + +def get_arch_mirrorconfig(cfg, mirrortype, arch): + """out of a list of potential mirror configurations select + and return the one matching the architecture (or default)""" + # select the mirror specification (if-any) + mirror_cfg_list = cfg.get(mirrortype, None) + if mirror_cfg_list is None: + return None + + # select the specification matching the target arch + default = None + for mirror_cfg_elem in mirror_cfg_list: + arches = mirror_cfg_elem.get("arches") + if arch in arches: + return mirror_cfg_elem + if "default" in arches: + default = mirror_cfg_elem + return default + + +def get_mirror(cfg, mirrortype, arch, cloud): + """pass the three potential stages of mirror specification + returns None is neither of them found anything otherwise the first + hit is returned""" + mcfg = get_arch_mirrorconfig(cfg, mirrortype, arch) + if mcfg is None: + return None + + # directly specified + mirror = mcfg.get("uri", None) + + # fallback to search if specified + if mirror is None: + # list of mirrors to try to resolve + mirror = search_for_mirror(mcfg.get("search", None)) + + # fallback to search_dns if specified + if mirror is None: + # list of mirrors to try to resolve + mirror = search_for_mirror_dns(mcfg.get("search_dns", None), + mirrortype, cfg, cloud) + + return mirror + + +def find_apt_mirror_info(cfg, cloud, arch=None): + """find_apt_mirror_info + find an apt_mirror given the cfg provided. + It can check for separate config of primary and security mirrors + If only primary is given security is assumed to be equal to primary + If the generic apt_mirror is given that is defining for both + """ - # this is a bit strange. - # if mirror is set, then one of the legacy options above set it - # but they do not cover security. so we need to get that from - # get_package_mirror_info - if mirror: - mirror_info.update({'primary': mirror}) + if arch is None: + arch = util.get_architecture() + LOG.debug("got arch for mirror selection: %s", arch) + pmirror = get_mirror(cfg, "primary", arch, cloud) + LOG.debug("got primary mirror: %s", pmirror) + smirror = get_mirror(cfg, "security", arch, cloud) + LOG.debug("got security mirror: %s", smirror) + + mirror_info = update_mirror_info(pmirror, smirror, arch, cloud) + + # less complex replacements use only MIRROR, derive from primary + mirror_info["MIRROR"] = mirror_info["PRIMARY"] return mirror_info def apply_apt_config(cfg, proxy_fname, config_fname): + """apply_apt_config + Applies any apt*proxy config from if specified + """ # Set up any apt proxy - cfgs = (('apt_proxy', 'Acquire::HTTP::Proxy "%s";'), - ('apt_http_proxy', 'Acquire::HTTP::Proxy "%s";'), - ('apt_ftp_proxy', 'Acquire::FTP::Proxy "%s";'), - ('apt_https_proxy', 'Acquire::HTTPS::Proxy "%s";')) + cfgs = (('proxy', 'Acquire::http::Proxy "%s";'), + ('http_proxy', 'Acquire::http::Proxy "%s";'), + ('ftp_proxy', 'Acquire::ftp::Proxy "%s";'), + ('https_proxy', 'Acquire::https::Proxy "%s";')) proxies = [fmt % cfg.get(name) for (name, fmt) in cfgs if cfg.get(name)] if len(proxies): + LOG.debug("write apt proxy info to %s", proxy_fname) util.write_file(proxy_fname, '\n'.join(proxies) + '\n') elif os.path.isfile(proxy_fname): util.del_file(proxy_fname) + LOG.debug("no apt proxy configured, removed %s", proxy_fname) - if cfg.get('apt_config', None): - util.write_file(config_fname, cfg.get('apt_config')) + if cfg.get('conf', None): + LOG.debug("write apt config info to %s", config_fname) + util.write_file(config_fname, cfg.get('conf')) elif os.path.isfile(config_fname): util.del_file(config_fname) + LOG.debug("no apt config configured, removed %s", config_fname) + + +CONFIG_CLEANERS = { + 'cloud-init': clean_cloud_init, +} + +# vi: ts=4 expandtab syntax=python diff --git a/cloudinit/gpg.py b/cloudinit/gpg.py index 6a76d785..5bbff513 100644 --- a/cloudinit/gpg.py +++ b/cloudinit/gpg.py @@ -36,11 +36,11 @@ def export_armour(key): return armour -def receive_key(key, keyserver): +def recv_key(key, keyserver): """Receive gpg key from the specified keyserver""" LOG.debug('Receive gpg key "%s"', key) try: - util.subp(["gpg", "--keyserver", keyserver, "--recv-keys", key], + util.subp(["gpg", "--keyserver", keyserver, "--recv", key], capture=True) except util.ProcessExecutionError as error: raise ValueError(('Failed to import key "%s" ' @@ -57,12 +57,12 @@ def delete_key(key): LOG.warn('Failed delete key "%s": %s', key, error) -def get_key_by_id(keyid, keyserver="keyserver.ubuntu.com"): +def getkeybyid(keyid, keyserver='keyserver.ubuntu.com'): """get gpg keyid from keyserver""" armour = export_armour(keyid) if not armour: try: - receive_key(keyid, keyserver=keyserver) + recv_key(keyid, keyserver=keyserver) armour = export_armour(keyid) except ValueError: LOG.exception('Failed to obtain gpg key %s', keyid) diff --git a/cloudinit/util.py b/cloudinit/util.py index 226628cc..db80ca96 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -61,6 +61,10 @@ from cloudinit import version from cloudinit.settings import (CFG_BUILTIN) +try: + string_types = (basestring,) +except NameError: + string_types = (str,) _DNS_REDIRECT_IP = None LOG = logging.getLogger(__name__) @@ -82,6 +86,71 @@ CONTAINER_TESTS = (['systemd-detect-virt', '--quiet', '--container'], PROC_CMDLINE = None +_LSB_RELEASE = {} + + +def get_architecture(target=None): + out, _ = subp(['dpkg', '--print-architecture'], capture=True, + target=target) + return out.strip() + + +def _lsb_release(target=None): + fmap = {'Codename': 'codename', 'Description': 'description', + 'Distributor ID': 'id', 'Release': 'release'} + + data = {} + try: + out, _ = subp(['lsb_release', '--all'], capture=True, target=target) + for line in out.splitlines(): + fname, _, val = line.partition(":") + if fname in fmap: + data[fmap[fname]] = val.strip() + missing = [k for k in fmap.values() if k not in data] + if len(missing): + LOG.warn("Missing fields in lsb_release --all output: %s", + ','.join(missing)) + + except ProcessExecutionError as err: + LOG.warn("Unable to get lsb_release --all: %s", err) + data = {v: "UNAVAILABLE" for v in fmap.values()} + + return data + + +def lsb_release(target=None): + if target_path(target) != "/": + # do not use or update cache if target is provided + return _lsb_release(target) + + global _LSB_RELEASE + if not _LSB_RELEASE: + data = _lsb_release() + _LSB_RELEASE.update(data) + return _LSB_RELEASE + + +def target_path(target, path=None): + # return 'path' inside target, accepting target as None + if target in (None, ""): + target = "/" + elif not isinstance(target, string_types): + raise ValueError("Unexpected input for target: %s" % target) + else: + target = os.path.abspath(target) + # abspath("//") returns "//" specifically for 2 slashes. + if target.startswith("//"): + target = target[1:] + + if not path: + return target + + # os.path.join("/etc", "/foo") returns "/foo". Chomp all leading /. + while len(path) and path[0] == "/": + path = path[1:] + + return os.path.join(target, path) + def decode_binary(blob, encoding='utf-8'): # Converts a binary type into a text type using given encoding. @@ -1688,10 +1757,20 @@ def delete_dir_contents(dirname): def subp(args, data=None, rcs=None, env=None, capture=True, shell=False, - logstring=False): + logstring=False, decode="replace", target=None): + + # not supported in cloud-init (yet), for now kept in the call signature + # to ease maintaining code shared between cloud-init and curtin + if target is not None: + raise ValueError("target arg not supported by cloud-init") + if rcs is None: rcs = [0] + + devnull_fp = None try: + if target_path(target) != "/": + args = ['chroot', target] + list(args) if not logstring: LOG.debug(("Running command %s with allowed return codes %s" @@ -1700,33 +1779,52 @@ def subp(args, data=None, rcs=None, env=None, capture=True, shell=False, LOG.debug(("Running hidden command to protect sensitive " "input/output logstring: %s"), logstring) - if not capture: - stdout = None - stderr = None - else: + stdin = None + stdout = None + stderr = None + if capture: stdout = subprocess.PIPE stderr = subprocess.PIPE - stdin = subprocess.PIPE - kws = dict(stdout=stdout, stderr=stderr, stdin=stdin, - env=env, shell=shell) - if six.PY3: - # Use this so subprocess output will be (Python 3) str, not bytes. - kws['universal_newlines'] = True - sp = subprocess.Popen(args, **kws) + if data is None: + # using devnull assures any reads get null, rather + # than possibly waiting on input. + devnull_fp = open(os.devnull) + stdin = devnull_fp + else: + stdin = subprocess.PIPE + if not isinstance(data, bytes): + data = data.encode() + + sp = subprocess.Popen(args, stdout=stdout, + stderr=stderr, stdin=stdin, + env=env, shell=shell) (out, err) = sp.communicate(data) + + # Just ensure blank instead of none. + if not out and capture: + out = b'' + if not err and capture: + err = b'' + if decode: + def ldecode(data, m='utf-8'): + if not isinstance(data, bytes): + return data + return data.decode(m, errors=decode) + + out = ldecode(out) + err = ldecode(err) except OSError as e: raise ProcessExecutionError(cmd=args, reason=e, errno=e.errno) + finally: + if devnull_fp: + devnull_fp.close() + rc = sp.returncode if rc not in rcs: raise ProcessExecutionError(stdout=out, stderr=err, exit_code=rc, cmd=args) - # Just ensure blank instead of none?? (iff capturing) - if not out and capture: - out = '' - if not err and capture: - err = '' return (out, err) @@ -2251,3 +2349,18 @@ def message_from_string(string): if sys.version_info[:2] < (2, 7): return email.message_from_file(six.StringIO(string)) return email.message_from_string(string) + + +def get_installed_packages(target=None): + (out, _) = subp(['dpkg-query', '--list'], target=target, capture=True) + + pkgs_inst = set() + for line in out.splitlines(): + try: + (state, pkg, _) = line.split(None, 2) + except ValueError: + continue + if state.startswith("hi") or state.startswith("ii"): + pkgs_inst.add(re.sub(":.*", "", pkg)) + + return pkgs_inst diff --git a/doc/examples/cloud-config-add-apt-repos.txt b/doc/examples/cloud-config-add-apt-repos.txt index be9d5472..22ef7612 100644 --- a/doc/examples/cloud-config-add-apt-repos.txt +++ b/doc/examples/cloud-config-add-apt-repos.txt @@ -4,18 +4,21 @@ # # Default: auto select based on cloud metadata # in ec2, the default is <region>.archive.ubuntu.com -# apt_mirror: -# use the provided mirror -# apt_mirror_search: -# search the list for the first mirror. -# this is currently very limited, only verifying that -# the mirror is dns resolvable or an IP address +# apt: +# primary: +# - arches [default] +# uri: +# use the provided mirror +# search: +# search the list for the first mirror. +# this is currently very limited, only verifying that +# the mirror is dns resolvable or an IP address # -# if neither apt_mirror nor apt_mirror search is set (the default) +# if neither mirror is set (the default) # then use the mirror provided by the DataSource found. # In EC2, that means using <region>.ec2.archive.ubuntu.com -# -# if no mirror is provided by the DataSource, and 'apt_mirror_search_dns' is +# +# if no mirror is provided by the DataSource, but 'search_dns' is # true, then search for dns names '<distro>-mirror' in each of # - fqdn of this host per cloud metadata # - localdomain @@ -27,8 +30,19 @@ # up and expose them only by creating dns entries. # # if none of that is found, then the default distro mirror is used -apt_mirror: http://us.archive.ubuntu.com/ubuntu/ -apt_mirror_search: - - http://local-mirror.mydomain - - http://archive.ubuntu.com -apt_mirror_search_dns: False +apt: + primary: + - arches: [default] + uri: http://us.archive.ubuntu.com/ubuntu/ +# or +apt: + primary: + - arches: [default] + search: + - http://local-mirror.mydomain + - http://archive.ubuntu.com +# or +apt: + primary: + - arches: [default] + search_dns: True diff --git a/doc/examples/cloud-config-apt.txt b/doc/examples/cloud-config-apt.txt new file mode 100644 index 00000000..1a0fc6f2 --- /dev/null +++ b/doc/examples/cloud-config-apt.txt @@ -0,0 +1,328 @@ +# apt_pipelining (configure Acquire::http::Pipeline-Depth) +# Default: disables HTTP pipelining. Certain web servers, such +# as S3 do not pipeline properly (LP: #948461). +# Valid options: +# False/default: Disables pipelining for APT +# None/Unchanged: Use OS default +# Number: Set pipelining to some number (not recommended) +apt_pipelining: False + +## apt config via system_info: +# under the 'system_info', you can customize cloud-init's interaction +# with apt. +# system_info: +# apt_get_command: [command, argument, argument] +# apt_get_upgrade_subcommand: dist-upgrade +# +# apt_get_command: +# To specify a different 'apt-get' command, set 'apt_get_command'. +# This must be a list, and the subcommand (update, upgrade) is appended to it. +# default is: +# ['apt-get', '--option=Dpkg::Options::=--force-confold', +# '--option=Dpkg::options::=--force-unsafe-io', '--assume-yes', '--quiet'] +# +# apt_get_upgrade_subcommand: "dist-upgrade" +# Specify a different subcommand for 'upgrade. The default is 'dist-upgrade'. +# This is the subcommand that is invoked for package_upgrade. +# +# apt_get_wrapper: +# command: eatmydata +# enabled: [True, False, "auto"] +# + +# Install additional packages on first boot +# +# Default: none +# +# if packages are specified, this apt_update will be set to true + +packages: ['pastebinit'] + +apt: + # The apt config consists of two major "areas". + # + # On one hand there is the global configuration for the apt feature. + # + # On one hand (down in this file) there is the source dictionary which allows + # to define various entries to be considered by apt. + + ############################################################################## + # Section 1: global apt configuration + # + # The following examples number the top keys to ease identification in + # discussions. + + # 1.1 preserve_sources_list + # + # Preserves the existing /etc/apt/sources.list + # Default: false - do overwrite sources_list. If set to true then any + # "mirrors" configuration will have no effect. + # Set to true to avoid affecting sources.list. In that case only + # "extra" source specifications will be written into + # /etc/apt/sources.list.d/* + preserve_sources_list: true + + # 1.2 disable_suites + # + # This is an empty list by default, so nothing is disabled. + # + # If given, those suites are removed from sources.list after all other + # modifications have been made. + # Suites are even disabled if no other modification was made, + # but not if is preserve_sources_list is active. + # There is a special alias “$RELEASE” as in the sources that will be replace + # by the matching release. + # + # To ease configuration and improve readability the following common ubuntu + # suites will be automatically mapped to their full definition. + # updates => $RELEASE-updates + # backports => $RELEASE-backports + # security => $RELEASE-security + # proposed => $RELEASE-proposed + # release => $RELEASE + # + # There is no harm in specifying a suite to be disabled that is not found in + # the source.list file (just a no-op then) + # + # Note: Lines don’t get deleted, but disabled by being converted to a comment. + # The following example disables all usual defaults except $RELEASE-security. + # On top it disables a custom suite called "mysuite" + disable_suites: [$RELEASE-updates, backports, $RELEASE, mysuite] + + # 1.3 primary/security archives + # + # Default: none - instead it is auto select based on cloud metadata + # so if neither "uri" nor "search", nor "search_dns" is set (the default) + # then use the mirror provided by the DataSource found. + # In EC2, that means using <region>.ec2.archive.ubuntu.com + # + # define a custom (e.g. localized) mirror that will be used in sources.list + # and any custom sources entries for deb / deb-src lines. + # + # One can set primary and security mirror to different uri's + # the child elements to the keys primary and secondary are equivalent + primary: + # arches is list of architectures the following config applies to + # the special keyword "default" applies to any architecture not explicitly + # listed. + - arches: [amd64, i386, default] + # uri is just defining the target as-is + uri: http://us.archive.ubuntu.com/ubuntu + # + # via search one can define lists that are tried one by one. + # The first with a working DNS resolution (or if it is an IP) will be + # picked. That way one can keep one configuration for multiple + # subenvironments that select the working one. + search: + - http://cool.but-sometimes-unreachable.com/ubuntu + - http://us.archive.ubuntu.com/ubuntu + # if no mirror is provided by uri or search but 'search_dns' is + # true, then search for dns names '<distro>-mirror' in each of + # - fqdn of this host per cloud metadata + # - localdomain + # - no domain (which would search domains listed in /etc/resolv.conf) + # If there is a dns entry for <distro>-mirror, then it is assumed that + # there is a distro mirror at http://<distro>-mirror.<domain>/<distro> + # + # That gives the cloud provider the opportunity to set mirrors of a distro + # up and expose them only by creating dns entries. + # + # if none of that is found, then the default distro mirror is used + search_dns: true + # + # If multiple of a category are given + # 1. uri + # 2. search + # 3. search_dns + # the first defining a valid mirror wins (in the order as defined here, + # not the order as listed in the config). + # + - arches: [s390x, arm64] + # as above, allowing to have one config for different per arch mirrors + # security is optional, if not defined it is set to the same value as primary + security: + uri: http://security.ubuntu.com/ubuntu + # If search_dns is set for security the searched pattern is: + # <distro>-security-mirror + + # if no mirrors are specified at all, or all lookups fail it will try + # to get them from the cloud datasource and if those neither provide one fall + # back to: + # primary: http://archive.ubuntu.com/ubuntu + # security: http://security.ubuntu.com/ubuntu + + # 1.4 sources_list + # + # Provide a custom template for rendering sources.list + # without one provided cloud-init uses builtin templates for + # ubuntu and debian. + # Within these sources.list templates you can use the following replacement + # variables (all have sane Ubuntu defaults, but mirrors can be overwritten + # as needed (see above)): + # => $RELEASE, $MIRROR, $PRIMARY, $SECURITY + sources_list: | # written by cloud-init custom template + deb $MIRROR $RELEASE main restricted + deb-src $MIRROR $RELEASE main restricted + deb $PRIMARY $RELEASE universe restricted + deb $SECURITY $RELEASE-security multiverse + + # 1.5 conf + # + # Any apt config string that will be made available to apt + # see the APT.CONF(5) man page for details what can be specified + conf: | # APT config + APT { + Get { + Assume-Yes "true"; + Fix-Broken "true"; + }; + }; + + # 1.6 (http_|ftp_|https_)proxy + # + # Proxies are the most common apt.conf option, so that for simplified use + # there is a shortcut for those. Those get automatically translated into the + # correct Acquire::*::Proxy statements. + # + # note: proxy actually being a short synonym to http_proxy + proxy: http://[[user][:pass]@]host[:port]/ + http_proxy: http://[[user][:pass]@]host[:port]/ + ftp_proxy: ftp://[[user][:pass]@]host[:port]/ + https_proxy: https://[[user][:pass]@]host[:port]/ + + # 1.7 add_apt_repo_match + # + # 'source' entries in apt-sources that match this python regex + # expression will be passed to add-apt-repository + # The following example is also the builtin default if nothing is specified + add_apt_repo_match: '^[\w-]+:\w' + + + ############################################################################## + # Section 2: source list entries + # + # This is a dictionary (unlike most block/net which are lists) + # + # The key of each source entry is the filename and will be prepended by + # /etc/apt/sources.list.d/ if it doesn't start with a '/'. + # If it doesn't end with .list it will be appended so that apt picks up it's + # configuration. + # + # Whenever there is no content to be written into such a file, the key is + # not used as filename - yet it can still be used as index for merging + # configuration. + # + # The values inside the entries consost of the following optional entries: + # 'source': a sources.list entry (some variable replacements apply) + # 'keyid': providing a key to import via shortid or fingerprint + # 'key': providing a raw PGP key + # 'keyserver': specify an alternate keyserver to pull keys from that + # were specified by keyid + + # This allows merging between multiple input files than a list like: + # cloud-config1 + # sources: + # s1: {'key': 'key1', 'source': 'source1'} + # cloud-config2 + # sources: + # s2: {'key': 'key2'} + # s1: {'keyserver': 'foo'} + # This would be merged to + # sources: + # s1: + # keyserver: foo + # key: key1 + # source: source1 + # s2: + # key: key2 + # + # The following examples number the subfeatures per sources entry to ease + # identification in discussions. + + + sources: + curtin-dev-ppa.list: + # 2.1 source + # + # Creates a file in /etc/apt/sources.list.d/ for the sources list entry + # based on the key: "/etc/apt/sources.list.d/curtin-dev-ppa.list" + source: "deb http://ppa.launchpad.net/curtin-dev/test-archive/ubuntu xenial main" + + # 2.2 keyid + # + # Importing a gpg key for a given key id. Used keyserver defaults to + # keyserver.ubuntu.com + keyid: F430BBA5 # GPG key ID published on a key server + + ignored1: + # 2.3 PPA shortcut + # + # Setup correct apt sources.list line and Auto-Import the signing key + # from LP + # + # See https://help.launchpad.net/Packaging/PPA for more information + # this requires 'add-apt-repository'. This will create a file in + # /etc/apt/sources.list.d automatically, therefore the key here is + # ignored as filename in those cases. + source: "ppa:curtin-dev/test-archive" # Quote the string + + my-repo2.list: + # 2.4 replacement variables + # + # sources can use $MIRROR, $PRIMARY, $SECURITY and $RELEASE replacement + # variables. + # They will be replaced with the default or specified mirrors and the + # running release. + # The entry below would be possibly turned into: + # source: deb http://archive.ubuntu.com/ubuntu xenial multiverse + source: deb $MIRROR $RELEASE multiverse + + my-repo3.list: + # this would have the same end effect as 'ppa:curtin-dev/test-archive' + source: "deb http://ppa.launchpad.net/curtin-dev/test-archive/ubuntu xenial main" + keyid: F430BBA5 # GPG key ID published on the key server + filename: curtin-dev-ppa.list + + ignored2: + # 2.5 key only + # + # this would only import the key without adding a ppa or other source spec + # since this doesn't generate a source.list file the filename key is ignored + keyid: F430BBA5 # GPG key ID published on a key server + + ignored3: + # 2.6 key id alternatives + # + # Keyid's can also be specified via their long fingerprints + keyid: B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77 + + ignored4: + # 2.7 alternative keyservers + # + # One can also specify alternative keyservers to fetch keys from. + keyid: B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77 + keyserver: pgp.mit.edu + + + my-repo4.list: + # 2.8 raw key + # + # The apt signing key can also be specified by providing a pgp public key + # block. Providing the PGP key this way is the most robust method for + # specifying a key, as it removes dependency on a remote key server. + # + # As with keyid's this can be specified with or without some actual source + # content. + key: | # The value needs to start with -----BEGIN PGP PUBLIC KEY BLOCK----- + -----BEGIN PGP PUBLIC KEY BLOCK----- + Version: SKS 1.0.10 + + mI0ESpA3UQEEALdZKVIMq0j6qWAXAyxSlF63SvPVIgxHPb9Nk0DZUixn+akqytxG4zKCONz6 + qLjoBBfHnynyVLfT4ihg9an1PqxRnTO+JKQxl8NgKGz6Pon569GtAOdWNKw15XKinJTDLjnj + 9y96ljJqRcpV9t/WsIcdJPcKFR5voHTEoABE2aEXABEBAAG0GUxhdW5jaHBhZCBQUEEgZm9y + IEFsZXN0aWOItgQTAQIAIAUCSpA3UQIbAwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAAoJEA7H + 5Qi+CcVxWZ8D/1MyYvfj3FJPZUm2Yo1zZsQ657vHI9+pPouqflWOayRR9jbiyUFIn0VdQBrP + t0FwvnOFArUovUWoKAEdqR8hPy3M3APUZjl5K4cMZR/xaMQeQRZ5CHpS4DBKURKAHC0ltS5o + uBJKQOZm5iltJp15cgyIkBkGe8Mx18VFyVglAZey + =Y2oI + -----END PGP PUBLIC KEY BLOCK----- diff --git a/doc/examples/cloud-config-chef-oneiric.txt b/doc/examples/cloud-config-chef-oneiric.txt index 2e5f4b16..75c9aeed 100644 --- a/doc/examples/cloud-config-chef-oneiric.txt +++ b/doc/examples/cloud-config-chef-oneiric.txt @@ -11,39 +11,40 @@ # The default is to install from packages. # Key from http://apt.opscode.com/packages@opscode.com.gpg.key -apt_sources: - - source: "deb http://apt.opscode.com/ $RELEASE-0.10 main" - key: | - -----BEGIN PGP PUBLIC KEY BLOCK----- - Version: GnuPG v1.4.9 (GNU/Linux) - - mQGiBEppC7QRBADfsOkZU6KZK+YmKw4wev5mjKJEkVGlus+NxW8wItX5sGa6kdUu - twAyj7Yr92rF+ICFEP3gGU6+lGo0Nve7KxkN/1W7/m3G4zuk+ccIKmjp8KS3qn99 - dxy64vcji9jIllVa+XXOGIp0G8GEaj7mbkixL/bMeGfdMlv8Gf2XPpp9vwCgn/GC - JKacfnw7MpLKUHOYSlb//JsEAJqao3ViNfav83jJKEkD8cf59Y8xKia5OpZqTK5W - ShVnNWS3U5IVQk10ZDH97Qn/YrK387H4CyhLE9mxPXs/ul18ioiaars/q2MEKU2I - XKfV21eMLO9LYd6Ny/Kqj8o5WQK2J6+NAhSwvthZcIEphcFignIuobP+B5wNFQpe - DbKfA/0WvN2OwFeWRcmmd3Hz7nHTpcnSF+4QX6yHRF/5BgxkG6IqBIACQbzPn6Hm - sMtm/SVf11izmDqSsQptCrOZILfLX/mE+YOl+CwWSHhl+YsFts1WOuh1EhQD26aO - Z84HuHV5HFRWjDLw9LriltBVQcXbpfSrRP5bdr7Wh8vhqJTPjrQnT3BzY29kZSBQ - YWNrYWdlcyA8cGFja2FnZXNAb3BzY29kZS5jb20+iGAEExECACAFAkppC7QCGwMG - CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRApQKupg++Caj8sAKCOXmdG36gWji/K - +o+XtBfvdMnFYQCfTCEWxRy2BnzLoBBFCjDSK6sJqCu5Ag0ESmkLtBAIAIO2SwlR - lU5i6gTOp42RHWW7/pmW78CwUqJnYqnXROrt3h9F9xrsGkH0Fh1FRtsnncgzIhvh - DLQnRHnkXm0ws0jV0PF74ttoUT6BLAUsFi2SPP1zYNJ9H9fhhK/pjijtAcQwdgxu - wwNJ5xCEscBZCjhSRXm0d30bK1o49Cow8ZIbHtnXVP41c9QWOzX/LaGZsKQZnaMx - EzDk8dyyctR2f03vRSVyTFGgdpUcpbr9eTFVgikCa6ODEBv+0BnCH6yGTXwBid9g - w0o1e/2DviKUWCC+AlAUOubLmOIGFBuI4UR+rux9affbHcLIOTiKQXv79lW3P7W8 - AAfniSQKfPWXrrcAAwUH/2XBqD4Uxhbs25HDUUiM/m6Gnlj6EsStg8n0nMggLhuN - QmPfoNByMPUqvA7sULyfr6xCYzbzRNxABHSpf85FzGQ29RF4xsA4vOOU8RDIYQ9X - Q8NqqR6pydprRFqWe47hsAN7BoYuhWqTtOLSBmnAnzTR5pURoqcquWYiiEavZixJ - 3ZRAq/HMGioJEtMFrvsZjGXuzef7f0ytfR1zYeLVWnL9Bd32CueBlI7dhYwkFe+V - Ep5jWOCj02C1wHcwt+uIRDJV6TdtbIiBYAdOMPk15+VBdweBXwMuYXr76+A7VeDL - zIhi7tKFo6WiwjKZq0dzctsJJjtIfr4K4vbiD9Ojg1iISQQYEQIACQUCSmkLtAIb - DAAKCRApQKupg++CauISAJ9CxYPOKhOxalBnVTLeNUkAHGg2gACeIsbobtaD4ZHG - 0GLl8EkfA8uhluM= - =zKAm - -----END PGP PUBLIC KEY BLOCK----- +apt: + sources: + - source: "deb http://apt.opscode.com/ $RELEASE-0.10 main" + key: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + Version: GnuPG v1.4.9 (GNU/Linux) + + mQGiBEppC7QRBADfsOkZU6KZK+YmKw4wev5mjKJEkVGlus+NxW8wItX5sGa6kdUu + twAyj7Yr92rF+ICFEP3gGU6+lGo0Nve7KxkN/1W7/m3G4zuk+ccIKmjp8KS3qn99 + dxy64vcji9jIllVa+XXOGIp0G8GEaj7mbkixL/bMeGfdMlv8Gf2XPpp9vwCgn/GC + JKacfnw7MpLKUHOYSlb//JsEAJqao3ViNfav83jJKEkD8cf59Y8xKia5OpZqTK5W + ShVnNWS3U5IVQk10ZDH97Qn/YrK387H4CyhLE9mxPXs/ul18ioiaars/q2MEKU2I + XKfV21eMLO9LYd6Ny/Kqj8o5WQK2J6+NAhSwvthZcIEphcFignIuobP+B5wNFQpe + DbKfA/0WvN2OwFeWRcmmd3Hz7nHTpcnSF+4QX6yHRF/5BgxkG6IqBIACQbzPn6Hm + sMtm/SVf11izmDqSsQptCrOZILfLX/mE+YOl+CwWSHhl+YsFts1WOuh1EhQD26aO + Z84HuHV5HFRWjDLw9LriltBVQcXbpfSrRP5bdr7Wh8vhqJTPjrQnT3BzY29kZSBQ + YWNrYWdlcyA8cGFja2FnZXNAb3BzY29kZS5jb20+iGAEExECACAFAkppC7QCGwMG + CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRApQKupg++Caj8sAKCOXmdG36gWji/K + +o+XtBfvdMnFYQCfTCEWxRy2BnzLoBBFCjDSK6sJqCu5Ag0ESmkLtBAIAIO2SwlR + lU5i6gTOp42RHWW7/pmW78CwUqJnYqnXROrt3h9F9xrsGkH0Fh1FRtsnncgzIhvh + DLQnRHnkXm0ws0jV0PF74ttoUT6BLAUsFi2SPP1zYNJ9H9fhhK/pjijtAcQwdgxu + wwNJ5xCEscBZCjhSRXm0d30bK1o49Cow8ZIbHtnXVP41c9QWOzX/LaGZsKQZnaMx + EzDk8dyyctR2f03vRSVyTFGgdpUcpbr9eTFVgikCa6ODEBv+0BnCH6yGTXwBid9g + w0o1e/2DviKUWCC+AlAUOubLmOIGFBuI4UR+rux9affbHcLIOTiKQXv79lW3P7W8 + AAfniSQKfPWXrrcAAwUH/2XBqD4Uxhbs25HDUUiM/m6Gnlj6EsStg8n0nMggLhuN + QmPfoNByMPUqvA7sULyfr6xCYzbzRNxABHSpf85FzGQ29RF4xsA4vOOU8RDIYQ9X + Q8NqqR6pydprRFqWe47hsAN7BoYuhWqTtOLSBmnAnzTR5pURoqcquWYiiEavZixJ + 3ZRAq/HMGioJEtMFrvsZjGXuzef7f0ytfR1zYeLVWnL9Bd32CueBlI7dhYwkFe+V + Ep5jWOCj02C1wHcwt+uIRDJV6TdtbIiBYAdOMPk15+VBdweBXwMuYXr76+A7VeDL + zIhi7tKFo6WiwjKZq0dzctsJJjtIfr4K4vbiD9Ojg1iISQQYEQIACQUCSmkLtAIb + DAAKCRApQKupg++CauISAJ9CxYPOKhOxalBnVTLeNUkAHGg2gACeIsbobtaD4ZHG + 0GLl8EkfA8uhluM= + =zKAm + -----END PGP PUBLIC KEY BLOCK----- chef: diff --git a/doc/examples/cloud-config-chef.txt b/doc/examples/cloud-config-chef.txt index b886cba2..75d78a15 100644 --- a/doc/examples/cloud-config-chef.txt +++ b/doc/examples/cloud-config-chef.txt @@ -11,39 +11,40 @@ # The default is to install from packages. # Key from http://apt.opscode.com/packages@opscode.com.gpg.key -apt_sources: - - source: "deb http://apt.opscode.com/ $RELEASE-0.10 main" - key: | - -----BEGIN PGP PUBLIC KEY BLOCK----- - Version: GnuPG v1.4.9 (GNU/Linux) - - mQGiBEppC7QRBADfsOkZU6KZK+YmKw4wev5mjKJEkVGlus+NxW8wItX5sGa6kdUu - twAyj7Yr92rF+ICFEP3gGU6+lGo0Nve7KxkN/1W7/m3G4zuk+ccIKmjp8KS3qn99 - dxy64vcji9jIllVa+XXOGIp0G8GEaj7mbkixL/bMeGfdMlv8Gf2XPpp9vwCgn/GC - JKacfnw7MpLKUHOYSlb//JsEAJqao3ViNfav83jJKEkD8cf59Y8xKia5OpZqTK5W - ShVnNWS3U5IVQk10ZDH97Qn/YrK387H4CyhLE9mxPXs/ul18ioiaars/q2MEKU2I - XKfV21eMLO9LYd6Ny/Kqj8o5WQK2J6+NAhSwvthZcIEphcFignIuobP+B5wNFQpe - DbKfA/0WvN2OwFeWRcmmd3Hz7nHTpcnSF+4QX6yHRF/5BgxkG6IqBIACQbzPn6Hm - sMtm/SVf11izmDqSsQptCrOZILfLX/mE+YOl+CwWSHhl+YsFts1WOuh1EhQD26aO - Z84HuHV5HFRWjDLw9LriltBVQcXbpfSrRP5bdr7Wh8vhqJTPjrQnT3BzY29kZSBQ - YWNrYWdlcyA8cGFja2FnZXNAb3BzY29kZS5jb20+iGAEExECACAFAkppC7QCGwMG - CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRApQKupg++Caj8sAKCOXmdG36gWji/K - +o+XtBfvdMnFYQCfTCEWxRy2BnzLoBBFCjDSK6sJqCu5Ag0ESmkLtBAIAIO2SwlR - lU5i6gTOp42RHWW7/pmW78CwUqJnYqnXROrt3h9F9xrsGkH0Fh1FRtsnncgzIhvh - DLQnRHnkXm0ws0jV0PF74ttoUT6BLAUsFi2SPP1zYNJ9H9fhhK/pjijtAcQwdgxu - wwNJ5xCEscBZCjhSRXm0d30bK1o49Cow8ZIbHtnXVP41c9QWOzX/LaGZsKQZnaMx - EzDk8dyyctR2f03vRSVyTFGgdpUcpbr9eTFVgikCa6ODEBv+0BnCH6yGTXwBid9g - w0o1e/2DviKUWCC+AlAUOubLmOIGFBuI4UR+rux9affbHcLIOTiKQXv79lW3P7W8 - AAfniSQKfPWXrrcAAwUH/2XBqD4Uxhbs25HDUUiM/m6Gnlj6EsStg8n0nMggLhuN - QmPfoNByMPUqvA7sULyfr6xCYzbzRNxABHSpf85FzGQ29RF4xsA4vOOU8RDIYQ9X - Q8NqqR6pydprRFqWe47hsAN7BoYuhWqTtOLSBmnAnzTR5pURoqcquWYiiEavZixJ - 3ZRAq/HMGioJEtMFrvsZjGXuzef7f0ytfR1zYeLVWnL9Bd32CueBlI7dhYwkFe+V - Ep5jWOCj02C1wHcwt+uIRDJV6TdtbIiBYAdOMPk15+VBdweBXwMuYXr76+A7VeDL - zIhi7tKFo6WiwjKZq0dzctsJJjtIfr4K4vbiD9Ojg1iISQQYEQIACQUCSmkLtAIb - DAAKCRApQKupg++CauISAJ9CxYPOKhOxalBnVTLeNUkAHGg2gACeIsbobtaD4ZHG - 0GLl8EkfA8uhluM= - =zKAm - -----END PGP PUBLIC KEY BLOCK----- +apt: + sources: + - source: "deb http://apt.opscode.com/ $RELEASE-0.10 main" + key: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + Version: GnuPG v1.4.9 (GNU/Linux) + + mQGiBEppC7QRBADfsOkZU6KZK+YmKw4wev5mjKJEkVGlus+NxW8wItX5sGa6kdUu + twAyj7Yr92rF+ICFEP3gGU6+lGo0Nve7KxkN/1W7/m3G4zuk+ccIKmjp8KS3qn99 + dxy64vcji9jIllVa+XXOGIp0G8GEaj7mbkixL/bMeGfdMlv8Gf2XPpp9vwCgn/GC + JKacfnw7MpLKUHOYSlb//JsEAJqao3ViNfav83jJKEkD8cf59Y8xKia5OpZqTK5W + ShVnNWS3U5IVQk10ZDH97Qn/YrK387H4CyhLE9mxPXs/ul18ioiaars/q2MEKU2I + XKfV21eMLO9LYd6Ny/Kqj8o5WQK2J6+NAhSwvthZcIEphcFignIuobP+B5wNFQpe + DbKfA/0WvN2OwFeWRcmmd3Hz7nHTpcnSF+4QX6yHRF/5BgxkG6IqBIACQbzPn6Hm + sMtm/SVf11izmDqSsQptCrOZILfLX/mE+YOl+CwWSHhl+YsFts1WOuh1EhQD26aO + Z84HuHV5HFRWjDLw9LriltBVQcXbpfSrRP5bdr7Wh8vhqJTPjrQnT3BzY29kZSBQ + YWNrYWdlcyA8cGFja2FnZXNAb3BzY29kZS5jb20+iGAEExECACAFAkppC7QCGwMG + CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRApQKupg++Caj8sAKCOXmdG36gWji/K + +o+XtBfvdMnFYQCfTCEWxRy2BnzLoBBFCjDSK6sJqCu5Ag0ESmkLtBAIAIO2SwlR + lU5i6gTOp42RHWW7/pmW78CwUqJnYqnXROrt3h9F9xrsGkH0Fh1FRtsnncgzIhvh + DLQnRHnkXm0ws0jV0PF74ttoUT6BLAUsFi2SPP1zYNJ9H9fhhK/pjijtAcQwdgxu + wwNJ5xCEscBZCjhSRXm0d30bK1o49Cow8ZIbHtnXVP41c9QWOzX/LaGZsKQZnaMx + EzDk8dyyctR2f03vRSVyTFGgdpUcpbr9eTFVgikCa6ODEBv+0BnCH6yGTXwBid9g + w0o1e/2DviKUWCC+AlAUOubLmOIGFBuI4UR+rux9affbHcLIOTiKQXv79lW3P7W8 + AAfniSQKfPWXrrcAAwUH/2XBqD4Uxhbs25HDUUiM/m6Gnlj6EsStg8n0nMggLhuN + QmPfoNByMPUqvA7sULyfr6xCYzbzRNxABHSpf85FzGQ29RF4xsA4vOOU8RDIYQ9X + Q8NqqR6pydprRFqWe47hsAN7BoYuhWqTtOLSBmnAnzTR5pURoqcquWYiiEavZixJ + 3ZRAq/HMGioJEtMFrvsZjGXuzef7f0ytfR1zYeLVWnL9Bd32CueBlI7dhYwkFe+V + Ep5jWOCj02C1wHcwt+uIRDJV6TdtbIiBYAdOMPk15+VBdweBXwMuYXr76+A7VeDL + zIhi7tKFo6WiwjKZq0dzctsJJjtIfr4K4vbiD9Ojg1iISQQYEQIACQUCSmkLtAIb + DAAKCRApQKupg++CauISAJ9CxYPOKhOxalBnVTLeNUkAHGg2gACeIsbobtaD4ZHG + 0GLl8EkfA8uhluM= + =zKAm + -----END PGP PUBLIC KEY BLOCK----- chef: diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index 3cc9c055..190029e4 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -18,256 +18,7 @@ package_upgrade: true # Aliases: apt_reboot_if_required package_reboot_if_required: true -# Add apt repositories -# -# Default: auto select based on cloud metadata -# in ec2, the default is <region>.archive.ubuntu.com -# apt_mirror: -# use the provided mirror -# apt_mirror_search: -# search the list for the first mirror. -# this is currently very limited, only verifying that -# the mirror is dns resolvable or an IP address -# -# if neither apt_mirror nor apt_mirror search is set (the default) -# then use the mirror provided by the DataSource found. -# In EC2, that means using <region>.ec2.archive.ubuntu.com -# -# if no mirror is provided by the DataSource, and 'apt_mirror_search_dns' is -# true, then search for dns names '<distro>-mirror' in each of -# - fqdn of this host per cloud metadata -# - localdomain -# - no domain (which would search domains listed in /etc/resolv.conf) -# If there is a dns entry for <distro>-mirror, then it is assumed that there -# is a distro mirror at http://<distro>-mirror.<domain>/<distro> -# -# That gives the cloud provider the opportunity to set mirrors of a distro -# up and expose them only by creating dns entries. -# -# if none of that is found, then the default distro mirror is used -apt_mirror: http://us.archive.ubuntu.com/ubuntu/ -apt_mirror_search: - - http://local-mirror.mydomain - - http://archive.ubuntu.com - -apt_mirror_search_dns: False - -# apt_proxy (configure Acquire::HTTP::Proxy) -# 'apt_http_proxy' is an alias for 'apt_proxy'. -# Also, available are 'apt_ftp_proxy' and 'apt_https_proxy'. -# These affect Acquire::FTP::Proxy and Acquire::HTTPS::Proxy respectively -apt_proxy: http://my.apt.proxy:3128 - -# apt_pipelining (configure Acquire::http::Pipeline-Depth) -# Default: disables HTTP pipelining. Certain web servers, such -# as S3 do not pipeline properly (LP: #948461). -# Valid options: -# False/default: Disables pipelining for APT -# None/Unchanged: Use OS default -# Number: Set pipelining to some number (not recommended) -apt_pipelining: False - -# Preserve existing /etc/apt/sources.list -# Default: overwrite sources_list with mirror. If this is true -# then apt_mirror above will have no effect -apt_preserve_sources_list: true - -# Provide a custom template for rendering sources.list -# Default: a default template for Ubuntu/Debain will be used as packaged in -# Ubuntu: /etc/cloud/templates/sources.list.ubuntu.tmpl -# Debian: /etc/cloud/templates/sources.list.debian.tmpl -# Others: n/a -# This will follow the normal mirror/codename replacement rules before -# being written to disk. -apt_custom_sources_list: | - ## template:jinja - ## Note, this file is written by cloud-init on first boot of an instance - ## modifications made here will not survive a re-bundle. - ## if you wish to make changes you can: - ## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg - ## or do the same in user-data - ## b.) add sources in /etc/apt/sources.list.d - ## c.) make changes to template file /etc/cloud/templates/sources.list.tmpl - deb {{mirror}} {{codename}} main restricted - deb-src {{mirror}} {{codename}} main restricted - - # could drop some of the usually used entries - - # could refer to other mirrors - deb http://ddebs.ubuntu.com {{codename}} main restricted universe multiverse - deb http://ddebs.ubuntu.com {{codename}}-updates main restricted universe multiverse - deb http://ddebs.ubuntu.com {{codename}}-proposed main restricted universe multiverse - - # or even more uncommon examples like local or NFS mounted repos, - # eventually whatever is compatible with sources.list syntax - deb file:/home/apt/debian unstable main contrib non-free - -# 'source' entries in apt-sources that match this python regex -# expression will be passed to add-apt-repository -add_apt_repo_match: '^[\w-]+:\w' - -# 'apt_sources' is a dictionary -# The key is the filename and will be prepended by /etc/apt/sources.list.d/ if -# it doesn't start with a '/'. -# There are certain cases - where no content is written into a source.list file -# where the filename will be ignored - yet it can still be used as index for -# merging. -# The value it maps to is a dictionary with the following optional entries: -# source: a sources.list entry (some variable replacements apply) -# keyid: providing a key to import via shortid or fingerprint -# key: providing a raw PGP key -# keyserver: keyserver to fetch keys from, default is keyserver.ubuntu.com -# filename: for compatibility with the older format (now the key to this -# dictionary is the filename). If specified this overwrites the -# filename given as key. - -# the new "filename: {specification-dictionary}, filename2: ..." format allows -# better merging between multiple input files than a list like: -# cloud-config1 -# sources: -# s1: {'key': 'key1', 'source': 'source1'} -# cloud-config2 -# sources: -# s2: {'key': 'key2'} -# s1: {filename: 'foo'} -# this would be merged to -#sources: -# s1: -# filename: foo -# key: key1 -# source: source1 -# s2: -# key: key2 -# Be aware that this style of merging is not the default (for backward -# compatibility reasons). You should specify the following merge_how to get -# this more complete and modern merging behaviour: -# merge_how: "list()+dict()+str()" -# This would then also be equivalent to the config merging used in curtin -# (https://launchpad.net/curtin). - -# for more details see below in the various examples - -apt_sources: - byobu-ppa.list: - source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main" - keyid: F430BBA5 # GPG key ID published on a key server - # adding a source.list line, importing a gpg key for a given key id and - # storing it in the file /etc/apt/sources.list.d/byobu-ppa.list - - # PPA shortcut: - # * Setup correct apt sources.list line - # * Import the signing key from LP - # - # See https://help.launchpad.net/Packaging/PPA for more information - # this requires 'add-apt-repository' - # due to that the filename key is ignored in this case - ignored1: - source: "ppa:smoser/ppa" # Quote the string - - # Custom apt repository: - # * all that is required is 'source' - # * Creates a file in /etc/apt/sources.list.d/ for the sources list entry - # * [optional] Import the apt signing key from the keyserver - # * Defaults: - # + keyserver: keyserver.ubuntu.com - # - # See sources.list man page for more information about the format - my-repo.list: - source: deb http://archive.ubuntu.com/ubuntu karmic-backports main universe multiverse restricted - - # sources can use $MIRROR and $RELEASE and they will be replaced - # with the local mirror for this cloud, and the running release - # the entry below would be possibly turned into: - # source: deb http://us-east-1.ec2.archive.ubuntu.com/ubuntu natty multiverse - my-repo.list: - source: deb $MIRROR $RELEASE multiverse - - # this would have the same end effect as 'ppa:byobu/ppa' - my-repo.list: - source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main" - keyid: F430BBA5 # GPG key ID published on a key server - filename: byobu-ppa.list - - # this would only import the key without adding a ppa or other source spec - # since this doesn't generate a source.list file the filename key is ignored - ignored2: - keyid: F430BBA5 # GPG key ID published on a key server - - # In general keyid's can also be specified via their long fingerprints - # since this doesn't generate a source.list file the filename key is ignored - ignored3: - keyid: B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77 - - # Custom apt repository: - # * The apt signing key can also be specified - # by providing a pgp public key block - # * Providing the PGP key here is the most robust method for - # specifying a key, as it removes dependency on a remote key server - my-repo.list: - source: deb http://ppa.launchpad.net/alestic/ppa/ubuntu karmic main - key: | # The value needs to start with -----BEGIN PGP PUBLIC KEY BLOCK----- - -----BEGIN PGP PUBLIC KEY BLOCK----- - Version: SKS 1.0.10 - - mI0ESpA3UQEEALdZKVIMq0j6qWAXAyxSlF63SvPVIgxHPb9Nk0DZUixn+akqytxG4zKCONz6 - qLjoBBfHnynyVLfT4ihg9an1PqxRnTO+JKQxl8NgKGz6Pon569GtAOdWNKw15XKinJTDLjnj - 9y96ljJqRcpV9t/WsIcdJPcKFR5voHTEoABE2aEXABEBAAG0GUxhdW5jaHBhZCBQUEEgZm9y - IEFsZXN0aWOItgQTAQIAIAUCSpA3UQIbAwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAAoJEA7H - 5Qi+CcVxWZ8D/1MyYvfj3FJPZUm2Yo1zZsQ657vHI9+pPouqflWOayRR9jbiyUFIn0VdQBrP - t0FwvnOFArUovUWoKAEdqR8hPy3M3APUZjl5K4cMZR/xaMQeQRZ5CHpS4DBKURKAHC0ltS5o - uBJKQOZm5iltJp15cgyIkBkGe8Mx18VFyVglAZey - =Y2oI - -----END PGP PUBLIC KEY BLOCK----- - - # Custom gpg key: - # * As with keyid, a key may also be specified without a related source. - # * all other facts mentioned above still apply - # since this doesn't generate a source.list file the filename key is ignored - ignored4: - key: | # The value needs to start with -----BEGIN PGP PUBLIC KEY BLOCK----- - -----BEGIN PGP PUBLIC KEY BLOCK----- - Version: SKS 1.0.10 - - mI0ESpA3UQEEALdZKVIMq0j6qWAXAyxSlF63SvPVIgxHPb9Nk0DZUixn+akqytxG4zKCONz6 - qLjoBBfHnynyVLfT4ihg9an1PqxRnTO+JKQxl8NgKGz6Pon569GtAOdWNKw15XKinJTDLjnj - 9y96ljJqRcpV9t/WsIcdJPcKFR5voHTEoABE2aEXABEBAAG0GUxhdW5jaHBhZCBQUEEgZm9y - IEFsZXN0aWOItgQTAQIAIAUCSpA3UQIbAwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAAoJEA7H - 5Qi+CcVxWZ8D/1MyYvfj3FJPZUm2Yo1zZsQ657vHI9+pPouqflWOayRR9jbiyUFIn0VdQBrP - t0FwvnOFArUovUWoKAEdqR8hPy3M3APUZjl5K4cMZR/xaMQeQRZ5CHpS4DBKURKAHC0ltS5o - uBJKQOZm5iltJp15cgyIkBkGe8Mx18VFyVglAZey - =Y2oI - -----END PGP PUBLIC KEY BLOCK----- - - -## apt config via system_info: -# under the 'system_info', you can further customize cloud-init's interaction -# with apt. -# system_info: -# apt_get_command: [command, argument, argument] -# apt_get_upgrade_subcommand: dist-upgrade -# -# apt_get_command: -# To specify a different 'apt-get' command, set 'apt_get_command'. -# This must be a list, and the subcommand (update, upgrade) is appended to it. -# default is: -# ['apt-get', '--option=Dpkg::Options::=--force-confold', -# '--option=Dpkg::options::=--force-unsafe-io', '--assume-yes', '--quiet'] -# -# apt_get_upgrade_subcommand: -# Specify a different subcommand for 'upgrade. The default is 'dist-upgrade'. -# This is the subcommand that is invoked if package_upgrade is set to true above. -# -# apt_get_wrapper: -# command: eatmydata -# enabled: [True, False, "auto"] -# - -# Install additional packages on first boot -# -# Default: none -# -# if packages are specified, this apt_update will be set to true -# +# For 'apt' specific config, see cloud-config-apt.txt packages: - pwgen - pastebinit diff --git a/tests/configs/sample1.yaml b/tests/configs/sample1.yaml index 6231f293..ae935cc0 100644 --- a/tests/configs/sample1.yaml +++ b/tests/configs/sample1.yaml @@ -3,9 +3,6 @@ #apt_upgrade: true packages: [ bzr, pastebinit, ubuntu-dev-tools, ccache, bzr-builddeb, vim-nox, git-core, lftp ] -#apt_sources: -# - source: ppa:smoser/ppa - #disable_root: False # mounts: diff --git a/tests/unittests/test_distros/test_generic.py b/tests/unittests/test_distros/test_generic.py index 96fa0811..24ad115f 100644 --- a/tests/unittests/test_distros/test_generic.py +++ b/tests/unittests/test_distros/test_generic.py @@ -226,8 +226,5 @@ class TestGenericDistro(helpers.FilesystemMockingTestCase): os.symlink('/', '/run/systemd/system') self.assertFalse(d.uses_systemd()) -# def _get_package_mirror_info(mirror_info, availability_zone=None, -# mirror_filter=util.search_for_mirror): - # vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_handler_apt_configure.py b/tests/unittests/test_handler/test_handler_apt_conf_v1.py index d1dca2c4..95fd1da2 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure.py +++ b/tests/unittests/test_handler/test_handler_apt_conf_v1.py @@ -27,7 +27,7 @@ class TestAptProxyConfig(TestCase): contents, flags=re.IGNORECASE) def test_apt_proxy_written(self): - cfg = {'apt_proxy': 'myproxy'} + cfg = {'proxy': 'myproxy'} cc_apt_configure.apply_apt_config(cfg, self.pfile, self.cfile) self.assertTrue(os.path.isfile(self.pfile)) @@ -37,7 +37,7 @@ class TestAptProxyConfig(TestCase): self.assertTrue(self._search_apt_config(contents, "http", "myproxy")) def test_apt_http_proxy_written(self): - cfg = {'apt_http_proxy': 'myproxy'} + cfg = {'http_proxy': 'myproxy'} cc_apt_configure.apply_apt_config(cfg, self.pfile, self.cfile) self.assertTrue(os.path.isfile(self.pfile)) @@ -47,13 +47,13 @@ class TestAptProxyConfig(TestCase): self.assertTrue(self._search_apt_config(contents, "http", "myproxy")) def test_apt_all_proxy_written(self): - cfg = {'apt_http_proxy': 'myproxy_http_proxy', - 'apt_https_proxy': 'myproxy_https_proxy', - 'apt_ftp_proxy': 'myproxy_ftp_proxy'} + cfg = {'http_proxy': 'myproxy_http_proxy', + 'https_proxy': 'myproxy_https_proxy', + 'ftp_proxy': 'myproxy_ftp_proxy'} - values = {'http': cfg['apt_http_proxy'], - 'https': cfg['apt_https_proxy'], - 'ftp': cfg['apt_ftp_proxy'], + values = {'http': cfg['http_proxy'], + 'https': cfg['https_proxy'], + 'ftp': cfg['ftp_proxy'], } cc_apt_configure.apply_apt_config(cfg, self.pfile, self.cfile) @@ -74,7 +74,7 @@ class TestAptProxyConfig(TestCase): def test_proxy_replaced(self): util.write_file(self.cfile, "content doesnt matter") - cc_apt_configure.apply_apt_config({'apt_proxy': "foo"}, + cc_apt_configure.apply_apt_config({'proxy': "foo"}, self.pfile, self.cfile) self.assertTrue(os.path.isfile(self.pfile)) contents = load_tfile_or_url(self.pfile) @@ -82,7 +82,7 @@ class TestAptProxyConfig(TestCase): def test_config_written(self): payload = 'this is my apt config' - cfg = {'apt_config': payload} + cfg = {'conf': payload} cc_apt_configure.apply_apt_config(cfg, self.pfile, self.cfile) @@ -93,13 +93,13 @@ class TestAptProxyConfig(TestCase): def test_config_replaced(self): util.write_file(self.pfile, "content doesnt matter") - cc_apt_configure.apply_apt_config({'apt_config': "foo"}, + cc_apt_configure.apply_apt_config({'conf': "foo"}, self.pfile, self.cfile) self.assertTrue(os.path.isfile(self.cfile)) self.assertEqual(load_tfile_or_url(self.cfile), "foo") def test_config_deleted(self): - # if no 'apt_config' is provided, delete any previously written file + # if no 'conf' is provided, delete any previously written file util.write_file(self.pfile, "content doesnt matter") cc_apt_configure.apply_apt_config({}, self.pfile, self.cfile) self.assertFalse(os.path.isfile(self.pfile)) diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py index acde0863..f4411869 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py @@ -79,6 +79,15 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): self.new_root = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.new_root) + rpatcher = mock.patch("cloudinit.util.lsb_release") + get_rel = rpatcher.start() + get_rel.return_value = {'codename': "fakerelease"} + self.addCleanup(rpatcher.stop) + apatcher = mock.patch("cloudinit.util.get_architecture") + get_arch = apatcher.start() + get_arch.return_value = 'amd64' + self.addCleanup(apatcher.stop) + def _get_cloud(self, distro, metadata=None): self.patchUtils(self.new_root) paths = helpers.Paths({}) @@ -102,25 +111,38 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): cfg = {'apt_mirror': mirror} mycloud = self._get_cloud(distro) - with mock.patch.object(templater, 'render_to_file') as mocktmpl: - with mock.patch.object(os.path, 'isfile', - return_value=True) as mockisfile: - with mock.patch.object(util, 'rename'): - cc_apt_configure.handle("notimportant", cfg, mycloud, - LOG, None) + with mock.patch.object(util, 'write_file') as mockwf: + with mock.patch.object(util, 'load_file', + return_value="faketmpl") as mocklf: + with mock.patch.object(os.path, 'isfile', + return_value=True) as mockisfile: + with mock.patch.object(templater, 'render_string', + return_value="fake") as mockrnd: + with mock.patch.object(util, 'rename'): + cc_apt_configure.handle("test", cfg, mycloud, + LOG, None) mockisfile.assert_any_call( ('/etc/cloud/templates/sources.list.%s.tmpl' % distro)) - mocktmpl.assert_called_once_with( - ('/etc/cloud/templates/sources.list.%s.tmpl' % distro), - '/etc/apt/sources.list', - {'codename': '', 'primary': mirrorcheck, 'mirror': mirrorcheck}) - - def test_apt_source_list_debian(self): + mocklf.assert_any_call( + ('/etc/cloud/templates/sources.list.%s.tmpl' % distro)) + mockrnd.assert_called_once_with('faketmpl', + {'RELEASE': 'fakerelease', + 'PRIMARY': mirrorcheck, + 'MIRROR': mirrorcheck, + 'SECURITY': mirrorcheck, + 'codename': 'fakerelease', + 'primary': mirrorcheck, + 'mirror': mirrorcheck, + 'security': mirrorcheck}) + mockwf.assert_called_once_with('/etc/apt/sources.list', 'fake', + mode=0o644) + + def test_apt_v1_source_list_debian(self): """Test rendering of a source.list from template for debian""" self.apt_source_list('debian', 'http://httpredir.debian.org/debian') - def test_apt_source_list_ubuntu(self): + def test_apt_v1_source_list_ubuntu(self): """Test rendering of a source.list from template for ubuntu""" self.apt_source_list('ubuntu', 'http://archive.ubuntu.com/ubuntu/') @@ -134,7 +156,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): print("Faking SUCCESS for '%s'" % name) return True - def test_apt_srcl_debian_mirrorfail(self): + def test_apt_v1_srcl_debian_mirrorfail(self): """Test rendering of a source.list from template for debian""" with mock.patch.object(util, 'is_resolvable', side_effect=self.myresolve) as mockresolve: @@ -145,7 +167,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): mockresolve.assert_any_call("does.not.exist") mockresolve.assert_any_call("httpredir.debian.org") - def test_apt_srcl_ubuntu_mirrorfail(self): + def test_apt_v1_srcl_ubuntu_mirrorfail(self): """Test rendering of a source.list from template for ubuntu""" with mock.patch.object(util, 'is_resolvable', side_effect=self.myresolve) as mockresolve: @@ -156,7 +178,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): mockresolve.assert_any_call("does.not.exist") mockresolve.assert_any_call("archive.ubuntu.com") - def test_apt_srcl_custom(self): + def test_apt_v1_srcl_custom(self): """Test rendering from a custom source.list template""" cfg = util.load_yaml(YAML_TEXT_CUSTOM_SL) mycloud = self._get_cloud('ubuntu') @@ -164,12 +186,10 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): # the second mock restores the original subp with mock.patch.object(util, 'write_file') as mockwrite: with mock.patch.object(util, 'subp', self.subp): - with mock.patch.object(cc_apt_configure, 'get_release', - return_value='fakerelease'): - with mock.patch.object(Distro, 'get_primary_arch', - return_value='amd64'): - cc_apt_configure.handle("notimportant", cfg, mycloud, - LOG, None) + with mock.patch.object(Distro, 'get_primary_arch', + return_value='amd64'): + cc_apt_configure.handle("notimportant", cfg, mycloud, + LOG, None) mockwrite.assert_called_once_with( '/etc/apt/sources.list', diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py new file mode 100644 index 00000000..e53b0450 --- /dev/null +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py @@ -0,0 +1,187 @@ +""" test_apt_custom_sources_list +Test templating of custom sources list +""" +import logging +import os +import shutil +import tempfile + +try: + from unittest import mock +except ImportError: + import mock +from mock import call + +from cloudinit import cloud +from cloudinit import distros +from cloudinit import helpers +from cloudinit import util + +from cloudinit.config import cc_apt_configure +from cloudinit.sources import DataSourceNone + +from cloudinit.distros.debian import Distro + +from .. import helpers as t_help + +LOG = logging.getLogger(__name__) + +TARGET = "/" + +# Input and expected output for the custom template +YAML_TEXT_CUSTOM_SL = """ +apt: + primary: + - arches: [default] + uri: http://test.ubuntu.com/ubuntu/ + security: + - arches: [default] + uri: http://testsec.ubuntu.com/ubuntu/ + sources_list: | + + # Note, this file is written by cloud-init at install time. It should not + # end up on the installed system itself. + # See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to + # newer versions of the distribution. + deb $MIRROR $RELEASE main restricted + deb-src $MIRROR $RELEASE main restricted + deb $PRIMARY $RELEASE universe restricted + deb $SECURITY $RELEASE-security multiverse + # FIND_SOMETHING_SPECIAL +""" + +EXPECTED_CONVERTED_CONTENT = """ +# Note, this file is written by cloud-init at install time. It should not +# end up on the installed system itself. +# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to +# newer versions of the distribution. +deb http://test.ubuntu.com/ubuntu/ fakerel main restricted +deb-src http://test.ubuntu.com/ubuntu/ fakerel main restricted +deb http://test.ubuntu.com/ubuntu/ fakerel universe restricted +deb http://testsec.ubuntu.com/ubuntu/ fakerel-security multiverse +# FIND_SOMETHING_SPECIAL +""" + +# mocked to be independent to the unittest system +MOCKED_APT_SRC_LIST = """ +deb http://test.ubuntu.com/ubuntu/ notouched main restricted +deb-src http://test.ubuntu.com/ubuntu/ notouched main restricted +deb http://test.ubuntu.com/ubuntu/ notouched-updates main restricted +deb http://testsec.ubuntu.com/ubuntu/ notouched-security main restricted +""" + +EXPECTED_BASE_CONTENT = (""" +deb http://test.ubuntu.com/ubuntu/ notouched main restricted +deb-src http://test.ubuntu.com/ubuntu/ notouched main restricted +deb http://test.ubuntu.com/ubuntu/ notouched-updates main restricted +deb http://testsec.ubuntu.com/ubuntu/ notouched-security main restricted +""") + +EXPECTED_MIRROR_CONTENT = (""" +deb http://test.ubuntu.com/ubuntu/ notouched main restricted +deb-src http://test.ubuntu.com/ubuntu/ notouched main restricted +deb http://test.ubuntu.com/ubuntu/ notouched-updates main restricted +deb http://test.ubuntu.com/ubuntu/ notouched-security main restricted +""") + +EXPECTED_PRIMSEC_CONTENT = (""" +deb http://test.ubuntu.com/ubuntu/ notouched main restricted +deb-src http://test.ubuntu.com/ubuntu/ notouched main restricted +deb http://test.ubuntu.com/ubuntu/ notouched-updates main restricted +deb http://testsec.ubuntu.com/ubuntu/ notouched-security main restricted +""") + + +class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): + """TestAptSourceConfigSourceList - Class to test sources list rendering""" + def setUp(self): + super(TestAptSourceConfigSourceList, self).setUp() + self.subp = util.subp + self.new_root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.new_root) + + rpatcher = mock.patch("cloudinit.util.lsb_release") + get_rel = rpatcher.start() + get_rel.return_value = {'codename': "fakerel"} + self.addCleanup(rpatcher.stop) + apatcher = mock.patch("cloudinit.util.get_architecture") + get_arch = apatcher.start() + get_arch.return_value = 'amd64' + self.addCleanup(apatcher.stop) + + def _get_cloud(self, distro, metadata=None): + self.patchUtils(self.new_root) + paths = helpers.Paths({}) + cls = distros.fetch(distro) + mydist = cls(distro, {}, paths) + myds = DataSourceNone.DataSourceNone({}, mydist, paths) + if metadata: + myds.metadata.update(metadata) + return cloud.Cloud(myds, paths, {}, mydist, None) + + def _apt_source_list(self, cfg, expected, distro): + "_apt_source_list - Test rendering from template (generic)" + + # entry at top level now, wrap in 'apt' key + cfg = {'apt': cfg} + mycloud = self._get_cloud(distro) + with mock.patch.object(util, 'write_file') as mockwf: + with mock.patch.object(util, 'load_file', + return_value=MOCKED_APT_SRC_LIST) as mocklf: + with mock.patch.object(os.path, 'isfile', + return_value=True) as mockisfile: + with mock.patch.object(util, 'rename'): + cc_apt_configure.handle("test", cfg, mycloud, + LOG, None) + + # check if it would have loaded the distro template + mockisfile.assert_any_call( + ('/etc/cloud/templates/sources.list.%s.tmpl' % distro)) + mocklf.assert_any_call( + ('/etc/cloud/templates/sources.list.%s.tmpl' % distro)) + # check expected content in result + mockwf.assert_called_once_with('/etc/apt/sources.list', expected, + mode=0o644) + + def test_apt_v3_source_list_debian(self): + """test_apt_v3_source_list_debian - without custom sources or parms""" + cfg = {} + self._apt_source_list(cfg, EXPECTED_BASE_CONTENT, 'debian') + + def test_apt_v3_source_list_ubuntu(self): + """test_apt_v3_source_list_ubuntu - without custom sources or parms""" + cfg = {} + self._apt_source_list(cfg, EXPECTED_BASE_CONTENT, 'ubuntu') + + def test_apt_v3_source_list_psm(self): + """test_apt_v3_source_list_psm - Test specifying prim+sec mirrors""" + pm = 'http://test.ubuntu.com/ubuntu/' + sm = 'http://testsec.ubuntu.com/ubuntu/' + cfg = {'preserve_sources_list': False, + 'primary': [{'arches': ["default"], + 'uri': pm}], + 'security': [{'arches': ["default"], + 'uri': sm}]} + + self._apt_source_list(cfg, EXPECTED_PRIMSEC_CONTENT, 'ubuntu') + + def test_apt_v3_srcl_custom(self): + """test_apt_v3_srcl_custom - Test rendering a custom source template""" + cfg = util.load_yaml(YAML_TEXT_CUSTOM_SL) + mycloud = self._get_cloud('ubuntu') + + # the second mock restores the original subp + with mock.patch.object(util, 'write_file') as mockwrite: + with mock.patch.object(util, 'subp', self.subp): + with mock.patch.object(Distro, 'get_primary_arch', + return_value='amd64'): + cc_apt_configure.handle("notimportant", cfg, mycloud, + LOG, None) + + calls = [call('/etc/apt/sources.list', + EXPECTED_CONVERTED_CONTENT, + mode=0o644)] + mockwrite.assert_has_calls(calls) + + +# vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source_v1.py index 99a4d860..d96779c5 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source_v1.py @@ -1,5 +1,7 @@ -""" test_handler_apt_source +""" test_handler_apt_source_v1 Testing various config variations of the apt_source config +This calls all things with v1 format to stress the conversion code on top of +the actually tested code. """ import os import re @@ -32,6 +34,8 @@ S0ORP6HXET3+jC8BMG4tBWCTK/XEZw== =ACB2 -----END PGP PUBLIC KEY BLOCK-----""" +ADD_APT_REPO_MATCH = r"^[\w-]+:\w" + def load_tfile_or_url(*args, **kwargs): """load_tfile_or_url @@ -40,6 +44,19 @@ def load_tfile_or_url(*args, **kwargs): return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents) +class FakeDistro(object): + """Fake Distro helper object""" + def update_package_sources(self): + """Fake update_package_sources helper method""" + return + + +class FakeCloud(object): + """Fake Cloud helper object""" + def __init__(self): + self.distro = FakeDistro() + + class TestAptSourceConfig(TestCase): """TestAptSourceConfig Main Class to test apt_source configs @@ -54,25 +71,39 @@ class TestAptSourceConfig(TestCase): self.aptlistfile2 = os.path.join(self.tmp, "single-deb2.list") self.aptlistfile3 = os.path.join(self.tmp, "single-deb3.list") self.join = os.path.join + self.matcher = re.compile(ADD_APT_REPO_MATCH).search # mock fallback filename into writable tmp dir self.fallbackfn = os.path.join(self.tmp, "etc/apt/sources.list.d/", "cloud_config_sources.list") - patcher = mock.patch("cloudinit.config.cc_apt_configure.get_release") - get_rel = patcher.start() - get_rel.return_value = self.release - self.addCleanup(patcher.stop) + self.fakecloud = FakeCloud() - @staticmethod - def _get_default_params(): + rpatcher = mock.patch("cloudinit.util.lsb_release") + get_rel = rpatcher.start() + get_rel.return_value = {'codename': self.release} + self.addCleanup(rpatcher.stop) + apatcher = mock.patch("cloudinit.util.get_architecture") + get_arch = apatcher.start() + get_arch.return_value = 'amd64' + self.addCleanup(apatcher.stop) + + def _get_default_params(self): """get_default_params Get the most basic default mrror and release info to be used in tests """ params = {} - params['RELEASE'] = cc_apt_configure.get_release() + params['RELEASE'] = self.release params['MIRROR'] = "http://archive.ubuntu.com/ubuntu" return params + def wrapv1conf(self, cfg): + params = self._get_default_params() + # old v1 list format under old keys, but callabe to main handler + # disable source.list rendering and set mirror to avoid other code + return {'apt_preserve_sources_list': True, + 'apt_mirror': params['MIRROR'], + 'apt_sources': cfg} + def myjoin(self, *args, **kwargs): """myjoin - redir into writable tmpdir""" if (args[0] == "/etc/apt/sources.list.d/" and @@ -86,9 +117,9 @@ class TestAptSourceConfig(TestCase): """apt_src_basic Test Fix deb source string, has to overwrite mirror conf in params """ - params = self._get_default_params() + cfg = self.wrapv1conf(cfg) - cc_apt_configure.add_apt_sources(cfg, params) + cc_apt_configure.handle("test", cfg, self.fakecloud, None, None) self.assertTrue(os.path.isfile(filename)) @@ -181,8 +212,9 @@ class TestAptSourceConfig(TestCase): """apt_src_replace Test Autoreplacement of MIRROR and RELEASE in source specs """ + cfg = self.wrapv1conf(cfg) params = self._get_default_params() - cc_apt_configure.add_apt_sources(cfg, params) + cc_apt_configure.handle("test", cfg, self.fakecloud, None, None) self.assertTrue(os.path.isfile(filename)) @@ -246,16 +278,18 @@ class TestAptSourceConfig(TestCase): """apt_src_keyid Test specification of a source + keyid """ - params = self._get_default_params() + cfg = self.wrapv1conf(cfg) with mock.patch.object(util, 'subp', return_value=('fakekey 1234', '')) as mockobj: - cc_apt_configure.add_apt_sources(cfg, params) + cc_apt_configure.handle("test", cfg, self.fakecloud, None, None) # check if it added the right ammount of keys calls = [] for _ in range(keynum): - calls.append(call(('apt-key', 'add', '-'), 'fakekey 1234')) + calls.append(call(['apt-key', 'add', '-'], + data=b'fakekey 1234', + target=None)) mockobj.assert_has_calls(calls, any_order=True) self.assertTrue(os.path.isfile(filename)) @@ -329,12 +363,13 @@ class TestAptSourceConfig(TestCase): """apt_src_key Test specification of a source + key """ - params = self._get_default_params() + cfg = self.wrapv1conf([cfg]) with mock.patch.object(util, 'subp') as mockobj: - cc_apt_configure.add_apt_sources([cfg], params) + cc_apt_configure.handle("test", cfg, self.fakecloud, None, None) - mockobj.assert_called_with(('apt-key', 'add', '-'), 'fakekey 4321') + mockobj.assert_called_with(['apt-key', 'add', '-'], + data=b'fakekey 4321', target=None) self.assertTrue(os.path.isfile(filename)) @@ -368,30 +403,31 @@ class TestAptSourceConfig(TestCase): def test_apt_src_keyonly(self): """Test specifying key without source""" - params = self._get_default_params() cfg = {'key': "fakekey 4242", 'filename': self.aptlistfile} + cfg = self.wrapv1conf([cfg]) with mock.patch.object(util, 'subp') as mockobj: - cc_apt_configure.add_apt_sources([cfg], params) + cc_apt_configure.handle("test", cfg, self.fakecloud, None, None) - mockobj.assert_called_once_with(('apt-key', 'add', '-'), - 'fakekey 4242') + mockobj.assert_called_once_with(['apt-key', 'add', '-'], + data=b'fakekey 4242', target=None) # filename should be ignored on key only self.assertFalse(os.path.isfile(self.aptlistfile)) def test_apt_src_keyidonly(self): """Test specification of a keyid without source""" - params = self._get_default_params() cfg = {'keyid': "03683F77", 'filename': self.aptlistfile} + cfg = self.wrapv1conf([cfg]) with mock.patch.object(util, 'subp', return_value=('fakekey 1212', '')) as mockobj: - cc_apt_configure.add_apt_sources([cfg], params) + cc_apt_configure.handle("test", cfg, self.fakecloud, None, None) - mockobj.assert_called_with(('apt-key', 'add', '-'), 'fakekey 1212') + mockobj.assert_called_with(['apt-key', 'add', '-'], + data=b'fakekey 1212', target=None) # filename should be ignored on key only self.assertFalse(os.path.isfile(self.aptlistfile)) @@ -402,17 +438,18 @@ class TestAptSourceConfig(TestCase): up to addition of the key (add_apt_key_raw mocked to keep the environment as is) """ - params = self._get_default_params() + key = cfg['keyid'] + keyserver = cfg.get('keyserver', 'keyserver.ubuntu.com') + cfg = self.wrapv1conf([cfg]) with mock.patch.object(cc_apt_configure, 'add_apt_key_raw') as mockkey: - with mock.patch.object(gpg, 'get_key_by_id', + with mock.patch.object(gpg, 'getkeybyid', return_value=expectedkey) as mockgetkey: - cc_apt_configure.add_apt_sources([cfg], params) + cc_apt_configure.handle("test", cfg, self.fakecloud, + None, None) - mockgetkey.assert_called_with(cfg['keyid'], - cfg.get('keyserver', - 'keyserver.ubuntu.com')) - mockkey.assert_called_with(expectedkey) + mockgetkey.assert_called_with(key, keyserver) + mockkey.assert_called_with(expectedkey, None) # filename should be ignored on key only self.assertFalse(os.path.isfile(self.aptlistfile)) @@ -444,41 +481,38 @@ class TestAptSourceConfig(TestCase): def test_apt_src_ppa(self): """Test adding a ppa""" - params = self._get_default_params() cfg = {'source': 'ppa:smoser/cloud-init-test', 'filename': self.aptlistfile} - - # default matcher needed for ppa - matcher = re.compile(r'^[\w-]+:\w').search + cfg = self.wrapv1conf([cfg]) with mock.patch.object(util, 'subp') as mockobj: - cc_apt_configure.add_apt_sources([cfg], params, - aa_repo_match=matcher) + cc_apt_configure.handle("test", cfg, self.fakecloud, None, None) mockobj.assert_called_once_with(['add-apt-repository', - 'ppa:smoser/cloud-init-test']) + 'ppa:smoser/cloud-init-test'], + target=None) # adding ppa should ignore filename (uses add-apt-repository) self.assertFalse(os.path.isfile(self.aptlistfile)) def test_apt_src_ppa_tri(self): """Test adding three ppa's""" - params = self._get_default_params() cfg1 = {'source': 'ppa:smoser/cloud-init-test', 'filename': self.aptlistfile} cfg2 = {'source': 'ppa:smoser/cloud-init-test2', 'filename': self.aptlistfile2} cfg3 = {'source': 'ppa:smoser/cloud-init-test3', 'filename': self.aptlistfile3} - - # default matcher needed for ppa - matcher = re.compile(r'^[\w-]+:\w').search + cfg = self.wrapv1conf([cfg1, cfg2, cfg3]) with mock.patch.object(util, 'subp') as mockobj: - cc_apt_configure.add_apt_sources([cfg1, cfg2, cfg3], params, - aa_repo_match=matcher) - calls = [call(['add-apt-repository', 'ppa:smoser/cloud-init-test']), - call(['add-apt-repository', 'ppa:smoser/cloud-init-test2']), - call(['add-apt-repository', 'ppa:smoser/cloud-init-test3'])] + cc_apt_configure.handle("test", cfg, self.fakecloud, + None, None) + calls = [call(['add-apt-repository', 'ppa:smoser/cloud-init-test'], + target=None), + call(['add-apt-repository', 'ppa:smoser/cloud-init-test2'], + target=None), + call(['add-apt-repository', 'ppa:smoser/cloud-init-test3'], + target=None)] mockobj.assert_has_calls(calls, any_order=True) # adding ppa should ignore all filenames (uses add-apt-repository) @@ -503,14 +537,15 @@ class TestAptSourceConfig(TestCase): 'source': 'deb $MIRROR $RELEASE ' 'universe'}} - newcfg = cc_apt_configure.convert_to_new_format([cfg1, cfg2, cfg3]) + newcfg = cc_apt_configure.convert_v1_to_v2_apt_format([cfg1, cfg2, + cfg3]) self.assertEqual(newcfg, checkcfg) - newcfg2 = cc_apt_configure.convert_to_new_format(newcfg) + newcfg2 = cc_apt_configure.convert_v1_to_v2_apt_format(newcfg) self.assertEqual(newcfg2, checkcfg) with self.assertRaises(ValueError): - cc_apt_configure.convert_to_new_format(5) + cc_apt_configure.convert_v1_to_v2_apt_format(5) # vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_handler_apt_source_v3.py b/tests/unittests/test_handler/test_handler_apt_source_v3.py new file mode 100644 index 00000000..75556b6d --- /dev/null +++ b/tests/unittests/test_handler/test_handler_apt_source_v3.py @@ -0,0 +1,1103 @@ +"""test_handler_apt_source_v3 +Testing various config variations of the apt_source custom config +This tries to call all in the new v3 format and cares about new features +""" +import glob +import os +import re +import shutil +import socket +import tempfile + +from unittest import TestCase + +try: + from unittest import mock +except ImportError: + import mock +from mock import call + +from cloudinit import cloud +from cloudinit import distros +from cloudinit import gpg +from cloudinit import helpers +from cloudinit import util + +from cloudinit.config import cc_apt_configure +from cloudinit.sources import DataSourceNone + +from .. import helpers as t_help + +EXPECTEDKEY = u"""-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1 + +mI0ESuZLUgEEAKkqq3idtFP7g9hzOu1a8+v8ImawQN4TrvlygfScMU1TIS1eC7UQ +NUA8Qqgr9iUaGnejb0VciqftLrU9D6WYHSKz+EITefgdyJ6SoQxjoJdsCpJ7o9Jy +8PQnpRttiFm4qHu6BVnKnBNxw/z3ST9YMqW5kbMQpfxbGe+obRox59NpABEBAAG0 +HUxhdW5jaHBhZCBQUEEgZm9yIFNjb3R0IE1vc2VyiLYEEwECACAFAkrmS1ICGwMG +CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRAGILvPA2g/d3aEA/9tVjc10HOZwV29 +OatVuTeERjjrIbxflO586GLA8cp0C9RQCwgod/R+cKYdQcHjbqVcP0HqxveLg0RZ +FJpWLmWKamwkABErwQLGlM/Hwhjfade8VvEQutH5/0JgKHmzRsoqfR+LMO6OS+Sm +S0ORP6HXET3+jC8BMG4tBWCTK/XEZw== +=ACB2 +-----END PGP PUBLIC KEY BLOCK-----""" + +ADD_APT_REPO_MATCH = r"^[\w-]+:\w" + +TARGET = None + + +def load_tfile(*args, **kwargs): + """load_tfile_or_url + load file and return content after decoding + """ + return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents) + + +class TestAptSourceConfig(t_help.FilesystemMockingTestCase): + """TestAptSourceConfig + Main Class to test apt configs + """ + def setUp(self): + super(TestAptSourceConfig, self).setUp() + self.tmp = tempfile.mkdtemp() + self.new_root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp) + self.aptlistfile = os.path.join(self.tmp, "single-deb.list") + self.aptlistfile2 = os.path.join(self.tmp, "single-deb2.list") + self.aptlistfile3 = os.path.join(self.tmp, "single-deb3.list") + self.join = os.path.join + self.matcher = re.compile(ADD_APT_REPO_MATCH).search + + @staticmethod + def _add_apt_sources(*args, **kwargs): + with mock.patch.object(cc_apt_configure, 'update_packages'): + cc_apt_configure.add_apt_sources(*args, **kwargs) + + @staticmethod + def _get_default_params(): + """get_default_params + Get the most basic default mrror and release info to be used in tests + """ + params = {} + params['RELEASE'] = util.lsb_release()['codename'] + arch = 'amd64' + params['MIRROR'] = cc_apt_configure.\ + get_default_mirrors(arch)["PRIMARY"] + return params + + def _myjoin(self, *args, **kwargs): + """_myjoin - redir into writable tmpdir""" + if (args[0] == "/etc/apt/sources.list.d/" and + args[1] == "cloud_config_sources.list" and + len(args) == 2): + return self.join(self.tmp, args[0].lstrip("/"), args[1]) + else: + return self.join(*args, **kwargs) + + def _get_cloud(self, distro, metadata=None): + self.patchUtils(self.new_root) + paths = helpers.Paths({}) + cls = distros.fetch(distro) + mydist = cls(distro, {}, paths) + myds = DataSourceNone.DataSourceNone({}, mydist, paths) + if metadata: + myds.metadata.update(metadata) + return cloud.Cloud(myds, paths, {}, mydist, None) + + def _apt_src_basic(self, filename, cfg): + """_apt_src_basic + Test Fix deb source string, has to overwrite mirror conf in params + """ + params = self._get_default_params() + + self._add_apt_sources(cfg, TARGET, template_params=params, + aa_repo_match=self.matcher) + + self.assertTrue(os.path.isfile(filename)) + + contents = load_tfile(filename) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", "http://test.ubuntu.com/ubuntu", + "karmic-backports", + "main universe multiverse restricted"), + contents, flags=re.IGNORECASE)) + + def test_apt_v3_src_basic(self): + """test_apt_v3_src_basic - Test fix deb source string""" + cfg = {self.aptlistfile: {'source': + ('deb http://test.ubuntu.com/ubuntu' + ' karmic-backports' + ' main universe multiverse restricted')}} + self._apt_src_basic(self.aptlistfile, cfg) + + def test_apt_v3_src_basic_tri(self): + """test_apt_v3_src_basic_tri - Test multiple fix deb source strings""" + cfg = {self.aptlistfile: {'source': + ('deb http://test.ubuntu.com/ubuntu' + ' karmic-backports' + ' main universe multiverse restricted')}, + self.aptlistfile2: {'source': + ('deb http://test.ubuntu.com/ubuntu' + ' precise-backports' + ' main universe multiverse restricted')}, + self.aptlistfile3: {'source': + ('deb http://test.ubuntu.com/ubuntu' + ' lucid-backports' + ' main universe multiverse restricted')}} + self._apt_src_basic(self.aptlistfile, cfg) + + # extra verify on two extra files of this test + contents = load_tfile(self.aptlistfile2) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", "http://test.ubuntu.com/ubuntu", + "precise-backports", + "main universe multiverse restricted"), + contents, flags=re.IGNORECASE)) + contents = load_tfile(self.aptlistfile3) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", "http://test.ubuntu.com/ubuntu", + "lucid-backports", + "main universe multiverse restricted"), + contents, flags=re.IGNORECASE)) + + def _apt_src_replacement(self, filename, cfg): + """apt_src_replace + Test Autoreplacement of MIRROR and RELEASE in source specs + """ + params = self._get_default_params() + self._add_apt_sources(cfg, TARGET, template_params=params, + aa_repo_match=self.matcher) + + self.assertTrue(os.path.isfile(filename)) + + contents = load_tfile(filename) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", params['MIRROR'], params['RELEASE'], + "multiverse"), + contents, flags=re.IGNORECASE)) + + def test_apt_v3_src_replace(self): + """test_apt_v3_src_replace - Test replacement of MIRROR & RELEASE""" + cfg = {self.aptlistfile: {'source': 'deb $MIRROR $RELEASE multiverse'}} + self._apt_src_replacement(self.aptlistfile, cfg) + + def test_apt_v3_src_replace_fn(self): + """test_apt_v3_src_replace_fn - Test filename overwritten in dict""" + cfg = {'ignored': {'source': 'deb $MIRROR $RELEASE multiverse', + 'filename': self.aptlistfile}} + # second file should overwrite the dict key + self._apt_src_replacement(self.aptlistfile, cfg) + + def _apt_src_replace_tri(self, cfg): + """_apt_src_replace_tri + Test three autoreplacements of MIRROR and RELEASE in source specs with + generic part + """ + self._apt_src_replacement(self.aptlistfile, cfg) + + # extra verify on two extra files of this test + params = self._get_default_params() + contents = load_tfile(self.aptlistfile2) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", params['MIRROR'], params['RELEASE'], + "main"), + contents, flags=re.IGNORECASE)) + contents = load_tfile(self.aptlistfile3) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", params['MIRROR'], params['RELEASE'], + "universe"), + contents, flags=re.IGNORECASE)) + + def test_apt_v3_src_replace_tri(self): + """test_apt_v3_src_replace_tri - Test multiple replace/overwrites""" + cfg = {self.aptlistfile: {'source': 'deb $MIRROR $RELEASE multiverse'}, + 'notused': {'source': 'deb $MIRROR $RELEASE main', + 'filename': self.aptlistfile2}, + self.aptlistfile3: {'source': 'deb $MIRROR $RELEASE universe'}} + self._apt_src_replace_tri(cfg) + + def _apt_src_keyid(self, filename, cfg, keynum): + """_apt_src_keyid + Test specification of a source + keyid + """ + params = self._get_default_params() + + with mock.patch("cloudinit.util.subp", + return_value=('fakekey 1234', '')) as mockobj: + self._add_apt_sources(cfg, TARGET, template_params=params, + aa_repo_match=self.matcher) + + # check if it added the right ammount of keys + calls = [] + for _ in range(keynum): + calls.append(call(['apt-key', 'add', '-'], data=b'fakekey 1234', + target=TARGET)) + mockobj.assert_has_calls(calls, any_order=True) + + self.assertTrue(os.path.isfile(filename)) + + contents = load_tfile(filename) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", + ('http://ppa.launchpad.net/smoser/' + 'cloud-init-test/ubuntu'), + "xenial", "main"), + contents, flags=re.IGNORECASE)) + + def test_apt_v3_src_keyid(self): + """test_apt_v3_src_keyid - Test source + keyid with filename""" + cfg = {self.aptlistfile: {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial main'), + 'keyid': "03683F77"}} + self._apt_src_keyid(self.aptlistfile, cfg, 1) + + def test_apt_v3_src_keyid_tri(self): + """test_apt_v3_src_keyid_tri - Test multiple src+key+filen writes""" + cfg = {self.aptlistfile: {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial main'), + 'keyid': "03683F77"}, + 'ignored': {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial universe'), + 'keyid': "03683F77", + 'filename': self.aptlistfile2}, + self.aptlistfile3: {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial multiverse'), + 'keyid': "03683F77"}} + + self._apt_src_keyid(self.aptlistfile, cfg, 3) + contents = load_tfile(self.aptlistfile2) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", + ('http://ppa.launchpad.net/smoser/' + 'cloud-init-test/ubuntu'), + "xenial", "universe"), + contents, flags=re.IGNORECASE)) + contents = load_tfile(self.aptlistfile3) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", + ('http://ppa.launchpad.net/smoser/' + 'cloud-init-test/ubuntu'), + "xenial", "multiverse"), + contents, flags=re.IGNORECASE)) + + def test_apt_v3_src_key(self): + """test_apt_v3_src_key - Test source + key""" + params = self._get_default_params() + cfg = {self.aptlistfile: {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial main'), + 'key': "fakekey 4321"}} + + with mock.patch.object(util, 'subp') as mockobj: + self._add_apt_sources(cfg, TARGET, template_params=params, + aa_repo_match=self.matcher) + + mockobj.assert_any_call(['apt-key', 'add', '-'], data=b'fakekey 4321', + target=TARGET) + + self.assertTrue(os.path.isfile(self.aptlistfile)) + + contents = load_tfile(self.aptlistfile) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", + ('http://ppa.launchpad.net/smoser/' + 'cloud-init-test/ubuntu'), + "xenial", "main"), + contents, flags=re.IGNORECASE)) + + def test_apt_v3_src_keyonly(self): + """test_apt_v3_src_keyonly - Test key without source""" + params = self._get_default_params() + cfg = {self.aptlistfile: {'key': "fakekey 4242"}} + + with mock.patch.object(util, 'subp') as mockobj: + self._add_apt_sources(cfg, TARGET, template_params=params, + aa_repo_match=self.matcher) + + mockobj.assert_any_call(['apt-key', 'add', '-'], data=b'fakekey 4242', + target=TARGET) + + # filename should be ignored on key only + self.assertFalse(os.path.isfile(self.aptlistfile)) + + def test_apt_v3_src_keyidonly(self): + """test_apt_v3_src_keyidonly - Test keyid without source""" + params = self._get_default_params() + cfg = {self.aptlistfile: {'keyid': "03683F77"}} + + with mock.patch.object(util, 'subp', + return_value=('fakekey 1212', '')) as mockobj: + self._add_apt_sources(cfg, TARGET, template_params=params, + aa_repo_match=self.matcher) + + mockobj.assert_any_call(['apt-key', 'add', '-'], data=b'fakekey 1212', + target=TARGET) + + # filename should be ignored on key only + self.assertFalse(os.path.isfile(self.aptlistfile)) + + def apt_src_keyid_real(self, cfg, expectedkey): + """apt_src_keyid_real + Test specification of a keyid without source including + up to addition of the key (add_apt_key_raw mocked to keep the + environment as is) + """ + params = self._get_default_params() + + with mock.patch.object(cc_apt_configure, 'add_apt_key_raw') as mockkey: + with mock.patch.object(gpg, 'getkeybyid', + return_value=expectedkey) as mockgetkey: + self._add_apt_sources(cfg, TARGET, template_params=params, + aa_repo_match=self.matcher) + + keycfg = cfg[self.aptlistfile] + mockgetkey.assert_called_with(keycfg['keyid'], + keycfg.get('keyserver', + 'keyserver.ubuntu.com')) + mockkey.assert_called_with(expectedkey, TARGET) + + # filename should be ignored on key only + self.assertFalse(os.path.isfile(self.aptlistfile)) + + def test_apt_v3_src_keyid_real(self): + """test_apt_v3_src_keyid_real - Test keyid including key add""" + keyid = "03683F77" + cfg = {self.aptlistfile: {'keyid': keyid}} + + self.apt_src_keyid_real(cfg, EXPECTEDKEY) + + def test_apt_v3_src_longkeyid_real(self): + """test_apt_v3_src_longkeyid_real Test long keyid including key add""" + keyid = "B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77" + cfg = {self.aptlistfile: {'keyid': keyid}} + + self.apt_src_keyid_real(cfg, EXPECTEDKEY) + + def test_apt_v3_src_longkeyid_ks_real(self): + """test_apt_v3_src_longkeyid_ks_real Test long keyid from other ks""" + keyid = "B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77" + cfg = {self.aptlistfile: {'keyid': keyid, + 'keyserver': 'keys.gnupg.net'}} + + self.apt_src_keyid_real(cfg, EXPECTEDKEY) + + def test_apt_v3_src_keyid_keyserver(self): + """test_apt_v3_src_keyid_keyserver - Test custom keyserver""" + keyid = "03683F77" + params = self._get_default_params() + cfg = {self.aptlistfile: {'keyid': keyid, + 'keyserver': 'test.random.com'}} + + # in some test environments only *.ubuntu.com is reachable + # so mock the call and check if the config got there + with mock.patch.object(gpg, 'getkeybyid', + return_value="fakekey") as mockgetkey: + with mock.patch.object(cc_apt_configure, + 'add_apt_key_raw') as mockadd: + self._add_apt_sources(cfg, TARGET, template_params=params, + aa_repo_match=self.matcher) + + mockgetkey.assert_called_with('03683F77', 'test.random.com') + mockadd.assert_called_with('fakekey', TARGET) + + # filename should be ignored on key only + self.assertFalse(os.path.isfile(self.aptlistfile)) + + def test_apt_v3_src_ppa(self): + """test_apt_v3_src_ppa - Test specification of a ppa""" + params = self._get_default_params() + cfg = {self.aptlistfile: {'source': 'ppa:smoser/cloud-init-test'}} + + with mock.patch("cloudinit.util.subp") as mockobj: + self._add_apt_sources(cfg, TARGET, template_params=params, + aa_repo_match=self.matcher) + mockobj.assert_any_call(['add-apt-repository', + 'ppa:smoser/cloud-init-test'], target=TARGET) + + # adding ppa should ignore filename (uses add-apt-repository) + self.assertFalse(os.path.isfile(self.aptlistfile)) + + def test_apt_v3_src_ppa_tri(self): + """test_apt_v3_src_ppa_tri - Test specification of multiple ppa's""" + params = self._get_default_params() + cfg = {self.aptlistfile: {'source': 'ppa:smoser/cloud-init-test'}, + self.aptlistfile2: {'source': 'ppa:smoser/cloud-init-test2'}, + self.aptlistfile3: {'source': 'ppa:smoser/cloud-init-test3'}} + + with mock.patch("cloudinit.util.subp") as mockobj: + self._add_apt_sources(cfg, TARGET, template_params=params, + aa_repo_match=self.matcher) + calls = [call(['add-apt-repository', 'ppa:smoser/cloud-init-test'], + target=TARGET), + call(['add-apt-repository', 'ppa:smoser/cloud-init-test2'], + target=TARGET), + call(['add-apt-repository', 'ppa:smoser/cloud-init-test3'], + target=TARGET)] + mockobj.assert_has_calls(calls, any_order=True) + + # adding ppa should ignore all filenames (uses add-apt-repository) + self.assertFalse(os.path.isfile(self.aptlistfile)) + self.assertFalse(os.path.isfile(self.aptlistfile2)) + self.assertFalse(os.path.isfile(self.aptlistfile3)) + + @mock.patch("cloudinit.config.cc_apt_configure.util.get_architecture") + def test_apt_v3_list_rename(self, m_get_architecture): + """test_apt_v3_list_rename - Test find mirror and apt list renaming""" + pre = "/var/lib/apt/lists" + # filenames are archive dependent + + arch = 's390x' + m_get_architecture.return_value = arch + component = "ubuntu-ports" + archive = "ports.ubuntu.com" + + cfg = {'primary': [{'arches': ["default"], + 'uri': + 'http://test.ubuntu.com/%s/' % component}], + 'security': [{'arches': ["default"], + 'uri': + 'http://testsec.ubuntu.com/%s/' % component}]} + post = ("%s_dists_%s-updates_InRelease" % + (component, util.lsb_release()['codename'])) + fromfn = ("%s/%s_%s" % (pre, archive, post)) + tofn = ("%s/test.ubuntu.com_%s" % (pre, post)) + + mirrors = cc_apt_configure.find_apt_mirror_info(cfg, None, arch) + + self.assertEqual(mirrors['MIRROR'], + "http://test.ubuntu.com/%s/" % component) + self.assertEqual(mirrors['PRIMARY'], + "http://test.ubuntu.com/%s/" % component) + self.assertEqual(mirrors['SECURITY'], + "http://testsec.ubuntu.com/%s/" % component) + + with mock.patch.object(os, 'rename') as mockren: + with mock.patch.object(glob, 'glob', + return_value=[fromfn]): + cc_apt_configure.rename_apt_lists(mirrors, TARGET) + + mockren.assert_any_call(fromfn, tofn) + + @mock.patch("cloudinit.config.cc_apt_configure.util.get_architecture") + def test_apt_v3_list_rename_non_slash(self, m_get_architecture): + target = os.path.join(self.tmp, "rename_non_slash") + apt_lists_d = os.path.join(target, "./" + cc_apt_configure.APT_LISTS) + + m_get_architecture.return_value = 'amd64' + + mirror_path = "some/random/path/" + primary = "http://test.ubuntu.com/" + mirror_path + security = "http://test-security.ubuntu.com/" + mirror_path + mirrors = {'PRIMARY': primary, 'SECURITY': security} + + # these match default archive prefixes + opri_pre = "archive.ubuntu.com_ubuntu_dists_xenial" + osec_pre = "security.ubuntu.com_ubuntu_dists_xenial" + # this one won't match and should not be renamed defaults. + other_pre = "dl.google.com_linux_chrome_deb_dists_stable" + # these are our new expected prefixes + npri_pre = "test.ubuntu.com_some_random_path_dists_xenial" + nsec_pre = "test-security.ubuntu.com_some_random_path_dists_xenial" + + files = [ + # orig prefix, new prefix, suffix + (opri_pre, npri_pre, "_main_binary-amd64_Packages"), + (opri_pre, npri_pre, "_main_binary-amd64_InRelease"), + (opri_pre, npri_pre, "-updates_main_binary-amd64_Packages"), + (opri_pre, npri_pre, "-updates_main_binary-amd64_InRelease"), + (other_pre, other_pre, "_main_binary-amd64_Packages"), + (other_pre, other_pre, "_Release"), + (other_pre, other_pre, "_Release.gpg"), + (osec_pre, nsec_pre, "_InRelease"), + (osec_pre, nsec_pre, "_main_binary-amd64_Packages"), + (osec_pre, nsec_pre, "_universe_binary-amd64_Packages"), + ] + + expected = sorted([npre + suff for opre, npre, suff in files]) + # create files + for (opre, npre, suff) in files: + fpath = os.path.join(apt_lists_d, opre + suff) + util.write_file(fpath, content=fpath) + + cc_apt_configure.rename_apt_lists(mirrors, target) + found = sorted(os.listdir(apt_lists_d)) + self.assertEqual(expected, found) + + @staticmethod + def test_apt_v3_proxy(): + """test_apt_v3_proxy - Test apt_*proxy configuration""" + cfg = {"proxy": "foobar1", + "http_proxy": "foobar2", + "ftp_proxy": "foobar3", + "https_proxy": "foobar4"} + + with mock.patch.object(util, 'write_file') as mockobj: + cc_apt_configure.apply_apt_config(cfg, "proxyfn", "notused") + + mockobj.assert_called_with('proxyfn', + ('Acquire::http::Proxy "foobar1";\n' + 'Acquire::http::Proxy "foobar2";\n' + 'Acquire::ftp::Proxy "foobar3";\n' + 'Acquire::https::Proxy "foobar4";\n')) + + def test_apt_v3_mirror(self): + """test_apt_v3_mirror - Test defining a mirror""" + pmir = "http://us.archive.ubuntu.com/ubuntu/" + smir = "http://security.ubuntu.com/ubuntu/" + cfg = {"primary": [{'arches': ["default"], + "uri": pmir}], + "security": [{'arches': ["default"], + "uri": smir}]} + + mirrors = cc_apt_configure.find_apt_mirror_info(cfg, None, 'amd64') + + self.assertEqual(mirrors['MIRROR'], + pmir) + self.assertEqual(mirrors['PRIMARY'], + pmir) + self.assertEqual(mirrors['SECURITY'], + smir) + + def test_apt_v3_mirror_default(self): + """test_apt_v3_mirror_default - Test without defining a mirror""" + arch = 'amd64' + default_mirrors = cc_apt_configure.get_default_mirrors(arch) + pmir = default_mirrors["PRIMARY"] + smir = default_mirrors["SECURITY"] + mycloud = self._get_cloud('ubuntu') + mirrors = cc_apt_configure.find_apt_mirror_info({}, mycloud, arch) + + self.assertEqual(mirrors['MIRROR'], + pmir) + self.assertEqual(mirrors['PRIMARY'], + pmir) + self.assertEqual(mirrors['SECURITY'], + smir) + + def test_apt_v3_mirror_arches(self): + """test_apt_v3_mirror_arches - Test arches selection of mirror""" + pmir = "http://my-primary.ubuntu.com/ubuntu/" + smir = "http://my-security.ubuntu.com/ubuntu/" + arch = 'ppc64el' + cfg = {"primary": [{'arches': ["default"], "uri": "notthis-primary"}, + {'arches': [arch], "uri": pmir}], + "security": [{'arches': ["default"], "uri": "nothis-security"}, + {'arches': [arch], "uri": smir}]} + + mirrors = cc_apt_configure.find_apt_mirror_info(cfg, None, arch) + + self.assertEqual(mirrors['PRIMARY'], pmir) + self.assertEqual(mirrors['MIRROR'], pmir) + self.assertEqual(mirrors['SECURITY'], smir) + + def test_apt_v3_mirror_arches_default(self): + """test_apt_v3_mirror_arches - Test falling back to default arch""" + pmir = "http://us.archive.ubuntu.com/ubuntu/" + smir = "http://security.ubuntu.com/ubuntu/" + cfg = {"primary": [{'arches': ["default"], + "uri": pmir}, + {'arches': ["thisarchdoesntexist"], + "uri": "notthis"}], + "security": [{'arches': ["thisarchdoesntexist"], + "uri": "nothat"}, + {'arches': ["default"], + "uri": smir}]} + + mirrors = cc_apt_configure.find_apt_mirror_info(cfg, None, 'amd64') + + self.assertEqual(mirrors['MIRROR'], + pmir) + self.assertEqual(mirrors['PRIMARY'], + pmir) + self.assertEqual(mirrors['SECURITY'], + smir) + + @mock.patch("cloudinit.config.cc_apt_configure.util.get_architecture") + def test_apt_v3_get_def_mir_non_intel_no_arch(self, m_get_architecture): + arch = 'ppc64el' + m_get_architecture.return_value = arch + expected = {'PRIMARY': 'http://ports.ubuntu.com/ubuntu-ports', + 'SECURITY': 'http://ports.ubuntu.com/ubuntu-ports'} + self.assertEqual(expected, cc_apt_configure.get_default_mirrors()) + + def test_apt_v3_get_default_mirrors_non_intel_with_arch(self): + found = cc_apt_configure.get_default_mirrors('ppc64el') + + expected = {'PRIMARY': 'http://ports.ubuntu.com/ubuntu-ports', + 'SECURITY': 'http://ports.ubuntu.com/ubuntu-ports'} + self.assertEqual(expected, found) + + def test_apt_v3_mirror_arches_sysdefault(self): + """test_apt_v3_mirror_arches - Test arches fallback to sys default""" + arch = 'amd64' + default_mirrors = cc_apt_configure.get_default_mirrors(arch) + pmir = default_mirrors["PRIMARY"] + smir = default_mirrors["SECURITY"] + mycloud = self._get_cloud('ubuntu') + cfg = {"primary": [{'arches': ["thisarchdoesntexist_64"], + "uri": "notthis"}, + {'arches': ["thisarchdoesntexist"], + "uri": "notthiseither"}], + "security": [{'arches': ["thisarchdoesntexist"], + "uri": "nothat"}, + {'arches': ["thisarchdoesntexist_64"], + "uri": "nothateither"}]} + + mirrors = cc_apt_configure.find_apt_mirror_info(cfg, mycloud, arch) + + self.assertEqual(mirrors['MIRROR'], pmir) + self.assertEqual(mirrors['PRIMARY'], pmir) + self.assertEqual(mirrors['SECURITY'], smir) + + def test_apt_v3_mirror_search(self): + """test_apt_v3_mirror_search - Test searching mirrors in a list + mock checks to avoid relying on network connectivity""" + pmir = "http://us.archive.ubuntu.com/ubuntu/" + smir = "http://security.ubuntu.com/ubuntu/" + cfg = {"primary": [{'arches': ["default"], + "search": ["pfailme", pmir]}], + "security": [{'arches': ["default"], + "search": ["sfailme", smir]}]} + + with mock.patch.object(cc_apt_configure, 'search_for_mirror', + side_effect=[pmir, smir]) as mocksearch: + mirrors = cc_apt_configure.find_apt_mirror_info(cfg, None, + 'amd64') + + calls = [call(["pfailme", pmir]), + call(["sfailme", smir])] + mocksearch.assert_has_calls(calls) + + self.assertEqual(mirrors['MIRROR'], + pmir) + self.assertEqual(mirrors['PRIMARY'], + pmir) + self.assertEqual(mirrors['SECURITY'], + smir) + + def test_apt_v3_mirror_search_many2(self): + """test_apt_v3_mirror_search_many3 - Test both mirrors specs at once""" + pmir = "http://us.archive.ubuntu.com/ubuntu/" + smir = "http://security.ubuntu.com/ubuntu/" + cfg = {"primary": [{'arches': ["default"], + "uri": pmir, + "search": ["pfailme", "foo"]}], + "security": [{'arches': ["default"], + "uri": smir, + "search": ["sfailme", "bar"]}]} + + arch = 'amd64' + + # should be called only once per type, despite two mirror configs + mycloud = None + with mock.patch.object(cc_apt_configure, 'get_mirror', + return_value="http://mocked/foo") as mockgm: + mirrors = cc_apt_configure.find_apt_mirror_info(cfg, mycloud, arch) + calls = [call(cfg, 'primary', arch, mycloud), + call(cfg, 'security', arch, mycloud)] + mockgm.assert_has_calls(calls) + + # should not be called, since primary is specified + with mock.patch.object(cc_apt_configure, + 'search_for_mirror') as mockse: + mirrors = cc_apt_configure.find_apt_mirror_info(cfg, None, arch) + mockse.assert_not_called() + + self.assertEqual(mirrors['MIRROR'], + pmir) + self.assertEqual(mirrors['PRIMARY'], + pmir) + self.assertEqual(mirrors['SECURITY'], + smir) + + def test_apt_v3_url_resolvable(self): + """test_apt_v3_url_resolvable - Test resolving urls""" + + with mock.patch.object(util, 'is_resolvable') as mockresolve: + util.is_resolvable_url("http://1.2.3.4/ubuntu") + mockresolve.assert_called_with("1.2.3.4") + + with mock.patch.object(util, 'is_resolvable') as mockresolve: + util.is_resolvable_url("http://us.archive.ubuntu.com/ubuntu") + mockresolve.assert_called_with("us.archive.ubuntu.com") + + # former tests can leave this set (or not if the test is ran directly) + # do a hard reset to ensure a stable result + util._DNS_REDIRECT_IP = None + bad = [(None, None, None, "badname", ["10.3.2.1"])] + good = [(None, None, None, "goodname", ["10.2.3.4"])] + with mock.patch.object(socket, 'getaddrinfo', + side_effect=[bad, bad, bad, good, + good]) as mocksock: + ret = util.is_resolvable_url("http://us.archive.ubuntu.com/ubuntu") + ret2 = util.is_resolvable_url("http://1.2.3.4/ubuntu") + mocksock.assert_any_call('does-not-exist.example.com.', None, + 0, 0, 1, 2) + mocksock.assert_any_call('example.invalid.', None, 0, 0, 1, 2) + mocksock.assert_any_call('us.archive.ubuntu.com', None) + mocksock.assert_any_call('1.2.3.4', None) + + self.assertTrue(ret) + self.assertTrue(ret2) + + # side effect need only bad ret after initial call + with mock.patch.object(socket, 'getaddrinfo', + side_effect=[bad]) as mocksock: + ret3 = util.is_resolvable_url("http://failme.com/ubuntu") + calls = [call('failme.com', None)] + mocksock.assert_has_calls(calls) + self.assertFalse(ret3) + + def test_apt_v3_disable_suites(self): + """test_disable_suites - disable_suites with many configurations""" + release = "xenial" + orig = """deb http://ubuntu.com//ubuntu xenial main +deb http://ubuntu.com//ubuntu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +deb http://ubuntu.com/ubuntu/ xenial-proposed main""" + + # disable nothing + disabled = [] + expect = """deb http://ubuntu.com//ubuntu xenial main +deb http://ubuntu.com//ubuntu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +deb http://ubuntu.com/ubuntu/ xenial-proposed main""" + result = cc_apt_configure.disable_suites(disabled, orig, release) + self.assertEqual(expect, result) + + # single disable release suite + disabled = ["$RELEASE"] + expect = """\ +# suite disabled by cloud-init: deb http://ubuntu.com//ubuntu xenial main +deb http://ubuntu.com//ubuntu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +deb http://ubuntu.com/ubuntu/ xenial-proposed main""" + result = cc_apt_configure.disable_suites(disabled, orig, release) + self.assertEqual(expect, result) + + # single disable other suite + disabled = ["$RELEASE-updates"] + expect = ("""deb http://ubuntu.com//ubuntu xenial main +# suite disabled by cloud-init: deb http://ubuntu.com//ubuntu""" + """ xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +deb http://ubuntu.com/ubuntu/ xenial-proposed main""") + result = cc_apt_configure.disable_suites(disabled, orig, release) + self.assertEqual(expect, result) + + # multi disable + disabled = ["$RELEASE-updates", "$RELEASE-security"] + expect = ("""deb http://ubuntu.com//ubuntu xenial main +# suite disabled by cloud-init: deb http://ubuntu.com//ubuntu """ + """xenial-updates main +# suite disabled by cloud-init: deb http://ubuntu.com//ubuntu """ + """xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +deb http://ubuntu.com/ubuntu/ xenial-proposed main""") + result = cc_apt_configure.disable_suites(disabled, orig, release) + self.assertEqual(expect, result) + + # multi line disable (same suite multiple times in input) + disabled = ["$RELEASE-updates", "$RELEASE-security"] + orig = """deb http://ubuntu.com//ubuntu xenial main +deb http://ubuntu.com//ubuntu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +deb http://UBUNTU.com//ubuntu xenial-updates main +deb http://UBUNTU.COM//ubuntu xenial-updates main +deb http://ubuntu.com/ubuntu/ xenial-proposed main""" + expect = ("""deb http://ubuntu.com//ubuntu xenial main +# suite disabled by cloud-init: deb http://ubuntu.com//ubuntu """ + """xenial-updates main +# suite disabled by cloud-init: deb http://ubuntu.com//ubuntu """ + """xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +# suite disabled by cloud-init: deb http://UBUNTU.com//ubuntu """ + """xenial-updates main +# suite disabled by cloud-init: deb http://UBUNTU.COM//ubuntu """ + """xenial-updates main +deb http://ubuntu.com/ubuntu/ xenial-proposed main""") + result = cc_apt_configure.disable_suites(disabled, orig, release) + self.assertEqual(expect, result) + + # comment in input + disabled = ["$RELEASE-updates", "$RELEASE-security"] + orig = """deb http://ubuntu.com//ubuntu xenial main +deb http://ubuntu.com//ubuntu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +#foo +#deb http://UBUNTU.com//ubuntu xenial-updates main +deb http://UBUNTU.COM//ubuntu xenial-updates main +deb http://ubuntu.com/ubuntu/ xenial-proposed main""" + expect = ("""deb http://ubuntu.com//ubuntu xenial main +# suite disabled by cloud-init: deb http://ubuntu.com//ubuntu """ + """xenial-updates main +# suite disabled by cloud-init: deb http://ubuntu.com//ubuntu """ + """xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +#foo +#deb http://UBUNTU.com//ubuntu xenial-updates main +# suite disabled by cloud-init: deb http://UBUNTU.COM//ubuntu """ + """xenial-updates main +deb http://ubuntu.com/ubuntu/ xenial-proposed main""") + result = cc_apt_configure.disable_suites(disabled, orig, release) + self.assertEqual(expect, result) + + # single disable custom suite + disabled = ["foobar"] + orig = """deb http://ubuntu.com//ubuntu xenial main +deb http://ubuntu.com//ubuntu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb http://ubuntu.com/ubuntu/ foobar main""" + expect = """deb http://ubuntu.com//ubuntu xenial main +deb http://ubuntu.com//ubuntu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +# suite disabled by cloud-init: deb http://ubuntu.com/ubuntu/ foobar main""" + result = cc_apt_configure.disable_suites(disabled, orig, release) + self.assertEqual(expect, result) + + # single disable non existing suite + disabled = ["foobar"] + orig = """deb http://ubuntu.com//ubuntu xenial main +deb http://ubuntu.com//ubuntu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb http://ubuntu.com/ubuntu/ notfoobar main""" + expect = """deb http://ubuntu.com//ubuntu xenial main +deb http://ubuntu.com//ubuntu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb http://ubuntu.com/ubuntu/ notfoobar main""" + result = cc_apt_configure.disable_suites(disabled, orig, release) + self.assertEqual(expect, result) + + # single disable suite with option + disabled = ["$RELEASE-updates"] + orig = """deb http://ubuntu.com//ubuntu xenial main +deb [a=b] http://ubu.com//ubu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +deb http://ubuntu.com/ubuntu/ xenial-proposed main""" + expect = ("""deb http://ubuntu.com//ubuntu xenial main +# suite disabled by cloud-init: deb [a=b] http://ubu.com//ubu """ + """xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +deb http://ubuntu.com/ubuntu/ xenial-proposed main""") + result = cc_apt_configure.disable_suites(disabled, orig, release) + self.assertEqual(expect, result) + + # single disable suite with more options and auto $RELEASE expansion + disabled = ["updates"] + orig = """deb http://ubuntu.com//ubuntu xenial main +deb [a=b c=d] http://ubu.com//ubu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +deb http://ubuntu.com/ubuntu/ xenial-proposed main""" + expect = """deb http://ubuntu.com//ubuntu xenial main +# suite disabled by cloud-init: deb [a=b c=d] \ +http://ubu.com//ubu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +deb http://ubuntu.com/ubuntu/ xenial-proposed main""" + result = cc_apt_configure.disable_suites(disabled, orig, release) + self.assertEqual(expect, result) + + # single disable suite while options at others + disabled = ["$RELEASE-security"] + orig = """deb http://ubuntu.com//ubuntu xenial main +deb [arch=foo] http://ubuntu.com//ubuntu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +deb http://ubuntu.com/ubuntu/ xenial-proposed main""" + expect = ("""deb http://ubuntu.com//ubuntu xenial main +deb [arch=foo] http://ubuntu.com//ubuntu xenial-updates main +# suite disabled by cloud-init: deb http://ubuntu.com//ubuntu """ + """xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +deb http://ubuntu.com/ubuntu/ xenial-proposed main""") + result = cc_apt_configure.disable_suites(disabled, orig, release) + self.assertEqual(expect, result) + + def test_disable_suites_blank_lines(self): + """test_disable_suites_blank_lines - ensure blank lines allowed""" + lines = ["deb %(repo)s %(rel)s main universe", + "", + "deb %(repo)s %(rel)s-updates main universe", + " # random comment", + "#comment here", + ""] + rel = "trusty" + repo = 'http://example.com/mirrors/ubuntu' + orig = "\n".join(lines) % {'repo': repo, 'rel': rel} + self.assertEqual( + orig, cc_apt_configure.disable_suites(["proposed"], orig, rel)) + + def test_apt_v3_mirror_search_dns(self): + """test_apt_v3_mirror_search_dns - Test searching dns patterns""" + pmir = "phit" + smir = "shit" + arch = 'amd64' + mycloud = self._get_cloud('ubuntu') + cfg = {"primary": [{'arches': ["default"], + "search_dns": True}], + "security": [{'arches': ["default"], + "search_dns": True}]} + + with mock.patch.object(cc_apt_configure, 'get_mirror', + return_value="http://mocked/foo") as mockgm: + mirrors = cc_apt_configure.find_apt_mirror_info(cfg, mycloud, arch) + calls = [call(cfg, 'primary', arch, mycloud), + call(cfg, 'security', arch, mycloud)] + mockgm.assert_has_calls(calls) + + with mock.patch.object(cc_apt_configure, 'search_for_mirror_dns', + return_value="http://mocked/foo") as mocksdns: + mirrors = cc_apt_configure.find_apt_mirror_info(cfg, mycloud, arch) + calls = [call(True, 'primary', cfg, mycloud), + call(True, 'security', cfg, mycloud)] + mocksdns.assert_has_calls(calls) + + # first return is for the non-dns call before + with mock.patch.object(cc_apt_configure, 'search_for_mirror', + side_effect=[None, pmir, None, smir]) as mockse: + mirrors = cc_apt_configure.find_apt_mirror_info(cfg, mycloud, arch) + + calls = [call(None), + call(['http://ubuntu-mirror.localdomain/ubuntu', + 'http://ubuntu-mirror/ubuntu']), + call(None), + call(['http://ubuntu-security-mirror.localdomain/ubuntu', + 'http://ubuntu-security-mirror/ubuntu'])] + mockse.assert_has_calls(calls) + + self.assertEqual(mirrors['MIRROR'], + pmir) + self.assertEqual(mirrors['PRIMARY'], + pmir) + self.assertEqual(mirrors['SECURITY'], + smir) + + +class TestDebconfSelections(TestCase): + + @mock.patch("cloudinit.config.cc_apt_configure.debconf_set_selections") + def test_no_set_sel_if_none_to_set(self, m_set_sel): + cc_apt_configure.apply_debconf_selections({'foo': 'bar'}) + m_set_sel.assert_not_called() + + @mock.patch("cloudinit.config.cc_apt_configure." + "debconf_set_selections") + @mock.patch("cloudinit.config.cc_apt_configure." + "util.get_installed_packages") + def test_set_sel_call_has_expected_input(self, m_get_inst, m_set_sel): + data = { + 'set1': 'pkga pkga/q1 mybool false', + 'set2': ('pkgb\tpkgb/b1\tstr\tthis is a string\n' + 'pkgc\tpkgc/ip\tstring\t10.0.0.1')} + lines = '\n'.join(data.values()).split('\n') + + m_get_inst.return_value = ["adduser", "apparmor"] + m_set_sel.return_value = None + + cc_apt_configure.apply_debconf_selections({'debconf_selections': data}) + self.assertTrue(m_get_inst.called) + self.assertEqual(m_set_sel.call_count, 1) + + # assumes called with *args value. + selections = m_set_sel.call_args_list[0][0][0].decode() + + missing = [l for l in lines if l not in selections.splitlines()] + self.assertEqual([], missing) + + @mock.patch("cloudinit.config.cc_apt_configure.dpkg_reconfigure") + @mock.patch("cloudinit.config.cc_apt_configure.debconf_set_selections") + @mock.patch("cloudinit.config.cc_apt_configure." + "util.get_installed_packages") + def test_reconfigure_if_intersection(self, m_get_inst, m_set_sel, + m_dpkg_r): + data = { + 'set1': 'pkga pkga/q1 mybool false', + 'set2': ('pkgb\tpkgb/b1\tstr\tthis is a string\n' + 'pkgc\tpkgc/ip\tstring\t10.0.0.1'), + 'cloud-init': ('cloud-init cloud-init/datasources' + 'multiselect MAAS')} + + m_set_sel.return_value = None + m_get_inst.return_value = ["adduser", "apparmor", "pkgb", + "cloud-init", 'zdog'] + + cc_apt_configure.apply_debconf_selections({'debconf_selections': data}) + + # reconfigure should be called with the intersection + # of (packages in config, packages installed) + self.assertEqual(m_dpkg_r.call_count, 1) + # assumes called with *args (dpkg_reconfigure([a,b,c], target=)) + packages = m_dpkg_r.call_args_list[0][0][0] + self.assertEqual(set(['cloud-init', 'pkgb']), set(packages)) + + @mock.patch("cloudinit.config.cc_apt_configure.dpkg_reconfigure") + @mock.patch("cloudinit.config.cc_apt_configure.debconf_set_selections") + @mock.patch("cloudinit.config.cc_apt_configure." + "util.get_installed_packages") + def test_reconfigure_if_no_intersection(self, m_get_inst, m_set_sel, + m_dpkg_r): + data = {'set1': 'pkga pkga/q1 mybool false'} + + m_get_inst.return_value = ["adduser", "apparmor", "pkgb", + "cloud-init", 'zdog'] + m_set_sel.return_value = None + + cc_apt_configure.apply_debconf_selections({'debconf_selections': data}) + + self.assertTrue(m_get_inst.called) + self.assertEqual(m_dpkg_r.call_count, 0) + + @mock.patch("cloudinit.config.cc_apt_configure.util.subp") + def test_dpkg_reconfigure_does_reconfigure(self, m_subp): + target = "/foo-target" + + # due to the way the cleaners are called (via dictionary reference) + # mocking clean_cloud_init directly does not work. So we mock + # the CONFIG_CLEANERS dictionary and assert our cleaner is called. + ci_cleaner = mock.MagicMock() + with mock.patch.dict(("cloudinit.config.cc_apt_configure." + "CONFIG_CLEANERS"), + values={'cloud-init': ci_cleaner}, clear=True): + cc_apt_configure.dpkg_reconfigure(['pkga', 'cloud-init'], + target=target) + # cloud-init is actually the only package we have a cleaner for + # so for now, its the only one that should reconfigured + self.assertTrue(m_subp.called) + ci_cleaner.assert_called_with(target) + self.assertEqual(m_subp.call_count, 1) + found = m_subp.call_args_list[0][0][0] + expected = ['dpkg-reconfigure', '--frontend=noninteractive', + 'cloud-init'] + self.assertEqual(expected, found) + + @mock.patch("cloudinit.config.cc_apt_configure.util.subp") + def test_dpkg_reconfigure_not_done_on_no_data(self, m_subp): + cc_apt_configure.dpkg_reconfigure([]) + m_subp.assert_not_called() + + @mock.patch("cloudinit.config.cc_apt_configure.util.subp") + def test_dpkg_reconfigure_not_done_if_no_cleaners(self, m_subp): + cc_apt_configure.dpkg_reconfigure(['pkgfoo', 'pkgbar']) + m_subp.assert_not_called() + +# +# vi: ts=4 expandtab diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 73369cd3..d2031f59 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -508,4 +508,73 @@ class TestReadSeeded(helpers.TestCase): self.assertEqual(found_md, {'key1': 'val1'}) self.assertEqual(found_ud, ud) + +class TestSubp(helpers.TestCase): + + stdin2err = ['bash', '-c', 'cat >&2'] + stdin2out = ['cat'] + utf8_invalid = b'ab\xaadef' + utf8_valid = b'start \xc3\xa9 end' + utf8_valid_2 = b'd\xc3\xa9j\xc8\xa7' + + def printf_cmd(self, *args): + # bash's printf supports \xaa. So does /usr/bin/printf + # but by using bash, we remove dependency on another program. + return(['bash', '-c', 'printf "$@"', 'printf'] + list(args)) + + def test_subp_handles_utf8(self): + # The given bytes contain utf-8 accented characters as seen in e.g. + # the "deja dup" package in Ubuntu. + cmd = self.printf_cmd(self.utf8_valid_2) + (out, _err) = util.subp(cmd, capture=True) + self.assertEqual(out, self.utf8_valid_2.decode('utf-8')) + + def test_subp_respects_decode_false(self): + (out, err) = util.subp(self.stdin2out, capture=True, decode=False, + data=self.utf8_valid) + self.assertTrue(isinstance(out, bytes)) + self.assertTrue(isinstance(err, bytes)) + self.assertEqual(out, self.utf8_valid) + + def test_subp_decode_ignore(self): + # this executes a string that writes invalid utf-8 to stdout + (out, _err) = util.subp(self.printf_cmd('abc\\xaadef'), + capture=True, decode='ignore') + self.assertEqual(out, 'abcdef') + + def test_subp_decode_strict_valid_utf8(self): + (out, _err) = util.subp(self.stdin2out, capture=True, + decode='strict', data=self.utf8_valid) + self.assertEqual(out, self.utf8_valid.decode('utf-8')) + + def test_subp_decode_invalid_utf8_replaces(self): + (out, _err) = util.subp(self.stdin2out, capture=True, + data=self.utf8_invalid) + expected = self.utf8_invalid.decode('utf-8', errors='replace') + self.assertEqual(out, expected) + + def test_subp_decode_strict_raises(self): + args = [] + kwargs = {'args': self.stdin2out, 'capture': True, + 'decode': 'strict', 'data': self.utf8_invalid} + self.assertRaises(UnicodeDecodeError, util.subp, *args, **kwargs) + + def test_subp_capture_stderr(self): + data = b'hello world' + (out, err) = util.subp(self.stdin2err, capture=True, + decode=False, data=data) + self.assertEqual(err, data) + self.assertEqual(out, b'') + + def test_returns_none_if_no_capture(self): + (out, err) = util.subp(self.stdin2out, data=b'', capture=False) + self.assertEqual(err, None) + self.assertEqual(out, None) + + def test_bunch_of_slashes_in_path(self): + self.assertEqual("/target/my/path/", + util.target_path("/target/", "//my/path/")) + self.assertEqual("/target/my/path/", + util.target_path("/target/", "///my/path/")) + # vi: ts=4 expandtab |