summaryrefslogtreecommitdiff
path: root/cloudinit/config/cc_apt_configure.py
diff options
context:
space:
mode:
Diffstat (limited to 'cloudinit/config/cc_apt_configure.py')
-rw-r--r--cloudinit/config/cc_apt_configure.py765
1 files changed, 340 insertions, 425 deletions
diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py
index 73d8719f..c558311a 100644
--- a/cloudinit/config/cc_apt_configure.py
+++ b/cloudinit/config/cc_apt_configure.py
@@ -10,16 +10,14 @@
import glob
import os
+import pathlib
import re
from textwrap import dedent
-from cloudinit.config.schema import (
- get_schema_doc, validate_cloudconfig_schema)
from cloudinit import gpg
from cloudinit import log as logging
-from cloudinit import subp
-from cloudinit import templater
-from cloudinit import util
+from cloudinit import subp, templater, util
+from cloudinit.config.schema import MetaSchema, get_meta_doc
from cloudinit.settings import PER_INSTANCE
LOG = logging.getLogger(__name__)
@@ -27,45 +25,19 @@ LOG = logging.getLogger(__name__)
# this will match 'XXX:YYY' (ie, 'cloud-archive:foo' or 'ppa:bar')
ADD_APT_REPO_MATCH = r"^[\w-]+:\w"
+APT_LOCAL_KEYS = "/etc/apt/trusted.gpg"
+APT_TRUSTED_GPG_DIR = "/etc/apt/trusted.gpg.d/"
+CLOUD_INIT_GPG_DIR = "/etc/apt/cloud-init.gpg.d/"
+
frequency = PER_INSTANCE
distros = ["ubuntu", "debian"]
-mirror_property = {
- 'type': 'array',
- 'item': {
- 'type': 'object',
- 'additionalProperties': False,
- 'required': ['arches'],
- 'properties': {
- 'arches': {
- 'type': 'array',
- 'item': {
- 'type': 'string'
- },
- 'minItems': 1
- },
- 'uri': {
- 'type': 'string',
- 'format': 'uri'
- },
- 'search': {
- 'type': 'array',
- 'item': {
- 'type': 'string',
- 'format': 'uri'
- },
- 'minItems': 1
- },
- 'search_dns': {
- 'type': 'boolean',
- }
- }
- }
-}
-schema = {
- 'id': 'cc_apt_configure',
- 'name': 'Apt Configure',
- 'title': 'Configure apt for the user',
- 'description': dedent("""\
+
+meta: MetaSchema = {
+ "id": "cc_apt_configure",
+ "name": "Apt Configure",
+ "title": "Configure apt for the user",
+ "description": dedent(
+ """\
This module handles both configuration of apt options and adding
source lists. There are configuration options such as
``apt_get_wrapper`` and ``apt_get_command`` that control how
@@ -80,9 +52,12 @@ schema = {
.. note::
For more information about apt configuration, see the
- ``Additional apt configuration`` example."""),
- 'distros': distros,
- 'examples': [dedent("""\
+ ``Additional apt configuration`` example."""
+ ),
+ "distros": distros,
+ "examples": [
+ dedent(
+ """\
apt:
preserve_sources_list: false
disable_suites:
@@ -99,11 +74,12 @@ schema = {
search:
- 'http://cool.but-sometimes-unreachable.com/ubuntu'
- 'http://us.archive.ubuntu.com/ubuntu'
- search_dns: <true/false>
+ search_dns: false
- arches:
- s390x
- arm64
uri: 'http://archive-to-use-for-arm64.example.com/ubuntu'
+
security:
- arches:
- default
@@ -130,7 +106,7 @@ schema = {
source1:
keyid: 'keyid'
keyserver: 'keyserverurl'
- source: 'deb http://<url>/ xenial main'
+ source: 'deb [signed-by=$KEY_FILE] http://<url>/ bionic main'
source2:
source: 'ppa:<ppa-name>'
source3:
@@ -138,239 +114,13 @@ schema = {
key: |
------BEGIN PGP PUBLIC KEY BLOCK-------
<key data>
- ------END PGP PUBLIC KEY BLOCK-------""")],
- 'frequency': frequency,
- 'type': 'object',
- 'properties': {
- 'apt': {
- 'type': 'object',
- 'additionalProperties': False,
- 'properties': {
- 'preserve_sources_list': {
- 'type': 'boolean',
- 'default': False,
- 'description': dedent("""\
- By default, cloud-init will generate a new sources
- list in ``/etc/apt/sources.list.d`` based on any
- changes specified in cloud config. To disable this
- behavior and preserve the sources list from the
- pristine image, set ``preserve_sources_list``
- to ``true``.
-
- The ``preserve_sources_list`` option overrides
- all other config keys that would alter
- ``sources.list`` or ``sources.list.d``,
- **except** for additional sources to be added
- to ``sources.list.d``.""")
- },
- 'disable_suites': {
- 'type': 'array',
- 'items': {
- 'type': 'string'
- },
- 'uniqueItems': True,
- 'description': dedent("""\
- Entries in the sources list can be disabled using
- ``disable_suites``, which takes a list of suites
- to be disabled. If the string ``$RELEASE`` is
- present in a suite in the ``disable_suites`` list,
- it will be replaced with the release name. If a
- suite specified in ``disable_suites`` is not
- present in ``sources.list`` it will be ignored.
- For convenience, several aliases are provided for
- ``disable_suites``:
-
- - ``updates`` => ``$RELEASE-updates``
- - ``backports`` => ``$RELEASE-backports``
- - ``security`` => ``$RELEASE-security``
- - ``proposed`` => ``$RELEASE-proposed``
- - ``release`` => ``$RELEASE``.
-
- When a suite is disabled using ``disable_suites``,
- its entry in ``sources.list`` is not deleted; it
- is just commented out.""")
- },
- 'primary': {
- **mirror_property,
- 'description': dedent("""\
- The primary and security archive mirrors can
- be specified using the ``primary`` and
- ``security`` keys, respectively. Both the
- ``primary`` and ``security`` keys take a list
- of configs, allowing mirrors to be specified
- on a per-architecture basis. Each config is a
- dictionary which must have an entry for
- ``arches``, specifying which architectures
- that config entry is for. The keyword
- ``default`` applies to any architecture not
- explicitly listed. The mirror url can be specified
- with the ``uri`` key, or a list of mirrors to
- check can be provided in order, with the first
- mirror that can be resolved being selected. This
- allows the same configuration to be used in
- different environment, with different hosts used
- for a local apt mirror. If no mirror is provided
- by ``uri`` or ``search``, ``search_dns`` may be
- used to search for dns names in the format
- ``<distro>-mirror`` in each of the following:
-
- - fqdn of this host per cloud metadata,
- - localdomain,
- - domains listed in ``/etc/resolv.conf``.
-
- If there is a dns entry for ``<distro>-mirror``,
- then it is assumed that there is a distro mirror
- at ``http://<distro>-mirror.<domain>/<distro>``.
- If the ``primary`` key is defined, but not the
- ``security`` key, then then configuration for
- ``primary`` is also used for ``security``.
- If ``search_dns`` is used for the ``security``
- key, the search pattern will be
- ``<distro>-security-mirror``.
-
- If no mirrors are specified, or all lookups fail,
- then default mirrors defined in the datasource
- are used. If none are present in the datasource
- either the following defaults are used:
-
- - ``primary`` => \
- ``http://archive.ubuntu.com/ubuntu``.
- - ``security`` => \
- ``http://security.ubuntu.com/ubuntu``
- """)},
- 'security': {
- **mirror_property,
- 'description': dedent("""\
- Please refer to the primary config documentation""")
- },
- 'add_apt_repo_match': {
- 'type': 'string',
- 'default': ADD_APT_REPO_MATCH,
- 'description': dedent("""\
- All source entries in ``apt-sources`` that match
- regex in ``add_apt_repo_match`` will be added to
- the system using ``add-apt-repository``. If
- ``add_apt_repo_match`` is not specified, it
- defaults to ``{}``""".format(ADD_APT_REPO_MATCH))
- },
- 'debconf_selections': {
- 'type': 'object',
- 'items': {'type': 'string'},
- 'description': dedent("""\
- Debconf additional configurations can be specified as a
- dictionary under the ``debconf_selections`` config
- key, with each key in the dict representing a
- different set of configurations. The value of each key
- must be a string containing all the debconf
- configurations that must be applied. We will bundle
- all of the values and pass them to
- ``debconf-set-selections``. Therefore, each value line
- must be a valid entry for ``debconf-set-selections``,
- meaning that they must possess for distinct fields:
-
- ``pkgname question type answer``
-
- Where:
-
- - ``pkgname`` is the name of the package.
- - ``question`` the name of the questions.
- - ``type`` is the type of question.
- - ``answer`` is the value used to ansert the \
- question.
-
- For example: \
- ``ippackage ippackage/ip string 127.0.01``
- """)
- },
- 'sources_list': {
- 'type': 'string',
- 'description': dedent("""\
- Specifies a custom template for rendering
- ``sources.list`` . If no ``sources_list`` template
- is given, cloud-init will use sane default. Within
- this template, the following strings will be
- replaced with the appropriate values:
-
- - ``$MIRROR``
- - ``$RELEASE``
- - ``$PRIMARY``
- - ``$SECURITY``""")
- },
- 'conf': {
- 'type': 'string',
- 'description': dedent("""\
- Specify configuration for apt, such as proxy
- configuration. This configuration is specified as a
- string. For multiline apt configuration, make sure
- to follow yaml syntax.""")
- },
- 'https_proxy': {
- 'type': 'string',
- 'description': dedent("""\
- More convenient way to specify https apt proxy.
- https proxy url is specified in the format
- ``https://[[user][:pass]@]host[:port]/``.""")
- },
- 'http_proxy': {
- 'type': 'string',
- 'description': dedent("""\
- More convenient way to specify http apt proxy.
- http proxy url is specified in the format
- ``http://[[user][:pass]@]host[:port]/``.""")
- },
- 'proxy': {
- 'type': 'string',
- 'description': 'Alias for defining a http apt proxy.'
- },
- 'ftp_proxy': {
- 'type': 'string',
- 'description': dedent("""\
- More convenient way to specify ftp apt proxy.
- ftp proxy url is specified in the format
- ``ftp://[[user][:pass]@]host[:port]/``.""")
- },
- 'sources': {
- 'type': 'object',
- 'items': {'type': 'string'},
- 'description': dedent("""\
- Source list entries can be specified as a
- dictionary under the ``sources`` config key, with
- each key in the dict representing a different source
- file. The key of each source entry will be used
- as an id that can be referenced in other config
- entries, as well as the filename for the source's
- configuration under ``/etc/apt/sources.list.d``.
- If the name does not end with ``.list``, it will
- be appended. If there is no configuration for a
- key in ``sources``, no file will be written, but
- the key may still be referred to as an id in other
- ``sources`` entries.
-
- Each entry under ``sources`` is a dictionary which
- may contain any of the following optional keys:
-
- - ``source``: a sources.list entry \
- (some variable replacements apply).
- - ``keyid``: a key to import via shortid or \
- fingerprint.
- - ``key``: a raw PGP key.
- - ``keyserver``: alternate keyserver to pull \
- ``keyid`` key from.
-
- The ``source`` key supports variable
- replacements for the following strings:
-
- - ``$MIRROR``
- - ``$PRIMARY``
- - ``$SECURITY``
- - ``$RELEASE``""")
- }
- }
- }
- }
+ ------END PGP PUBLIC KEY BLOCK-------"""
+ )
+ ],
+ "frequency": frequency,
}
-__doc__ = get_schema_doc(schema)
+__doc__ = get_meta_doc(meta)
# place where apt stores cached repository data
@@ -384,18 +134,22 @@ APT_PROXY_FN = "/etc/apt/apt.conf.d/90cloud-init-aptproxy"
DEFAULT_KEYSERVER = "keyserver.ubuntu.com"
# Default archive mirrors
-PRIMARY_ARCH_MIRRORS = {"PRIMARY": "http://archive.ubuntu.com/ubuntu/",
- "SECURITY": "http://security.ubuntu.com/ubuntu/"}
-PORTS_MIRRORS = {"PRIMARY": "http://ports.ubuntu.com/ubuntu-ports",
- "SECURITY": "http://ports.ubuntu.com/ubuntu-ports"}
-PRIMARY_ARCHES = ['amd64', 'i386']
-PORTS_ARCHES = ['s390x', 'arm64', 'armhf', 'powerpc', 'ppc64el']
+PRIMARY_ARCH_MIRRORS = {
+ "PRIMARY": "http://archive.ubuntu.com/ubuntu/",
+ "SECURITY": "http://security.ubuntu.com/ubuntu/",
+}
+PORTS_MIRRORS = {
+ "PRIMARY": "http://ports.ubuntu.com/ubuntu-ports",
+ "SECURITY": "http://ports.ubuntu.com/ubuntu-ports",
+}
+PRIMARY_ARCHES = ["amd64", "i386"]
+PORTS_ARCHES = ["s390x", "arm64", "armhf", "powerpc", "ppc64el", "riscv64"]
def get_default_mirrors(arch=None, target=None):
"""returns the default mirrors for the target. These depend on the
- architecture, for more see:
- https://wiki.ubuntu.com/UbuntuDevelopment/PackageArchive#Ports"""
+ architecture, for more see:
+ https://wiki.ubuntu.com/UbuntuDevelopment/PackageArchive#Ports"""
if arch is None:
arch = util.get_dpkg_architecture(target)
if arch in PRIMARY_ARCHES:
@@ -407,8 +161,8 @@ def get_default_mirrors(arch=None, target=None):
def handle(name, ocfg, cloud, log, _):
"""process the config for apt_config. This can be called from
- curthooks if a global apt config was provided or via the "apt"
- standalone command."""
+ curthooks if a global apt config was provided or via the "apt"
+ standalone command."""
# keeping code close to curtin codebase via entry handler
target = None
if log is not None:
@@ -416,14 +170,15 @@ def handle(name, ocfg, cloud, log, _):
LOG = log
# feed back converted config, but only work on the subset under 'apt'
ocfg = convert_to_v3_apt_format(ocfg)
- cfg = ocfg.get('apt', {})
+ cfg = ocfg.get("apt", {})
if not isinstance(cfg, dict):
raise ValueError(
"Expected dictionary for 'apt' config, found {config_type}".format(
- config_type=type(cfg)))
+ config_type=type(cfg)
+ )
+ )
- validate_cloudconfig_schema(cfg, schema)
apply_debconf_selections(cfg, target)
apply_apt(cfg, cloud, target)
@@ -432,7 +187,7 @@ def _should_configure_on_empty_apt():
# if no config was provided, should apt configuration be done?
if util.system_is_snappy():
return False, "system is snappy."
- if not (subp.which('apt-get') or subp.which('apt')):
+ if not (subp.which("apt-get") or subp.which("apt")):
return False, "no apt commands."
return True, "Apt is available."
@@ -447,12 +202,13 @@ def apply_apt(cfg, cloud, target):
LOG.debug("handling apt config: %s", cfg)
- release = util.lsb_release(target=target)['codename']
+ release = util.lsb_release(target=target)["codename"]
arch = util.get_dpkg_architecture(target)
mirrors = find_apt_mirror_info(cfg, cloud, arch=arch)
LOG.debug("Apt Mirror info: %s", mirrors)
- if util.is_false(cfg.get('preserve_sources_list', False)):
+ if util.is_false(cfg.get("preserve_sources_list", False)):
+ add_mirror_keys(cfg, target)
generate_sources_list(cfg, release, mirrors, cloud)
rename_apt_lists(mirrors, target, arch)
@@ -462,25 +218,34 @@ def apply_apt(cfg, cloud, target):
LOG.exception("Failed to apply proxy or apt config info:")
# Process 'apt_source -> sources {dict}'
- if 'sources' in cfg:
+ if "sources" in cfg:
params = mirrors
- params['RELEASE'] = release
- params['MIRROR'] = mirrors["MIRROR"]
+ params["RELEASE"] = release
+ params["MIRROR"] = mirrors["MIRROR"]
matcher = None
- matchcfg = cfg.get('add_apt_repo_match', ADD_APT_REPO_MATCH)
+ matchcfg = cfg.get("add_apt_repo_match", ADD_APT_REPO_MATCH)
if matchcfg:
matcher = re.compile(matchcfg).search
- add_apt_sources(cfg['sources'], cloud, target=target,
- template_params=params, aa_repo_match=matcher)
+ add_apt_sources(
+ cfg["sources"],
+ cloud,
+ target=target,
+ template_params=params,
+ aa_repo_match=matcher,
+ )
def debconf_set_selections(selections, target=None):
- if not selections.endswith(b'\n'):
- selections += b'\n'
- subp.subp(['debconf-set-selections'], data=selections, target=target,
- capture=True)
+ if not selections.endswith(b"\n"):
+ selections += b"\n"
+ subp.subp(
+ ["debconf-set-selections"],
+ data=selections,
+ target=target,
+ capture=True,
+ )
def dpkg_reconfigure(packages, target=None):
@@ -500,12 +265,20 @@ def dpkg_reconfigure(packages, target=None):
unhandled.append(pkg)
if len(unhandled):
- LOG.warning("The following packages were installed and preseeded, "
- "but cannot be unconfigured: %s", unhandled)
+ LOG.warning(
+ "The following packages were installed and preseeded, "
+ "but cannot be unconfigured: %s",
+ unhandled,
+ )
if len(to_config):
- subp.subp(['dpkg-reconfigure', '--frontend=noninteractive'] +
- list(to_config), data=None, target=target, capture=True)
+ subp.subp(
+ ["dpkg-reconfigure", "--frontend=noninteractive"]
+ + list(to_config),
+ data=None,
+ target=target,
+ capture=True,
+ )
def apply_debconf_selections(cfg, target=None):
@@ -514,13 +287,12 @@ def apply_debconf_selections(cfg, target=None):
# set1: |
# cloud-init cloud-init/datasources multiselect MAAS
# set2: pkg pkg/value string bar
- selsets = cfg.get('debconf_selections')
+ selsets = cfg.get("debconf_selections")
if not selsets:
LOG.debug("debconf_selections was not set in config")
return
- selections = '\n'.join(
- [selsets[key] for key in sorted(selsets.keys())])
+ selections = "\n".join([selsets[key] for key in sorted(selsets.keys())])
debconf_set_selections(selections.encode(), target=target)
# get a complete list of packages listed in input
@@ -547,7 +319,8 @@ def apply_debconf_selections(cfg, target=None):
def clean_cloud_init(target):
"""clean out any local cloud-init config"""
flist = glob.glob(
- subp.target_path(target, "/etc/cloud/cloud.cfg.d/*dpkg*"))
+ subp.target_path(target, "/etc/cloud/cloud.cfg.d/*dpkg*")
+ )
LOG.debug("cleaning cloud-init config from: %s", flist)
for dpkg_cfg in flist:
@@ -556,18 +329,18 @@ def clean_cloud_init(target):
def mirrorurl_to_apt_fileprefix(mirror):
"""mirrorurl_to_apt_fileprefix
- Convert a mirror url to the file prefix used by apt on disk to
- store cache information for that mirror.
- To do so do:
- - take off ???://
- - drop tailing /
- - convert in string / to _"""
+ Convert a mirror url to the file prefix used by apt on disk to
+ store cache information for that mirror.
+ To do so do:
+ - take off ???://
+ - drop tailing /
+ - convert in string / to _"""
string = mirror
if string.endswith("/"):
string = string[0:-1]
pos = string.find("://")
if pos >= 0:
- string = string[pos + 3:]
+ string = string[pos + 3 :]
string = string.replace("/", "_")
return string
@@ -599,8 +372,8 @@ def rename_apt_lists(new_mirrors, target, arch):
def mirror_to_placeholder(tmpl, mirror, placeholder):
"""mirror_to_placeholder
- replace the specified mirror in a template with a placeholder string
- Checks for existance of the expected mirror and warns if not found"""
+ replace the specified mirror in a template with a placeholder string
+ Checks for existance of the expected mirror and warns if not found"""
if mirror not in tmpl:
LOG.warning("Expected mirror '%s' not found in: %s", mirror, tmpl)
return tmpl.replace(mirror, placeholder)
@@ -608,13 +381,15 @@ def mirror_to_placeholder(tmpl, mirror, placeholder):
def map_known_suites(suite):
"""there are a few default names which will be auto-extended.
- This comes at the inability to use those names literally as suites,
- but on the other hand increases readability of the cfg quite a lot"""
- mapping = {'updates': '$RELEASE-updates',
- 'backports': '$RELEASE-backports',
- 'security': '$RELEASE-security',
- 'proposed': '$RELEASE-proposed',
- 'release': '$RELEASE'}
+ This comes at the inability to use those names literally as suites,
+ but on the other hand increases readability of the cfg quite a lot"""
+ mapping = {
+ "updates": "$RELEASE-updates",
+ "backports": "$RELEASE-backports",
+ "security": "$RELEASE-security",
+ "proposed": "$RELEASE-proposed",
+ "release": "$RELEASE",
+ }
try:
retsuite = mapping[suite]
except KeyError:
@@ -624,14 +399,14 @@ def map_known_suites(suite):
def disable_suites(disabled, src, release):
"""reads the config for suites to be disabled and removes those
- from the template"""
+ from the template"""
if not disabled:
return src
retsrc = src
for suite in disabled:
suite = map_known_suites(suite)
- releasesuite = templater.render_string(suite, {'RELEASE': release})
+ releasesuite = templater.render_string(suite, {"RELEASE": release})
LOG.debug("Disabling suite %s as %s", suite, releasesuite)
newsrc = ""
@@ -653,109 +428,146 @@ def disable_suites(disabled, src, release):
break
if cols[pcol] == releasesuite:
- line = '# suite disabled by cloud-init: %s' % line
+ line = "# suite disabled by cloud-init: %s" % line
newsrc += line
retsrc = newsrc
return retsrc
+def add_mirror_keys(cfg, target):
+ """Adds any keys included in the primary/security mirror clauses"""
+ for key in ("primary", "security"):
+ for mirror in cfg.get(key, []):
+ add_apt_key(mirror, target, file_name=key)
+
+
def generate_sources_list(cfg, release, mirrors, cloud):
"""generate_sources_list
- create a source.list file based on a custom or default template
- by replacing mirrors and release in the template"""
+ create a source.list file based on a custom or default template
+ by replacing mirrors and release in the template"""
aptsrc = "/etc/apt/sources.list"
- params = {'RELEASE': release, 'codename': release}
+ params = {"RELEASE": release, "codename": release}
for k in mirrors:
params[k] = mirrors[k]
params[k.lower()] = mirrors[k]
- tmpl = cfg.get('sources_list', None)
+ tmpl = cfg.get("sources_list", None)
if tmpl is None:
LOG.info("No custom template provided, fall back to builtin")
- template_fn = cloud.get_template_filename('sources.list.%s' %
- (cloud.distro.name))
+ template_fn = cloud.get_template_filename(
+ "sources.list.%s" % (cloud.distro.name)
+ )
if not template_fn:
- template_fn = cloud.get_template_filename('sources.list')
+ template_fn = cloud.get_template_filename("sources.list")
if not template_fn:
- LOG.warning("No template found, "
- "not rendering /etc/apt/sources.list")
+ LOG.warning(
+ "No template found, not rendering /etc/apt/sources.list"
+ )
return
tmpl = util.load_file(template_fn)
rendered = templater.render_string(tmpl, params)
- disabled = disable_suites(cfg.get('disable_suites'), rendered, release)
+ disabled = disable_suites(cfg.get("disable_suites"), rendered, release)
util.write_file(aptsrc, disabled, mode=0o644)
-def add_apt_key_raw(key, target=None):
+def add_apt_key_raw(key, file_name, hardened=False, target=None):
"""
actual adding of a key as defined in key argument
to the system
"""
LOG.debug("Adding key:\n'%s'", key)
try:
- subp.subp(['apt-key', 'add', '-'], data=key.encode(), target=target)
+ name = pathlib.Path(file_name).stem
+ return apt_key("add", output_file=name, data=key, hardened=hardened)
except subp.ProcessExecutionError:
LOG.exception("failed to add apt GPG Key to apt keyring")
raise
-def add_apt_key(ent, target=None):
+def add_apt_key(ent, target=None, hardened=False, file_name=None):
"""
Add key to the system as defined in ent (if any).
Supports raw keys or keyid's
The latter will as a first step fetched to get the raw key
"""
- if 'keyid' in ent and 'key' not in ent:
+ if "keyid" in ent and "key" not in ent:
keyserver = DEFAULT_KEYSERVER
- if 'keyserver' in ent:
- keyserver = ent['keyserver']
+ if "keyserver" in ent:
+ keyserver = ent["keyserver"]
- ent['key'] = gpg.getkeybyid(ent['keyid'], keyserver)
+ ent["key"] = gpg.getkeybyid(ent["keyid"], keyserver)
- if 'key' in ent:
- add_apt_key_raw(ent['key'], target)
+ if "key" in ent:
+ return add_apt_key_raw(
+ ent["key"], file_name or ent["filename"], hardened=hardened
+ )
def update_packages(cloud):
cloud.distro.update_package_sources()
-def add_apt_sources(srcdict, cloud, target=None, template_params=None,
- aa_repo_match=None):
+def add_apt_sources(
+ srcdict, cloud, target=None, template_params=None, aa_repo_match=None
+):
"""
- add entries in /etc/apt/sources.list.d for each abbreviated
- sources.list entry in 'srcdict'. When rendering template, also
- include the values in dictionary searchList
+ install keys and repo source .list files defined in 'sources'
+
+ for each 'source' entry in the config:
+ 1. expand template variables and write source .list file in
+ /etc/apt/sources.list.d/
+ 2. install defined keys
+ 3. update packages via distro-specific method (i.e. apt-key update)
+
+
+ @param srcdict: a dict containing elements required
+ @param cloud: cloud instance object
+
+ Example srcdict value:
+ {
+ 'rio-grande-repo': {
+ 'source': 'deb [signed-by=$KEY_FILE] $MIRROR $RELEASE main',
+ 'keyid': 'B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77',
+ 'keyserver': 'pgp.mit.edu'
+ }
+ }
+
+ Note: Deb822 format is not supported
"""
if template_params is None:
template_params = {}
if aa_repo_match is None:
- raise ValueError('did not get a valid repo matcher')
+ raise ValueError("did not get a valid repo matcher")
if not isinstance(srcdict, dict):
- raise TypeError('unknown apt format: %s' % (srcdict))
+ raise TypeError("unknown apt format: %s" % (srcdict))
for filename in srcdict:
ent = srcdict[filename]
LOG.debug("adding source/key '%s'", ent)
- if 'filename' not in ent:
- ent['filename'] = filename
+ if "filename" not in ent:
+ ent["filename"] = filename
- add_apt_key(ent, target)
+ if "source" in ent and "$KEY_FILE" in ent["source"]:
+ key_file = add_apt_key(ent, target, hardened=True)
+ template_params["KEY_FILE"] = key_file
+ else:
+ key_file = add_apt_key(ent, target)
- if 'source' not in ent:
+ if "source" not in ent:
continue
- source = ent['source']
+ source = ent["source"]
source = templater.render_string(source, template_params)
- if not ent['filename'].startswith("/"):
- ent['filename'] = os.path.join("/etc/apt/sources.list.d/",
- ent['filename'])
- if not ent['filename'].endswith(".list"):
- ent['filename'] += ".list"
+ if not ent["filename"].startswith("/"):
+ ent["filename"] = os.path.join(
+ "/etc/apt/sources.list.d/", ent["filename"]
+ )
+ if not ent["filename"].endswith(".list"):
+ ent["filename"] += ".list"
if aa_repo_match(source):
try:
@@ -765,7 +577,7 @@ def add_apt_sources(srcdict, cloud, target=None, template_params=None,
raise
continue
- sourcefn = subp.target_path(target, ent['filename'])
+ sourcefn = subp.target_path(target, ent["filename"])
try:
contents = "%s\n" % (source)
util.write_file(sourcefn, contents, omode="a")
@@ -781,17 +593,21 @@ def add_apt_sources(srcdict, cloud, target=None, template_params=None,
def convert_v1_to_v2_apt_format(srclist):
"""convert v1 apt format to v2 (dict in apt_sources)"""
srcdict = {}
+ LOG.warning(
+ "DEPRECATION: 'apt_sources' deprecated config key found."
+ " Use 'apt' instead"
+ )
if isinstance(srclist, list):
LOG.debug("apt config: convert V1 to V2 format (source list to dict)")
for srcent in srclist:
- if 'filename' not in srcent:
+ if "filename" not in srcent:
# file collides for multiple !filename cases for compatibility
# yet we need them all processed, so not same dictionary key
- srcent['filename'] = "cloud_config_sources.list"
+ srcent["filename"] = "cloud_config_sources.list"
key = util.rand_dict_key(srcdict, "cloud_config_sources.list")
else:
# all with filename use that as key (matching new format)
- key = srcent['filename']
+ key = srcent["filename"]
srcdict[key] = srcent
elif isinstance(srclist, dict):
srcdict = srclist
@@ -803,7 +619,7 @@ def convert_v1_to_v2_apt_format(srclist):
def convert_key(oldcfg, aptcfg, oldkey, newkey):
"""convert an old key to the new one if the old one exists
- returns true if a key was found and converted"""
+ returns true if a key was found and converted"""
if oldcfg.get(oldkey, None) is not None:
aptcfg[newkey] = oldcfg.get(oldkey)
del oldcfg[oldkey]
@@ -813,33 +629,37 @@ def convert_key(oldcfg, aptcfg, oldkey, newkey):
def convert_mirror(oldcfg, aptcfg):
"""convert old apt_mirror keys into the new more advanced mirror spec"""
- keymap = [('apt_mirror', 'uri'),
- ('apt_mirror_search', 'search'),
- ('apt_mirror_search_dns', 'search_dns')]
+ keymap = [
+ ("apt_mirror", "uri"),
+ ("apt_mirror_search", "search"),
+ ("apt_mirror_search_dns", "search_dns"),
+ ]
converted = False
- newmcfg = {'arches': ['default']}
+ newmcfg = {"arches": ["default"]}
for oldkey, newkey in keymap:
if convert_key(oldcfg, newmcfg, oldkey, newkey):
converted = True
# only insert new style config if anything was converted
if converted:
- aptcfg['primary'] = [newmcfg]
+ aptcfg["primary"] = [newmcfg]
def convert_v2_to_v3_apt_format(oldcfg):
"""convert old to new keys and adapt restructured mirror spec"""
- mapoldkeys = {'apt_sources': 'sources',
- 'apt_mirror': None,
- 'apt_mirror_search': None,
- 'apt_mirror_search_dns': None,
- 'apt_proxy': 'proxy',
- 'apt_http_proxy': 'http_proxy',
- 'apt_ftp_proxy': 'https_proxy',
- 'apt_https_proxy': 'ftp_proxy',
- 'apt_preserve_sources_list': 'preserve_sources_list',
- 'apt_custom_sources_list': 'sources_list',
- 'add_apt_repo_match': 'add_apt_repo_match'}
+ mapoldkeys = {
+ "apt_sources": "sources",
+ "apt_mirror": None,
+ "apt_mirror_search": None,
+ "apt_mirror_search_dns": None,
+ "apt_proxy": "proxy",
+ "apt_http_proxy": "http_proxy",
+ "apt_ftp_proxy": "https_proxy",
+ "apt_https_proxy": "ftp_proxy",
+ "apt_preserve_sources_list": "preserve_sources_list",
+ "apt_custom_sources_list": "sources_list",
+ "add_apt_repo_match": "add_apt_repo_match",
+ }
needtoconvert = []
for oldkey in mapoldkeys:
if oldkey in oldcfg:
@@ -851,13 +671,19 @@ def convert_v2_to_v3_apt_format(oldcfg):
# no old config, so no new one to be created
if not needtoconvert:
return oldcfg
- LOG.debug("apt config: convert V2 to V3 format for keys '%s'",
- ", ".join(needtoconvert))
+ LOG.warning(
+ "DEPRECATION apt: converted deprecated config V2 to V3 format for"
+ " keys '%s'. Use updated config keys.",
+ ", ".join(needtoconvert),
+ )
# if old AND new config are provided, prefer the new one (LP #1616831)
- newaptcfg = oldcfg.get('apt', None)
+ newaptcfg = oldcfg.get("apt", None)
if newaptcfg is not None:
- LOG.debug("apt config: V1/2 and V3 format specified, preferring V3")
+ LOG.warning(
+ "DEPRECATION: apt config: deprecated V1/2 and V3 format specified,"
+ " preferring V3"
+ )
for oldkey in needtoconvert:
newkey = mapoldkeys[oldkey]
verify = oldcfg[oldkey] # drop, but keep a ref for verification
@@ -866,10 +692,11 @@ def convert_v2_to_v3_apt_format(oldcfg):
# no simple mapping or no collision on this particular key
continue
if verify != newaptcfg[newkey]:
- raise ValueError("Old and New apt format defined with unequal "
- "values %s vs %s @ %s" % (verify,
- newaptcfg[newkey],
- oldkey))
+ raise ValueError(
+ "Old and New apt format defined with unequal "
+ "values %s vs %s @ %s"
+ % (verify, newaptcfg[newkey], oldkey)
+ )
# return conf after clearing conflicting V1/2 keys
return oldcfg
@@ -889,17 +716,17 @@ def convert_v2_to_v3_apt_format(oldcfg):
raise ValueError("old apt key '%s' left after conversion" % oldkey)
# insert new format into config and return full cfg with only v3 content
- oldcfg['apt'] = aptcfg
+ oldcfg["apt"] = aptcfg
return oldcfg
def convert_to_v3_apt_format(cfg):
"""convert the old list based format to the new dict based one. After that
- convert the old dict keys/format to v3 a.k.a 'new apt config'"""
+ convert the old dict keys/format to v3 a.k.a 'new apt config'"""
# V1 -> V2, the apt_sources entry from list to dict
- apt_sources = cfg.get('apt_sources', None)
+ apt_sources = cfg.get("apt_sources", None)
if apt_sources is not None:
- cfg['apt_sources'] = convert_v1_to_v2_apt_format(apt_sources)
+ cfg["apt_sources"] = convert_v1_to_v2_apt_format(apt_sources)
# V2 -> V3, move all former globals under the "apt" key
# Restructure into new key names and mirror hierarchy
@@ -931,7 +758,12 @@ def search_for_mirror_dns(configured, mirrortype, cfg, cloud):
if mydom:
doms.append(".%s" % mydom)
- doms.extend((".localdomain", "",))
+ doms.extend(
+ (
+ ".localdomain",
+ "",
+ )
+ )
mirror_list = []
distro = cloud.distro.name
@@ -946,12 +778,11 @@ def search_for_mirror_dns(configured, mirrortype, cfg, cloud):
def update_mirror_info(pmirror, smirror, arch, cloud):
"""sets security mirror to primary if not defined.
- returns defaults if no mirrors are defined"""
+ returns defaults if no mirrors are defined"""
if pmirror is not None:
if smirror is None:
smirror = pmirror
- return {'PRIMARY': pmirror,
- 'SECURITY': smirror}
+ return {"PRIMARY": pmirror, "SECURITY": smirror}
# None specified at all, get default mirrors from cloud
mirror_info = cloud.datasource.get_package_mirror_info()
@@ -960,8 +791,8 @@ def update_mirror_info(pmirror, smirror, arch, cloud):
# arbitrary key/value pairs including 'primary' and 'security' keys.
# caller expects dict with PRIMARY and SECURITY.
m = mirror_info.copy()
- m['PRIMARY'] = m['primary']
- m['SECURITY'] = m['security']
+ m["PRIMARY"] = m["primary"]
+ m["SECURITY"] = m["security"]
return m
@@ -971,7 +802,7 @@ def update_mirror_info(pmirror, smirror, arch, cloud):
def get_arch_mirrorconfig(cfg, mirrortype, arch):
"""out of a list of potential mirror configurations select
- and return the one matching the architecture (or default)"""
+ and return the one matching the architecture (or default)"""
# select the mirror specification (if-any)
mirror_cfg_list = cfg.get(mirrortype, None)
if mirror_cfg_list is None:
@@ -980,7 +811,7 @@ def get_arch_mirrorconfig(cfg, mirrortype, arch):
# select the specification matching the target arch
default = None
for mirror_cfg_elem in mirror_cfg_list:
- arches = mirror_cfg_elem.get("arches")
+ arches = mirror_cfg_elem.get("arches") or []
if arch in arches:
return mirror_cfg_elem
if "default" in arches:
@@ -990,8 +821,8 @@ def get_arch_mirrorconfig(cfg, mirrortype, arch):
def get_mirror(cfg, mirrortype, arch, cloud):
"""pass the three potential stages of mirror specification
- returns None is neither of them found anything otherwise the first
- hit is returned"""
+ returns None is neither of them found anything otherwise the first
+ hit is returned"""
mcfg = get_arch_mirrorconfig(cfg, mirrortype, arch)
if mcfg is None:
return None
@@ -1007,18 +838,19 @@ def get_mirror(cfg, mirrortype, arch, cloud):
# fallback to search_dns if specified
if mirror is None:
# list of mirrors to try to resolve
- mirror = search_for_mirror_dns(mcfg.get("search_dns", None),
- mirrortype, cfg, cloud)
+ mirror = search_for_mirror_dns(
+ mcfg.get("search_dns", None), mirrortype, cfg, cloud
+ )
return mirror
def find_apt_mirror_info(cfg, cloud, arch=None):
"""find_apt_mirror_info
- find an apt_mirror given the cfg provided.
- It can check for separate config of primary and security mirrors
- If only primary is given security is assumed to be equal to primary
- If the generic apt_mirror is given that is defining for both
+ find an apt_mirror given the cfg provided.
+ It can check for separate config of primary and security mirrors
+ If only primary is given security is assumed to be equal to primary
+ If the generic apt_mirror is given that is defining for both
"""
if arch is None:
@@ -1039,32 +871,115 @@ def find_apt_mirror_info(cfg, cloud, arch=None):
def apply_apt_config(cfg, proxy_fname, config_fname):
"""apply_apt_config
- Applies any apt*proxy config from if specified
+ Applies any apt*proxy config from if specified
"""
# Set up any apt proxy
- cfgs = (('proxy', 'Acquire::http::Proxy "%s";'),
- ('http_proxy', 'Acquire::http::Proxy "%s";'),
- ('ftp_proxy', 'Acquire::ftp::Proxy "%s";'),
- ('https_proxy', 'Acquire::https::Proxy "%s";'))
+ cfgs = (
+ ("proxy", 'Acquire::http::Proxy "%s";'),
+ ("http_proxy", 'Acquire::http::Proxy "%s";'),
+ ("ftp_proxy", 'Acquire::ftp::Proxy "%s";'),
+ ("https_proxy", 'Acquire::https::Proxy "%s";'),
+ )
proxies = [fmt % cfg.get(name) for (name, fmt) in cfgs if cfg.get(name)]
if len(proxies):
LOG.debug("write apt proxy info to %s", proxy_fname)
- util.write_file(proxy_fname, '\n'.join(proxies) + '\n')
+ util.write_file(proxy_fname, "\n".join(proxies) + "\n")
elif os.path.isfile(proxy_fname):
util.del_file(proxy_fname)
LOG.debug("no apt proxy configured, removed %s", proxy_fname)
- if cfg.get('conf', None):
+ if cfg.get("conf", None):
LOG.debug("write apt config info to %s", config_fname)
- util.write_file(config_fname, cfg.get('conf'))
+ util.write_file(config_fname, cfg.get("conf"))
elif os.path.isfile(config_fname):
util.del_file(config_fname)
LOG.debug("no apt config configured, removed %s", config_fname)
+def apt_key(
+ command, output_file=None, data=None, hardened=False, human_output=True
+):
+ """apt-key replacement
+
+ commands implemented: 'add', 'list', 'finger'
+
+ @param output_file: name of output gpg file (without .gpg or .asc)
+ @param data: key contents
+ @param human_output: list keys formatted for human parsing
+ @param hardened: write keys to to /etc/apt/cloud-init.gpg.d/ (referred to
+ with [signed-by] in sources file)
+ """
+
+ def _get_key_files():
+ """return all apt keys
+
+ /etc/apt/trusted.gpg (if it exists) and all keyfiles (and symlinks to
+ keyfiles) in /etc/apt/trusted.gpg.d/ are returned
+
+ based on apt-key implementation
+ """
+ key_files = [APT_LOCAL_KEYS] if os.path.isfile(APT_LOCAL_KEYS) else []
+
+ for file in os.listdir(APT_TRUSTED_GPG_DIR):
+ if file.endswith(".gpg") or file.endswith(".asc"):
+ key_files.append(APT_TRUSTED_GPG_DIR + file)
+ return key_files if key_files else ""
+
+ def apt_key_add():
+ """apt-key add <file>
+
+ returns filepath to new keyring, or '/dev/null' when an error occurs
+ """
+ file_name = "/dev/null"
+ if not output_file:
+ util.logexc(
+ LOG, 'Unknown filename, failed to add key: "{}"'.format(data)
+ )
+ else:
+ try:
+ key_dir = (
+ CLOUD_INIT_GPG_DIR if hardened else APT_TRUSTED_GPG_DIR
+ )
+ stdout = gpg.dearmor(data)
+ file_name = "{}{}.gpg".format(key_dir, output_file)
+ util.write_file(file_name, stdout)
+ except subp.ProcessExecutionError:
+ util.logexc(
+ LOG, "Gpg error, failed to add key: {}".format(data)
+ )
+ except UnicodeDecodeError:
+ util.logexc(
+ LOG, "Decode error, failed to add key: {}".format(data)
+ )
+ return file_name
+
+ def apt_key_list():
+ """apt-key list
+
+ returns string of all trusted keys (in /etc/apt/trusted.gpg and
+ /etc/apt/trusted.gpg.d/)
+ """
+ key_list = []
+ for key_file in _get_key_files():
+ try:
+ key_list.append(gpg.list(key_file, human_output=human_output))
+ except subp.ProcessExecutionError as error:
+ LOG.warning('Failed to list key "%s": %s', key_file, error)
+ return "\n".join(key_list)
+
+ if command == "add":
+ return apt_key_add()
+ elif command == "finger" or command == "list":
+ return apt_key_list()
+ else:
+ raise ValueError(
+ "apt_key() commands add, list, and finger are currently supported"
+ )
+
+
CONFIG_CLEANERS = {
- 'cloud-init': clean_cloud_init,
+ "cloud-init": clean_cloud_init,
}
# vi: ts=4 expandtab
generated by cgit v1.2.3 (git 2.39.1) at 2024-09-19 18:48:03 +0000