Add LXD datasource (#1040)

Add DataSourceLXD which knows how to talk to the dev-lxd socket to
obtain all instance metadata API:
https://linuxcontainers.org/lxd/docs/master/dev-lxd.

This first branch is to deliver feature parity with the existing
NoCloud datasource which is currently used to intialize LXC instances
on first boot.

Introduce a SocketConnectionPool and LXDSocketAdapter to support
performing HTTP GETs on the following routes which are surfaced by the
LXD host to all containers:
http://unix.socket/1.0/meta-data
http://unix.socket/1.0/config/user.user-data
http://unix.socket/1.0/config/user.network-config
http://unix.socket/1.0/config/user.vendor-data
These 4 routes minimally replace the static content provided in the
following nocloud-net seed files:
/var/lib/cloud/nocloud-net/{meta-data,vendor-data,user-data,network-config}

The intent of this commit is to set a foundation for LXD socket
communication that will allow us to build network hot-plug features
by eventually consuming LXD's websocket upgrade route 1.0/events to
react to network, meta-data and user-data config changes over time.

In the event that no custom network-config is provided, default to the
same network-config definition provided by LXD to the NoCloud
network-config seed file.

Supplemental features above NoCloud datasource:
surface all custom instance data config keys via cloud-init query ds
which aids in discoverability of features/tags/labels as well as
conditional #cloud-config jinja templates operations based on custom
config options.
TBD: better cloud-init query support for dot-delimited keys

3 files changed, 544 insertions, 0 deletions

diff --git a/cloudinit/settings.py b/cloudinit/settings.py
index f69005ea..43c8fa24 100644
--- a/cloudinit/settings.py
+++ b/cloudinit/settings.py
@@ -21,6 +21,7 @@ CFG_BUILTIN = {
     'datasource_list': [
         'NoCloud',
         'ConfigDrive',
+        'LXD',
         'OpenNebula',
         'DigitalOcean',
         'Azure',
diff --git a/cloudinit/sources/DataSourceLXD.py b/cloudinit/sources/DataSourceLXD.py
new file mode 100644
index 00000000..732b32ff
--- /dev/null
+++ b/cloudinit/sources/DataSourceLXD.py
@@ -0,0 +1,358 @@
+
+"""Datasource for LXD, reads /dev/lxd/sock representaton of instance data.
+
+Notes:
+ * This datasource replaces previous NoCloud datasource for LXD.
+ * Older LXD images may not have updates for cloud-init so NoCloud may
+   still be detected on those images.
+ * Detect LXD datasource when /dev/lxd/sock is an active socket file.
+ * Info on dev-lxd API: https://linuxcontainers.org/lxd/docs/master/dev-lxd
+ * TODO( Hotplug support using websockets API 1.0/events )
+"""
+
+import os
+
+import requests
+from requests.adapters import HTTPAdapter
+
+# pylint fails to import the two modules below.
+# These are imported via requests.packages rather than urllib3 because:
+#  a.) the provider of the requests package should ensure that urllib3
+#      contained in it is consistent/correct.
+#  b.) cloud-init does not specifically have a dependency on urllib3
+#
+# For future reference, see:
+#   https://github.com/kennethreitz/requests/pull/2375
+#   https://github.com/requests/requests/issues/4104
+# pylint: disable=E0401
+from requests.packages.urllib3.connection import HTTPConnection
+from requests.packages.urllib3.connectionpool import HTTPConnectionPool
+
+import socket
+import stat
+
+from cloudinit import log as logging
+from cloudinit import sources, subp, util
+
+LOG = logging.getLogger(__name__)
+
+LXD_SOCKET_PATH = "/dev/lxd/sock"
+LXD_SOCKET_API_VERSION = "1.0"
+
+# Config key mappings to alias as top-level instance data keys
+CONFIG_KEY_ALIASES = {
+    "user.user-data": "user-data",
+    "user.network-config": "network-config",
+    "user.network_mode": "network_mode",
+    "user.vendor-data": "vendor-data"
+}
+
+
+def generate_fallback_network_config(network_mode: str = "") -> dict:
+    """Return network config V1 dict representing instance network config."""
+    network_v1 = {
+        "version": 1,
+        "config": [
+            {
+                "type": "physical", "name": "eth0",
+                "subnets": [{"type": "dhcp", "control": "auto"}]
+            }
+        ]
+    }
+    if subp.which("systemd-detect-virt"):
+        try:
+            virt_type, _ = subp.subp(['systemd-detect-virt'])
+        except subp.ProcessExecutionError as err:
+            LOG.warning(
+                "Unable to run systemd-detect-virt: %s."
+                " Rendering default network config.", err
+            )
+            return network_v1
+        if virt_type.strip() == "kvm":  # instance.type VIRTUAL-MACHINE
+            arch = util.system_info()["uname"][4]
+            if arch == "ppc64le":
+                network_v1["config"][0]["name"] = "enp0s5"
+            elif arch == "s390x":
+                network_v1["config"][0]["name"] = "enc9"
+            else:
+                network_v1["config"][0]["name"] = "enp5s0"
+    if network_mode == "link-local":
+        network_v1["config"][0]["subnets"][0]["control"] = "manual"
+    elif network_mode not in ("", "dhcp"):
+        LOG.warning(
+            "Ignoring unexpected value user.network_mode: %s", network_mode
+        )
+    return network_v1
+
+
+class SocketHTTPConnection(HTTPConnection):
+    def __init__(self, socket_path):
+        super().__init__('localhost')
+        self.socket_path = socket_path
+
+    def connect(self):
+        self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        self.sock.connect(self.socket_path)
+
+
+class SocketConnectionPool(HTTPConnectionPool):
+    def __init__(self, socket_path):
+        self.socket_path = socket_path
+        super().__init__('localhost')
+
+    def _new_conn(self):
+        return SocketHTTPConnection(self.socket_path)
+
+
+class LXDSocketAdapter(HTTPAdapter):
+    def get_connection(self, url, proxies=None):
+        return SocketConnectionPool(LXD_SOCKET_PATH)
+
+
+def _maybe_remove_top_network(cfg):
+    """If network-config contains top level 'network' key, then remove it.
+
+    Some providers of network configuration may provide a top level
+    'network' key (LP: #1798117) even though it is not necessary.
+
+    Be friendly and remove it if it really seems so.
+
+    Return the original value if no change or the updated value if changed."""
+    if "network" not in cfg:
+        return cfg
+    network_val = cfg["network"]
+    bmsg = 'Top level network key in network-config %s: %s'
+    if not isinstance(network_val, dict):
+        LOG.debug(bmsg, "was not a dict", cfg)
+        return cfg
+    if len(list(cfg.keys())) != 1:
+        LOG.debug(bmsg, "had multiple top level keys", cfg)
+        return cfg
+    if network_val.get('config') == "disabled":
+        LOG.debug(bmsg, "was config/disabled", cfg)
+    elif not all(('config' in network_val, 'version' in network_val)):
+        LOG.debug(bmsg, "but missing 'config' or 'version'", cfg)
+        return cfg
+    LOG.debug(bmsg, "fixed by removing shifting network.", cfg)
+    return network_val
+
+
+def _raw_instance_data_to_dict(metadata_type: str, metadata_value) -> dict:
+    """Convert raw instance data from str, bytes, YAML to dict
+
+    :param metadata_type: string, one of as: meta-data, vendor-data, user-data
+        network-config
+
+    :param metadata_value: str, bytes or dict representing or instance-data.
+
+    :raises: InvalidMetaDataError on invalid instance-data content.
+    """
+    if isinstance(metadata_value, dict):
+        return metadata_value
+    if metadata_value is None:
+        return {}
+    try:
+        parsed_metadata = util.load_yaml(metadata_value)
+    except AttributeError as exc:  # not str or bytes
+        raise sources.InvalidMetaDataException(
+            "Invalid {md_type}. Expected str, bytes or dict but found:"
+            " {value}".format(md_type=metadata_type, value=metadata_value)
+        ) from exc
+    if parsed_metadata is None:
+        raise sources.InvalidMetaDataException(
+            "Invalid {md_type} format. Expected YAML but found:"
+            " {value}".format(md_type=metadata_type, value=metadata_value)
+        )
+    return parsed_metadata
+
+
+class DataSourceLXD(sources.DataSource):
+
+    dsname = 'LXD'
+
+    _network_config = sources.UNSET
+    _crawled_metadata = sources.UNSET
+
+    sensitive_metadata_keys = (
+        'merged_cfg', 'user.meta-data', 'user.vendor-data', 'user.user-data',
+    )
+
+    def _is_platform_viable(self) -> bool:
+        """Check platform environment to report if this datasource may run."""
+        return is_platform_viable()
+
+    def _get_data(self) -> bool:
+        """Crawl LXD socket API instance data and return True on success"""
+        if not self._is_platform_viable():
+            LOG.debug("Not an LXD datasource: No LXD socket found.")
+            return False
+
+        self._crawled_metadata = util.log_time(
+            logfunc=LOG.debug, msg='Crawl of metadata service',
+            func=read_metadata)
+        self.metadata = _raw_instance_data_to_dict(
+            "meta-data", self._crawled_metadata.get("meta-data")
+        )
+        if LXD_SOCKET_API_VERSION in self._crawled_metadata:
+            config = self._crawled_metadata[LXD_SOCKET_API_VERSION].get(
+                "config", {}
+            )
+            user_metadata = config.get("user.meta-data", {})
+            if user_metadata:
+                user_metadata = _raw_instance_data_to_dict(
+                    "user.meta-data", user_metadata
+                )
+            if not isinstance(self.metadata, dict):
+                self.metadata = util.mergemanydict(
+                    [util.load_yaml(self.metadata), user_metadata]
+                )
+        if "user-data" in self._crawled_metadata:
+            self.userdata_raw = self._crawled_metadata["user-data"]
+        if "network-config" in self._crawled_metadata:
+            self._network_config = _maybe_remove_top_network(
+                _raw_instance_data_to_dict(
+                    "network-config", self._crawled_metadata["network-config"]
+                )
+            )
+        if "vendor-data" in self._crawled_metadata:
+            self.vendordata_raw = self._crawled_metadata["vendor-data"]
+        return True
+
+    def _get_subplatform(self) -> str:
+        """Return subplatform details for this datasource"""
+        return "LXD socket API v. {ver} ({socket})".format(
+            ver=LXD_SOCKET_API_VERSION, socket=LXD_SOCKET_PATH
+        )
+
+    def check_instance_id(self, sys_cfg) -> str:
+        """Return True if instance_id unchanged."""
+        response = read_metadata(metadata_only=True)
+        md = response.get("meta-data", {})
+        if not isinstance(md, dict):
+            md = util.load_yaml(md)
+        return md.get("instance-id") == self.metadata.get("instance-id")
+
+    @property
+    def network_config(self) -> dict:
+        """Network config read from LXD socket config/user.network-config.
+
+        If none is present, then we generate fallback configuration.
+        """
+        if self._network_config == sources.UNSET:
+            if self._crawled_metadata.get("network-config"):
+                self._network_config = self._crawled_metadata.get(
+                    "network-config"
+                )
+            else:
+                network_mode = self._crawled_metadata.get("network_mode", "")
+                self._network_config = generate_fallback_network_config(
+                    network_mode
+                )
+        return self._network_config
+
+
+def is_platform_viable() -> bool:
+    """Return True when this platform appears to have an LXD socket."""
+    if os.path.exists(LXD_SOCKET_PATH):
+        return stat.S_ISSOCK(os.lstat(LXD_SOCKET_PATH).st_mode)
+    return False
+
+
+def read_metadata(
+    api_version: str = LXD_SOCKET_API_VERSION, metadata_only: bool = False
+) -> dict:
+    """Fetch metadata from the /dev/lxd/socket routes.
+
+    Perform a number of HTTP GETs on known routes on the devlxd socket API.
+    Minimally all containers must respond to http://lxd/1.0/meta-data when
+    the LXD configuration setting `security.devlxd` is true.
+
+    When `security.devlxd` is false, no /dev/lxd/socket file exists. This
+    datasource will return False from `is_platform_viable` in that case.
+
+    Perform a GET of <LXD_SOCKET_API_VERSION>/config` and walk all `user.*`
+    configuration keys, storing all keys and values under a dict key
+        LXD_SOCKET_API_VERSION: config {...}.
+
+    In the presence of the following optional user config keys,
+    create top level aliases:
+      - user.user-data -> user-data
+      - user.vendor-data -> vendor-data
+      - user.network-config -> network-config
+
+    :return:
+        A dict with the following mandatory key: meta-data.
+        Optional keys: user-data, vendor-data, network-config, network_mode
+
+        Below <LXD_SOCKET_API_VERSION> is a dict representation of all raw
+        configuration keys and values provided to the container surfaced by
+        the socket under the /1.0/config/ route.
+    """
+    md = {}
+    lxd_url = "http://lxd"
+    version_url = lxd_url + "/" + api_version + "/"
+    with requests.Session() as session:
+        session.mount(version_url, LXDSocketAdapter())
+        # Raw meta-data as text
+        md_route = "{route}/meta-data".format(route=version_url)
+        response = session.get(md_route)
+        LOG.debug("[GET] [HTTP:%d] %s", response.status_code, md_route)
+        if not response.ok:
+            raise sources.InvalidMetaDataException(
+                "Invalid HTTP response [{code}] from {route}: {resp}".format(
+                    code=response.status_code,
+                    route=md_route,
+                    resp=response.txt
+                )
+            )
+
+        md["meta-data"] = response.text
+        if metadata_only:
+            return md  # Skip network-data, vendor-data, user-data
+
+        config_url = version_url + "config"
+        # Represent all advertized/available config routes under
+        # the dict path {LXD_SOCKET_API_VERSION: {config: {...}}.
+        LOG.debug("[GET] %s", config_url)
+        config_routes = session.get(config_url).json()
+        md[LXD_SOCKET_API_VERSION] = {
+            "config": {},
+            "meta-data": md["meta-data"]
+        }
+        for config_route in config_routes:
+            url = "http://lxd{route}".format(route=config_route)
+            LOG.debug("[GET] %s", url)
+            response = session.get(url)
+            if response.ok:
+                cfg_key = config_route.rpartition("/")[-1]
+                # Leave raw data values/format unchanged to represent it in
+                # instance-data.json for cloud-init query or jinja template
+                # use.
+                md[LXD_SOCKET_API_VERSION]["config"][cfg_key] = response.text
+                # Promote common CONFIG_KEY_ALIASES to top-level keys.
+                if cfg_key in CONFIG_KEY_ALIASES:
+                    md[CONFIG_KEY_ALIASES[cfg_key]] = response.text
+            else:
+                LOG.debug("Skipping %s on invalid response", url)
+    return md
+
+
+# Used to match classes to dependencies
+datasources = [
+    (DataSourceLXD, (sources.DEP_FILESYSTEM,)),
+]
+
+
+# Return a list of data sources that match this set of dependencies
+def get_datasource_list(depends):
+    return sources.list_from_depends(depends, datasources)
+
+
+if __name__ == "__main__":
+    import argparse
+
+    description = """Query LXD metadata and emit a JSON object."""
+    parser = argparse.ArgumentParser(description=description)
+    parser.parse_args()
+    print(util.json_dumps(read_metadata()))
+# vi: ts=4 expandtab
diff --git a/cloudinit/sources/tests/test_lxd.py b/cloudinit/sources/tests/test_lxd.py
new file mode 100644
index 00000000..c2027616
--- /dev/null
+++ b/cloudinit/sources/tests/test_lxd.py
@@ -0,0 +1,185 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+from collections import namedtuple
+from copy import deepcopy
+import stat
+from unittest import mock
+import yaml
+
+import pytest
+
+from cloudinit.sources import DataSourceLXD as lxd, UNSET
+DS_PATH = "cloudinit.sources.DataSourceLXD."
+
+
+LStatResponse = namedtuple("lstatresponse", "st_mode")
+
+
+NETWORK_V1 = {
+    "version": 1,
+    "config": [
+        {
+            "type": "physical", "name": "eth0",
+            "subnets": [{"type": "dhcp", "control": "auto"}]
+        }
+    ]
+}
+NETWORK_V1_MANUAL = deepcopy(NETWORK_V1)
+NETWORK_V1_MANUAL["config"][0]["subnets"][0]["control"] = "manual"
+
+
+def _add_network_v1_device(devname) -> dict:
+    """Helper to inject device name into default network v1 config."""
+    network_cfg = deepcopy(NETWORK_V1)
+    network_cfg["config"][0]["name"] = devname
+    return network_cfg
+
+
+LXD_V1_METADATA = {
+    "meta-data": "instance-id: my-lxc\nlocal-hostname: my-lxc\n\n",
+    "network-config": NETWORK_V1,
+    "user-data": "#cloud-config\npackages: [sl]\n",
+    "vendor-data": "#cloud-config\nruncmd: ['echo vendor-data']\n",
+    "1.0": {
+        "meta-data": "instance-id: my-lxc\nlocal-hostname: my-lxc\n\n",
+        "config": {
+            "user.user-data":
+                "instance-id: my-lxc\nlocal-hostname: my-lxc\n\n",
+            "user.vendor-data":
+                "#cloud-config\nruncmd: ['echo vendor-data']\n",
+            "user.network-config": yaml.safe_dump(NETWORK_V1),
+        }
+    }
+}
+
+
+@pytest.fixture
+def lxd_metadata():
+    return LXD_V1_METADATA
+
+
+@pytest.yield_fixture
+def lxd_ds(request, paths, lxd_metadata):
+    """
+    Return an instantiated DataSourceLXD.
+
+    This also performs the mocking required for the default test case:
+        * ``is_platform_viable`` returns True,
+        * ``read_metadata`` returns ``LXD_V1_METADATA``
+
+    (This uses the paths fixture for the required helpers.Paths object)
+    """
+    with mock.patch(DS_PATH + "is_platform_viable", return_value=True):
+        with mock.patch(DS_PATH + "read_metadata", return_value=lxd_metadata):
+            yield lxd.DataSourceLXD(
+                sys_cfg={}, distro=mock.Mock(), paths=paths
+            )
+
+
+class TestGenerateFallbackNetworkConfig:
+
+    @pytest.mark.parametrize(
+        "uname_machine,systemd_detect_virt,network_mode,expected", (
+            # None for systemd_detect_virt returns None from which
+            ({}, None, "", NETWORK_V1),
+            ({}, None, "dhcp", NETWORK_V1),
+            # invalid network_mode logs warning
+            ({}, None, "bogus", NETWORK_V1),
+            ({}, None, "link-local", NETWORK_V1_MANUAL),
+            ("anything", "lxc\n", "", NETWORK_V1),
+            # `uname -m` on kvm determines devname
+            ("x86_64", "kvm\n", "", _add_network_v1_device("enp5s0")),
+            ("ppc64le", "kvm\n", "", _add_network_v1_device("enp0s5")),
+            ("s390x", "kvm\n", "", _add_network_v1_device("enc9"))
+        )
+    )
+    @mock.patch(DS_PATH + "util.system_info")
+    @mock.patch(DS_PATH + "subp.subp")
+    @mock.patch(DS_PATH + "subp.which")
+    def test_net_v2_based_on_network_mode_virt_type_and_uname_machine(
+        self,
+        m_which,
+        m_subp,
+        m_system_info,
+        uname_machine,
+        systemd_detect_virt,
+        network_mode,
+        expected,
+        caplog
+    ):
+        """Return network config v2 based on uname -m, systemd-detect-virt.
+
+        LXC config network_mode of "link-local" will determine whether to set
+        "activation-mode: manual", leaving the interface down.
+        """
+        if systemd_detect_virt is None:
+            m_which.return_value = None
+        m_system_info.return_value = {"uname": ["", "", "", "", uname_machine]}
+        m_subp.return_value = (systemd_detect_virt, "")
+        assert expected == lxd.generate_fallback_network_config(
+            network_mode=network_mode
+        )
+        if systemd_detect_virt is None:
+            assert 0 == m_subp.call_count
+            assert 0 == m_system_info.call_count
+        else:
+            assert [
+                mock.call(["systemd-detect-virt"])
+            ] == m_subp.call_args_list
+            if systemd_detect_virt != "kvm\n":
+                assert 0 == m_system_info.call_count
+            else:
+                assert 1 == m_system_info.call_count
+        if network_mode not in ("dhcp", "", "link-local"):
+            assert "Ignoring unexpected value user.network_mode: {}".format(
+                network_mode
+            ) in caplog.text
+
+
+class TestDataSourceLXD:
+    def test_platform_info(self, lxd_ds):
+        assert "LXD" == lxd_ds.dsname
+        assert "lxd" == lxd_ds.cloud_name
+        assert "lxd" == lxd_ds.platform_type
+
+    def test_subplatform(self, lxd_ds):
+        assert "LXD socket API v. 1.0 (/dev/lxd/sock)" == lxd_ds.subplatform
+
+    def test__get_data(self, lxd_ds):
+        """get_data calls read_metadata, setting appropiate instance attrs."""
+        assert UNSET == lxd_ds._crawled_metadata
+        assert UNSET == lxd_ds._network_config
+        assert None is lxd_ds.userdata_raw
+        assert True is lxd_ds._get_data()
+        assert LXD_V1_METADATA == lxd_ds._crawled_metadata
+        # network-config is dumped from YAML
+        assert NETWORK_V1 == lxd_ds._network_config
+        # Any user-data and vendor-data are saved as raw
+        assert LXD_V1_METADATA["user-data"] == lxd_ds.userdata_raw
+        assert LXD_V1_METADATA["vendor-data"] == lxd_ds.vendordata_raw
+
+
+class TestIsPlatformViable:
+    @pytest.mark.parametrize(
+        "exists,lstat_mode,expected", (
+            (False, None, False),
+            (True, stat.S_IFREG, False),
+            (True, stat.S_IFSOCK, True),
+        )
+    )
+    @mock.patch(DS_PATH + "os.lstat")
+    @mock.patch(DS_PATH + "os.path.exists")
+    def test_expected_viable(
+        self, m_exists, m_lstat, exists, lstat_mode, expected
+    ):
+        """Return True only when LXD_SOCKET_PATH exists and is a socket."""
+        m_exists.return_value = exists
+        m_lstat.return_value = LStatResponse(lstat_mode)
+        assert expected is lxd.is_platform_viable()
+        m_exists.assert_has_calls([mock.call(lxd.LXD_SOCKET_PATH)])
+        if exists:
+            m_lstat.assert_has_calls([mock.call(lxd.LXD_SOCKET_PATH)])
+        else:
+            assert 0 == m_lstat.call_count
+
+# vi: ts=4 expandtab