Add LXD datasource (#1040)

Add DataSourceLXD which knows how to talk to the dev-lxd socket to
obtain all instance metadata API:
https://linuxcontainers.org/lxd/docs/master/dev-lxd.

This first branch is to deliver feature parity with the existing
NoCloud datasource which is currently used to intialize LXC instances
on first boot.

Introduce a SocketConnectionPool and LXDSocketAdapter to support
performing HTTP GETs on the following routes which are surfaced by the
LXD host to all containers:
http://unix.socket/1.0/meta-data
http://unix.socket/1.0/config/user.user-data
http://unix.socket/1.0/config/user.network-config
http://unix.socket/1.0/config/user.vendor-data
These 4 routes minimally replace the static content provided in the
following nocloud-net seed files:
/var/lib/cloud/nocloud-net/{meta-data,vendor-data,user-data,network-config}

The intent of this commit is to set a foundation for LXD socket
communication that will allow us to build network hot-plug features
by eventually consuming LXD's websocket upgrade route 1.0/events to
react to network, meta-data and user-data config changes over time.

In the event that no custom network-config is provided, default to the
same network-config definition provided by LXD to the NoCloud
network-config seed file.

Supplemental features above NoCloud datasource:
surface all custom instance data config keys via cloud-init query ds
which aids in discoverability of features/tags/labels as well as
conditional #cloud-config jinja templates operations based on custom
config options.
TBD: better cloud-init query support for dot-delimited keys

2 files changed, 88 insertions, 1 deletions

diff --git a/tests/integration_tests/clouds.py b/tests/integration_tests/clouds.py
index 32fdc91e..dee2adff 100644
--- a/tests/integration_tests/clouds.py
+++ b/tests/integration_tests/clouds.py
@@ -1,7 +1,10 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 from abc import ABC, abstractmethod
+import datetime
 import logging
 import os.path
+import random
+import string
 from uuid import UUID
 
 from pycloudlib import (
@@ -309,8 +312,12 @@ class _LxdIntegrationCloud(IntegrationCloud):
         except KeyError:
             profile_list = self._get_or_set_profile_list(release)
 
+        prefix = datetime.datetime.utcnow().strftime("cloudinit-%m%d-%H%M%S")
+        default_name = prefix + "".join(
+            random.choices(string.ascii_lowercase + string.digits, k=8)
+        )
         pycloudlib_instance = self.cloud_instance.init(
-            launch_kwargs.pop('name', None),
+            launch_kwargs.pop('name', default_name),
             release,
             profile_list=profile_list,
             **launch_kwargs
diff --git a/tests/integration_tests/datasources/test_lxd_discovery.py b/tests/integration_tests/datasources/test_lxd_discovery.py
new file mode 100644
index 00000000..f7d8bfc3
--- /dev/null
+++ b/tests/integration_tests/datasources/test_lxd_discovery.py
@@ -0,0 +1,80 @@
+import json
+import pytest
+import yaml
+
+from tests.integration_tests.clouds import IntegrationCloud
+from tests.integration_tests.conftest import get_validated_source
+from tests.integration_tests.util import verify_clean_log
+
+
+def _setup_custom_image(session_cloud: IntegrationCloud):
+    """Like `setup_image` in conftest.py, but with customized content."""
+    source = get_validated_source(session_cloud)
+    if not source.installs_new_version():
+        return
+    client = session_cloud.launch()
+
+    # Insert our "detect_lxd_ds" file here
+    client.write_to_file(
+        '/etc/cloud/cloud.cfg.d/99-detect-lxd.cfg',
+        'datasource_list: [LXD]\n',
+    )
+
+    client.execute('rm -f /etc/netplan/50-cloud-init.yaml')
+
+    client.install_new_cloud_init(source)
+    # Even if we're keeping instances, we don't want to keep this
+    # one around as it was just for image creation
+    client.destroy()
+
+
+# This test should be able to work on any cloud whose datasource specifies
+# a NETWORK dependency
+@pytest.mark.lxd_container
+@pytest.mark.lxd_vm
+@pytest.mark.ubuntu  # Because netplan
+def test_lxd_datasource_discovery(session_cloud: IntegrationCloud):
+    """Test that DataSourceLXD is detected instead of NoCloud."""
+    _setup_custom_image(session_cloud)
+    nic_dev = "eth0"
+    if session_cloud.settings.PLATFORM == "lxd_vm":
+        nic_dev = "enp5s0"
+
+    with session_cloud.launch() as client:
+        result = client.execute('cloud-init status --long')
+        if not result.ok:
+            raise AssertionError('cloud-init failed:\n%s',
+                                 result.stderr)
+        if "DataSourceLXD" not in result.stdout:
+            raise AssertionError(
+                'cloud-init did not discover DataSourceLXD', result.stdout
+            )
+        netplan_yaml = client.execute('cat /etc/netplan/50-cloud-init.yaml')
+        netplan_cfg = yaml.safe_load(netplan_yaml)
+        assert {
+            'network': {'ethernets': {nic_dev: {'dhcp4': True}}, 'version': 2}
+        } == netplan_cfg
+        log = client.read_from_file('/var/log/cloud-init.log')
+        verify_clean_log(log)
+        result = client.execute('cloud-id')
+        if "lxd" != result.stdout:
+            raise AssertionError(
+                "cloud-id didn't report lxd. Result: %s", result.stdout
+            )
+        # Validate config instance data represented
+        data = json.loads(client.read_from_file(
+            '/run/cloud-init/instance-data.json')
+        )
+        v1 = data["v1"]
+        ds_cfg = data["ds"]
+        assert "lxd" == v1["platform"]
+        assert "LXD socket API v. 1.0 (/dev/lxd/sock)" == v1["subplatform"]
+        ds_cfg = json.loads(client.execute('cloud-init query ds').stdout)
+        assert ["config", "meta_data"] == sorted(list(ds_cfg["1.0"].keys()))
+        assert ["user.meta_data"] == list(ds_cfg["1.0"]["config"].keys())
+        assert {"public-keys": v1["public_ssh_keys"][0]} == (
+            yaml.safe_load(ds_cfg["1.0"]["config"]["user.meta_data"])
+        )
+        assert (
+            "#cloud-config\ninstance-id" in ds_cfg["1.0"]["meta_data"]
+        )