From 40a400e42603aa1b80d9f623bc779799b370c091 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 21 Sep 2016 15:45:45 -0400 Subject: subp: add 'update_env' argument In order for a caller to use 'env' argument of subp, they will realistically do: env = os.environ.copy() env['FOO'] = 'BZR' subp(cmd, env=env) This shortens that to be: subp(cmd, update_env={'FOO': 'BZR'}) Add tests, and update growpart tests to use mock when playing with os.environ. --- tests/unittests/test_handler/test_handler_growpart.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'tests/unittests/test_handler') diff --git a/tests/unittests/test_handler/test_handler_growpart.py b/tests/unittests/test_handler/test_handler_growpart.py index e653488a..e28067de 100644 --- a/tests/unittests/test_handler/test_handler_growpart.py +++ b/tests/unittests/test_handler/test_handler_growpart.py @@ -81,11 +81,11 @@ class TestConfig(TestCase): self.cloud = cloud.Cloud(None, self.paths, None, None, None) self.log = logging.getLogger("TestConfig") self.args = [] - os.environ = {} self.cloud_init = None self.handle = cc_growpart.handle + @mock.patch.dict("os.environ", clear=True) def test_no_resizers_auto_is_fine(self): with mock.patch.object( util, 'subp', @@ -98,6 +98,7 @@ class TestConfig(TestCase): mockobj.assert_called_once_with( ['growpart', '--help'], env={'LANG': 'C'}) + @mock.patch.dict("os.environ", clear=True) def test_no_resizers_mode_growpart_is_exception(self): with mock.patch.object( util, 'subp', @@ -110,6 +111,7 @@ class TestConfig(TestCase): mockobj.assert_called_once_with( ['growpart', '--help'], env={'LANG': 'C'}) + @mock.patch.dict("os.environ", clear=True) def test_mode_auto_prefers_growpart(self): with mock.patch.object( util, 'subp', -- cgit v1.2.3 From 02f6c4bb8cef17b3fe04ef4dc1ef199e20aeb4d9 Mon Sep 17 00:00:00 2001 From: Stéphane Graber Date: Thu, 29 Sep 2016 01:40:32 -0400 Subject: lxd: Update network config for LXD 2.3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prior to LXD 2.3, the bridge configuration was done through distro packaging. Thus, lxd module interacted with debconf. With 2.3 and higher, this is now done inside LXD itself, so we need to use "lxc network" there. For now, this perfectly matches what we had before with debconf and doesn't cover any of the new options. We can always add those later. A set of tests similar to what we had for debconf has been added to make sure things look good. This is tested in Yakkety container running LXD 2.3 and all options seem to be passed through as expected, giving me the bridge I defined. Signed-off-by: Stéphane Graber --- cloudinit/config/cc_lxd.py | 107 +++++++++++++++++++---- tests/unittests/test_handler/test_handler_lxd.py | 51 +++++++++++ 2 files changed, 140 insertions(+), 18 deletions(-) (limited to 'tests/unittests/test_handler') diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py index 0086840f..cead2c95 100644 --- a/cloudinit/config/cc_lxd.py +++ b/cloudinit/config/cc_lxd.py @@ -46,6 +46,7 @@ Example config: """ from cloudinit import util +import os distros = ['ubuntu'] @@ -105,25 +106,43 @@ def handle(name, cfg, cloud, log, args): # Set up lxd-bridge if bridge config is given dconf_comm = "debconf-communicate" - if bridge_cfg and util.which(dconf_comm): - debconf = bridge_to_debconf(bridge_cfg) + if bridge_cfg: + if os.path.exists("/etc/default/lxd-bridge") \ + and util.which(dconf_comm): + # Bridge configured through packaging + + debconf = bridge_to_debconf(bridge_cfg) + + # Update debconf database + try: + log.debug("Setting lxd debconf via " + dconf_comm) + data = "\n".join(["set %s %s" % (k, v) + for k, v in debconf.items()]) + "\n" + util.subp(['debconf-communicate'], data) + except Exception: + util.logexc(log, "Failed to run '%s' for lxd with" % + dconf_comm) + + # Remove the existing configuration file (forces re-generation) + util.del_file("/etc/default/lxd-bridge") + + # Run reconfigure + log.debug("Running dpkg-reconfigure for lxd") + util.subp(['dpkg-reconfigure', 'lxd', + '--frontend=noninteractive']) + else: + # Built-in LXD bridge support + cmd_create, cmd_attach = bridge_to_cmd(bridge_cfg) + if cmd_create: + log.debug("Creating lxd bridge: %s" % + " ".join(cmd_create)) + util.subp(cmd_create) + + if cmd_attach: + log.debug("Setting up default lxd bridge: %s" % + " ".join(cmd_create)) + util.subp(cmd_attach) - # Update debconf database - try: - log.debug("Setting lxd debconf via " + dconf_comm) - data = "\n".join(["set %s %s" % (k, v) - for k, v in debconf.items()]) + "\n" - util.subp(['debconf-communicate'], data) - except Exception: - util.logexc(log, "Failed to run '%s' for lxd with" % dconf_comm) - - # Remove the existing configuration file (forces re-generation) - util.del_file("/etc/default/lxd-bridge") - - # Run reconfigure - log.debug("Running dpkg-reconfigure for lxd") - util.subp(['dpkg-reconfigure', 'lxd', - '--frontend=noninteractive']) elif bridge_cfg: raise RuntimeError( "Unable to configure lxd bridge without %s." + dconf_comm) @@ -177,3 +196,55 @@ def bridge_to_debconf(bridge_cfg): raise Exception("invalid bridge mode \"%s\"" % bridge_cfg.get("mode")) return debconf + + +def bridge_to_cmd(bridge_cfg): + if bridge_cfg.get("mode") == "none": + return None, None + + bridge_name = bridge_cfg.get("name", "lxdbr0") + cmd_create = [] + cmd_attach = ["lxc", "network", "attach-profile", bridge_name, + "default", "eth0", "--force-local"] + + if bridge_cfg.get("mode") == "existing": + return None, cmd_attach + + if bridge_cfg.get("mode") != "new": + raise Exception("invalid bridge mode \"%s\"" % bridge_cfg.get("mode")) + + cmd_create = ["lxc", "network", "create", bridge_name] + + if bridge_cfg.get("ipv4_address") and bridge_cfg.get("ipv4_netmask"): + cmd_create.append("ipv4.address=%s/%s" % + (bridge_cfg.get("ipv4_address"), + bridge_cfg.get("ipv4_netmask"))) + + if bridge_cfg.get("ipv4_nat", "true") == "true": + cmd_create.append("ipv4.nat=true") + + if bridge_cfg.get("ipv4_dhcp_first") and \ + bridge_cfg.get("ipv4_dhcp_last"): + dhcp_range = "%s-%s" % (bridge_cfg.get("ipv4_dhcp_first"), + bridge_cfg.get("ipv4_dhcp_last")) + cmd_create.append("ipv4.dhcp.ranges=%s" % dhcp_range) + else: + cmd_create.append("ipv4.address=none") + + if bridge_cfg.get("ipv6_address") and bridge_cfg.get("ipv6_netmask"): + cmd_create.append("ipv6.address=%s/%s" % + (bridge_cfg.get("ipv6_address"), + bridge_cfg.get("ipv6_netmask"))) + + if bridge_cfg.get("ipv6_nat", "false") == "true": + cmd_create.append("ipv6.nat=true") + + else: + cmd_create.append("ipv6.address=none") + + if bridge_cfg.get("domain"): + cmd_create.append("dns.domain=%s" % bridge_cfg.get("domain")) + + cmd_create.append("--force-local") + + return cmd_create, cmd_attach diff --git a/tests/unittests/test_handler/test_handler_lxd.py b/tests/unittests/test_handler/test_handler_lxd.py index 6f90defb..14366a10 100644 --- a/tests/unittests/test_handler/test_handler_lxd.py +++ b/tests/unittests/test_handler/test_handler_lxd.py @@ -132,3 +132,54 @@ class TestLxd(t_help.TestCase): cc_lxd.bridge_to_debconf(data), {"lxd/setup-bridge": "false", "lxd/bridge-name": ""}) + + def test_lxd_cmd_new_full(self): + data = {"mode": "new", + "name": "testbr0", + "ipv4_address": "10.0.8.1", + "ipv4_netmask": "24", + "ipv4_dhcp_first": "10.0.8.2", + "ipv4_dhcp_last": "10.0.8.254", + "ipv4_dhcp_leases": "250", + "ipv4_nat": "true", + "ipv6_address": "fd98:9e0:3744::1", + "ipv6_netmask": "64", + "ipv6_nat": "true", + "domain": "lxd"} + self.assertEqual( + cc_lxd.bridge_to_cmd(data), + (["lxc", "network", "create", "testbr0", + "ipv4.address=10.0.8.1/24", "ipv4.nat=true", + "ipv4.dhcp.ranges=10.0.8.2-10.0.8.254", + "ipv6.address=fd98:9e0:3744::1/64", + "ipv6.nat=true", "dns.domain=lxd", + "--force-local"], + ["lxc", "network", "attach-profile", + "testbr0", "default", "eth0", "--force-local"])) + + def test_lxd_cmd_new_partial(self): + data = {"mode": "new", + "ipv6_address": "fd98:9e0:3744::1", + "ipv6_netmask": "64", + "ipv6_nat": "true"} + self.assertEqual( + cc_lxd.bridge_to_cmd(data), + (["lxc", "network", "create", "lxdbr0", "ipv4.address=none", + "ipv6.address=fd98:9e0:3744::1/64", "ipv6.nat=true", + "--force-local"], + ["lxc", "network", "attach-profile", + "lxdbr0", "default", "eth0", "--force-local"])) + + def test_lxd_cmd_existing(self): + data = {"mode": "existing", + "name": "testbr0"} + self.assertEqual( + cc_lxd.bridge_to_cmd(data), + (None, ["lxc", "network", "attach-profile", + "testbr0", "default", "eth0", "--force-local"])) + + def test_lxd_cmd_none(self): + data = {"mode": "none"} + self.assertEqual( + cc_lxd.bridge_to_cmd(data), + (None, None)) -- cgit v1.2.3 From e8730078df8c99696b1b684e09c803eef7c4926c Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 30 Sep 2016 15:53:42 -0400 Subject: Fix python2.6 things found running in centos 6. This gets the tests running in centos 6. * ProcessExecutionError: remove setting of .message Nothing in cloud-init seems to use .message anywhere, so it does not seem necessary. The reason to change it is that on 2.6 it spits out: cloudinit/util.py:286: DeprecationWarning: BaseException.message * tox.ini: add a centos6 environment the tox versions listed here replicate a centos6 install with packages from EPEL. You will still need a python2.6 to run this env so we do not enable it by default. --- cloudinit/sources/DataSourceAltCloud.py | 6 ++---- cloudinit/sources/helpers/azure.py | 2 +- cloudinit/util.py | 7 ++----- tests/unittests/test_handler/test_handler_apt_conf_v1.py | 2 +- tests/unittests/test_util.py | 2 +- tox.ini | 16 ++++++++++++++++ 6 files changed, 23 insertions(+), 12 deletions(-) (limited to 'tests/unittests/test_handler') diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py index 48136f7c..20345389 100644 --- a/cloudinit/sources/DataSourceAltCloud.py +++ b/cloudinit/sources/DataSourceAltCloud.py @@ -195,8 +195,7 @@ class DataSourceAltCloud(sources.DataSource): (cmd_out, _err) = util.subp(cmd) LOG.debug(('Command: %s\nOutput%s') % (' '.join(cmd), cmd_out)) except ProcessExecutionError as _err: - util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), - _err.message) + util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err) return False except OSError as _err: util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err) @@ -211,8 +210,7 @@ class DataSourceAltCloud(sources.DataSource): (cmd_out, _err) = util.subp(cmd) LOG.debug(('Command: %s\nOutput%s') % (' '.join(cmd), cmd_out)) except ProcessExecutionError as _err: - util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), - _err.message) + util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err) return False except OSError as _err: util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 689ed4cc..1b3e9b70 100644 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -232,7 +232,7 @@ class WALinuxAgentShim(object): def _get_value_from_leases_file(fallback_lease_file): leases = [] content = util.load_file(fallback_lease_file) - LOG.debug("content is {}".format(content)) + LOG.debug("content is %s", content) for line in content.splitlines(): if 'unknown-245' in line: # Example line from Ubuntu diff --git a/cloudinit/util.py b/cloudinit/util.py index eb3e5899..4cff83c5 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -199,7 +199,7 @@ def fully_decoded_payload(part): encoding = charset.input_codec else: encoding = 'utf-8' - return cte_payload.decode(encoding, errors='surrogateescape') + return cte_payload.decode(encoding, 'surrogateescape') return cte_payload @@ -282,9 +282,6 @@ class ProcessExecutionError(IOError): 'reason': self.reason, } IOError.__init__(self, message) - # For backward compatibility with Python 2. - if not hasattr(self, 'message'): - self.message = message class SeLinuxGuard(object): @@ -1821,7 +1818,7 @@ def subp(args, data=None, rcs=None, env=None, capture=True, shell=False, def ldecode(data, m='utf-8'): if not isinstance(data, bytes): return data - return data.decode(m, errors=decode) + return data.decode(m, decode) out = ldecode(out) err = ldecode(err) diff --git a/tests/unittests/test_handler/test_handler_apt_conf_v1.py b/tests/unittests/test_handler/test_handler_apt_conf_v1.py index 45714efd..64acc3e0 100644 --- a/tests/unittests/test_handler/test_handler_apt_conf_v1.py +++ b/tests/unittests/test_handler/test_handler_apt_conf_v1.py @@ -118,7 +118,7 @@ class TestConversion(TestCase): def test_convert_with_apt_mirror(self): mirror = 'http://my.mirror/ubuntu' f = cc_apt_configure.convert_to_v3_apt_format({'apt_mirror': mirror}) - self.assertIn(mirror, {m['uri'] for m in f['apt']['primary']}) + self.assertIn(mirror, set(m['uri'] for m in f['apt']['primary'])) def test_no_old_content(self): mirror = 'http://my.mirror/ubuntu' diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index fc6b9d40..881509aa 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -553,7 +553,7 @@ class TestSubp(helpers.TestCase): def test_subp_decode_invalid_utf8_replaces(self): (out, _err) = util.subp(self.stdin2out, capture=True, data=self.utf8_invalid) - expected = self.utf8_invalid.decode('utf-8', errors='replace') + expected = self.utf8_invalid.decode('utf-8', 'replace') self.assertEqual(out, expected) def test_subp_decode_strict_raises(self): diff --git a/tox.ini b/tox.ini index 729de2a6..277858ed 100644 --- a/tox.ini +++ b/tox.ini @@ -59,3 +59,19 @@ deps = pyflakes==1.1.0 flake8==2.5.4 hacking==0.10.2 + +[testenv:centos6] +basepython = python2.6 +commands = nosetests {posargs:tests} +deps = + # requirements + argparse==1.2.1 + jinja2==2.2.1 + pyyaml==3.10 + PrettyTable==0.7.2 + oauthlib==0.6.0 + configobj==4.6.0 + requests==2.6.0 + jsonpatch==1.2 + six==1.9.0 + -r{toxinidir}/test-requirements.txt -- cgit v1.2.3 From d8534561ba76db25b6fc0044eb1bfda63686e859 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Thu, 1 Sep 2016 15:49:20 -0500 Subject: Add support for snap create-user on Ubuntu Core images. Ubuntu Core images use the `snap create-user` to add users to an Ubuntu Core system. Add support for creating snap users by adding a key to the users dictionary. users: - name: bob snapuser: bob@bobcom.io Or via the 'snappy' dictionary: snappy: email: bob@bobcom.io Users may also create a snap user without contacting the SSO by providing a 'system-user' assertion by importing them into snapd. Additionally, Ubuntu Core systems have a read-only /etc/passwd such that the normal useradd/groupadd commands do not function without an additional flag, '--extrausers', which redirects the pwd to /var/lib/extrausers. Move the system_is_snappy() check from cc_snappy module to util for re-use and then update the Distro class to append '--extrausers' if the system is Ubuntu Core. --- cloudinit/config/cc_snap_config.py | 184 +++++++++++++ cloudinit/config/cc_snappy.py | 18 +- cloudinit/distros/__init__.py | 35 +++ cloudinit/util.py | 12 + config/cloud.cfg | 1 + doc/examples/cloud-config-user-groups.txt | 8 + .../test_distros/test_user_data_normalize.py | 65 +++++ .../unittests/test_handler/test_handler_snappy.py | 293 ++++++++++++++++++++- 8 files changed, 601 insertions(+), 15 deletions(-) create mode 100644 cloudinit/config/cc_snap_config.py (limited to 'tests/unittests/test_handler') diff --git a/cloudinit/config/cc_snap_config.py b/cloudinit/config/cc_snap_config.py new file mode 100644 index 00000000..275a2d09 --- /dev/null +++ b/cloudinit/config/cc_snap_config.py @@ -0,0 +1,184 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2016 Canonical Ltd. +# +# Author: Ryan Harper +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +Snappy +------ +**Summary:** snap_config modules allows configuration of snapd. + +This module uses the same ``snappy`` namespace for configuration but +acts only only a subset of the configuration. + +If ``assertions`` is set and the user has included a list of assertions +then cloud-init will collect the assertions into a single assertion file +and invoke ``snap ack `` which will attempt +to load the provided assertions into the snapd assertion database. + +If ``email`` is set, this value is used to create an authorized user for +contacting and installing snaps from the Ubuntu Store. This is done by +calling ``snap create-user`` command. + +If ``known`` is set to True, then it is expected the user also included +an assertion of type ``system-user``. When ``snap create-user`` is called +cloud-init will append '--known' flag which instructs snapd to look for +a system-user assertion with the details. If ``known`` is not set, then +``snap create-user`` will contact the Ubuntu SSO for validating and importing +a system-user for the instance. + +.. note:: + If the system is already managed, then cloud-init will not attempt to + create a system-user. + +**Internal name:** ``cc_snap_config`` + +**Module frequency:** per instance + +**Supported distros:** any with 'snapd' available + +**Config keys**:: + + #cloud-config + snappy: + assertions: + - | + + - | + + email: user@user.org + known: true + +""" + +from cloudinit import log as logging +from cloudinit.settings import PER_INSTANCE +from cloudinit import util + +LOG = logging.getLogger(__name__) + +frequency = PER_INSTANCE +SNAPPY_CMD = "snap" +ASSERTIONS_FILE = "/var/lib/cloud/instance/snapd.assertions" + + +""" +snappy: + assertions: + - | + + - | + + email: foo@foo.io + known: true +""" + + +def add_assertions(assertions=None): + """Import list of assertions. + + Import assertions by concatenating each assertion into a + string separated by a '\n'. Write this string to a instance file and + then invoke `snap ack /path/to/file` and check for errors. + If snap exits 0, then all assertions are imported. + """ + if not assertions: + assertions = [] + + if not isinstance(assertions, list): + raise ValueError('assertion parameter was not a list: %s', assertions) + + snap_cmd = [SNAPPY_CMD, 'ack'] + combined = "\n".join(assertions) + if len(combined) == 0: + raise ValueError("Assertion list is empty") + + for asrt in assertions: + LOG.debug('Acking: %s', asrt.split('\n')[0:2]) + + util.write_file(ASSERTIONS_FILE, combined.encode('utf-8')) + util.subp(snap_cmd + [ASSERTIONS_FILE], capture=True) + + +def add_snap_user(cfg=None): + """Add a snap system-user if provided with email under snappy config. + + - Check that system is not already managed. + - Check that if using a system-user assertion, that it's + imported into snapd. + + Returns a dictionary to be passed to Distro.create_user + """ + + if not cfg: + cfg = {} + + if not isinstance(cfg, dict): + raise ValueError('configuration parameter was not a dict: %s', cfg) + + snapuser = cfg.get('email', None) + if not snapuser: + return + + usercfg = { + 'snapuser': snapuser, + 'known': cfg.get('known', False), + } + + # query if we're already registered + out, _ = util.subp([SNAPPY_CMD, 'managed'], capture=True) + if out.strip() == "true": + LOG.warning('This device is already managed. ' + 'Skipping system-user creation') + return + + if usercfg.get('known'): + # Check that we imported a system-user assertion + out, _ = util.subp([SNAPPY_CMD, 'known', 'system-user'], + capture=True) + if len(out) == 0: + LOG.error('Missing "system-user" assertion. ' + 'Check "snappy" user-data assertions.') + return + + return usercfg + + +def handle(name, cfg, cloud, log, args): + cfgin = cfg.get('snappy') + if not cfgin: + LOG.debug('No snappy config provided, skipping') + return + + if not(util.system_is_snappy()): + LOG.debug("%s: system not snappy", name) + return + + assertions = cfgin.get('assertions', []) + if len(assertions) > 0: + LOG.debug('Importing user-provided snap assertions') + add_assertions(assertions) + + # Create a snap user if requested. + # Snap systems contact the store with a user's email + # and extract information needed to create a local user. + # A user may provide a 'system-user' assertion which includes + # the required information. Using such an assertion to create + # a local user requires specifying 'known: true' in the supplied + # user-data. + usercfg = add_snap_user(cfg=cfgin) + if usercfg: + cloud.distro.create_user(usercfg.get('snapuser'), **usercfg) diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index 36db9e67..e03ec483 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -257,24 +257,14 @@ def disable_enable_ssh(enabled): util.write_file(not_to_be_run, "cloud-init\n") -def system_is_snappy(): - # channel.ini is configparser loadable. - # snappy will move to using /etc/system-image/config.d/*.ini - # this is certainly not a perfect test, but good enough for now. - content = util.load_file("/etc/system-image/channel.ini", quiet=True) - if 'ubuntu-core' in content.lower(): - return True - if os.path.isdir("/etc/system-image/config.d/"): - return True - return False - - def set_snappy_command(): global SNAPPY_CMD if util.which("snappy-go"): SNAPPY_CMD = "snappy-go" - else: + elif util.which("snappy"): SNAPPY_CMD = "snappy" + else: + SNAPPY_CMD = "snap" LOG.debug("snappy command is '%s'", SNAPPY_CMD) @@ -289,7 +279,7 @@ def handle(name, cfg, cloud, log, args): LOG.debug("%s: System is not snappy. disabling", name) return - if sys_snappy.lower() == "auto" and not(system_is_snappy()): + if sys_snappy.lower() == "auto" and not(util.system_is_snappy()): LOG.debug("%s: 'auto' mode, and system not snappy", name) return diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 78adf5f9..4a726430 100755 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -367,6 +367,9 @@ class Distro(object): adduser_cmd = ['useradd', name] log_adduser_cmd = ['useradd', name] + if util.system_is_snappy(): + adduser_cmd.append('--extrausers') + log_adduser_cmd.append('--extrausers') # Since we are creating users, we want to carefully validate the # inputs. If something goes wrong, we can end up with a system @@ -445,6 +448,32 @@ class Distro(object): util.logexc(LOG, "Failed to create user %s", name) raise e + def add_snap_user(self, name, **kwargs): + """ + Add a snappy user to the system using snappy tools + """ + + snapuser = kwargs.get('snapuser') + known = kwargs.get('known', False) + adduser_cmd = ["snap", "create-user", "--sudoer", "--json"] + if known: + adduser_cmd.append("--known") + adduser_cmd.append(snapuser) + + # Run the command + LOG.debug("Adding snap user %s", name) + try: + (out, err) = util.subp(adduser_cmd, logstring=adduser_cmd, + capture=True) + LOG.debug("snap create-user returned: %s:%s", out, err) + jobj = util.load_json(out) + username = jobj.get('username', None) + except Exception as e: + util.logexc(LOG, "Failed to create snap user %s", name) + raise e + + return username + def create_user(self, name, **kwargs): """ Creates users for the system using the GNU passwd tools. This @@ -452,6 +481,10 @@ class Distro(object): distros where useradd is not desirable or not available. """ + # Add a snap user, if requested + if 'snapuser' in kwargs: + return self.add_snap_user(name, **kwargs) + # Add the user self.add_user(name, **kwargs) @@ -602,6 +635,8 @@ class Distro(object): def create_group(self, name, members=None): group_add_cmd = ['groupadd', name] + if util.system_is_snappy(): + group_add_cmd.append('--extrausers') if not members: members = [] diff --git a/cloudinit/util.py b/cloudinit/util.py index 4cff83c5..4b3fd0cb 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2374,3 +2374,15 @@ def get_installed_packages(target=None): pkgs_inst.add(re.sub(":.*", "", pkg)) return pkgs_inst + + +def system_is_snappy(): + # channel.ini is configparser loadable. + # snappy will move to using /etc/system-image/config.d/*.ini + # this is certainly not a perfect test, but good enough for now. + content = load_file("/etc/system-image/channel.ini", quiet=True) + if 'ubuntu-core' in content.lower(): + return True + if os.path.isdir("/etc/system-image/config.d/"): + return True + return False diff --git a/config/cloud.cfg b/config/cloud.cfg index d608dc86..1b93e7f9 100644 --- a/config/cloud.cfg +++ b/config/cloud.cfg @@ -45,6 +45,7 @@ cloud_config_modules: # Emit the cloud config ready event # this can be used by upstart jobs for 'start on cloud-config'. - emit_upstart + - snap_config - ssh-import-id - locale - set-passwords diff --git a/doc/examples/cloud-config-user-groups.txt b/doc/examples/cloud-config-user-groups.txt index 0e8ed243..9c5202f5 100644 --- a/doc/examples/cloud-config-user-groups.txt +++ b/doc/examples/cloud-config-user-groups.txt @@ -30,6 +30,7 @@ users: gecos: Magic Cloud App Daemon User inactive: true system: true + - snapuser: joe@joeuser.io # Valid Values: # name: The user's login name @@ -80,6 +81,13 @@ users: # cloud-init does not parse/check the syntax of the sudo # directive. # system: Create the user as a system user. This means no home directory. +# snapuser: Create a Snappy (Ubuntu-Core) user via the snap create-user +# command available on Ubuntu systems. If the user has an account +# on the Ubuntu SSO, specifying the email will allow snap to +# request a username and any public ssh keys and will import +# these into the system with username specifed by SSO account. +# If 'username' is not set in SSO, then username will be the +# shortname before the email domain. # # Default user creation: diff --git a/tests/unittests/test_distros/test_user_data_normalize.py b/tests/unittests/test_distros/test_user_data_normalize.py index b24888fc..33bf922d 100755 --- a/tests/unittests/test_distros/test_user_data_normalize.py +++ b/tests/unittests/test_distros/test_user_data_normalize.py @@ -4,6 +4,7 @@ from cloudinit import helpers from cloudinit import settings from ..helpers import TestCase +import mock bcfg = { @@ -296,3 +297,67 @@ class TestUGNormalize(TestCase): self.assertIn('bob', users) self.assertEqual({'default': False}, users['joe']) self.assertEqual({'default': False}, users['bob']) + + @mock.patch('cloudinit.util.subp') + def test_create_snap_user(self, mock_subp): + mock_subp.side_effect = [('{"username": "joe", "ssh-key-count": 1}\n', + '')] + distro = self._make_distro('ubuntu') + ug_cfg = { + 'users': [ + {'name': 'joe', 'snapuser': 'joe@joe.com'}, + ], + } + (users, _groups) = self._norm(ug_cfg, distro) + for (user, config) in users.items(): + print('user=%s config=%s' % (user, config)) + username = distro.create_user(user, **config) + + snapcmd = ['snap', 'create-user', '--sudoer', '--json', 'joe@joe.com'] + mock_subp.assert_called_with(snapcmd, capture=True, logstring=snapcmd) + self.assertEqual(username, 'joe') + + @mock.patch('cloudinit.util.subp') + def test_create_snap_user_known(self, mock_subp): + mock_subp.side_effect = [('{"username": "joe", "ssh-key-count": 1}\n', + '')] + distro = self._make_distro('ubuntu') + ug_cfg = { + 'users': [ + {'name': 'joe', 'snapuser': 'joe@joe.com', 'known': True}, + ], + } + (users, _groups) = self._norm(ug_cfg, distro) + for (user, config) in users.items(): + print('user=%s config=%s' % (user, config)) + username = distro.create_user(user, **config) + + snapcmd = ['snap', 'create-user', '--sudoer', '--json', '--known', + 'joe@joe.com'] + mock_subp.assert_called_with(snapcmd, capture=True, logstring=snapcmd) + self.assertEqual(username, 'joe') + + @mock.patch('cloudinit.util.system_is_snappy') + @mock.patch('cloudinit.util.is_group') + @mock.patch('cloudinit.util.subp') + def test_add_user_on_snappy_system(self, mock_subp, mock_isgrp, + mock_snappy): + mock_isgrp.return_value = False + mock_subp.return_value = True + mock_snappy.return_value = True + distro = self._make_distro('ubuntu') + ug_cfg = { + 'users': [ + {'name': 'joe', 'groups': 'users', 'create_groups': True}, + ], + } + (users, _groups) = self._norm(ug_cfg, distro) + for (user, config) in users.items(): + print('user=%s config=%s' % (user, config)) + distro.add_user(user, **config) + + groupcmd = ['groupadd', 'users', '--extrausers'] + addcmd = ['useradd', 'joe', '--extrausers', '--groups', 'users', '-m'] + + mock_subp.assert_any_call(groupcmd) + mock_subp.assert_any_call(addcmd, logstring=addcmd) diff --git a/tests/unittests/test_handler/test_handler_snappy.py b/tests/unittests/test_handler/test_handler_snappy.py index 57dce1bc..e320dd82 100644 --- a/tests/unittests/test_handler/test_handler_snappy.py +++ b/tests/unittests/test_handler/test_handler_snappy.py @@ -1,14 +1,22 @@ from cloudinit.config.cc_snappy import ( makeop, get_package_ops, render_snap_op) -from cloudinit import util +from cloudinit.config.cc_snap_config import ( + add_assertions, add_snap_user, ASSERTIONS_FILE) +from cloudinit import (distros, helpers, cloud, util) +from cloudinit.config.cc_snap_config import handle as snap_handle +from cloudinit.sources import DataSourceNone +from ..helpers import FilesystemMockingTestCase, mock from .. import helpers as t_help +import logging import os import shutil import tempfile +import textwrap import yaml +LOG = logging.getLogger(__name__) ALLOWED = (dict, list, int, str) @@ -287,6 +295,289 @@ class TestInstallPackages(t_help.TestCase): self.assertEqual(yaml.safe_load(mydata), data_found) +class TestSnapConfig(FilesystemMockingTestCase): + + SYSTEM_USER_ASSERTION = textwrap.dedent(""" + type: system-user + authority-id: LqvZQdfyfGlYvtep4W6Oj6pFXP9t1Ksp + brand-id: LqvZQdfyfGlYvtep4W6Oj6pFXP9t1Ksp + email: foo@bar.com + password: $6$E5YiAuMIPAwX58jG$miomhVNui/vf7f/3ctB/f0RWSKFxG0YXzrJ9rtJ1ikvzt + series: + - 16 + since: 2016-09-10T16:34:00+03:00 + until: 2017-11-10T16:34:00+03:00 + username: baz + sign-key-sha3-384: RuVvnp4n52GilycjfbbTCI3_L8Y6QlIE75wxMc0KzGV3AUQqVd9GuXoj + + AcLBXAQAAQoABgUCV/UU1wAKCRBKnlMoJQLkZVeLD/9/+hIeVywtzsDA3oxl+P+u9D13y9s6svP + Jd6Wnf4FTw6sq1GjBE4ZA7lrwSaRCUJ9Vcsvf2q9OGPY7mOb2TBxaDe0PbUMjrSrqllSSQwhpNI + zG+NxkkKuxsUmLzFa+k9m6cyojNbw5LFhQZBQCGlr3JYqC0tIREq/UsZxj+90TUC87lDJwkU8GF + s4CR+rejZj4itIcDcVxCSnJH6hv6j2JrJskJmvObqTnoOlcab+JXdamXqbldSP3UIhWoyVjqzkj + +to7mXgx+cCUA9+ngNCcfUG+1huGGTWXPCYkZ78HvErcRlIdeo4d3xwtz1cl/w3vYnq9og1XwsP + Yfetr3boig2qs1Y+j/LpsfYBYncgWjeDfAB9ZZaqQz/oc8n87tIPZDJHrusTlBfop8CqcM4xsKS + d+wnEY8e/F24mdSOYmS1vQCIDiRU3MKb6x138Ud6oHXFlRBbBJqMMctPqWDunWzb5QJ7YR0I39q + BrnEqv5NE0G7w6HOJ1LSPG5Hae3P4T2ea+ATgkb03RPr3KnXnzXg4TtBbW1nytdlgoNc/BafE1H + f3NThcq9gwX4xWZ2PAWnqVPYdDMyCtzW3Ck+o6sIzx+dh4gDLPHIi/6TPe/pUuMop9CBpWwez7V + v1z+1+URx6Xlq3Jq18y5pZ6fY3IDJ6km2nQPMzcm4Q==""") + + ACCOUNT_ASSERTION = textwrap.dedent(""" + type: account-key + authority-id: canonical + revision: 2 + public-key-sha3-384: BWDEoaqyr25nF5SNCvEv2v7QnM9QsfCc0PBMYD_i2NGSQ32EF2d4D0 + account-id: canonical + name: store + since: 2016-04-01T00:00:00.0Z + body-length: 717 + sign-key-sha3-384: -CvQKAwRQ5h3Ffn10FILJoEZUXOv6km9FwA80-Rcj-f-6jadQ89VRswH + + AcbBTQRWhcGAARAA0KKYYQWuHOrsFVi4p4l7ZzSvX7kLgJFFeFgOkzdWKBTHEnsMKjl5mefFe9j + qe8NlmJdfY7BenP7XeBtwKp700H/t9lLrZbpTNAPHXYxEWFJp5bPqIcJYBZ+29oLVLN1Tc5X482 + vCiDqL8+pPYqBrK2fNlyPlNNSum9wI70rDDL4r6FVvr+osTnGejibdV8JphWX+lrSQDnRSdM8KJ + UM43vTgLGTi9W54oRhsA2OFexRfRksTrnqGoonCjqX5wO3OFSaMDzMsO2MJ/hPfLgDqw53qjzuK + Iec9OL3k5basvu2cj5u9tKwVFDsCKK2GbKUsWWpx2KTpOifmhmiAbzkTHbH9KaoMS7p0kJwhTQG + o9aJ9VMTWHJc/NCBx7eu451u6d46sBPCXS/OMUh2766fQmoRtO1OwCTxsRKG2kkjbMn54UdFULl + VfzvyghMNRKIezsEkmM8wueTqGUGZWa6CEZqZKwhe/PROxOPYzqtDH18XZknbU1n5lNb7vNfem9 + 2ai+3+JyFnW9UhfvpVF7gzAgdyCqNli4C6BIN43uwoS8HkykocZS/+Gv52aUQ/NZ8BKOHLw+7an + Q0o8W9ltSLZbEMxFIPSN0stiZlkXAp6DLyvh1Y4wXSynDjUondTpej2fSvSlCz/W5v5V7qA4nIc + vUvV7RjVzv17ut0AEQEAAQ== + + AcLDXAQAAQoABgUCV83k9QAKCRDUpVvql9g3IBT8IACKZ7XpiBZ3W4lqbPssY6On81WmxQLtvsM + WTp6zZpl/wWOSt2vMNUk9pvcmrNq1jG9CuhDfWFLGXEjcrrmVkN3YuCOajMSPFCGrxsIBLSRt/b + nrKykdLAAzMfG8rP1d82bjFFiIieE+urQ0Kcv09Jtdvavq3JT1Tek5mFyyfhHNlQEKOzWqmRWiL + 3c3VOZUs1ZD8TSlnuq/x+5T0X0YtOyGjSlVxk7UybbyMNd6MZfNaMpIG4x+mxD3KHFtBAC7O6kL + eX3i6j5nCY5UABfA3DZEAkWP4zlmdBEOvZ9t293NaDdOpzsUHRkoi0Zez/9BHQ/kwx/uNc2WqrY + inCmu16JGNeXqsyinnLl7Ghn2RwhvDMlLxF6RTx8xdx1yk6p3PBTwhZMUvuZGjUtN/AG8BmVJQ1 + rsGSRkkSywvnhVJRB2sudnrMBmNS2goJbzSbmJnOlBrd2WsV0T9SgNMWZBiov3LvU4o2SmAb6b+ + rYwh8H5QHcuuYJuxDjFhPswIp6Wes5T6hUicf3SWtObcDS4HSkVS4ImBjjX9YgCuFy7QdnooOWE + aPvkRw3XCVeYq0K6w9GRsk1YFErD4XmXXZjDYY650MX9v42Sz5MmphHV8jdIY5ssbadwFSe2rCQ + 6UX08zy7RsIb19hTndE6ncvSNDChUR9eEnCm73eYaWTWTnq1cxdVP/s52r8uss++OYOkPWqh5nO + haRn7INjH/yZX4qXjNXlTjo0PnHH0q08vNKDwLhxS+D9du+70FeacXFyLIbcWllSbJ7DmbumGpF + yYbtj3FDDPzachFQdIG3lSt+cSUGeyfSs6wVtc3cIPka/2Urx7RprfmoWSI6+a5NcLdj0u2z8O9 + HxeIgxDpg/3gT8ZIuFKePMcLDM19Fh/p0ysCsX+84B9chNWtsMSmIaE57V+959MVtsLu7SLb9gi + skrju0pQCwsu2wHMLTNd1f3PTHmrr49hxetTus07HSQUApMtAGKzQilF5zqFjbyaTd4xgQbd+PK + CjFyzQTDOcUhXpuUGt/IzlqiFfsCsmbj2K4KdSNYMlqIgZ3Azu8KvZLIhsyN7v5vNIZSPfEbjde + ClU9r0VRiJmtYBUjcSghD9LWn+yRLwOxhfQVjm0cBwIt5R/yPF/qC76yIVuWUtM5Y2/zJR1J8OF + qWchvlImHtvDzS9FQeLyzJAOjvZ2CnWp2gILgUz0WQdOk1Dq8ax7KS9BQ42zxw9EZAEPw3PEFqR + IQsRTONp+iVS8YxSmoYZjDlCgRMWUmawez/Fv5b9Fb/XkO5Eq4e+KfrpUujXItaipb+tV8h5v3t + oG3Ie3WOHrVjCLXIdYslpL1O4nadqR6Xv58pHj6k""") + + test_assertions = [ACCOUNT_ASSERTION, SYSTEM_USER_ASSERTION] + + def setUp(self): + super(TestSnapConfig, self).setUp() + self.subp = util.subp + self.new_root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.new_root) + + def _get_cloud(self, distro, metadata=None): + self.patchUtils(self.new_root) + paths = helpers.Paths({}) + cls = distros.fetch(distro) + mydist = cls(distro, {}, paths) + myds = DataSourceNone.DataSourceNone({}, mydist, paths) + if metadata: + myds.metadata.update(metadata) + return cloud.Cloud(myds, paths, {}, mydist, None) + + @mock.patch('cloudinit.util.write_file') + @mock.patch('cloudinit.util.subp') + def test_snap_config_add_assertions(self, msubp, mwrite): + add_assertions(self.test_assertions) + + combined = "\n".join(self.test_assertions) + mwrite.assert_any_call(ASSERTIONS_FILE, combined.encode('utf-8')) + msubp.assert_called_with(['snap', 'ack', ASSERTIONS_FILE], + capture=True) + + def test_snap_config_add_assertions_empty(self): + self.assertRaises(ValueError, add_assertions, []) + + def test_add_assertions_nonlist(self): + self.assertRaises(ValueError, add_assertions, {}) + + @mock.patch('cloudinit.util.write_file') + @mock.patch('cloudinit.util.subp') + def test_snap_config_add_assertions_ack_fails(self, msubp, mwrite): + msubp.side_effect = [util.ProcessExecutionError("Invalid assertion")] + self.assertRaises(util.ProcessExecutionError, add_assertions, + self.test_assertions) + + @mock.patch('cloudinit.config.cc_snap_config.add_assertions') + @mock.patch('cloudinit.config.cc_snap_config.util') + def test_snap_config_handle_no_config(self, mock_util, mock_add): + cfg = {} + cc = self._get_cloud('ubuntu') + cc.distro = mock.MagicMock() + cc.distro.name = 'ubuntu' + mock_util.which.return_value = None + snap_handle('snap_config', cfg, cc, LOG, None) + mock_add.assert_not_called() + + def test_snap_config_add_snap_user_no_config(self): + usercfg = add_snap_user(cfg=None) + self.assertEqual(usercfg, None) + + def test_snap_config_add_snap_user_not_dict(self): + cfg = ['foobar'] + self.assertRaises(ValueError, add_snap_user, cfg) + + def test_snap_config_add_snap_user_no_email(self): + cfg = {'assertions': [], 'known': True} + usercfg = add_snap_user(cfg=cfg) + self.assertEqual(usercfg, None) + + @mock.patch('cloudinit.config.cc_snap_config.util') + def test_snap_config_add_snap_user_email_only(self, mock_util): + email = 'janet@planetjanet.org' + cfg = {'email': email} + mock_util.which.return_value = None + mock_util.system_is_snappy.return_value = True + mock_util.subp.side_effect = [ + ("false\n", ""), # snap managed + ] + + usercfg = add_snap_user(cfg=cfg) + + self.assertEqual(usercfg, {'snapuser': email, 'known': False}) + + @mock.patch('cloudinit.config.cc_snap_config.util') + def test_snap_config_add_snap_user_email_known(self, mock_util): + email = 'janet@planetjanet.org' + known = True + cfg = {'email': email, 'known': known} + mock_util.which.return_value = None + mock_util.system_is_snappy.return_value = True + mock_util.subp.side_effect = [ + ("false\n", ""), # snap managed + (self.SYSTEM_USER_ASSERTION, ""), # snap known system-user + ] + + usercfg = add_snap_user(cfg=cfg) + + self.assertEqual(usercfg, {'snapuser': email, 'known': known}) + + @mock.patch('cloudinit.config.cc_snap_config.add_assertions') + @mock.patch('cloudinit.config.cc_snap_config.util') + def test_snap_config_handle_system_not_snappy(self, mock_util, mock_add): + cfg = {'snappy': {'assertions': self.test_assertions}} + cc = self._get_cloud('ubuntu') + cc.distro = mock.MagicMock() + cc.distro.name = 'ubuntu' + mock_util.which.return_value = None + mock_util.system_is_snappy.return_value = False + + snap_handle('snap_config', cfg, cc, LOG, None) + + mock_add.assert_not_called() + + @mock.patch('cloudinit.config.cc_snap_config.add_assertions') + @mock.patch('cloudinit.config.cc_snap_config.util') + def test_snap_config_handle_snapuser(self, mock_util, mock_add): + email = 'janet@planetjanet.org' + cfg = { + 'snappy': { + 'assertions': self.test_assertions, + 'email': email, + } + } + cc = self._get_cloud('ubuntu') + cc.distro = mock.MagicMock() + cc.distro.name = 'ubuntu' + mock_util.which.return_value = None + mock_util.system_is_snappy.return_value = True + mock_util.subp.side_effect = [ + ("false\n", ""), # snap managed + ] + + snap_handle('snap_config', cfg, cc, LOG, None) + + mock_add.assert_called_with(self.test_assertions) + usercfg = {'snapuser': email, 'known': False} + cc.distro.create_user.assert_called_with(email, **usercfg) + + @mock.patch('cloudinit.config.cc_snap_config.add_assertions') + @mock.patch('cloudinit.config.cc_snap_config.util') + def test_snap_config_handle_snapuser_known(self, mock_util, mock_add): + email = 'janet@planetjanet.org' + cfg = { + 'snappy': { + 'assertions': self.test_assertions, + 'email': email, + 'known': True, + } + } + cc = self._get_cloud('ubuntu') + cc.distro = mock.MagicMock() + cc.distro.name = 'ubuntu' + mock_util.which.return_value = None + mock_util.system_is_snappy.return_value = True + mock_util.subp.side_effect = [ + ("false\n", ""), # snap managed + (self.SYSTEM_USER_ASSERTION, ""), # snap known system-user + ] + + snap_handle('snap_config', cfg, cc, LOG, None) + + mock_add.assert_called_with(self.test_assertions) + usercfg = {'snapuser': email, 'known': True} + cc.distro.create_user.assert_called_with(email, **usercfg) + + @mock.patch('cloudinit.config.cc_snap_config.add_assertions') + @mock.patch('cloudinit.config.cc_snap_config.util') + def test_snap_config_handle_snapuser_known_managed(self, mock_util, + mock_add): + email = 'janet@planetjanet.org' + cfg = { + 'snappy': { + 'assertions': self.test_assertions, + 'email': email, + 'known': True, + } + } + cc = self._get_cloud('ubuntu') + cc.distro = mock.MagicMock() + cc.distro.name = 'ubuntu' + mock_util.which.return_value = None + mock_util.system_is_snappy.return_value = True + mock_util.subp.side_effect = [ + ("true\n", ""), # snap managed + ] + + snap_handle('snap_config', cfg, cc, LOG, None) + + mock_add.assert_called_with(self.test_assertions) + cc.distro.create_user.assert_not_called() + + @mock.patch('cloudinit.config.cc_snap_config.add_assertions') + @mock.patch('cloudinit.config.cc_snap_config.util') + def test_snap_config_handle_snapuser_known_no_assertion(self, mock_util, + mock_add): + email = 'janet@planetjanet.org' + cfg = { + 'snappy': { + 'assertions': [self.ACCOUNT_ASSERTION], + 'email': email, + 'known': True, + } + } + cc = self._get_cloud('ubuntu') + cc.distro = mock.MagicMock() + cc.distro.name = 'ubuntu' + mock_util.which.return_value = None + mock_util.system_is_snappy.return_value = True + mock_util.subp.side_effect = [ + ("true\n", ""), # snap managed + ("", ""), # snap known system-user + ] + + snap_handle('snap_config', cfg, cc, LOG, None) + + mock_add.assert_called_with([self.ACCOUNT_ASSERTION]) + cc.distro.create_user.assert_not_called() + + def makeop_tmpd(tmpd, op, name, config=None, path=None, cfgfile=None): if cfgfile: cfgfile = os.path.sep.join([tmpd, cfgfile]) -- cgit v1.2.3