ubuntu_advantage: rewrite cloud-config module

ubuntu-advantage-tools version 19 has a different command line
interface. Update cloud-init's config module to accept new
ubuntu_advantage configuration settings.

* Underscores better than hyphens: deprecate 'ubuntu-advantage'
  cloud-config key in favor of 'ubuntu_advantage'
* Attach machines with either sso credentials of UA user_token
* Services are enabled by name though an 'enable' list
* Raise warnings if deprecated ubuntu-advantage config keys are
  present, or errors if its config we cannott adapt to

Ubuntu Advantage support can now be configured via #cloud-config
with the following yaml:

ubuntu_advantage:
  token: 'thisismyubuntuadvantagetoken'
  enable: [esm, fips, livepatch]

Co-Authored-By: Daniel Watkins <daniel.watkins@canonical.com>

1 files changed, 191 insertions, 156 deletions

diff --git a/cloudinit/config/tests/test_ubuntu_advantage.py b/cloudinit/config/tests/test_ubuntu_advantage.py
index b7cf9bee..8c4161ef 100644
--- a/cloudinit/config/tests/test_ubuntu_advantage.py
+++ b/cloudinit/config/tests/test_ubuntu_advantage.py
@@ -1,10 +1,7 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
-import re
-from six import StringIO
-
 from cloudinit.config.cc_ubuntu_advantage import (
-    handle, maybe_install_ua_tools, run_commands, schema)
+    configure_ua, handle, maybe_install_ua_tools, schema)
 from cloudinit.config.schema import validate_cloudconfig_schema
 from cloudinit import util
 from cloudinit.tests.helpers import (
@@ -20,90 +17,120 @@ class FakeCloud(object):
         self.distro = distro
 
 
-class TestRunCommands(CiTestCase):
+class TestConfigureUA(CiTestCase):
 
     with_logs = True
     allowed_subp = [CiTestCase.SUBP_SHELL_TRUE]
 
     def setUp(self):
-        super(TestRunCommands, self).setUp()
+        super(TestConfigureUA, self).setUp()
         self.tmp = self.tmp_dir()
 
     @mock.patch('%s.util.subp' % MPATH)
-    def test_run_commands_on_empty_list(self, m_subp):
-        """When provided with an empty list, run_commands does nothing."""
-        run_commands([])
-        self.assertEqual('', self.logs.getvalue())
-        m_subp.assert_not_called()
-
-    def test_run_commands_on_non_list_or_dict(self):
-        """When provided an invalid type, run_commands raises an error."""
-        with self.assertRaises(TypeError) as context_manager:
-            run_commands(commands="I'm Not Valid")
+    def test_configure_ua_attach_error(self, m_subp):
+        """Errors from ua attach command are raised."""
+        m_subp.side_effect = util.ProcessExecutionError(
+            'Invalid token SomeToken')
+        with self.assertRaises(RuntimeError) as context_manager:
+            configure_ua(token='SomeToken')
         self.assertEqual(
-            "commands parameter was not a list or dict: I'm Not Valid",
+            'Failure attaching Ubuntu Advantage:\nUnexpected error while'
+            ' running command.\nCommand: -\nExit code: -\nReason: -\n'
+            'Stdout: Invalid token SomeToken\nStderr: -',
             str(context_manager.exception))
 
-    def test_run_command_logs_commands_and_exit_codes_to_stderr(self):
-        """All exit codes are logged to stderr."""
-        outfile = self.tmp_path('output.log', dir=self.tmp)
-
-        cmd1 = 'echo "HI" >> %s' % outfile
-        cmd2 = 'bogus command'
-        cmd3 = 'echo "MOM" >> %s' % outfile
-        commands = [cmd1, cmd2, cmd3]
-
-        mock_path = '%s.sys.stderr' % MPATH
-        with mock.patch(mock_path, new_callable=StringIO) as m_stderr:
-            with self.assertRaises(RuntimeError) as context_manager:
-                run_commands(commands=commands)
-
-        self.assertIsNotNone(
-            re.search(r'bogus: (command )?not found',
-                      str(context_manager.exception)),
-            msg='Expected bogus command not found')
-        expected_stderr_log = '\n'.join([
-            'Begin run command: {cmd}'.format(cmd=cmd1),
-            'End run command: exit(0)',
-            'Begin run command: {cmd}'.format(cmd=cmd2),
-            'ERROR: End run command: exit(127)',
-            'Begin run command: {cmd}'.format(cmd=cmd3),
-            'End run command: exit(0)\n'])
-        self.assertEqual(expected_stderr_log, m_stderr.getvalue())
-
-    def test_run_command_as_lists(self):
-        """When commands are specified as a list, run them in order."""
-        outfile = self.tmp_path('output.log', dir=self.tmp)
-
-        cmd1 = 'echo "HI" >> %s' % outfile
-        cmd2 = 'echo "MOM" >> %s' % outfile
-        commands = [cmd1, cmd2]
-        with mock.patch('%s.sys.stderr' % MPATH, new_callable=StringIO):
-            run_commands(commands=commands)
+    @mock.patch('%s.util.subp' % MPATH)
+    def test_configure_ua_attach_with_token(self, m_subp):
+        """When token is provided, attach the machine to ua using the token."""
+        configure_ua(token='SomeToken')
+        m_subp.assert_called_once_with(['ua', 'attach', 'SomeToken'])
+        self.assertEqual(
+            'DEBUG: Attaching to Ubuntu Advantage. ua attach SomeToken\n',
+            self.logs.getvalue())
+
+    @mock.patch('%s.util.subp' % MPATH)
+    def test_configure_ua_attach_on_service_error(self, m_subp):
+        """all services should be enabled and then any failures raised"""
 
+        def fake_subp(cmd, capture=None):
+            fail_cmds = [['ua', 'enable', svc] for svc in ['esm', 'cc']]
+            if cmd in fail_cmds and capture:
+                svc = cmd[-1]
+                raise util.ProcessExecutionError(
+                    'Invalid {} credentials'.format(svc.upper()))
+
+        m_subp.side_effect = fake_subp
+
+        with self.assertRaises(RuntimeError) as context_manager:
+            configure_ua(token='SomeToken', enable=['esm', 'cc', 'fips'])
+        self.assertEqual(
+            m_subp.call_args_list,
+            [mock.call(['ua', 'attach', 'SomeToken']),
+             mock.call(['ua', 'enable', 'esm'], capture=True),
+             mock.call(['ua', 'enable', 'cc'], capture=True),
+             mock.call(['ua', 'enable', 'fips'], capture=True)])
         self.assertIn(
-            'DEBUG: Running user-provided ubuntu-advantage commands',
+            'WARNING: Failure enabling "esm":\nUnexpected error'
+            ' while running command.\nCommand: -\nExit code: -\nReason: -\n'
+            'Stdout: Invalid ESM credentials\nStderr: -\n',
             self.logs.getvalue())
-        self.assertEqual('HI\nMOM\n', util.load_file(outfile))
         self.assertIn(
-            'WARNING: Non-ubuntu-advantage commands in ubuntu-advantage'
-            ' config:',
+            'WARNING: Failure enabling "cc":\nUnexpected error'
+            ' while running command.\nCommand: -\nExit code: -\nReason: -\n'
+            'Stdout: Invalid CC credentials\nStderr: -\n',
+            self.logs.getvalue())
+        self.assertEqual(
+            'Failure enabling Ubuntu Advantage service(s): "esm", "cc"',
+            str(context_manager.exception))
+
+    @mock.patch('%s.util.subp' % MPATH)
+    def test_configure_ua_attach_with_empty_services(self, m_subp):
+        """When services is an empty list, do not auto-enable attach."""
+        configure_ua(token='SomeToken', enable=[])
+        m_subp.assert_called_once_with(['ua', 'attach', 'SomeToken'])
+        self.assertEqual(
+            'DEBUG: Attaching to Ubuntu Advantage. ua attach SomeToken\n',
             self.logs.getvalue())
 
-    def test_run_command_dict_sorted_as_command_script(self):
-        """When commands are a dict, sort them and run."""
-        outfile = self.tmp_path('output.log', dir=self.tmp)
-        cmd1 = 'echo "HI" >> %s' % outfile
-        cmd2 = 'echo "MOM" >> %s' % outfile
-        commands = {'02': cmd1, '01': cmd2}
-        with mock.patch('%s.sys.stderr' % MPATH, new_callable=StringIO):
-            run_commands(commands=commands)
+    @mock.patch('%s.util.subp' % MPATH)
+    def test_configure_ua_attach_with_specific_services(self, m_subp):
+        """When services a list, only enable specific services."""
+        configure_ua(token='SomeToken', enable=['fips'])
+        self.assertEqual(
+            m_subp.call_args_list,
+            [mock.call(['ua', 'attach', 'SomeToken']),
+             mock.call(['ua', 'enable', 'fips'], capture=True)])
+        self.assertEqual(
+            'DEBUG: Attaching to Ubuntu Advantage. ua attach SomeToken\n',
+            self.logs.getvalue())
+
+    @mock.patch('%s.maybe_install_ua_tools' % MPATH, mock.MagicMock())
+    @mock.patch('%s.util.subp' % MPATH)
+    def test_configure_ua_attach_with_string_services(self, m_subp):
+        """When services a string, treat as singleton list and warn"""
+        configure_ua(token='SomeToken', enable='fips')
+        self.assertEqual(
+            m_subp.call_args_list,
+            [mock.call(['ua', 'attach', 'SomeToken']),
+             mock.call(['ua', 'enable', 'fips'], capture=True)])
+        self.assertEqual(
+            'WARNING: ubuntu_advantage: enable should be a list, not a'
+            ' string; treating as a single enable\n'
+            'DEBUG: Attaching to Ubuntu Advantage. ua attach SomeToken\n',
+            self.logs.getvalue())
 
-        expected_messages = [
-            'DEBUG: Running user-provided ubuntu-advantage commands']
-        for message in expected_messages:
-            self.assertIn(message, self.logs.getvalue())
-        self.assertEqual('MOM\nHI\n', util.load_file(outfile))
+    @mock.patch('%s.util.subp' % MPATH)
+    def test_configure_ua_attach_with_weird_services(self, m_subp):
+        """When services not string or list, warn but still attach"""
+        configure_ua(token='SomeToken', enable={'deffo': 'wont work'})
+        self.assertEqual(
+            m_subp.call_args_list,
+            [mock.call(['ua', 'attach', 'SomeToken'])])
+        self.assertEqual(
+            'WARNING: ubuntu_advantage: enable should be a list, not a'
+            ' dict; skipping enabling services\n'
+            'DEBUG: Attaching to Ubuntu Advantage. ua attach SomeToken\n',
+            self.logs.getvalue())
 
 
 @skipUnlessJsonSchema()
@@ -112,90 +139,50 @@ class TestSchema(CiTestCase, SchemaTestCaseMixin):
     with_logs = True
     schema = schema
 
-    def test_schema_warns_on_ubuntu_advantage_not_as_dict(self):
-        """If ubuntu-advantage configuration is not a dict, emit a warning."""
-        validate_cloudconfig_schema({'ubuntu-advantage': 'wrong type'}, schema)
+    @mock.patch('%s.maybe_install_ua_tools' % MPATH)
+    @mock.patch('%s.configure_ua' % MPATH)
+    def test_schema_warns_on_ubuntu_advantage_not_dict(self, _cfg, _):
+        """If ubuntu_advantage configuration is not a dict, emit a warning."""
+        validate_cloudconfig_schema({'ubuntu_advantage': 'wrong type'}, schema)
         self.assertEqual(
-            "WARNING: Invalid config:\nubuntu-advantage: 'wrong type' is not"
+            "WARNING: Invalid config:\nubuntu_advantage: 'wrong type' is not"
             " of type 'object'\n",
             self.logs.getvalue())
 
-    @mock.patch('%s.run_commands' % MPATH)
-    def test_schema_disallows_unknown_keys(self, _):
-        """Unknown keys in ubuntu-advantage configuration emit warnings."""
+    @mock.patch('%s.maybe_install_ua_tools' % MPATH)
+    @mock.patch('%s.configure_ua' % MPATH)
+    def test_schema_disallows_unknown_keys(self, _cfg, _):
+        """Unknown keys in ubuntu_advantage configuration emit warnings."""
         validate_cloudconfig_schema(
-            {'ubuntu-advantage': {'commands': ['ls'], 'invalid-key': ''}},
+            {'ubuntu_advantage': {'token': 'winner', 'invalid-key': ''}},
             schema)
         self.assertIn(
-            'WARNING: Invalid config:\nubuntu-advantage: Additional properties'
+            'WARNING: Invalid config:\nubuntu_advantage: Additional properties'
             " are not allowed ('invalid-key' was unexpected)",
             self.logs.getvalue())
 
-    def test_warn_schema_requires_commands(self):
-        """Warn when ubuntu-advantage configuration lacks commands."""
-        validate_cloudconfig_schema(
-            {'ubuntu-advantage': {}}, schema)
-        self.assertEqual(
-            "WARNING: Invalid config:\nubuntu-advantage: 'commands' is a"
-            " required property\n",
-            self.logs.getvalue())
-
-    @mock.patch('%s.run_commands' % MPATH)
-    def test_warn_schema_commands_is_not_list_or_dict(self, _):
-        """Warn when ubuntu-advantage:commands config is not a list or dict."""
+    @mock.patch('%s.maybe_install_ua_tools' % MPATH)
+    @mock.patch('%s.configure_ua' % MPATH)
+    def test_warn_schema_requires_token(self, _cfg, _):
+        """Warn if ubuntu_advantage configuration lacks token."""
         validate_cloudconfig_schema(
-            {'ubuntu-advantage': {'commands': 'broken'}}, schema)
+            {'ubuntu_advantage': {'enable': ['esm']}}, schema)
         self.assertEqual(
-            "WARNING: Invalid config:\nubuntu-advantage.commands: 'broken' is"
-            " not of type 'object', 'array'\n",
-            self.logs.getvalue())
+            "WARNING: Invalid config:\nubuntu_advantage:"
+            " 'token' is a required property\n", self.logs.getvalue())
 
-    @mock.patch('%s.run_commands' % MPATH)
-    def test_warn_schema_when_commands_is_empty(self, _):
-        """Emit warnings when ubuntu-advantage:commands is empty."""
-        validate_cloudconfig_schema(
-            {'ubuntu-advantage': {'commands': []}}, schema)
+    @mock.patch('%s.maybe_install_ua_tools' % MPATH)
+    @mock.patch('%s.configure_ua' % MPATH)
+    def test_warn_schema_services_is_not_list_or_dict(self, _cfg, _):
+        """Warn when ubuntu_advantage:enable config is not a list."""
         validate_cloudconfig_schema(
-            {'ubuntu-advantage': {'commands': {}}}, schema)
+            {'ubuntu_advantage': {'enable': 'needslist'}}, schema)
         self.assertEqual(
-            "WARNING: Invalid config:\nubuntu-advantage.commands: [] is too"
-            " short\nWARNING: Invalid config:\nubuntu-advantage.commands: {}"
-            " does not have enough properties\n",
+            "WARNING: Invalid config:\nubuntu_advantage: 'token' is a"
+            " required property\nubuntu_advantage.enable: 'needslist'"
+            " is not of type 'array'\n",
             self.logs.getvalue())
 
-    @mock.patch('%s.run_commands' % MPATH)
-    def test_schema_when_commands_are_list_or_dict(self, _):
-        """No warnings when ubuntu-advantage:commands are a list or dict."""
-        validate_cloudconfig_schema(
-            {'ubuntu-advantage': {'commands': ['valid']}}, schema)
-        validate_cloudconfig_schema(
-            {'ubuntu-advantage': {'commands': {'01': 'also valid'}}}, schema)
-        self.assertEqual('', self.logs.getvalue())
-
-    def test_duplicates_are_fine_array_array(self):
-        """Duplicated commands array/array entries are allowed."""
-        self.assertSchemaValid(
-            {'commands': [["echo", "bye"], ["echo" "bye"]]},
-            "command entries can be duplicate.")
-
-    def test_duplicates_are_fine_array_string(self):
-        """Duplicated commands array/string entries are allowed."""
-        self.assertSchemaValid(
-            {'commands': ["echo bye", "echo bye"]},
-            "command entries can be duplicate.")
-
-    def test_duplicates_are_fine_dict_array(self):
-        """Duplicated commands dict/array entries are allowed."""
-        self.assertSchemaValid(
-            {'commands': {'00': ["echo", "bye"], '01': ["echo", "bye"]}},
-            "command entries can be duplicate.")
-
-    def test_duplicates_are_fine_dict_string(self):
-        """Duplicated commands dict/string entries are allowed."""
-        self.assertSchemaValid(
-            {'commands': {'00': "echo bye", '01': "echo bye"}},
-            "command entries can be duplicate.")
-
 
 class TestHandle(CiTestCase):
 
@@ -205,41 +192,89 @@ class TestHandle(CiTestCase):
         super(TestHandle, self).setUp()
         self.tmp = self.tmp_dir()
 
-    @mock.patch('%s.run_commands' % MPATH)
     @mock.patch('%s.validate_cloudconfig_schema' % MPATH)
-    def test_handle_no_config(self, m_schema, m_run):
+    def test_handle_no_config(self, m_schema):
         """When no ua-related configuration is provided, nothing happens."""
         cfg = {}
         handle('ua-test', cfg=cfg, cloud=None, log=self.logger, args=None)
         self.assertIn(
-            "DEBUG: Skipping module named ua-test, no 'ubuntu-advantage' key"
-            " in config",
+            "DEBUG: Skipping module named ua-test, no 'ubuntu_advantage'"
+            ' configuration found',
             self.logs.getvalue())
         m_schema.assert_not_called()
-        m_run.assert_not_called()
 
+    @mock.patch('%s.configure_ua' % MPATH)
     @mock.patch('%s.maybe_install_ua_tools' % MPATH)
-    def test_handle_tries_to_install_ubuntu_advantage_tools(self, m_install):
+    def test_handle_tries_to_install_ubuntu_advantage_tools(
+            self, m_install, m_cfg):
         """If ubuntu_advantage is provided, try installing ua-tools package."""
-        cfg = {'ubuntu-advantage': {}}
+        cfg = {'ubuntu_advantage': {'token': 'valid'}}
         mycloud = FakeCloud(None)
         handle('nomatter', cfg=cfg, cloud=mycloud, log=self.logger, args=None)
         m_install.assert_called_once_with(mycloud)
 
+    @mock.patch('%s.configure_ua' % MPATH)
     @mock.patch('%s.maybe_install_ua_tools' % MPATH)
-    def test_handle_runs_commands_provided(self, m_install):
-        """When commands are specified as a list, run them."""
-        outfile = self.tmp_path('output.log', dir=self.tmp)
+    def test_handle_passes_credentials_and_services_to_configure_ua(
+            self, m_install, m_configure_ua):
+        """All ubuntu_advantage config keys are passed to configure_ua."""
+        cfg = {'ubuntu_advantage': {'token': 'token', 'enable': ['esm']}}
+        handle('nomatter', cfg=cfg, cloud=None, log=self.logger, args=None)
+        m_configure_ua.assert_called_once_with(
+            token='token', enable=['esm'])
+
+    @mock.patch('%s.maybe_install_ua_tools' % MPATH, mock.MagicMock())
+    @mock.patch('%s.configure_ua' % MPATH)
+    def test_handle_warns_on_deprecated_ubuntu_advantage_key_w_config(
+            self, m_configure_ua):
+        """Warning when ubuntu-advantage key is present with new config"""
+        cfg = {'ubuntu-advantage': {'token': 'token', 'enable': ['esm']}}
+        handle('nomatter', cfg=cfg, cloud=None, log=self.logger, args=None)
+        self.assertEqual(
+            'WARNING: Deprecated configuration key "ubuntu-advantage"'
+            ' provided. Expected underscore delimited "ubuntu_advantage";'
+            ' will attempt to continue.',
+            self.logs.getvalue().splitlines()[0])
+        m_configure_ua.assert_called_once_with(
+            token='token', enable=['esm'])
+
+    def test_handle_error_on_deprecated_commands_key_dashed(self):
+        """Error when commands is present in ubuntu-advantage key."""
+        cfg = {'ubuntu-advantage': {'commands': 'nogo'}}
+        with self.assertRaises(RuntimeError) as context_manager:
+            handle('nomatter', cfg=cfg, cloud=None, log=self.logger, args=None)
+        self.assertEqual(
+            'Deprecated configuration "ubuntu-advantage: commands" provided.'
+            ' Expected "token"',
+            str(context_manager.exception))
+
+    def test_handle_error_on_deprecated_commands_key_underscored(self):
+        """Error when commands is present in ubuntu_advantage key."""
+        cfg = {'ubuntu_advantage': {'commands': 'nogo'}}
+        with self.assertRaises(RuntimeError) as context_manager:
+            handle('nomatter', cfg=cfg, cloud=None, log=self.logger, args=None)
+        self.assertEqual(
+            'Deprecated configuration "ubuntu-advantage: commands" provided.'
+            ' Expected "token"',
+            str(context_manager.exception))
 
+    @mock.patch('%s.maybe_install_ua_tools' % MPATH, mock.MagicMock())
+    @mock.patch('%s.configure_ua' % MPATH)
+    def test_handle_prefers_new_style_config(
+            self, m_configure_ua):
+        """ubuntu_advantage should be preferred over ubuntu-advantage"""
         cfg = {
-            'ubuntu-advantage': {'commands': ['echo "HI" >> %s' % outfile,
-                                              'echo "MOM" >> %s' % outfile]}}
-        mock_path = '%s.sys.stderr' % MPATH
-        with self.allow_subp([CiTestCase.SUBP_SHELL_TRUE]):
-            with mock.patch(mock_path, new_callable=StringIO):
-                handle('nomatter', cfg=cfg, cloud=None, log=self.logger,
-                       args=None)
-        self.assertEqual('HI\nMOM\n', util.load_file(outfile))
+            'ubuntu-advantage': {'token': 'nope', 'enable': ['wrong']},
+            'ubuntu_advantage': {'token': 'token', 'enable': ['esm']},
+        }
+        handle('nomatter', cfg=cfg, cloud=None, log=self.logger, args=None)
+        self.assertEqual(
+            'WARNING: Deprecated configuration key "ubuntu-advantage"'
+            ' provided. Expected underscore delimited "ubuntu_advantage";'
+            ' will attempt to continue.',
+            self.logs.getvalue().splitlines()[0])
+        m_configure_ua.assert_called_once_with(
+            token='token', enable=['esm'])
 
 
 class TestMaybeInstallUATools(CiTestCase):
@@ -253,7 +288,7 @@ class TestMaybeInstallUATools(CiTestCase):
     @mock.patch('%s.util.which' % MPATH)
     def test_maybe_install_ua_tools_noop_when_ua_tools_present(self, m_which):
         """Do nothing if ubuntu-advantage-tools already exists."""
-        m_which.return_value = '/usr/bin/ubuntu-advantage'  # already installed
+        m_which.return_value = '/usr/bin/ua'  # already installed
         distro = mock.MagicMock()
         distro.update_package_sources.side_effect = RuntimeError(
             'Some apt error')