From 4292a47b3cefed2f6023e6007277fcaf2deb0f3e Mon Sep 17 00:00:00 2001 From: Dimitri John Ledkov <19779+xnox@users.noreply.github.com> Date: Wed, 25 Nov 2020 15:32:35 +0000 Subject: cla: add xnox (#692) --- tools/.github-cla-signers | 1 + 1 file changed, 1 insertion(+) (limited to 'tools/.github-cla-signers') diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 1e0c3ea4..00d02a2d 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -31,3 +31,4 @@ TheRealFalcon tomponline tsanghan WebSpider +xnox -- cgit v1.2.3 From e5f745954b273fbf1f917f034669c15720aa0f89 Mon Sep 17 00:00:00 2001 From: Olivier Lemasle Date: Wed, 16 Dec 2020 20:44:31 +0100 Subject: doc: fix CloudStack configuration example (#707) This commit removes lines which are invalid configuration. --- doc/rtd/topics/datasources/cloudstack.rst | 2 -- tools/.github-cla-signers | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) (limited to 'tools/.github-cla-signers') diff --git a/doc/rtd/topics/datasources/cloudstack.rst b/doc/rtd/topics/datasources/cloudstack.rst index a24de34f..325aeeaf 100644 --- a/doc/rtd/topics/datasources/cloudstack.rst +++ b/doc/rtd/topics/datasources/cloudstack.rst @@ -46,8 +46,6 @@ An example configuration with the default values is provided below: CloudStack: max_wait: 120 timeout: 50 - datasource_list: - - CloudStack .. _Apache CloudStack: http://cloudstack.apache.org/ diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 00d02a2d..c278b032 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -21,6 +21,7 @@ manuelisimo marlluslustosa matthewruffell nishigori +olivierlemasle omBratteng onitake riedel -- cgit v1.2.3 From a5484d02973e5710442c11e1dc6b1153695c9a59 Mon Sep 17 00:00:00 2001 From: cawamata <1749824+cawamata@users.noreply.github.com> Date: Fri, 18 Dec 2020 00:59:48 +0900 Subject: cc_ca_certs: add RHEL support (#633) This refactors cc_ca_certs to support non-ca-certificates distros, and adds RHEL support. --- cloudinit/config/cc_ca_certs.py | 123 ++++++--- .../test_handler/test_handler_ca_certs.py | 292 +++++++++++++-------- tools/.github-cla-signers | 1 + 3 files changed, 267 insertions(+), 149 deletions(-) (limited to 'tools/.github-cla-signers') diff --git a/cloudinit/config/cc_ca_certs.py b/cloudinit/config/cc_ca_certs.py index 3c453d91..bd7bead9 100644 --- a/cloudinit/config/cc_ca_certs.py +++ b/cloudinit/config/cc_ca_certs.py @@ -25,7 +25,7 @@ can be removed from the system with the configuration option **Module frequency:** per instance -**Supported distros:** alpine, debian, ubuntu +**Supported distros:** alpine, debian, ubuntu, rhel **Config keys**:: @@ -44,60 +44,104 @@ import os from cloudinit import subp from cloudinit import util -CA_CERT_PATH = "/usr/share/ca-certificates/" -CA_CERT_FILENAME = "cloud-init-ca-certs.crt" -CA_CERT_CONFIG = "/etc/ca-certificates.conf" -CA_CERT_SYSTEM_PATH = "/etc/ssl/certs/" -CA_CERT_FULL_PATH = os.path.join(CA_CERT_PATH, CA_CERT_FILENAME) +DEFAULT_CONFIG = { + 'ca_cert_path': '/usr/share/ca-certificates/', + 'ca_cert_filename': 'cloud-init-ca-certs.crt', + 'ca_cert_config': '/etc/ca-certificates.conf', + 'ca_cert_system_path': '/etc/ssl/certs/', + 'ca_cert_update_cmd': ['update-ca-certificates'] +} +DISTRO_OVERRIDES = { + 'rhel': { + 'ca_cert_path': '/usr/share/pki/ca-trust-source/', + 'ca_cert_filename': 'anchors/cloud-init-ca-certs.crt', + 'ca_cert_config': None, + 'ca_cert_system_path': '/etc/pki/ca-trust/', + 'ca_cert_update_cmd': ['update-ca-trust'] + } +} -distros = ['alpine', 'debian', 'ubuntu'] +distros = ['alpine', 'debian', 'ubuntu', 'rhel'] -def update_ca_certs(): + +def _distro_ca_certs_configs(distro_name): + """Return a distro-specific ca_certs config dictionary + + @param distro_name: String providing the distro class name. + @returns: Dict of distro configurations for ca-cert. + """ + cfg = DISTRO_OVERRIDES.get(distro_name, DEFAULT_CONFIG) + cfg['ca_cert_full_path'] = os.path.join(cfg['ca_cert_path'], + cfg['ca_cert_filename']) + return cfg + + +def update_ca_certs(distro_cfg): """ Updates the CA certificate cache on the current machine. + + @param distro_cfg: A hash providing _distro_ca_certs_configs function. """ - subp.subp(["update-ca-certificates"], capture=False) + subp.subp(distro_cfg['ca_cert_update_cmd'], capture=False) -def add_ca_certs(certs): +def add_ca_certs(distro_cfg, certs): """ Adds certificates to the system. To actually apply the new certificates you must also call L{update_ca_certs}. + @param distro_cfg: A hash providing _distro_ca_certs_configs function. @param certs: A list of certificate strings. """ - if certs: - # First ensure they are strings... - cert_file_contents = "\n".join([str(c) for c in certs]) - util.write_file(CA_CERT_FULL_PATH, cert_file_contents, mode=0o644) - - if os.stat(CA_CERT_CONFIG).st_size == 0: - # If the CA_CERT_CONFIG file is empty (i.e. all existing - # CA certs have been deleted) then simply output a single - # line with the cloud-init cert filename. - out = "%s\n" % CA_CERT_FILENAME - else: - # Append cert filename to CA_CERT_CONFIG file. - # We have to strip the content because blank lines in the file - # causes subsequent entries to be ignored. (LP: #1077020) - orig = util.load_file(CA_CERT_CONFIG) - cur_cont = '\n'.join([line for line in orig.splitlines() - if line != CA_CERT_FILENAME]) - out = "%s\n%s\n" % (cur_cont.rstrip(), CA_CERT_FILENAME) - util.write_file(CA_CERT_CONFIG, out, omode="wb") - - -def remove_default_ca_certs(distro_name): + if not certs: + return + # First ensure they are strings... + cert_file_contents = "\n".join([str(c) for c in certs]) + util.write_file(distro_cfg['ca_cert_full_path'], + cert_file_contents, + mode=0o644) + update_cert_config(distro_cfg) + + +def update_cert_config(distro_cfg): + """ + Update Certificate config file to add the file path managed cloud-init + + @param distro_cfg: A hash providing _distro_ca_certs_configs function. + """ + if distro_cfg['ca_cert_config'] is None: + return + if os.stat(distro_cfg['ca_cert_config']).st_size == 0: + # If the CA_CERT_CONFIG file is empty (i.e. all existing + # CA certs have been deleted) then simply output a single + # line with the cloud-init cert filename. + out = "%s\n" % distro_cfg['ca_cert_filename'] + else: + # Append cert filename to CA_CERT_CONFIG file. + # We have to strip the content because blank lines in the file + # causes subsequent entries to be ignored. (LP: #1077020) + orig = util.load_file(distro_cfg['ca_cert_config']) + cr_cont = '\n'.join([line for line in orig.splitlines() + if line != distro_cfg['ca_cert_filename']]) + out = "%s\n%s\n" % (cr_cont.rstrip(), + distro_cfg['ca_cert_filename']) + util.write_file(distro_cfg['ca_cert_config'], out, omode="wb") + + +def remove_default_ca_certs(distro_name, distro_cfg): """ Removes all default trusted CA certificates from the system. To actually apply the change you must also call L{update_ca_certs}. + + @param distro_name: String providing the distro class name. + @param distro_cfg: A hash providing _distro_ca_certs_configs function. """ - util.delete_dir_contents(CA_CERT_PATH) - util.delete_dir_contents(CA_CERT_SYSTEM_PATH) - util.write_file(CA_CERT_CONFIG, "", mode=0o644) + util.delete_dir_contents(distro_cfg['ca_cert_path']) + util.delete_dir_contents(distro_cfg['ca_cert_system_path']) + util.write_file(distro_cfg['ca_cert_config'], "", mode=0o644) - if distro_name != 'alpine': + if distro_name in ['debian', 'ubuntu']: debconf_sel = ( "ca-certificates ca-certificates/trust_new_crts " + "select no") subp.subp(('debconf-set-selections', '-'), debconf_sel) @@ -120,22 +164,23 @@ def handle(name, cfg, cloud, log, _args): return ca_cert_cfg = cfg['ca-certs'] + distro_cfg = _distro_ca_certs_configs(cloud.distro.name) # If there is a remove-defaults option set to true, remove the system # default trusted CA certs first. if ca_cert_cfg.get("remove-defaults", False): log.debug("Removing default certificates") - remove_default_ca_certs(cloud.distro.name) + remove_default_ca_certs(cloud.distro.name, distro_cfg) # If we are given any new trusted CA certs to add, add them. if "trusted" in ca_cert_cfg: trusted_certs = util.get_cfg_option_list(ca_cert_cfg, "trusted") if trusted_certs: log.debug("Adding %d certificates" % len(trusted_certs)) - add_ca_certs(trusted_certs) + add_ca_certs(distro_cfg, trusted_certs) # Update the system with the new cert configuration. log.debug("Updating certificates") - update_ca_certs() + update_ca_certs(distro_cfg) # vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_handler_ca_certs.py b/tests/unittests/test_handler/test_handler_ca_certs.py index a16430d5..6e3831ed 100644 --- a/tests/unittests/test_handler/test_handler_ca_certs.py +++ b/tests/unittests/test_handler/test_handler_ca_certs.py @@ -47,12 +47,20 @@ class TestConfig(TestCase): def setUp(self): super(TestConfig, self).setUp() self.name = "ca-certs" - distro = self._fetch_distro('ubuntu') self.paths = None - self.cloud = cloud.Cloud(None, self.paths, None, distro, None) self.log = logging.getLogger("TestNoConfig") self.args = [] + def _fetch_distro(self, kind): + cls = distros.fetch(kind) + paths = helpers.Paths({}) + return cls(kind, {}, paths) + + def _get_cloud(self, kind): + distro = self._fetch_distro(kind) + return cloud.Cloud(None, self.paths, None, distro, None) + + def _mock_init(self): self.mocks = ExitStack() self.addCleanup(self.mocks.close) @@ -64,11 +72,6 @@ class TestConfig(TestCase): self.mock_remove = self.mocks.enter_context( mock.patch.object(cc_ca_certs, 'remove_default_ca_certs')) - def _fetch_distro(self, kind): - cls = distros.fetch(kind) - paths = helpers.Paths({}) - return cls(kind, {}, paths) - def test_no_trusted_list(self): """ Test that no certificates are written if the 'trusted' key is not @@ -76,71 +79,95 @@ class TestConfig(TestCase): """ config = {"ca-certs": {}} - cc_ca_certs.handle(self.name, config, self.cloud, self.log, self.args) + for distro_name in cc_ca_certs.distros: + self._mock_init() + cloud = self._get_cloud(distro_name) + cc_ca_certs.handle(self.name, config, cloud, self.log, self.args) - self.assertEqual(self.mock_add.call_count, 0) - self.assertEqual(self.mock_update.call_count, 1) - self.assertEqual(self.mock_remove.call_count, 0) + self.assertEqual(self.mock_add.call_count, 0) + self.assertEqual(self.mock_update.call_count, 1) + self.assertEqual(self.mock_remove.call_count, 0) def test_empty_trusted_list(self): """Test that no certificate are written if 'trusted' list is empty.""" config = {"ca-certs": {"trusted": []}} - cc_ca_certs.handle(self.name, config, self.cloud, self.log, self.args) + for distro_name in cc_ca_certs.distros: + self._mock_init() + cloud = self._get_cloud(distro_name) + cc_ca_certs.handle(self.name, config, cloud, self.log, self.args) - self.assertEqual(self.mock_add.call_count, 0) - self.assertEqual(self.mock_update.call_count, 1) - self.assertEqual(self.mock_remove.call_count, 0) + self.assertEqual(self.mock_add.call_count, 0) + self.assertEqual(self.mock_update.call_count, 1) + self.assertEqual(self.mock_remove.call_count, 0) def test_single_trusted(self): """Test that a single cert gets passed to add_ca_certs.""" config = {"ca-certs": {"trusted": ["CERT1"]}} - cc_ca_certs.handle(self.name, config, self.cloud, self.log, self.args) + for distro_name in cc_ca_certs.distros: + self._mock_init() + cloud = self._get_cloud(distro_name) + conf = cc_ca_certs._distro_ca_certs_configs(distro_name) + cc_ca_certs.handle(self.name, config, cloud, self.log, self.args) - self.mock_add.assert_called_once_with(['CERT1']) - self.assertEqual(self.mock_update.call_count, 1) - self.assertEqual(self.mock_remove.call_count, 0) + self.mock_add.assert_called_once_with(conf, ['CERT1']) + self.assertEqual(self.mock_update.call_count, 1) + self.assertEqual(self.mock_remove.call_count, 0) def test_multiple_trusted(self): """Test that multiple certs get passed to add_ca_certs.""" config = {"ca-certs": {"trusted": ["CERT1", "CERT2"]}} - cc_ca_certs.handle(self.name, config, self.cloud, self.log, self.args) + for distro_name in cc_ca_certs.distros: + self._mock_init() + cloud = self._get_cloud(distro_name) + conf = cc_ca_certs._distro_ca_certs_configs(distro_name) + cc_ca_certs.handle(self.name, config, cloud, self.log, self.args) - self.mock_add.assert_called_once_with(['CERT1', 'CERT2']) - self.assertEqual(self.mock_update.call_count, 1) - self.assertEqual(self.mock_remove.call_count, 0) + self.mock_add.assert_called_once_with(conf, ['CERT1', 'CERT2']) + self.assertEqual(self.mock_update.call_count, 1) + self.assertEqual(self.mock_remove.call_count, 0) def test_remove_default_ca_certs(self): """Test remove_defaults works as expected.""" config = {"ca-certs": {"remove-defaults": True}} - cc_ca_certs.handle(self.name, config, self.cloud, self.log, self.args) + for distro_name in cc_ca_certs.distros: + self._mock_init() + cloud = self._get_cloud(distro_name) + cc_ca_certs.handle(self.name, config, cloud, self.log, self.args) - self.assertEqual(self.mock_add.call_count, 0) - self.assertEqual(self.mock_update.call_count, 1) - self.assertEqual(self.mock_remove.call_count, 1) + self.assertEqual(self.mock_add.call_count, 0) + self.assertEqual(self.mock_update.call_count, 1) + self.assertEqual(self.mock_remove.call_count, 1) def test_no_remove_defaults_if_false(self): """Test remove_defaults is not called when config value is False.""" config = {"ca-certs": {"remove-defaults": False}} - cc_ca_certs.handle(self.name, config, self.cloud, self.log, self.args) + for distro_name in cc_ca_certs.distros: + self._mock_init() + cloud = self._get_cloud(distro_name) + cc_ca_certs.handle(self.name, config, cloud, self.log, self.args) - self.assertEqual(self.mock_add.call_count, 0) - self.assertEqual(self.mock_update.call_count, 1) - self.assertEqual(self.mock_remove.call_count, 0) + self.assertEqual(self.mock_add.call_count, 0) + self.assertEqual(self.mock_update.call_count, 1) + self.assertEqual(self.mock_remove.call_count, 0) def test_correct_order_for_remove_then_add(self): """Test remove_defaults is not called when config value is False.""" config = {"ca-certs": {"remove-defaults": True, "trusted": ["CERT1"]}} - cc_ca_certs.handle(self.name, config, self.cloud, self.log, self.args) + for distro_name in cc_ca_certs.distros: + self._mock_init() + cloud = self._get_cloud(distro_name) + conf = cc_ca_certs._distro_ca_certs_configs(distro_name) + cc_ca_certs.handle(self.name, config, cloud, self.log, self.args) - self.mock_add.assert_called_once_with(['CERT1']) - self.assertEqual(self.mock_update.call_count, 1) - self.assertEqual(self.mock_remove.call_count, 1) + self.mock_add.assert_called_once_with(conf, ['CERT1']) + self.assertEqual(self.mock_update.call_count, 1) + self.assertEqual(self.mock_remove.call_count, 1) class TestAddCaCerts(TestCase): @@ -154,11 +181,18 @@ class TestAddCaCerts(TestCase): }) self.add_patch("cloudinit.config.cc_ca_certs.os.stat", "m_stat") + def _fetch_distro(self, kind): + cls = distros.fetch(kind) + paths = helpers.Paths({}) + return cls(kind, {}, paths) + def test_no_certs_in_list(self): """Test that no certificate are written if not provided.""" - with mock.patch.object(util, 'write_file') as mockobj: - cc_ca_certs.add_ca_certs([]) - self.assertEqual(mockobj.call_count, 0) + for distro_name in cc_ca_certs.distros: + conf = cc_ca_certs._distro_ca_certs_configs(distro_name) + with mock.patch.object(util, 'write_file') as mockobj: + cc_ca_certs.add_ca_certs(conf, []) + self.assertEqual(mockobj.call_count, 0) def test_single_cert_trailing_cr(self): """Test adding a single certificate to the trusted CAs @@ -168,20 +202,28 @@ class TestAddCaCerts(TestCase): ca_certs_content = "line1\nline2\ncloud-init-ca-certs.crt\nline3\n" expected = "line1\nline2\nline3\ncloud-init-ca-certs.crt\n" - with ExitStack() as mocks: - mock_write = mocks.enter_context( - mock.patch.object(util, 'write_file')) - mock_load = mocks.enter_context( - mock.patch.object(util, 'load_file', - return_value=ca_certs_content)) + self.m_stat.return_value.st_size = 1 - cc_ca_certs.add_ca_certs([cert]) + for distro_name in cc_ca_certs.distros: + conf = cc_ca_certs._distro_ca_certs_configs(distro_name) - mock_write.assert_has_calls([ - mock.call("/usr/share/ca-certificates/cloud-init-ca-certs.crt", - cert, mode=0o644), - mock.call("/etc/ca-certificates.conf", expected, omode="wb")]) - mock_load.assert_called_once_with("/etc/ca-certificates.conf") + with ExitStack() as mocks: + mock_write = mocks.enter_context( + mock.patch.object(util, 'write_file')) + mock_load = mocks.enter_context( + mock.patch.object(util, 'load_file', + return_value=ca_certs_content)) + + cc_ca_certs.add_ca_certs(conf, [cert]) + + mock_write.assert_has_calls([ + mock.call(conf['ca_cert_full_path'], + cert, mode=0o644)]) + if conf['ca_cert_config'] is not None: + mock_write.assert_has_calls([ + mock.call(conf['ca_cert_config'], + expected, omode="wb")]) + mock_load.assert_called_once_with(conf['ca_cert_config']) def test_single_cert_no_trailing_cr(self): """Test adding a single certificate to the trusted CAs @@ -190,24 +232,31 @@ class TestAddCaCerts(TestCase): ca_certs_content = "line1\nline2\nline3" - with ExitStack() as mocks: - mock_write = mocks.enter_context( - mock.patch.object(util, 'write_file')) - mock_load = mocks.enter_context( - mock.patch.object(util, 'load_file', - return_value=ca_certs_content)) + self.m_stat.return_value.st_size = 1 + + for distro_name in cc_ca_certs.distros: + conf = cc_ca_certs._distro_ca_certs_configs(distro_name) - cc_ca_certs.add_ca_certs([cert]) + with ExitStack() as mocks: + mock_write = mocks.enter_context( + mock.patch.object(util, 'write_file')) + mock_load = mocks.enter_context( + mock.patch.object(util, 'load_file', + return_value=ca_certs_content)) - mock_write.assert_has_calls([ - mock.call("/usr/share/ca-certificates/cloud-init-ca-certs.crt", - cert, mode=0o644), - mock.call("/etc/ca-certificates.conf", - "%s\n%s\n" % (ca_certs_content, - "cloud-init-ca-certs.crt"), - omode="wb")]) + cc_ca_certs.add_ca_certs(conf, [cert]) - mock_load.assert_called_once_with("/etc/ca-certificates.conf") + mock_write.assert_has_calls([ + mock.call(conf['ca_cert_full_path'], + cert, mode=0o644)]) + if conf['ca_cert_config'] is not None: + mock_write.assert_has_calls([ + mock.call(conf['ca_cert_config'], + "%s\n%s\n" % (ca_certs_content, + conf['ca_cert_filename']), + omode="wb")]) + + mock_load.assert_called_once_with(conf['ca_cert_config']) def test_single_cert_to_empty_existing_ca_file(self): """Test adding a single certificate to the trusted CAs @@ -216,15 +265,22 @@ class TestAddCaCerts(TestCase): expected = "cloud-init-ca-certs.crt\n" - with mock.patch.object(util, 'write_file', autospec=True) as m_write: - self.m_stat.return_value.st_size = 0 + self.m_stat.return_value.st_size = 0 + + for distro_name in cc_ca_certs.distros: + conf = cc_ca_certs._distro_ca_certs_configs(distro_name) + with mock.patch.object(util, 'write_file', + autospec=True) as m_write: - cc_ca_certs.add_ca_certs([cert]) + cc_ca_certs.add_ca_certs(conf, [cert]) - m_write.assert_has_calls([ - mock.call("/usr/share/ca-certificates/cloud-init-ca-certs.crt", - cert, mode=0o644), - mock.call("/etc/ca-certificates.conf", expected, omode="wb")]) + m_write.assert_has_calls([ + mock.call(conf['ca_cert_full_path'], + cert, mode=0o644)]) + if conf['ca_cert_config'] is not None: + m_write.assert_has_calls([ + mock.call(conf['ca_cert_config'], + expected, omode="wb")]) def test_multiple_certs(self): """Test adding multiple certificates to the trusted CAs.""" @@ -232,32 +288,41 @@ class TestAddCaCerts(TestCase): expected_cert_file = "\n".join(certs) ca_certs_content = "line1\nline2\nline3" - with ExitStack() as mocks: - mock_write = mocks.enter_context( - mock.patch.object(util, 'write_file')) - mock_load = mocks.enter_context( - mock.patch.object(util, 'load_file', - return_value=ca_certs_content)) + self.m_stat.return_value.st_size = 1 - cc_ca_certs.add_ca_certs(certs) + for distro_name in cc_ca_certs.distros: + conf = cc_ca_certs._distro_ca_certs_configs(distro_name) - mock_write.assert_has_calls([ - mock.call("/usr/share/ca-certificates/cloud-init-ca-certs.crt", - expected_cert_file, mode=0o644), - mock.call("/etc/ca-certificates.conf", - "%s\n%s\n" % (ca_certs_content, - "cloud-init-ca-certs.crt"), - omode='wb')]) + with ExitStack() as mocks: + mock_write = mocks.enter_context( + mock.patch.object(util, 'write_file')) + mock_load = mocks.enter_context( + mock.patch.object(util, 'load_file', + return_value=ca_certs_content)) - mock_load.assert_called_once_with("/etc/ca-certificates.conf") + cc_ca_certs.add_ca_certs(conf, certs) + + mock_write.assert_has_calls([ + mock.call(conf['ca_cert_full_path'], + expected_cert_file, mode=0o644)]) + if conf['ca_cert_config'] is not None: + mock_write.assert_has_calls([ + mock.call(conf['ca_cert_config'], + "%s\n%s\n" % (ca_certs_content, + conf['ca_cert_filename']), + omode='wb')]) + + mock_load.assert_called_once_with(conf['ca_cert_config']) class TestUpdateCaCerts(unittest.TestCase): def test_commands(self): - with mock.patch.object(subp, 'subp') as mockobj: - cc_ca_certs.update_ca_certs() - mockobj.assert_called_once_with( - ["update-ca-certificates"], capture=False) + for distro_name in cc_ca_certs.distros: + conf = cc_ca_certs._distro_ca_certs_configs(distro_name) + with mock.patch.object(subp, 'subp') as mockobj: + cc_ca_certs.update_ca_certs(conf) + mockobj.assert_called_once_with( + conf['ca_cert_update_cmd'], capture=False) class TestRemoveDefaultCaCerts(TestCase): @@ -271,24 +336,31 @@ class TestRemoveDefaultCaCerts(TestCase): }) def test_commands(self): - with ExitStack() as mocks: - mock_delete = mocks.enter_context( - mock.patch.object(util, 'delete_dir_contents')) - mock_write = mocks.enter_context( - mock.patch.object(util, 'write_file')) - mock_subp = mocks.enter_context(mock.patch.object(subp, 'subp')) - - cc_ca_certs.remove_default_ca_certs('ubuntu') - - mock_delete.assert_has_calls([ - mock.call("/usr/share/ca-certificates/"), - mock.call("/etc/ssl/certs/")]) - - mock_write.assert_called_once_with( - "/etc/ca-certificates.conf", "", mode=0o644) - - mock_subp.assert_called_once_with( - ('debconf-set-selections', '-'), - "ca-certificates ca-certificates/trust_new_crts select no") + for distro_name in cc_ca_certs.distros: + conf = cc_ca_certs._distro_ca_certs_configs(distro_name) + + with ExitStack() as mocks: + mock_delete = mocks.enter_context( + mock.patch.object(util, 'delete_dir_contents')) + mock_write = mocks.enter_context( + mock.patch.object(util, 'write_file')) + mock_subp = mocks.enter_context( + mock.patch.object(subp, 'subp')) + + cc_ca_certs.remove_default_ca_certs(distro_name, conf) + + mock_delete.assert_has_calls([ + mock.call(conf['ca_cert_path']), + mock.call(conf['ca_cert_system_path'])]) + + if conf['ca_cert_config'] is not None: + mock_write.assert_called_once_with( + conf['ca_cert_config'], "", mode=0o644) + + if distro_name in ['debian', 'ubuntu']: + mock_subp.assert_called_once_with( + ('debconf-set-selections', '-'), + "ca-certificates \ +ca-certificates/trust_new_crts select no") # vi: ts=4 expandtab diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index c278b032..c843e475 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -6,6 +6,7 @@ beezly bipinbachhao BirknerAlex candlerb +cawamata dermotbradley dhensby eandersson -- cgit v1.2.3 From 48b2c5f16bd4ef754fef137ea19894908d4bf1db Mon Sep 17 00:00:00 2001 From: Amy Chen <66719270+xiachen-rh@users.noreply.github.com> Date: Wed, 6 Jan 2021 22:37:02 +0800 Subject: fix a typo in man page cloud-init.1 (#752) 1. fix a typo in cloud-init.1 2. add xiachen-rh as contributor --- doc/man/cloud-init.1 | 2 +- tools/.github-cla-signers | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'tools/.github-cla-signers') diff --git a/doc/man/cloud-init.1 b/doc/man/cloud-init.1 index 9b52dc8d..3fde4148 100644 --- a/doc/man/cloud-init.1 +++ b/doc/man/cloud-init.1 @@ -10,7 +10,7 @@ cloud-init \- Cloud instance initialization Cloud-init provides a mechanism for cloud instance initialization. This is done by identifying the cloud platform that is in use, reading provided cloud metadata and optional vendor and user -data, and then intializing the instance as requested. +data, and then initializing the instance as requested. Generally, this command is not normally meant to be run directly by the user. However, some subcommands may useful for development or diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index c843e475..24a38f09 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -33,4 +33,5 @@ TheRealFalcon tomponline tsanghan WebSpider +xiachen-rh xnox -- cgit v1.2.3 From 83ba4d18b383d5dff1a897a8506fd90408d3efe2 Mon Sep 17 00:00:00 2001 From: Antti Myyrä Date: Fri, 8 Jan 2021 19:53:11 +0200 Subject: Add ajmyyra as contributor (#742) --- tools/.github-cla-signers | 1 + 1 file changed, 1 insertion(+) (limited to 'tools/.github-cla-signers') diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 24a38f09..d6ca6d1b 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -1,4 +1,5 @@ ader1990 +ajmyyra AlexBaranowski Aman306 aswinrajamannar -- cgit v1.2.3 From 2b2c7f391140cdf268a62ddca0c40bc2e01017ea Mon Sep 17 00:00:00 2001 From: Bao Trinh Date: Tue, 12 Jan 2021 10:47:37 -0600 Subject: archlinux: fix package upgrade command handling (#768) pacman uses `-u` instead of `upgrade` to trigger a system upgrade, fix the command handling so this is properly accounted for. as is, the resulting command attempts to install a (non-existent) `upgrade` package Co-authored-by: Rick Harding --- cloudinit/distros/arch.py | 2 ++ tools/.github-cla-signers | 1 + 2 files changed, 3 insertions(+) (limited to 'tools/.github-cla-signers') diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py index 967be168..378a6daa 100644 --- a/cloudinit/distros/arch.py +++ b/cloudinit/distros/arch.py @@ -152,6 +152,8 @@ class Distro(distros.Distro): elif args and isinstance(args, list): cmd.extend(args) + if command == "upgrade": + command = "-u" if command: cmd.append(command) diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index d6ca6d1b..e6e3bdd1 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -26,6 +26,7 @@ nishigori olivierlemasle omBratteng onitake +qubidt riedel slyon smoser -- cgit v1.2.3 From 9a258eebd96aa5ad4486dba1fe86bea5bcf00c2f Mon Sep 17 00:00:00 2001 From: Pavel Abalikhin Date: Thu, 14 Jan 2021 01:19:17 +0300 Subject: net: Fix static routes to host in eni renderer (#668) Route '-net' parameter is incompatible with /32 IPv4 addresses so we have to use '-host' in that case. --- cloudinit/net/eni.py | 2 ++ tests/integration_tests/bugs/test_gh668.py | 37 ++++++++++++++++++++++++++++++ tests/unittests/test_net.py | 7 ++++++ tools/.github-cla-signers | 1 + 4 files changed, 47 insertions(+) create mode 100644 tests/integration_tests/bugs/test_gh668.py (limited to 'tools/.github-cla-signers') diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index 0074691b..a89e5ad2 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -387,6 +387,8 @@ class Renderer(renderer.Renderer): if k == 'network': if ':' in route[k]: route_line += ' -A inet6' + elif route.get('prefix') == 32: + route_line += ' -host' else: route_line += ' -net' if 'prefix' in route: diff --git a/tests/integration_tests/bugs/test_gh668.py b/tests/integration_tests/bugs/test_gh668.py new file mode 100644 index 00000000..a3a0c374 --- /dev/null +++ b/tests/integration_tests/bugs/test_gh668.py @@ -0,0 +1,37 @@ +"""Integration test for gh-668. + +Ensure that static route to host is working correctly. +The original problem is specific to the ENI renderer but that test is suitable +for all network configuration outputs. +""" + +import pytest + +from tests.integration_tests.instances import IntegrationInstance + + +DESTINATION_IP = "172.16.0.10" +GATEWAY_IP = "10.0.0.100" + +NETWORK_CONFIG = """\ +version: 2 +ethernets: + eth0: + addresses: [10.0.0.10/8] + dhcp4: false + routes: + - to: {}/32 + via: {} +""".format(DESTINATION_IP, GATEWAY_IP) + +EXPECTED_ROUTE = "{} via {}".format(DESTINATION_IP, GATEWAY_IP) + + +@pytest.mark.lxd_container +@pytest.mark.lxd_vm +@pytest.mark.lxd_config_dict({ + "user.network-config": NETWORK_CONFIG, +}) +def test_static_route_to_host(client: IntegrationInstance): + route = client.execute("ip route | grep {}".format(DESTINATION_IP)) + assert route.startswith(EXPECTED_ROUTE) diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index bf0cdabb..38d934d4 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -4820,6 +4820,9 @@ class TestEniRoundTrip(CiTestCase): {'type': 'route', 'id': 6, 'metric': 1, 'destination': '10.0.200.0/16', 'gateway': '172.23.31.1'}, + {'type': 'route', 'id': 7, + 'metric': 1, 'destination': '10.0.0.100/32', + 'gateway': '172.23.31.1'}, ] files = self._render_and_read( @@ -4843,6 +4846,10 @@ class TestEniRoundTrip(CiTestCase): '172.23.31.1 metric 1 || true'), ('pre-down route del -net 10.0.200.0/16 gw ' '172.23.31.1 metric 1 || true'), + ('post-up route add -host 10.0.0.100/32 gw ' + '172.23.31.1 metric 1 || true'), + ('pre-down route del -host 10.0.0.100/32 gw ' + '172.23.31.1 metric 1 || true'), ] found = files['/etc/network/interfaces'].splitlines() diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index e6e3bdd1..ac95b422 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -32,6 +32,7 @@ slyon smoser sshedi TheRealFalcon +tnt-dev tomponline tsanghan WebSpider -- cgit v1.2.3 From ad23441d82cb844828022b8b3f55857ca9e4baf8 Mon Sep 17 00:00:00 2001 From: Anton Chaporgin Date: Tue, 19 Jan 2021 20:57:25 +0300 Subject: Add antonyc to .github-cla-signers (#747) --- tools/.github-cla-signers | 1 + 1 file changed, 1 insertion(+) (limited to 'tools/.github-cla-signers') diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index ac95b422..4cee82ac 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -2,6 +2,7 @@ ader1990 ajmyyra AlexBaranowski Aman306 +antonyc aswinrajamannar beezly bipinbachhao -- cgit v1.2.3 From e051d54227b153e19774775edf475425e6ebabd8 Mon Sep 17 00:00:00 2001 From: Dan Kenigsberg Date: Thu, 21 Jan 2021 17:17:57 +0200 Subject: doc: avoid two warnings (#781) Two shell code blocks are not marked as such, confusing rst to consider them as yaml. Be explicit about their syntax, and use $ prompt to match elsewhere in the docs. /home/travis/build/canonical/cloud-init/doc/rtd/topics/format.rst:28: WARNING: Could not lex literal_block as "yaml". Highlighting skipped. /home/travis/build/canonical/cloud-init/doc/rtd/topics/format.rst:52: WARNING: Could not lex literal_block as "yaml". Highlighting skipped. Signed-off-by: Dan Kenigsberg --- doc/rtd/topics/format.rst | 14 +++++++++----- tools/.github-cla-signers | 1 + 2 files changed, 10 insertions(+), 5 deletions(-) (limited to 'tools/.github-cla-signers') diff --git a/doc/rtd/topics/format.rst b/doc/rtd/topics/format.rst index d03e4caf..fa8aa925 100644 --- a/doc/rtd/topics/format.rst +++ b/doc/rtd/topics/format.rst @@ -23,9 +23,11 @@ Using a mime-multi part file, the user can specify more than one type of data. For example, both a user data script and a cloud-config type could be specified. -Supported content-types are listed from the cloud-init subcommand make-mime:: +Supported content-types are listed from the cloud-init subcommand make-mime: - % cloud-init devel make-mime --list-types +.. code-block:: shell-session + + $ cloud-init devel make-mime --list-types cloud-boothook cloud-config cloud-config-archive @@ -47,9 +49,11 @@ The cloud-init subcommand can generate MIME multi-part files: `make-mime`_. separated by a colon (e.g. ``config.yaml:cloud-config``) and emits a MIME multipart message to stdout. An example invocation, assuming you have your cloud config in ``config.yaml`` and a shell script in ``script.sh`` and want -to store the multipart message in ``user-data``:: +to store the multipart message in ``user-data``: + +.. code-block:: shell-session - % cloud-init devel make-mime -a config.yaml:cloud-config -a script.sh:x-shellscript > user-data + $ cloud-init devel make-mime -a config.yaml:cloud-config -a script.sh:x-shellscript > user-data .. _make-mime: https://github.com/canonical/cloud-init/blob/master/cloudinit/cmd/devel/make_mime.py @@ -70,7 +74,7 @@ archive. Example ------- -:: +.. code-block:: shell-session $ cat myscript.sh diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 4cee82ac..a2354594 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -9,6 +9,7 @@ bipinbachhao BirknerAlex candlerb cawamata +dankenigsberg dermotbradley dhensby eandersson -- cgit v1.2.3 From 2d0b63681ca411a606b7d4f4d316320395d30e82 Mon Sep 17 00:00:00 2001 From: Andrew Bogott Date: Thu, 21 Jan 2021 13:50:25 -0600 Subject: Adding self to cla signers (#776) --- tools/.github-cla-signers | 1 + 1 file changed, 1 insertion(+) (limited to 'tools/.github-cla-signers') diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index a2354594..48b4b96d 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -2,6 +2,7 @@ ader1990 ajmyyra AlexBaranowski Aman306 +andrewbogott antonyc aswinrajamannar beezly -- cgit v1.2.3 From a0fdd85664e1e5474bbab1767150cc05ac6ff9b3 Mon Sep 17 00:00:00 2001 From: Rick Harding Date: Tue, 26 Jan 2021 16:19:23 -0500 Subject: Add Rick Harding to CLA signers (#792) --- tools/.github-cla-signers | 1 + 1 file changed, 1 insertion(+) (limited to 'tools/.github-cla-signers') diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 48b4b96d..4708bb46 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -25,6 +25,7 @@ lungj manuelisimo marlluslustosa matthewruffell +mitechie nishigori olivierlemasle omBratteng -- cgit v1.2.3 From 5ccc8cbadb6b30ecbfe6e448ed5f7f000f05c7f1 Mon Sep 17 00:00:00 2001 From: Jordi Massaguer Pla Date: Tue, 26 Jan 2021 22:36:16 +0100 Subject: Add jordimassaguerpla as contributor (#787) Signed-off-by: Jordi Massaguer Pla --- tools/.github-cla-signers | 1 + 1 file changed, 1 insertion(+) (limited to 'tools/.github-cla-signers') diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 4708bb46..689d7902 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -17,6 +17,7 @@ eandersson emmanuelthome izzyleung johnsonshi +jordimassaguerpla jqueuniet jsf9k landon912 -- cgit v1.2.3 From 695c4f8f46585dd7feac2fdc0729f410c539d3bc Mon Sep 17 00:00:00 2001 From: Toshi Aoyama Date: Wed, 24 Feb 2021 01:09:23 +0900 Subject: Update cc_set_hostname documentation (#818) It is distro dependent whether hostname or fqdn is used --- cloudinit/config/cc_set_hostname.py | 4 ++-- tools/.github-cla-signers | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'tools/.github-cla-signers') diff --git a/cloudinit/config/cc_set_hostname.py b/cloudinit/config/cc_set_hostname.py index 1d23d80d..d4017478 100644 --- a/cloudinit/config/cc_set_hostname.py +++ b/cloudinit/config/cc_set_hostname.py @@ -18,8 +18,8 @@ A hostname and fqdn can be provided by specifying a full domain name under the ``fqdn`` key. Alternatively, a hostname can be specified using the ``hostname`` key, and the fqdn of the cloud wil be used. If a fqdn specified with the ``hostname`` key, it will be handled properly, although it is better to use -the ``fqdn`` config key. If both ``fqdn`` and ``hostname`` are set, ``fqdn`` -will be used. +the ``fqdn`` config key. If both ``fqdn`` and ``hostname`` are set, +it is distro dependent whether ``hostname`` or ``fqdn`` is used. This module will run in the init-local stage before networking is configured if the hostname is set by metadata or user data on the local system. diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 689d7902..5dfa2eb3 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -37,6 +37,7 @@ slyon smoser sshedi TheRealFalcon +taoyama tnt-dev tomponline tsanghan -- cgit v1.2.3 From 2757333e844f597b85980093bddc52552ef73aa5 Mon Sep 17 00:00:00 2001 From: Kristian Klausen Date: Thu, 25 Feb 2021 17:12:17 +0100 Subject: archlinux: Use hostnamectl to set the transient hostname (#797) hostname (inetutils) isn't installed per default on arch, so switch to hostnamectl which is installed per default (systemd). --- cloudinit/distros/arch.py | 11 +++++++++++ tools/.github-cla-signers | 1 + 2 files changed, 12 insertions(+) (limited to 'tools/.github-cla-signers') diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py index 378a6daa..5f42a24c 100644 --- a/cloudinit/distros/arch.py +++ b/cloudinit/distros/arch.py @@ -137,6 +137,17 @@ class Distro(distros.Distro): return default return hostname + # hostname (inetutils) isn't installed per default on arch, so we use + # hostnamectl which is installed per default (systemd). + def _apply_hostname(self, hostname): + LOG.debug("Non-persistently setting the system hostname to %s", + hostname) + try: + subp.subp(['hostnamectl', '--transient', 'set-hostname', hostname]) + except subp.ProcessExecutionError: + util.logexc(LOG, "Failed to non-persistently adjust the system " + "hostname to %s", hostname) + def set_timezone(self, tz): distros.set_etc_timezone(tz=tz, tz_file=self._find_tz_file(tz)) diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 5dfa2eb3..aca0ee5e 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -20,6 +20,7 @@ johnsonshi jordimassaguerpla jqueuniet jsf9k +klausenbusk landon912 lucasmoura lungj -- cgit v1.2.3 From f35181fa970453ba6c7c14575b12185533391b97 Mon Sep 17 00:00:00 2001 From: eb3095 <45504889+eb3095@users.noreply.github.com> Date: Tue, 16 Mar 2021 12:35:05 -0400 Subject: Fix stack trace if vendordata_raw contained an array (#837) The implementation in existing datasources means that vendordata_raw is not "raw" as it ideally would be. Instead, actual values may include bytes, string or list. If the value was a list, then the attempt to persist that data to a file in '_store_rawdata' would raise a TypeError. The change is to encode with util.json_dumps (which is safe for binary data) before writing. --- cloudinit/stages.py | 18 ++++++++++++++---- tools/.github-cla-signers | 1 + 2 files changed, 15 insertions(+), 4 deletions(-) (limited to 'tools/.github-cla-signers') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 3ef4491c..5bacc85d 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -364,12 +364,12 @@ class Init(object): 'userdata') self._store_processeddata(self.datasource.get_userdata(), 'userdata') - self._store_rawdata(self.datasource.get_vendordata_raw(), - 'vendordata') + self._store_raw_vendordata(self.datasource.get_vendordata_raw(), + 'vendordata') self._store_processeddata(self.datasource.get_vendordata(), 'vendordata') - self._store_rawdata(self.datasource.get_vendordata2_raw(), - 'vendordata2') + self._store_raw_vendordata(self.datasource.get_vendordata2_raw(), + 'vendordata2') self._store_processeddata(self.datasource.get_vendordata2(), 'vendordata2') @@ -397,6 +397,16 @@ class Init(object): data = b'' util.write_file(self._get_ipath('%s_raw' % datasource), data, 0o600) + def _store_raw_vendordata(self, data, datasource): + # Only these data types + if data is not None and type(data) not in [bytes, str, list]: + raise TypeError("vendordata_raw is unsupported type '%s'" % + str(type(data))) + # This data may be a list, convert it to a string if so + if isinstance(data, list): + data = util.json_dumps(data) + self._store_rawdata(data, datasource) + def _store_processeddata(self, processed_data, datasource): # processed is a Mime message, so write as string. if processed_data is None: diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index aca0ee5e..5c57acac 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -14,6 +14,7 @@ dankenigsberg dermotbradley dhensby eandersson +eb3095 emmanuelthome izzyleung johnsonshi -- cgit v1.2.3 From d4acc0920cdc19e5be3e1054a9a5fbc20e954492 Mon Sep 17 00:00:00 2001 From: timothegenzmer Date: Wed, 24 Mar 2021 20:57:53 +0100 Subject: Fix chef apt source example (#826) key is a property of source1 and not sources --- doc/examples/cloud-config-chef.txt | 70 +++++++++++++++++++------------------- tools/.github-cla-signers | 1 + 2 files changed, 36 insertions(+), 35 deletions(-) (limited to 'tools/.github-cla-signers') diff --git a/doc/examples/cloud-config-chef.txt b/doc/examples/cloud-config-chef.txt index 8cebfd80..414111a1 100644 --- a/doc/examples/cloud-config-chef.txt +++ b/doc/examples/cloud-config-chef.txt @@ -15,41 +15,41 @@ apt: sources: source1: source: "deb http://packages.chef.io/repos/apt/stable $RELEASE main" - key: | - -----BEGIN PGP PUBLIC KEY BLOCK----- - Version: GnuPG v1.4.12 (Darwin) - Comment: GPGTools - http://gpgtools.org - - mQGiBEppC7QRBADfsOkZU6KZK+YmKw4wev5mjKJEkVGlus+NxW8wItX5sGa6kdUu - twAyj7Yr92rF+ICFEP3gGU6+lGo0Nve7KxkN/1W7/m3G4zuk+ccIKmjp8KS3qn99 - dxy64vcji9jIllVa+XXOGIp0G8GEaj7mbkixL/bMeGfdMlv8Gf2XPpp9vwCgn/GC - JKacfnw7MpLKUHOYSlb//JsEAJqao3ViNfav83jJKEkD8cf59Y8xKia5OpZqTK5W - ShVnNWS3U5IVQk10ZDH97Qn/YrK387H4CyhLE9mxPXs/ul18ioiaars/q2MEKU2I - XKfV21eMLO9LYd6Ny/Kqj8o5WQK2J6+NAhSwvthZcIEphcFignIuobP+B5wNFQpe - DbKfA/0WvN2OwFeWRcmmd3Hz7nHTpcnSF+4QX6yHRF/5BgxkG6IqBIACQbzPn6Hm - sMtm/SVf11izmDqSsQptCrOZILfLX/mE+YOl+CwWSHhl+YsFts1WOuh1EhQD26aO - Z84HuHV5HFRWjDLw9LriltBVQcXbpfSrRP5bdr7Wh8vhqJTPjrQnT3BzY29kZSBQ - YWNrYWdlcyA8cGFja2FnZXNAb3BzY29kZS5jb20+iGAEExECACAFAkppC7QCGwMG - CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRApQKupg++Caj8sAKCOXmdG36gWji/K - +o+XtBfvdMnFYQCfTCEWxRy2BnzLoBBFCjDSK6sJqCu0IENIRUYgUGFja2FnZXMg - PHBhY2thZ2VzQGNoZWYuaW8+iGIEExECACIFAlQwYFECGwMGCwkIBwMCBhUIAgkK - CwQWAgMBAh4BAheAAAoJEClAq6mD74JqX94An26z99XOHWpLN8ahzm7cp13t4Xid - AJ9wVcgoUBzvgg91lKfv/34cmemZn7kCDQRKaQu0EAgAg7ZLCVGVTmLqBM6njZEd - Zbv+mZbvwLBSomdiqddE6u3eH0X3GuwaQfQWHUVG2yedyDMiG+EMtCdEeeRebTCz - SNXQ8Xvi22hRPoEsBSwWLZI8/XNg0n0f1+GEr+mOKO0BxDB2DG7DA0nnEISxwFkK - OFJFebR3fRsrWjj0KjDxkhse2ddU/jVz1BY7Nf8toZmwpBmdozETMOTx3LJy1HZ/ - Te9FJXJMUaB2lRyluv15MVWCKQJro4MQG/7QGcIfrIZNfAGJ32DDSjV7/YO+IpRY - IL4CUBQ65suY4gYUG4jhRH6u7H1p99sdwsg5OIpBe/v2Vbc/tbwAB+eJJAp89Zeu - twADBQf/ZcGoPhTGFuzbkcNRSIz+boaeWPoSxK2DyfScyCAuG41CY9+g0HIw9Sq8 - DuxQvJ+vrEJjNvNE3EAEdKl/zkXMZDb1EXjGwDi845TxEMhhD1dDw2qpHqnJ2mtE - WpZ7juGwA3sGhi6FapO04tIGacCfNNHmlRGipyq5ZiKIRq9mLEndlECr8cwaKgkS - 0wWu+xmMZe7N5/t/TK19HXNh4tVacv0F3fYK54GUjt2FjCQV75USnmNY4KPTYLXA - dzC364hEMlXpN21siIFgB04w+TXn5UF3B4FfAy5hevvr4DtV4MvMiGLu0oWjpaLC - MpmrR3Ny2wkmO0h+vgri9uIP06ODWIhJBBgRAgAJBQJKaQu0AhsMAAoJEClAq6mD - 74Jq4hIAoJ5KrYS8kCwj26SAGzglwggpvt3CAJ0bekyky56vNqoegB+y4PQVDv4K - zA== - =IxPr - -----END PGP PUBLIC KEY BLOCK----- + key: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + Version: GnuPG v1.4.12 (Darwin) + Comment: GPGTools - http://gpgtools.org + + mQGiBEppC7QRBADfsOkZU6KZK+YmKw4wev5mjKJEkVGlus+NxW8wItX5sGa6kdUu + twAyj7Yr92rF+ICFEP3gGU6+lGo0Nve7KxkN/1W7/m3G4zuk+ccIKmjp8KS3qn99 + dxy64vcji9jIllVa+XXOGIp0G8GEaj7mbkixL/bMeGfdMlv8Gf2XPpp9vwCgn/GC + JKacfnw7MpLKUHOYSlb//JsEAJqao3ViNfav83jJKEkD8cf59Y8xKia5OpZqTK5W + ShVnNWS3U5IVQk10ZDH97Qn/YrK387H4CyhLE9mxPXs/ul18ioiaars/q2MEKU2I + XKfV21eMLO9LYd6Ny/Kqj8o5WQK2J6+NAhSwvthZcIEphcFignIuobP+B5wNFQpe + DbKfA/0WvN2OwFeWRcmmd3Hz7nHTpcnSF+4QX6yHRF/5BgxkG6IqBIACQbzPn6Hm + sMtm/SVf11izmDqSsQptCrOZILfLX/mE+YOl+CwWSHhl+YsFts1WOuh1EhQD26aO + Z84HuHV5HFRWjDLw9LriltBVQcXbpfSrRP5bdr7Wh8vhqJTPjrQnT3BzY29kZSBQ + YWNrYWdlcyA8cGFja2FnZXNAb3BzY29kZS5jb20+iGAEExECACAFAkppC7QCGwMG + CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRApQKupg++Caj8sAKCOXmdG36gWji/K + +o+XtBfvdMnFYQCfTCEWxRy2BnzLoBBFCjDSK6sJqCu0IENIRUYgUGFja2FnZXMg + PHBhY2thZ2VzQGNoZWYuaW8+iGIEExECACIFAlQwYFECGwMGCwkIBwMCBhUIAgkK + CwQWAgMBAh4BAheAAAoJEClAq6mD74JqX94An26z99XOHWpLN8ahzm7cp13t4Xid + AJ9wVcgoUBzvgg91lKfv/34cmemZn7kCDQRKaQu0EAgAg7ZLCVGVTmLqBM6njZEd + Zbv+mZbvwLBSomdiqddE6u3eH0X3GuwaQfQWHUVG2yedyDMiG+EMtCdEeeRebTCz + SNXQ8Xvi22hRPoEsBSwWLZI8/XNg0n0f1+GEr+mOKO0BxDB2DG7DA0nnEISxwFkK + OFJFebR3fRsrWjj0KjDxkhse2ddU/jVz1BY7Nf8toZmwpBmdozETMOTx3LJy1HZ/ + Te9FJXJMUaB2lRyluv15MVWCKQJro4MQG/7QGcIfrIZNfAGJ32DDSjV7/YO+IpRY + IL4CUBQ65suY4gYUG4jhRH6u7H1p99sdwsg5OIpBe/v2Vbc/tbwAB+eJJAp89Zeu + twADBQf/ZcGoPhTGFuzbkcNRSIz+boaeWPoSxK2DyfScyCAuG41CY9+g0HIw9Sq8 + DuxQvJ+vrEJjNvNE3EAEdKl/zkXMZDb1EXjGwDi845TxEMhhD1dDw2qpHqnJ2mtE + WpZ7juGwA3sGhi6FapO04tIGacCfNNHmlRGipyq5ZiKIRq9mLEndlECr8cwaKgkS + 0wWu+xmMZe7N5/t/TK19HXNh4tVacv0F3fYK54GUjt2FjCQV75USnmNY4KPTYLXA + dzC364hEMlXpN21siIFgB04w+TXn5UF3B4FfAy5hevvr4DtV4MvMiGLu0oWjpaLC + MpmrR3Ny2wkmO0h+vgri9uIP06ODWIhJBBgRAgAJBQJKaQu0AhsMAAoJEClAq6mD + 74Jq4hIAoJ5KrYS8kCwj26SAGzglwggpvt3CAJ0bekyky56vNqoegB+y4PQVDv4K + zA== + =IxPr + -----END PGP PUBLIC KEY BLOCK----- chef: diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 5c57acac..6661c250 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -40,6 +40,7 @@ smoser sshedi TheRealFalcon taoyama +timothegenzmer tnt-dev tomponline tsanghan -- cgit v1.2.3 From fc5d541529d9f4a076998b7b4a3c90bb4be0000d Mon Sep 17 00:00:00 2001 From: hamalq <81582959+hamalq@users.noreply.github.com> Date: Tue, 30 Mar 2021 10:52:54 -0700 Subject: add hamalq user (#860) --- tools/.github-cla-signers | 1 + 1 file changed, 1 insertion(+) (limited to 'tools/.github-cla-signers') diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 6661c250..0a573f9e 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -47,3 +47,4 @@ tsanghan WebSpider xiachen-rh xnox +hamalq -- cgit v1.2.3 From 15dd3601c484e189ea82917600322b7d0e25c088 Mon Sep 17 00:00:00 2001 From: Petr Fedchenkov Date: Wed, 7 Apr 2021 19:16:30 +0300 Subject: bringup_static_routes: fix gateway check (#850) When bringing up DHCP-provided static routes, we check for "0.0.0.0/0" to indicate an unspecified gateway. However, when parsing the static route in `parse_static_routes`, the gateway is never specified with a net length, so the "/0" will never happen. This change updates the gateway check to check only for "0.0.0.0". --- cloudinit/net/__init__.py | 2 +- cloudinit/net/tests/test_dhcp.py | 5 +++++ cloudinit/net/tests/test_init.py | 15 +++++++++++---- tools/.github-cla-signers | 1 + 4 files changed, 18 insertions(+), 5 deletions(-) (limited to 'tools/.github-cla-signers') diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 385b7bcc..6b3b84f7 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -1135,7 +1135,7 @@ class EphemeralIPv4Network(object): # ("0.0.0.0/0", "130.56.240.1")] for net_address, gateway in self.static_routes: via_arg = [] - if gateway != "0.0.0.0/0": + if gateway != "0.0.0.0": via_arg = ['via', gateway] subp.subp( ['ip', '-4', 'route', 'add', net_address] + via_arg + diff --git a/cloudinit/net/tests/test_dhcp.py b/cloudinit/net/tests/test_dhcp.py index 74cf4b94..6f9a02de 100644 --- a/cloudinit/net/tests/test_dhcp.py +++ b/cloudinit/net/tests/test_dhcp.py @@ -194,6 +194,11 @@ class TestDHCPParseStaticRoutes(CiTestCase): self.assertEqual([('0.0.0.0/0', '130.56.240.1')], parse_static_routes(rfc3442)) + def test_unspecified_gateway(self): + rfc3442 = "32,169,254,169,254,0,0,0,0" + self.assertEqual([('169.254.169.254/32', '0.0.0.0')], + parse_static_routes(rfc3442)) + def test_parse_static_routes_class_c_b_a(self): class_c = "24,192,168,74,192,168,0,4" class_b = "16,172,16,172,16,0,4" diff --git a/cloudinit/net/tests/test_init.py b/cloudinit/net/tests/test_init.py index 946f8ee2..ad9c90ff 100644 --- a/cloudinit/net/tests/test_init.py +++ b/cloudinit/net/tests/test_init.py @@ -706,18 +706,22 @@ class TestEphemeralIPV4Network(CiTestCase): def test_ephemeral_ipv4_network_with_rfc3442_static_routes(self, m_subp): params = { 'interface': 'eth0', 'ip': '192.168.2.2', - 'prefix_or_mask': '255.255.255.0', 'broadcast': '192.168.2.255', - 'static_routes': [('169.254.169.254/32', '192.168.2.1'), + 'prefix_or_mask': '255.255.255.255', 'broadcast': '192.168.2.255', + 'static_routes': [('192.168.2.1/32', '0.0.0.0'), + ('169.254.169.254/32', '192.168.2.1'), ('0.0.0.0/0', '192.168.2.1')], 'router': '192.168.2.1'} expected_setup_calls = [ mock.call( - ['ip', '-family', 'inet', 'addr', 'add', '192.168.2.2/24', + ['ip', '-family', 'inet', 'addr', 'add', '192.168.2.2/32', 'broadcast', '192.168.2.255', 'dev', 'eth0'], capture=True, update_env={'LANG': 'C'}), mock.call( ['ip', '-family', 'inet', 'link', 'set', 'dev', 'eth0', 'up'], capture=True), + mock.call( + ['ip', '-4', 'route', 'add', '192.168.2.1/32', + 'dev', 'eth0'], capture=True), mock.call( ['ip', '-4', 'route', 'add', '169.254.169.254/32', 'via', '192.168.2.1', 'dev', 'eth0'], capture=True), @@ -731,12 +735,15 @@ class TestEphemeralIPV4Network(CiTestCase): mock.call( ['ip', '-4', 'route', 'del', '169.254.169.254/32', 'via', '192.168.2.1', 'dev', 'eth0'], capture=True), + mock.call( + ['ip', '-4', 'route', 'del', '192.168.2.1/32', + 'dev', 'eth0'], capture=True), mock.call( ['ip', '-family', 'inet', 'link', 'set', 'dev', 'eth0', 'down'], capture=True), mock.call( ['ip', '-family', 'inet', 'addr', 'del', - '192.168.2.2/24', 'dev', 'eth0'], capture=True) + '192.168.2.2/32', 'dev', 'eth0'], capture=True) ] with net.EphemeralIPv4Network(**params): self.assertEqual(expected_setup_calls, m_subp.call_args_list) diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 0a573f9e..b39f4198 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -16,6 +16,7 @@ dhensby eandersson eb3095 emmanuelthome +giggsoff izzyleung johnsonshi jordimassaguerpla -- cgit v1.2.3 From 0ae0b1d4336acdcab12bd49e9bddb46922fb19c7 Mon Sep 17 00:00:00 2001 From: David Dymko Date: Tue, 13 Apr 2021 14:15:34 -0400 Subject: Add Vultr support (#827) This PR adds in support so that cloud-init can run on instances deployed on Vultr cloud. This was originally brought up in #628. Co-authored-by: Eric Benner --- README.md | 2 +- cloudinit/apport.py | 1 + cloudinit/settings.py | 1 + cloudinit/sources/DataSourceVultr.py | 147 +++++++++++ cloudinit/sources/helpers/vultr.py | 242 +++++++++++++++++ doc/rtd/topics/availability.rst | 1 + doc/rtd/topics/datasources.rst | 2 +- doc/rtd/topics/datasources/vultr.rst | 35 +++ doc/rtd/topics/network-config.rst | 5 + tests/unittests/test_datasource/test_common.py | 2 + tests/unittests/test_datasource/test_vultr.py | 343 +++++++++++++++++++++++++ tools/.github-cla-signers | 1 + tools/ds-identify | 16 +- 13 files changed, 795 insertions(+), 3 deletions(-) create mode 100644 cloudinit/sources/DataSourceVultr.py create mode 100644 cloudinit/sources/helpers/vultr.py create mode 100644 doc/rtd/topics/datasources/vultr.rst create mode 100644 tests/unittests/test_datasource/test_vultr.py (limited to 'tools/.github-cla-signers') diff --git a/README.md b/README.md index 435405da..aa6d84ae 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ get in contact with that distribution and send them our way! | Supported OSes | Supported Public Clouds | Supported Private Clouds | | --- | --- | --- | -| Alpine Linux
ArchLinux
Debian
Fedora
FreeBSD
Gentoo Linux
NetBSD
OpenBSD
RHEL/CentOS
SLES/openSUSE
Ubuntu










| Amazon Web Services
Microsoft Azure
Google Cloud Platform
Oracle Cloud Infrastructure
Softlayer
Rackspace Public Cloud
IBM Cloud
Digital Ocean
Bigstep
Hetzner
Joyent
CloudSigma
Alibaba Cloud
OVH
OpenNebula
Exoscale
Scaleway
CloudStack
AltCloud
SmartOS
HyperOne
Rootbox
| Bare metal installs
OpenStack
LXD
KVM
Metal-as-a-Service (MAAS)















| +| Alpine Linux
ArchLinux
Debian
Fedora
FreeBSD
Gentoo Linux
NetBSD
OpenBSD
RHEL/CentOS
SLES/openSUSE
Ubuntu










| Amazon Web Services
Microsoft Azure
Google Cloud Platform
Oracle Cloud Infrastructure
Softlayer
Rackspace Public Cloud
IBM Cloud
Digital Ocean
Bigstep
Hetzner
Joyent
CloudSigma
Alibaba Cloud
OVH
OpenNebula
Exoscale
Scaleway
CloudStack
AltCloud
SmartOS
HyperOne
Vultr
Rootbox
| Bare metal installs
OpenStack
LXD
KVM
Metal-as-a-Service (MAAS)















| ## To start developing cloud-init diff --git a/cloudinit/apport.py b/cloudinit/apport.py index 25f254e3..aadc638f 100644 --- a/cloudinit/apport.py +++ b/cloudinit/apport.py @@ -41,6 +41,7 @@ KNOWN_CLOUD_NAMES = [ 'SmartOS', 'UpCloud', 'VMware', + 'Vultr', 'ZStack', 'Other' ] diff --git a/cloudinit/settings.py b/cloudinit/settings.py index 91e1bfe7..23e4c0ad 100644 --- a/cloudinit/settings.py +++ b/cloudinit/settings.py @@ -30,6 +30,7 @@ CFG_BUILTIN = { 'GCE', 'OpenStack', 'AliYun', + 'Vultr', 'Ec2', 'CloudSigma', 'CloudStack', diff --git a/cloudinit/sources/DataSourceVultr.py b/cloudinit/sources/DataSourceVultr.py new file mode 100644 index 00000000..c08ff848 --- /dev/null +++ b/cloudinit/sources/DataSourceVultr.py @@ -0,0 +1,147 @@ +# Author: Eric Benner +# +# This file is part of cloud-init. See LICENSE file for license information. + +# Vultr Metadata API: +# https://www.vultr.com/metadata/ + +from cloudinit import log as log +from cloudinit import sources +from cloudinit import util + +import cloudinit.sources.helpers.vultr as vultr + +LOG = log.getLogger(__name__) +BUILTIN_DS_CONFIG = { + 'url': 'http://169.254.169.254', + 'retries': 30, + 'timeout': 2, + 'wait': 2 +} + + +class DataSourceVultr(sources.DataSource): + + dsname = 'Vultr' + + def __init__(self, sys_cfg, distro, paths): + super(DataSourceVultr, self).__init__(sys_cfg, distro, paths) + self.ds_cfg = util.mergemanydict([ + util.get_cfg_by_path(sys_cfg, ["datasource", "Vultr"], {}), + BUILTIN_DS_CONFIG]) + + # Initiate data and check if Vultr + def _get_data(self): + LOG.debug("Detecting if machine is a Vultr instance") + if not vultr.is_vultr(): + LOG.debug("Machine is not a Vultr instance") + return False + + LOG.debug("Machine is a Vultr instance") + + # Fetch metadata + md = self.get_metadata() + + self.metadata_full = md + self.metadata['instanceid'] = md['instanceid'] + self.metadata['local-hostname'] = md['hostname'] + self.metadata['public-keys'] = md["public-keys"] + self.userdata_raw = md["user-data"] + + # Generate config and process data + self.get_datasource_data(md) + + # Dump some data so diagnosing failures is manageable + LOG.debug("Vultr Vendor Config:") + LOG.debug(md['vendor-data']['config']) + LOG.debug("SUBID: %s", self.metadata['instanceid']) + LOG.debug("Hostname: %s", self.metadata['local-hostname']) + if self.userdata_raw is not None: + LOG.debug("User-Data:") + LOG.debug(self.userdata_raw) + + return True + + # Process metadata + def get_datasource_data(self, md): + # Grab config + config = md['vendor-data']['config'] + + # Generate network config + self.netcfg = vultr.generate_network_config(md['interfaces']) + + # This requires info generated in the vendor config + user_scripts = vultr.generate_user_scripts(md, self.netcfg['config']) + + # Default hostname is "guest" for whitelabel + if self.metadata['local-hostname'] == "": + self.metadata['local-hostname'] = "guest" + + self.userdata_raw = md["user-data"] + if self.userdata_raw == "": + self.userdata_raw = None + + # Assemble vendor-data + # This adds provided scripts and the config + self.vendordata_raw = [] + self.vendordata_raw.extend(user_scripts) + self.vendordata_raw.append("#cloud-config\n%s" % config) + + # Get the metadata by flag + def get_metadata(self): + return vultr.get_metadata(self.ds_cfg['url'], + self.ds_cfg['timeout'], + self.ds_cfg['retries'], + self.ds_cfg['wait']) + + # Compare subid as instance id + def check_instance_id(self, sys_cfg): + if not vultr.is_vultr(): + return False + + # Baremetal has no way to implement this in local + if vultr.is_baremetal(): + return False + + subid = vultr.get_sysinfo()['subid'] + return sources.instance_id_matches_system_uuid(subid) + + # Currently unsupported + @property + def launch_index(self): + return None + + @property + def network_config(self): + return self.netcfg + + +# Used to match classes to dependencies +datasources = [ + (DataSourceVultr, (sources.DEP_FILESYSTEM, )), +] + + +# Return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return sources.list_from_depends(depends, datasources) + + +if __name__ == "__main__": + import sys + + if not vultr.is_vultr(): + print("Machine is not a Vultr instance") + sys.exit(1) + + md = vultr.get_metadata(BUILTIN_DS_CONFIG['url'], + BUILTIN_DS_CONFIG['timeout'], + BUILTIN_DS_CONFIG['retries'], + BUILTIN_DS_CONFIG['wait']) + config = md['vendor-data']['config'] + sysinfo = vultr.get_sysinfo() + + print(util.json_dumps(sysinfo)) + print(config) + +# vi: ts=4 expandtab diff --git a/cloudinit/sources/helpers/vultr.py b/cloudinit/sources/helpers/vultr.py new file mode 100644 index 00000000..c22cd0b1 --- /dev/null +++ b/cloudinit/sources/helpers/vultr.py @@ -0,0 +1,242 @@ +# Author: Eric Benner +# +# This file is part of cloud-init. See LICENSE file for license information. + +import json + +from cloudinit import log as log +from cloudinit import url_helper +from cloudinit import dmi +from cloudinit import util +from cloudinit import net +from cloudinit.net.dhcp import EphemeralDHCPv4, NoDHCPLeaseError +from functools import lru_cache + +# Get LOG +LOG = log.getLogger(__name__) + + +@lru_cache() +def get_metadata(url, timeout, retries, sec_between): + # Bring up interface + try: + with EphemeralDHCPv4(connectivity_url=url): + # Fetch the metadata + v1 = read_metadata(url, timeout, retries, sec_between) + except (NoDHCPLeaseError) as exc: + LOG.error("Bailing, DHCP Exception: %s", exc) + raise + + v1_json = json.loads(v1) + metadata = v1_json + + return metadata + + +# Read the system information from SMBIOS +def get_sysinfo(): + return { + 'manufacturer': dmi.read_dmi_data("system-manufacturer"), + 'subid': dmi.read_dmi_data("system-serial-number") + } + + +# Assumes is Vultr is already checked +def is_baremetal(): + if get_sysinfo()['manufacturer'] != "Vultr": + return True + return False + + +# Confirm is Vultr +def is_vultr(): + # VC2, VDC, and HFC use DMI + sysinfo = get_sysinfo() + + if sysinfo['manufacturer'] == "Vultr": + return True + + # Baremetal requires a kernel parameter + if "vultr" in util.get_cmdline().split(): + return True + + return False + + +# Read Metadata endpoint +def read_metadata(url, timeout, retries, sec_between): + url = "%s/v1.json" % url + response = url_helper.readurl(url, + timeout=timeout, + retries=retries, + headers={'Metadata-Token': 'vultr'}, + sec_between=sec_between) + + if not response.ok(): + raise RuntimeError("Failed to connect to %s: Code: %s" % + url, response.code) + + return response.contents.decode() + + +# Wrapped for caching +@lru_cache() +def get_interface_map(): + return net.get_interfaces_by_mac() + + +# Convert macs to nics +def get_interface_name(mac): + macs_to_nic = get_interface_map() + + if mac not in macs_to_nic: + return None + + return macs_to_nic.get(mac) + + +# Generate network configs +def generate_network_config(interfaces): + network = { + "version": 1, + "config": [ + { + "type": "nameserver", + "address": [ + "108.61.10.10" + ] + } + ] + } + + # Prepare interface 0, public + if len(interfaces) > 0: + public = generate_public_network_interface(interfaces[0]) + network['config'].append(public) + + # Prepare interface 1, private + if len(interfaces) > 1: + private = generate_private_network_interface(interfaces[1]) + network['config'].append(private) + + return network + + +# Input Metadata and generate public network config part +def generate_public_network_interface(interface): + interface_name = get_interface_name(interface['mac']) + if not interface_name: + raise RuntimeError( + "Interface: %s could not be found on the system" % + interface['mac']) + + netcfg = { + "name": interface_name, + "type": "physical", + "mac_address": interface['mac'], + "accept-ra": 1, + "subnets": [ + { + "type": "dhcp", + "control": "auto" + }, + { + "type": "dhcp6", + "control": "auto" + }, + ] + } + + # Check for additional IP's + additional_count = len(interface['ipv4']['additional']) + if "ipv4" in interface and additional_count > 0: + for additional in interface['ipv4']['additional']: + add = { + "type": "static", + "control": "auto", + "address": additional['address'], + "netmask": additional['netmask'] + } + netcfg['subnets'].append(add) + + # Check for additional IPv6's + additional_count = len(interface['ipv6']['additional']) + if "ipv6" in interface and additional_count > 0: + for additional in interface['ipv6']['additional']: + add = { + "type": "static6", + "control": "auto", + "address": additional['address'], + "netmask": additional['netmask'] + } + netcfg['subnets'].append(add) + + # Add config to template + return netcfg + + +# Input Metadata and generate private network config part +def generate_private_network_interface(interface): + interface_name = get_interface_name(interface['mac']) + if not interface_name: + raise RuntimeError( + "Interface: %s could not be found on the system" % + interface['mac']) + + netcfg = { + "name": interface_name, + "type": "physical", + "mac_address": interface['mac'], + "accept-ra": 1, + "subnets": [ + { + "type": "static", + "control": "auto", + "address": interface['ipv4']['address'], + "netmask": interface['ipv4']['netmask'] + } + ] + } + + return netcfg + + +# This is for the vendor and startup scripts +def generate_user_scripts(md, network_config): + user_scripts = [] + + # Raid 1 script + if md['vendor-data']['raid1-script']: + user_scripts.append(md['vendor-data']['raid1-script']) + + # Enable multi-queue on linux + if util.is_Linux() and md['vendor-data']['ethtool-script']: + ethtool_script = md['vendor-data']['ethtool-script'] + + # Tool location + tool = "/opt/vultr/ethtool" + + # Go through the interfaces + for netcfg in network_config: + # If the interface has a mac and is physical + if "mac_address" in netcfg and netcfg['type'] == "physical": + # Set its multi-queue to num of cores as per RHEL Docs + name = netcfg['name'] + command = "%s -L %s combined $(nproc --all)" % (tool, name) + ethtool_script = '%s\n%s' % (ethtool_script, command) + + user_scripts.append(ethtool_script) + + # This is for vendor scripts + if md['vendor-data']['vendor-script']: + user_scripts.append(md['vendor-data']['vendor-script']) + + # Startup script + script = md['startup-script'] + if script and script != "echo No configured startup script": + user_scripts.append(script) + + return user_scripts + + +# vi: ts=4 expandtab diff --git a/doc/rtd/topics/availability.rst b/doc/rtd/topics/availability.rst index f58b2b38..f3e13edc 100644 --- a/doc/rtd/topics/availability.rst +++ b/doc/rtd/topics/availability.rst @@ -56,6 +56,7 @@ environments in the public cloud: - AltCloud - SmartOS - UpCloud +- Vultr Additionally, cloud-init is supported on these private clouds: diff --git a/doc/rtd/topics/datasources.rst b/doc/rtd/topics/datasources.rst index 228173d2..497b1467 100644 --- a/doc/rtd/topics/datasources.rst +++ b/doc/rtd/topics/datasources.rst @@ -49,7 +49,7 @@ The following is a list of documents for each supported datasource: datasources/smartos.rst datasources/upcloud.rst datasources/zstack.rst - + datasources/vultr.rst Creation ======== diff --git a/doc/rtd/topics/datasources/vultr.rst b/doc/rtd/topics/datasources/vultr.rst new file mode 100644 index 00000000..e73406a8 --- /dev/null +++ b/doc/rtd/topics/datasources/vultr.rst @@ -0,0 +1,35 @@ +.. _datasource_vultr: + +Vultr +===== + +The `Vultr`_ datasource retrieves basic configuration values from the locally +accessible `metadata service`_. All data is served over HTTP from the address +169.254.169.254. The endpoints are documented in, +`https://www.vultr.com/metadata/ +`_ + +Configuration +------------- + +Vultr's datasource can be configured as follows: + + datasource: + Vultr: + url: 'http://169.254.169.254' + retries: 3 + timeout: 2 + wait: 2 + +- *url*: The URL used to aquire the metadata configuration from +- *retries*: Determines the number of times to attempt to connect to the + metadata service +- *timeout*: Determines the timeout in seconds to wait for a response from the + metadata service +- *wait*: Determines the timeout in seconds to wait before retrying after + accessible failure + +.. _Vultr: https://www.vultr.com/ +.. _metadata service: https://www.vultr.com/metadata/ + +.. vi: textwidth=78 diff --git a/doc/rtd/topics/network-config.rst b/doc/rtd/topics/network-config.rst index 07cad765..5f7a74f8 100644 --- a/doc/rtd/topics/network-config.rst +++ b/doc/rtd/topics/network-config.rst @@ -148,6 +148,10 @@ The following Datasources optionally provide network configuration: - `UpCloud JSON metadata`_ +- :ref:`datasource_vultr` + + - `Vultr JSON metadata`_ + For more information on network configuration formats .. toctree:: @@ -262,5 +266,6 @@ Example output converting V2 to sysconfig: .. _OpenStack Metadata Service Network: https://specs.openstack.org/openstack/nova-specs/specs/liberty/implemented/metadata-service-network-info.html .. _SmartOS JSON Metadata: https://eng.joyent.com/mdata/datadict.html .. _UpCloud JSON metadata: https://developers.upcloud.com/1.3/8-servers/#metadata-service +.. _Vultr JSON metadata: https://www.vultr.com/metadata/ .. vi: textwidth=78 diff --git a/tests/unittests/test_datasource/test_common.py b/tests/unittests/test_datasource/test_common.py index 5912f7ee..5e9c547a 100644 --- a/tests/unittests/test_datasource/test_common.py +++ b/tests/unittests/test_datasource/test_common.py @@ -28,6 +28,7 @@ from cloudinit.sources import ( DataSourceScaleway as Scaleway, DataSourceSmartOS as SmartOS, DataSourceUpCloud as UpCloud, + DataSourceVultr as Vultr, ) from cloudinit.sources import DataSourceNone as DSNone @@ -45,6 +46,7 @@ DEFAULT_LOCAL = [ Oracle.DataSourceOracle, OVF.DataSourceOVF, SmartOS.DataSourceSmartOS, + Vultr.DataSourceVultr, Ec2.DataSourceEc2Local, OpenStack.DataSourceOpenStackLocal, RbxCloud.DataSourceRbxCloud, diff --git a/tests/unittests/test_datasource/test_vultr.py b/tests/unittests/test_datasource/test_vultr.py new file mode 100644 index 00000000..bbea2aa3 --- /dev/null +++ b/tests/unittests/test_datasource/test_vultr.py @@ -0,0 +1,343 @@ +# Author: Eric Benner +# +# This file is part of cloud-init. See LICENSE file for license information. + +# Vultr Metadata API: +# https://www.vultr.com/metadata/ + +import json + +from cloudinit import helpers +from cloudinit import settings +from cloudinit.sources import DataSourceVultr +from cloudinit.sources.helpers import vultr + +from cloudinit.tests.helpers import mock, CiTestCase + +# Vultr metadata test data +VULTR_V1_1 = { + 'bgp': { + 'ipv4': { + 'my-address': '', + 'my-asn': '', + 'peer-address': '', + 'peer-asn': '' + }, + 'ipv6': { + 'my-address': '', + 'my-asn': '', + 'peer-address': '', + 'peer-asn': '' + } + }, + 'hostname': 'CLOUDINIT_1', + 'instanceid': '42506325', + 'interfaces': [ + { + 'ipv4': { + 'additional': [ + ], + 'address': '108.61.89.242', + 'gateway': '108.61.89.1', + 'netmask': '255.255.255.0' + }, + 'ipv6': { + 'additional': [ + ], + 'address': '2001:19f0:5:56c2:5400:03ff:fe15:c465', + 'network': '2001:19f0:5:56c2::', + 'prefix': '64' + }, + 'mac': '56:00:03:15:c4:65', + 'network-type': 'public' + } + ], + 'public-keys': [ + 'ssh-rsa AAAAB3NzaC1y...IQQhv5PAOKaIl+mM3c= test3@key' + ], + 'region': { + 'regioncode': 'EWR' + }, + 'user-defined': [ + ], + 'startup-script': 'echo No configured startup script', + 'raid1-script': '', + 'user-data': [ + ], + 'vendor-data': { + 'vendor-script': '', + 'ethtool-script': '', + 'config': { + 'package_upgrade': 'true', + 'disable_root': 0, + 'ssh_pwauth': 1, + 'chpasswd': { + 'expire': False, + 'list': [ + 'root:$6$S2Smuj.../VqxmIR9Urw0jPZ88i4yvB/' + ] + }, + 'system_info': { + 'default_user': { + 'name': 'root' + } + } + } + } +} + +VULTR_V1_2 = { + 'bgp': { + 'ipv4': { + 'my-address': '', + 'my-asn': '', + 'peer-address': '', + 'peer-asn': '' + }, + 'ipv6': { + 'my-address': '', + 'my-asn': '', + 'peer-address': '', + 'peer-asn': '' + } + }, + 'hostname': 'CLOUDINIT_2', + 'instance-v2-id': '29bea708-2e6e-480a-90ad-0e6b5d5ad62f', + 'instanceid': '42872224', + 'interfaces': [ + { + 'ipv4': { + 'additional': [ + ], + 'address':'45.76.7.171', + 'gateway':'45.76.6.1', + 'netmask':'255.255.254.0' + }, + 'ipv6':{ + 'additional': [ + ], + 'address':'2001:19f0:5:28a7:5400:03ff:fe1b:4eca', + 'network':'2001:19f0:5:28a7::', + 'prefix':'64' + }, + 'mac':'56:00:03:1b:4e:ca', + 'network-type':'public' + }, + { + 'ipv4': { + 'additional': [ + ], + 'address':'10.1.112.3', + 'gateway':'', + 'netmask':'255.255.240.0' + }, + 'ipv6':{ + 'additional': [ + ], + 'network':'', + 'prefix':'' + }, + 'mac':'5a:00:03:1b:4e:ca', + 'network-type':'private', + 'network-v2-id':'fbbe2b5b-b986-4396-87f5-7246660ccb64', + 'networkid':'net5e7155329d730' + } + ], + 'public-keys': [ + 'ssh-rsa AAAAB3NzaC1y...IQQhv5PAOKaIl+mM3c= test3@key' + ], + 'region': { + 'regioncode': 'EWR' + }, + 'user-defined': [ + ], + 'startup-script': 'echo No configured startup script', + 'user-data': [ + ], + + 'vendor-data': { + 'vendor-script': '', + 'ethtool-script': '', + 'raid1-script': '', + 'config': { + 'package_upgrade': 'true', + 'disable_root': 0, + 'ssh_pwauth': 1, + 'chpasswd': { + 'expire': False, + 'list': [ + 'root:$6$SxXx...k2mJNIzZB5vMCDBlYT1' + ] + }, + 'system_info': { + 'default_user': { + 'name': 'root' + } + } + } + } +} + +SSH_KEYS_1 = [ + "ssh-rsa AAAAB3NzaC1y...IQQhv5PAOKaIl+mM3c= test3@key" +] + +# Expected generated objects + +# Expected config +EXPECTED_VULTR_CONFIG = { + 'package_upgrade': 'true', + 'disable_root': 0, + 'ssh_pwauth': 1, + 'chpasswd': { + 'expire': False, + 'list': [ + 'root:$6$SxXx...k2mJNIzZB5vMCDBlYT1' + ] + }, + 'system_info': { + 'default_user': { + 'name': 'root' + } + } +} + +# Expected network config object from generator +EXPECTED_VULTR_NETWORK_1 = { + 'version': 1, + 'config': [ + { + 'type': 'nameserver', + 'address': ['108.61.10.10'] + }, + { + 'name': 'eth0', + 'type': 'physical', + 'mac_address': '56:00:03:15:c4:65', + 'accept-ra': 1, + 'subnets': [ + {'type': 'dhcp', 'control': 'auto'}, + {'type': 'dhcp6', 'control': 'auto'} + ], + } + ] +} + +EXPECTED_VULTR_NETWORK_2 = { + 'version': 1, + 'config': [ + { + 'type': 'nameserver', + 'address': ['108.61.10.10'] + }, + { + 'name': 'eth0', + 'type': 'physical', + 'mac_address': '56:00:03:1b:4e:ca', + 'accept-ra': 1, + 'subnets': [ + {'type': 'dhcp', 'control': 'auto'}, + {'type': 'dhcp6', 'control': 'auto'} + ], + }, + { + 'name': 'eth1', + 'type': 'physical', + 'mac_address': '5a:00:03:1b:4e:ca', + 'accept-ra': 1, + 'subnets': [ + { + "type": "static", + "control": "auto", + "address": "10.1.112.3", + "netmask": "255.255.240.0" + } + ], + } + ] +} + + +INTERFACE_MAP = { + '56:00:03:15:c4:65': 'eth0', + '56:00:03:1b:4e:ca': 'eth0', + '5a:00:03:1b:4e:ca': 'eth1' +} + + +class TestDataSourceVultr(CiTestCase): + def setUp(self): + super(TestDataSourceVultr, self).setUp() + + # Stored as a dict to make it easier to maintain + raw1 = json.dumps(VULTR_V1_1['vendor-data']['config']) + raw2 = json.dumps(VULTR_V1_2['vendor-data']['config']) + + # Make expected format + VULTR_V1_1['vendor-data']['config'] = raw1 + VULTR_V1_2['vendor-data']['config'] = raw2 + + self.tmp = self.tmp_dir() + + # Test the datasource itself + @mock.patch('cloudinit.net.get_interfaces_by_mac') + @mock.patch('cloudinit.sources.helpers.vultr.is_vultr') + @mock.patch('cloudinit.sources.helpers.vultr.get_metadata') + def test_datasource(self, + mock_getmeta, + mock_isvultr, + mock_netmap): + mock_getmeta.return_value = VULTR_V1_2 + mock_isvultr.return_value = True + mock_netmap.return_value = INTERFACE_MAP + + source = DataSourceVultr.DataSourceVultr( + settings.CFG_BUILTIN, None, helpers.Paths({'run_dir': self.tmp})) + + # Test for failure + self.assertEqual(True, source._get_data()) + + # Test instance id + self.assertEqual("42872224", source.metadata['instanceid']) + + # Test hostname + self.assertEqual("CLOUDINIT_2", source.metadata['local-hostname']) + + # Test ssh keys + self.assertEqual(SSH_KEYS_1, source.metadata['public-keys']) + + # Test vendor data generation + orig_val = self.maxDiff + self.maxDiff = None + + vendordata = source.vendordata_raw + + # Test vendor config + self.assertEqual( + EXPECTED_VULTR_CONFIG, + json.loads(vendordata[0].replace("#cloud-config", ""))) + + self.maxDiff = orig_val + + # Test network config generation + self.assertEqual(EXPECTED_VULTR_NETWORK_2, source.network_config) + + # Test network config generation + @mock.patch('cloudinit.net.get_interfaces_by_mac') + def test_network_config(self, mock_netmap): + mock_netmap.return_value = INTERFACE_MAP + interf = VULTR_V1_1['interfaces'] + + self.assertEqual(EXPECTED_VULTR_NETWORK_1, + vultr.generate_network_config(interf)) + + # Test Private Networking config generation + @mock.patch('cloudinit.net.get_interfaces_by_mac') + def test_private_network_config(self, mock_netmap): + mock_netmap.return_value = INTERFACE_MAP + interf = VULTR_V1_2['interfaces'] + + self.assertEqual(EXPECTED_VULTR_NETWORK_2, + vultr.generate_network_config(interf)) + +# vi: ts=4 expandtab diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index b39f4198..d6212d1d 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -11,6 +11,7 @@ BirknerAlex candlerb cawamata dankenigsberg +ddymko dermotbradley dhensby eandersson diff --git a/tools/ds-identify b/tools/ds-identify index 2f2486f7..73e27c71 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -124,7 +124,7 @@ DI_DSNAME="" # this has to match the builtin list in cloud-init, it is what will # be searched if there is no setting found in config. DI_DSLIST_DEFAULT="MAAS ConfigDrive NoCloud AltCloud Azure Bigstep \ -CloudSigma CloudStack DigitalOcean AliYun Ec2 GCE OpenNebula OpenStack \ +CloudSigma CloudStack DigitalOcean Vultr AliYun Ec2 GCE OpenNebula OpenStack \ OVF SmartOS Scaleway Hetzner IBMCloud Oracle Exoscale RbxCloud UpCloud" DI_DSLIST="" DI_MODE="" @@ -1350,6 +1350,20 @@ dscheck_IBMCloud() { return ${DS_NOT_FOUND} } +dscheck_Vultr() { + dmi_sys_vendor_is Vultr && return $DS_FOUND + + case " $DI_KERNEL_CMDLINE " in + *\ vultr\ *) return $DS_FOUND ;; + esac + + if [ -f "${PATH_ROOT}/etc/vultr" ]; then + return $DS_FOUND + fi + + return $DS_NOT_FOUND +} + collect_info() { read_uname_info read_virt -- cgit v1.2.3 From ced836e69274af905bbc1e5f5fde71de4066c86c Mon Sep 17 00:00:00 2001 From: Nicolas Bock Date: Thu, 22 Apr 2021 12:22:54 -0600 Subject: Use `partprobe` to re-read partition table if available (#856) The blkdev command is fragile re-reading partition tables if a partition is mounted. This change instead uses the partprobe if it is available. LP: #1920939 --- cloudinit/config/cc_disk_setup.py | 13 +++++++++---- tools/.github-cla-signers | 1 + 2 files changed, 10 insertions(+), 4 deletions(-) (limited to 'tools/.github-cla-signers') diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index d1200694..a582924b 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -113,6 +113,7 @@ SGDISK_CMD = subp.which("sgdisk") LSBLK_CMD = subp.which("lsblk") BLKID_CMD = subp.which("blkid") BLKDEV_CMD = subp.which("blockdev") +PARTPROBE_CMD = subp.which("partprobe") WIPEFS_CMD = subp.which("wipefs") LANG_C_ENV = {'LANG': 'C'} @@ -685,13 +686,17 @@ def get_partition_layout(table_type, size, layout): def read_parttbl(device): """ - Use partprobe instead of 'udevadm'. Partprobe is the only - reliable way to probe the partition table. + Use `partprobe` or `blkdev` instead of `udevadm`. `Partprobe` is + preferred over `blkdev` since it is more reliably able to probe + the partition table. """ - blkdev_cmd = [BLKDEV_CMD, '--rereadpt', device] + if PARTPROBE_CMD is not None: + probe_cmd = [PARTPROBE_CMD, device] + else: + probe_cmd = [BLKDEV_CMD, '--rereadpt', device] util.udevadm_settle() try: - subp.subp(blkdev_cmd) + subp.subp(probe_cmd) except Exception as e: util.logexc(LOG, "Failed reading the partition table %s" % e) diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index d6212d1d..38e1ad74 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -31,6 +31,7 @@ manuelisimo marlluslustosa matthewruffell mitechie +nicolasbock nishigori olivierlemasle omBratteng -- cgit v1.2.3 From 02db2c3ecf29924690d4c4adf6ec059f36f31103 Mon Sep 17 00:00:00 2001 From: Ben Hughes Date: Mon, 26 Apr 2021 21:19:26 +0100 Subject: Fix chef module run failure when chef_license is set (#868) Move chef_license from TPL_PATH_KEYS to TPL_KEYS as the chef license setting is not a path but must be added to the client config template. Fixes file or folder not found exception raised from ensure_dirs. --- cloudinit/config/cc_chef.py | 2 +- templates/chef_client.rb.tmpl | 2 +- tools/.github-cla-signers | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) (limited to 'tools/.github-cla-signers') diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index aaf71366..7b20222e 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -70,7 +70,6 @@ CHEF_RB_TPL_PATH_KEYS = frozenset([ 'json_attribs', 'pid_file', 'encrypted_data_bag_secret', - 'chef_license', ]) CHEF_RB_TPL_KEYS = list(CHEF_RB_TPL_DEFAULTS.keys()) CHEF_RB_TPL_KEYS.extend(CHEF_RB_TPL_BOOL_KEYS) @@ -80,6 +79,7 @@ CHEF_RB_TPL_KEYS.extend([ 'node_name', 'environment', 'validation_name', + 'chef_license', ]) CHEF_RB_TPL_KEYS = frozenset(CHEF_RB_TPL_KEYS) CHEF_RB_PATH = '/etc/chef/client.rb' diff --git a/templates/chef_client.rb.tmpl b/templates/chef_client.rb.tmpl index 0a759b04..b9d58172 100644 --- a/templates/chef_client.rb.tmpl +++ b/templates/chef_client.rb.tmpl @@ -15,7 +15,7 @@ The reason these are not in quotes is because they are ruby symbols that will be placed inside here, and not actual strings... #} {% if chef_license %} -chef_license "{{chef_license}}" +chef_license "{{chef_license}}" {% endif%} {% if log_level %} log_level {{log_level}} diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 38e1ad74..d7151a59 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -51,3 +51,4 @@ WebSpider xiachen-rh xnox hamalq +bmhughes -- cgit v1.2.3 From 9ea643ff35382ab8ff1e6fe514cf21eb6640ef8c Mon Sep 17 00:00:00 2001 From: Paul Goins Date: Thu, 29 Apr 2021 09:07:17 -0700 Subject: Add Vultaire as contributor (#881) --- tools/.github-cla-signers | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'tools/.github-cla-signers') diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index d7151a59..607f1d8e 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -8,6 +8,7 @@ aswinrajamannar beezly bipinbachhao BirknerAlex +bmhughes candlerb cawamata dankenigsberg @@ -18,6 +19,7 @@ eandersson eb3095 emmanuelthome giggsoff +hamalq izzyleung johnsonshi jordimassaguerpla @@ -47,8 +49,7 @@ timothegenzmer tnt-dev tomponline tsanghan +Vultaire WebSpider xiachen-rh xnox -hamalq -bmhughes -- cgit v1.2.3 From 8cf40a73646a9448746fc4043c6410ae92172e7c Mon Sep 17 00:00:00 2001 From: Andrew Lukoshko Date: Sat, 8 May 2021 00:31:14 +0300 Subject: Add AlmaLinux OS support (#872) AlmaLinux OS is RHEL-compatible so all the changes needed are trivial. --- README.md | 2 +- cloudinit/config/cc_ntp.py | 4 ++-- cloudinit/config/cc_yum_add_repo.py | 4 ++-- cloudinit/distros/__init__.py | 2 +- cloudinit/distros/almalinux.py | 9 +++++++++ cloudinit/net/sysconfig.py | 2 +- cloudinit/tests/test_util.py | 35 +++++++++++++++++++++++++++++++++++ cloudinit/util.py | 4 ++-- config/cloud.cfg.tmpl | 6 +++--- systemd/cloud-init-generator.tmpl | 2 +- systemd/cloud-init.service.tmpl | 2 +- tests/unittests/test_cli.py | 2 +- tools/.github-cla-signers | 1 + tools/render-cloudcfg | 4 ++-- 14 files changed, 62 insertions(+), 17 deletions(-) create mode 100644 cloudinit/distros/almalinux.py (limited to 'tools/.github-cla-signers') diff --git a/README.md b/README.md index aa6d84ae..01fd3b07 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ get in contact with that distribution and send them our way! | Supported OSes | Supported Public Clouds | Supported Private Clouds | | --- | --- | --- | -| Alpine Linux
ArchLinux
Debian
Fedora
FreeBSD
Gentoo Linux
NetBSD
OpenBSD
RHEL/CentOS
SLES/openSUSE
Ubuntu










| Amazon Web Services
Microsoft Azure
Google Cloud Platform
Oracle Cloud Infrastructure
Softlayer
Rackspace Public Cloud
IBM Cloud
Digital Ocean
Bigstep
Hetzner
Joyent
CloudSigma
Alibaba Cloud
OVH
OpenNebula
Exoscale
Scaleway
CloudStack
AltCloud
SmartOS
HyperOne
Vultr
Rootbox
| Bare metal installs
OpenStack
LXD
KVM
Metal-as-a-Service (MAAS)















| +| Alpine Linux
ArchLinux
Debian
Fedora
FreeBSD
Gentoo Linux
NetBSD
OpenBSD
RHEL/CentOS/AlmaLinux
SLES/openSUSE
Ubuntu










| Amazon Web Services
Microsoft Azure
Google Cloud Platform
Oracle Cloud Infrastructure
Softlayer
Rackspace Public Cloud
IBM Cloud
Digital Ocean
Bigstep
Hetzner
Joyent
CloudSigma
Alibaba Cloud
OVH
OpenNebula
Exoscale
Scaleway
CloudStack
AltCloud
SmartOS
HyperOne
Vultr
Rootbox
| Bare metal installs
OpenStack
LXD
KVM
Metal-as-a-Service (MAAS)















| ## To start developing cloud-init diff --git a/cloudinit/config/cc_ntp.py b/cloudinit/config/cc_ntp.py index e183993f..41c278ff 100644 --- a/cloudinit/config/cc_ntp.py +++ b/cloudinit/config/cc_ntp.py @@ -24,8 +24,8 @@ LOG = logging.getLogger(__name__) frequency = PER_INSTANCE NTP_CONF = '/etc/ntp.conf' NR_POOL_SERVERS = 4 -distros = ['alpine', 'centos', 'debian', 'fedora', 'opensuse', 'rhel', - 'sles', 'ubuntu'] +distros = ['almalinux', 'alpine', 'centos', 'debian', 'fedora', 'opensuse', + 'rhel', 'sles', 'ubuntu'] NTP_CLIENT_CONFIG = { 'chrony': { diff --git a/cloudinit/config/cc_yum_add_repo.py b/cloudinit/config/cc_yum_add_repo.py index 01fe683c..db513ed7 100644 --- a/cloudinit/config/cc_yum_add_repo.py +++ b/cloudinit/config/cc_yum_add_repo.py @@ -18,7 +18,7 @@ entry, the config entry will be skipped. **Module frequency:** per always -**Supported distros:** centos, fedora, rhel +**Supported distros:** almalinux, centos, fedora, rhel **Config keys**:: @@ -36,7 +36,7 @@ from configparser import ConfigParser from cloudinit import util -distros = ['centos', 'fedora', 'rhel'] +distros = ['almalinux', 'centos', 'fedora', 'rhel'] def _canonicalize_id(repo_id): diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 8b8a647d..107b928c 100755 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -46,7 +46,7 @@ OSFAMILIES = { 'debian': ['debian', 'ubuntu'], 'freebsd': ['freebsd'], 'gentoo': ['gentoo'], - 'redhat': ['amazon', 'centos', 'fedora', 'rhel'], + 'redhat': ['almalinux', 'amazon', 'centos', 'fedora', 'rhel'], 'suse': ['opensuse', 'sles'], } diff --git a/cloudinit/distros/almalinux.py b/cloudinit/distros/almalinux.py new file mode 100644 index 00000000..edb3165d --- /dev/null +++ b/cloudinit/distros/almalinux.py @@ -0,0 +1,9 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit.distros import rhel + + +class Distro(rhel.Distro): + pass + +# vi: ts=4 expandtab diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index e4607804..089b44b2 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -19,7 +19,7 @@ from .network_state import ( LOG = logging.getLogger(__name__) NM_CFG_FILE = "/etc/NetworkManager/NetworkManager.conf" -KNOWN_DISTROS = ['centos', 'fedora', 'rhel', 'suse'] +KNOWN_DISTROS = ['almalinux', 'centos', 'fedora', 'rhel', 'suse'] def _make_header(sep='#'): diff --git a/cloudinit/tests/test_util.py b/cloudinit/tests/test_util.py index e811917e..a4c02877 100644 --- a/cloudinit/tests/test_util.py +++ b/cloudinit/tests/test_util.py @@ -107,12 +107,31 @@ OS_RELEASE_REDHAT_7 = dedent("""\ REDHAT_SUPPORT_PRODUCT_VERSION="7.5" """) +OS_RELEASE_ALMALINUX_8 = dedent("""\ + NAME="AlmaLinux" + VERSION="8.3 (Purple Manul)" + ID="almalinux" + ID_LIKE="rhel centos fedora" + VERSION_ID="8.3" + PLATFORM_ID="platform:el8" + PRETTY_NAME="AlmaLinux 8.3 (Purple Manul)" + ANSI_COLOR="0;34" + CPE_NAME="cpe:/o:almalinux:almalinux:8.3:GA" + HOME_URL="https://almalinux.org/" + BUG_REPORT_URL="https://bugs.almalinux.org/" + + ALMALINUX_MANTISBT_PROJECT="AlmaLinux-8" + ALMALINUX_MANTISBT_PROJECT_VERSION="8.3" +""") + REDHAT_RELEASE_CENTOS_6 = "CentOS release 6.10 (Final)" REDHAT_RELEASE_CENTOS_7 = "CentOS Linux release 7.5.1804 (Core)" REDHAT_RELEASE_REDHAT_6 = ( "Red Hat Enterprise Linux Server release 6.10 (Santiago)") REDHAT_RELEASE_REDHAT_7 = ( "Red Hat Enterprise Linux Server release 7.5 (Maipo)") +REDHAT_RELEASE_ALMALINUX_8 = ( + "AlmaLinux release 8.3 (Purple Manul)") OS_RELEASE_DEBIAN = dedent("""\ @@ -502,6 +521,22 @@ class TestGetLinuxDistro(CiTestCase): dist = util.get_linux_distro() self.assertEqual(('centos', '7', 'Core'), dist) + @mock.patch('cloudinit.util.load_file') + def test_get_linux_almalinux8_rhrelease(self, m_os_release, m_path_exists): + """Verify almalinux 8 read from redhat-release.""" + m_os_release.return_value = REDHAT_RELEASE_ALMALINUX_8 + m_path_exists.side_effect = TestGetLinuxDistro.redhat_release_exists + dist = util.get_linux_distro() + self.assertEqual(('almalinux', '8.3', 'Purple Manul'), dist) + + @mock.patch('cloudinit.util.load_file') + def test_get_linux_almalinux8_osrelease(self, m_os_release, m_path_exists): + """Verify almalinux 8 read from os-release.""" + m_os_release.return_value = OS_RELEASE_ALMALINUX_8 + m_path_exists.side_effect = TestGetLinuxDistro.os_release_exists + dist = util.get_linux_distro() + self.assertEqual(('almalinux', '8.3', 'Purple Manul'), dist) + @mock.patch('cloudinit.util.load_file') def test_get_linux_debian(self, m_os_release, m_path_exists): """Verify we get the correct name and release name on Debian.""" diff --git a/cloudinit/util.py b/cloudinit/util.py index 4e0a72db..fdea1181 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -521,8 +521,8 @@ def system_info(): if system == "linux": linux_dist = info['dist'][0].lower() if linux_dist in ( - 'alpine', 'arch', 'centos', 'debian', 'fedora', 'rhel', - 'suse'): + 'almalinux', 'alpine', 'arch', 'centos', 'debian', 'fedora', + 'rhel', 'suse'): var = linux_dist elif linux_dist in ('ubuntu', 'linuxmint', 'mint'): var = 'ubuntu' diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl index 7171aaa5..8656daa7 100644 --- a/config/cloud.cfg.tmpl +++ b/config/cloud.cfg.tmpl @@ -21,7 +21,7 @@ disable_root: false disable_root: true {% endif %} -{% if variant in ["alpine", "amazon", "centos", "fedora", "rhel"] %} +{% if variant in ["almalinux", "alpine", "amazon", "centos", "fedora", "rhel"] %} mount_default_fields: [~, ~, 'auto', 'defaults,nofail', '0', '2'] {% if variant == "amazon" %} resize_rootfs: noblock @@ -153,7 +153,7 @@ cloud_final_modules: # (not accessible to handlers/transforms) system_info: # This will affect which distro class gets used -{% if variant in ["alpine", "amazon", "arch", "centos", "debian", +{% if variant in ["almalinux", "alpine", "amazon", "arch", "centos", "debian", "fedora", "freebsd", "netbsd", "openbsd", "rhel", "suse", "ubuntu"] %} distro: {{ variant }} @@ -206,7 +206,7 @@ system_info: primary: http://ports.ubuntu.com/ubuntu-ports security: http://ports.ubuntu.com/ubuntu-ports ssh_svcname: ssh -{% elif variant in ["alpine", "amazon", "arch", "centos", "fedora", +{% elif variant in ["almalinux", "alpine", "amazon", "arch", "centos", "fedora", "rhel", "suse"] %} # Default user name + that default users groups (if added/used) default_user: diff --git a/systemd/cloud-init-generator.tmpl b/systemd/cloud-init-generator.tmpl index 0773356b..9b103ef9 100755 --- a/systemd/cloud-init-generator.tmpl +++ b/systemd/cloud-init-generator.tmpl @@ -83,7 +83,7 @@ default() { check_for_datasource() { local ds_rc="" -{% if variant in ["rhel", "fedora", "centos"] %} +{% if variant in ["almalinux", "rhel", "fedora", "centos"] %} local dsidentify="/usr/libexec/cloud-init/ds-identify" {% else %} local dsidentify="/usr/lib/cloud-init/ds-identify" diff --git a/systemd/cloud-init.service.tmpl b/systemd/cloud-init.service.tmpl index f140344d..a5c51277 100644 --- a/systemd/cloud-init.service.tmpl +++ b/systemd/cloud-init.service.tmpl @@ -10,7 +10,7 @@ After=systemd-networkd-wait-online.service {% if variant in ["ubuntu", "unknown", "debian"] %} After=networking.service {% endif %} -{% if variant in ["centos", "fedora", "rhel"] %} +{% if variant in ["almalinux", "centos", "fedora", "rhel"] %} After=network.service After=NetworkManager.service {% endif %} diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py index 74f85959..f5cf514d 100644 --- a/tests/unittests/test_cli.py +++ b/tests/unittests/test_cli.py @@ -224,7 +224,7 @@ class TestCLI(test_helpers.FilesystemMockingTestCase): self._call_main(['cloud-init', 'devel', 'schema', '--docs', 'all']) expected_doc_sections = [ '**Supported distros:** all', - '**Supported distros:** alpine, centos, debian, fedora', + '**Supported distros:** almalinux, alpine, centos, debian, fedora', '**Config schema**:\n **resize_rootfs:** (true/false/noblock)', '**Examples**::\n\n runcmd:\n - [ ls, -l, / ]\n' ] diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 607f1d8e..48995057 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -3,6 +3,7 @@ ajmyyra AlexBaranowski Aman306 andrewbogott +andrewlukoshko antonyc aswinrajamannar beezly diff --git a/tools/render-cloudcfg b/tools/render-cloudcfg index ed454840..f5990748 100755 --- a/tools/render-cloudcfg +++ b/tools/render-cloudcfg @@ -4,8 +4,8 @@ import argparse import os import sys -VARIANTS = ["alpine", "amazon", "arch", "centos", "debian", "fedora", - "freebsd", "netbsd", "openbsd", "rhel", "suse", "ubuntu", +VARIANTS = ["almalinux", "alpine", "amazon", "arch", "centos", "debian", + "fedora", "freebsd", "netbsd", "openbsd", "rhel", "suse", "ubuntu", "unknown"] -- cgit v1.2.3 From 21a0b12052691d6634d0848dfa353c12939945e9 Mon Sep 17 00:00:00 2001 From: Geert Stappers Date: Fri, 14 May 2021 22:25:55 +0200 Subject: [examples] config-user-groups expire in the future (#902) Changed year 2012 into 2032 --- doc/examples/cloud-config-user-groups.txt | 2 +- tools/.github-cla-signers | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'tools/.github-cla-signers') diff --git a/doc/examples/cloud-config-user-groups.txt b/doc/examples/cloud-config-user-groups.txt index 4a5a7e20..1faecf75 100644 --- a/doc/examples/cloud-config-user-groups.txt +++ b/doc/examples/cloud-config-user-groups.txt @@ -19,7 +19,7 @@ users: primary_group: foobar groups: users selinux_user: staff_u - expiredate: '2012-09-01' + expiredate: '2032-09-01' ssh_import_id: foobar lock_passwd: false passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/ diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 48995057..d5eb3b06 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -44,6 +44,7 @@ riedel slyon smoser sshedi +stappersg TheRealFalcon taoyama timothegenzmer -- cgit v1.2.3 From c8db603218dd56123e43857d81f5eae5f66618f1 Mon Sep 17 00:00:00 2001 From: Emanuele Giuseppe Esposito Date: Fri, 21 May 2021 18:00:39 +0200 Subject: Add "esposem" as contributor (#907) --- tools/.github-cla-signers | 1 + 1 file changed, 1 insertion(+) (limited to 'tools/.github-cla-signers') diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index d5eb3b06..6f626643 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -19,6 +19,7 @@ dhensby eandersson eb3095 emmanuelthome +esposem giggsoff hamalq izzyleung -- cgit v1.2.3 From 7c1d27b8836c266f6db1e179b98ef32effeb750e Mon Sep 17 00:00:00 2001 From: Louis Abel Date: Tue, 25 May 2021 06:25:41 -0700 Subject: Add Rocky Linux support to cloud-init (#906) Rocky Linux is a RHEL-compatible distribution so all changes that have been made should be trivial. --- README.md | 2 +- cloudinit/config/cc_ntp.py | 2 +- cloudinit/config/cc_yum_add_repo.py | 4 ++-- cloudinit/distros/__init__.py | 2 +- cloudinit/distros/rocky.py | 9 +++++++++ cloudinit/net/sysconfig.py | 2 +- cloudinit/tests/.test_util.py.swp | Bin 0 -> 16384 bytes cloudinit/tests/test_util.py | 35 ++++++++++++++++++++++++++++++++++- cloudinit/util.py | 2 +- config/cloud.cfg.tmpl | 7 ++++--- systemd/cloud-init-generator.tmpl | 2 +- systemd/cloud-init.service.tmpl | 2 +- tests/unittests/test_cli.py | 3 ++- tools/.github-cla-signers | 1 + tools/render-cloudcfg | 4 ++-- 15 files changed, 61 insertions(+), 16 deletions(-) create mode 100644 cloudinit/distros/rocky.py create mode 100644 cloudinit/tests/.test_util.py.swp (limited to 'tools/.github-cla-signers') diff --git a/README.md b/README.md index 01fd3b07..bf232eba 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ get in contact with that distribution and send them our way! | Supported OSes | Supported Public Clouds | Supported Private Clouds | | --- | --- | --- | -| Alpine Linux
ArchLinux
Debian
Fedora
FreeBSD
Gentoo Linux
NetBSD
OpenBSD
RHEL/CentOS/AlmaLinux
SLES/openSUSE
Ubuntu










| Amazon Web Services
Microsoft Azure
Google Cloud Platform
Oracle Cloud Infrastructure
Softlayer
Rackspace Public Cloud
IBM Cloud
Digital Ocean
Bigstep
Hetzner
Joyent
CloudSigma
Alibaba Cloud
OVH
OpenNebula
Exoscale
Scaleway
CloudStack
AltCloud
SmartOS
HyperOne
Vultr
Rootbox
| Bare metal installs
OpenStack
LXD
KVM
Metal-as-a-Service (MAAS)















| +| Alpine Linux
ArchLinux
Debian
Fedora
FreeBSD
Gentoo Linux
NetBSD
OpenBSD
RHEL/CentOS/AlmaLinux/Rocky
SLES/openSUSE
Ubuntu










| Amazon Web Services
Microsoft Azure
Google Cloud Platform
Oracle Cloud Infrastructure
Softlayer
Rackspace Public Cloud
IBM Cloud
Digital Ocean
Bigstep
Hetzner
Joyent
CloudSigma
Alibaba Cloud
OVH
OpenNebula
Exoscale
Scaleway
CloudStack
AltCloud
SmartOS
HyperOne
Vultr
Rootbox
| Bare metal installs
OpenStack
LXD
KVM
Metal-as-a-Service (MAAS)















| ## To start developing cloud-init diff --git a/cloudinit/config/cc_ntp.py b/cloudinit/config/cc_ntp.py index 41c278ff..70c24610 100644 --- a/cloudinit/config/cc_ntp.py +++ b/cloudinit/config/cc_ntp.py @@ -25,7 +25,7 @@ frequency = PER_INSTANCE NTP_CONF = '/etc/ntp.conf' NR_POOL_SERVERS = 4 distros = ['almalinux', 'alpine', 'centos', 'debian', 'fedora', 'opensuse', - 'rhel', 'sles', 'ubuntu'] + 'rhel', 'rocky', 'sles', 'ubuntu'] NTP_CLIENT_CONFIG = { 'chrony': { diff --git a/cloudinit/config/cc_yum_add_repo.py b/cloudinit/config/cc_yum_add_repo.py index db513ed7..7daa6bd9 100644 --- a/cloudinit/config/cc_yum_add_repo.py +++ b/cloudinit/config/cc_yum_add_repo.py @@ -18,7 +18,7 @@ entry, the config entry will be skipped. **Module frequency:** per always -**Supported distros:** almalinux, centos, fedora, rhel +**Supported distros:** almalinux, centos, fedora, rhel, rocky **Config keys**:: @@ -36,7 +36,7 @@ from configparser import ConfigParser from cloudinit import util -distros = ['almalinux', 'centos', 'fedora', 'rhel'] +distros = ['almalinux', 'centos', 'fedora', 'rhel', 'rocky'] def _canonicalize_id(repo_id): diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 107b928c..57e33621 100755 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -46,7 +46,7 @@ OSFAMILIES = { 'debian': ['debian', 'ubuntu'], 'freebsd': ['freebsd'], 'gentoo': ['gentoo'], - 'redhat': ['almalinux', 'amazon', 'centos', 'fedora', 'rhel'], + 'redhat': ['almalinux', 'amazon', 'centos', 'fedora', 'rhel', 'rocky'], 'suse': ['opensuse', 'sles'], } diff --git a/cloudinit/distros/rocky.py b/cloudinit/distros/rocky.py new file mode 100644 index 00000000..edb3165d --- /dev/null +++ b/cloudinit/distros/rocky.py @@ -0,0 +1,9 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit.distros import rhel + + +class Distro(rhel.Distro): + pass + +# vi: ts=4 expandtab diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index 089b44b2..3a433c99 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -19,7 +19,7 @@ from .network_state import ( LOG = logging.getLogger(__name__) NM_CFG_FILE = "/etc/NetworkManager/NetworkManager.conf" -KNOWN_DISTROS = ['almalinux', 'centos', 'fedora', 'rhel', 'suse'] +KNOWN_DISTROS = ['almalinux', 'centos', 'fedora', 'rhel', 'rocky', 'suse'] def _make_header(sep='#'): diff --git a/cloudinit/tests/.test_util.py.swp b/cloudinit/tests/.test_util.py.swp new file mode 100644 index 00000000..78ef5865 Binary files /dev/null and b/cloudinit/tests/.test_util.py.swp differ diff --git a/cloudinit/tests/test_util.py b/cloudinit/tests/test_util.py index a4c02877..f9bc31be 100644 --- a/cloudinit/tests/test_util.py +++ b/cloudinit/tests/test_util.py @@ -124,6 +124,22 @@ OS_RELEASE_ALMALINUX_8 = dedent("""\ ALMALINUX_MANTISBT_PROJECT_VERSION="8.3" """) +OS_RELEASE_ROCKY_8 = dedent("""\ + NAME="Rocky Linux" + VERSION="8.3 (Green Obsidian)" + ID="rocky" + ID_LIKE="rhel fedora" + VERSION_ID="8.3" + PLATFORM_ID="platform:el8" + PRETTY_NAME="Rocky Linux 8.3 (Green Obsidian)" + ANSI_COLOR="0;31" + CPE_NAME="cpe:/o:rocky:rocky:8" + HOME_URL="https://rockylinux.org/" + BUG_REPORT_URL="https://bugs.rockylinux.org/" + ROCKY_SUPPORT_PRODUCT="Rocky Linux" + ROCKY_SUPPORT_PRODUCT_VERSION="8" +""") + REDHAT_RELEASE_CENTOS_6 = "CentOS release 6.10 (Final)" REDHAT_RELEASE_CENTOS_7 = "CentOS Linux release 7.5.1804 (Core)" REDHAT_RELEASE_REDHAT_6 = ( @@ -132,7 +148,8 @@ REDHAT_RELEASE_REDHAT_7 = ( "Red Hat Enterprise Linux Server release 7.5 (Maipo)") REDHAT_RELEASE_ALMALINUX_8 = ( "AlmaLinux release 8.3 (Purple Manul)") - +REDHAT_RELEASE_ROCKY_8 = ( + "Rocky Linux release 8.3 (Green Obsidian)") OS_RELEASE_DEBIAN = dedent("""\ PRETTY_NAME="Debian GNU/Linux 9 (stretch)" @@ -537,6 +554,22 @@ class TestGetLinuxDistro(CiTestCase): dist = util.get_linux_distro() self.assertEqual(('almalinux', '8.3', 'Purple Manul'), dist) + @mock.patch('cloudinit.util.load_file') + def test_get_linux_rocky8_rhrelease(self, m_os_release, m_path_exists): + """Verify rocky linux 8 read from redhat-release.""" + m_os_release.return_value = REDHAT_RELEASE_ROCKY_8 + m_path_exists.side_effect = TestGetLinuxDistro.redhat_release_exists + dist = util.get_linux_distro() + self.assertEqual(('rocky', '8.3', 'Green Obsidian'), dist) + + @mock.patch('cloudinit.util.load_file') + def test_get_linux_rocky8_osrelease(self, m_os_release, m_path_exists): + """Verify rocky linux 8 read from os-release.""" + m_os_release.return_value = OS_RELEASE_ROCKY_8 + m_path_exists.side_effect = TestGetLinuxDistro.os_release_exists + dist = util.get_linux_distro() + self.assertEqual(('rocky', '8.3', 'Green Obsidian'), dist) + @mock.patch('cloudinit.util.load_file') def test_get_linux_debian(self, m_os_release, m_path_exists): """Verify we get the correct name and release name on Debian.""" diff --git a/cloudinit/util.py b/cloudinit/util.py index fdea1181..2de1123e 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -522,7 +522,7 @@ def system_info(): linux_dist = info['dist'][0].lower() if linux_dist in ( 'almalinux', 'alpine', 'arch', 'centos', 'debian', 'fedora', - 'rhel', 'suse'): + 'rhel', 'rocky', 'suse'): var = linux_dist elif linux_dist in ('ubuntu', 'linuxmint', 'mint'): var = 'ubuntu' diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl index 8656daa7..2f6c3a7d 100644 --- a/config/cloud.cfg.tmpl +++ b/config/cloud.cfg.tmpl @@ -21,7 +21,8 @@ disable_root: false disable_root: true {% endif %} -{% if variant in ["almalinux", "alpine", "amazon", "centos", "fedora", "rhel"] %} +{% if variant in ["almalinux", "alpine", "amazon", "centos", "fedora", + "rhel", "rocky"] %} mount_default_fields: [~, ~, 'auto', 'defaults,nofail', '0', '2'] {% if variant == "amazon" %} resize_rootfs: noblock @@ -154,7 +155,7 @@ cloud_final_modules: system_info: # This will affect which distro class gets used {% if variant in ["almalinux", "alpine", "amazon", "arch", "centos", "debian", - "fedora", "freebsd", "netbsd", "openbsd", "rhel", + "fedora", "freebsd", "netbsd", "openbsd", "rhel", "rocky", "suse", "ubuntu"] %} distro: {{ variant }} {% else %} @@ -207,7 +208,7 @@ system_info: security: http://ports.ubuntu.com/ubuntu-ports ssh_svcname: ssh {% elif variant in ["almalinux", "alpine", "amazon", "arch", "centos", "fedora", - "rhel", "suse"] %} + "rhel", "rocky", "suse"] %} # Default user name + that default users groups (if added/used) default_user: {% if variant == "amazon" %} diff --git a/systemd/cloud-init-generator.tmpl b/systemd/cloud-init-generator.tmpl index 9b103ef9..0713db16 100755 --- a/systemd/cloud-init-generator.tmpl +++ b/systemd/cloud-init-generator.tmpl @@ -83,7 +83,7 @@ default() { check_for_datasource() { local ds_rc="" -{% if variant in ["almalinux", "rhel", "fedora", "centos"] %} +{% if variant in ["almalinux", "rhel", "fedora", "centos", "rocky"] %} local dsidentify="/usr/libexec/cloud-init/ds-identify" {% else %} local dsidentify="/usr/lib/cloud-init/ds-identify" diff --git a/systemd/cloud-init.service.tmpl b/systemd/cloud-init.service.tmpl index a5c51277..4da1a905 100644 --- a/systemd/cloud-init.service.tmpl +++ b/systemd/cloud-init.service.tmpl @@ -10,7 +10,7 @@ After=systemd-networkd-wait-online.service {% if variant in ["ubuntu", "unknown", "debian"] %} After=networking.service {% endif %} -{% if variant in ["almalinux", "centos", "fedora", "rhel"] %} +{% if variant in ["almalinux", "centos", "fedora", "rhel", "rocky"] %} After=network.service After=NetworkManager.service {% endif %} diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py index f5cf514d..fbc6ec11 100644 --- a/tests/unittests/test_cli.py +++ b/tests/unittests/test_cli.py @@ -224,7 +224,8 @@ class TestCLI(test_helpers.FilesystemMockingTestCase): self._call_main(['cloud-init', 'devel', 'schema', '--docs', 'all']) expected_doc_sections = [ '**Supported distros:** all', - '**Supported distros:** almalinux, alpine, centos, debian, fedora', + ('**Supported distros:** almalinux, alpine, centos, debian, ' + 'fedora, opensuse, rhel, rocky, sles, ubuntu'), '**Config schema**:\n **resize_rootfs:** (true/false/noblock)', '**Examples**::\n\n runcmd:\n - [ ls, -l, / ]\n' ] diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 6f626643..14916d31 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -35,6 +35,7 @@ manuelisimo marlluslustosa matthewruffell mitechie +nazunalika nicolasbock nishigori olivierlemasle diff --git a/tools/render-cloudcfg b/tools/render-cloudcfg index f5990748..9ec554bd 100755 --- a/tools/render-cloudcfg +++ b/tools/render-cloudcfg @@ -5,8 +5,8 @@ import os import sys VARIANTS = ["almalinux", "alpine", "amazon", "arch", "centos", "debian", - "fedora", "freebsd", "netbsd", "openbsd", "rhel", "suse", "ubuntu", - "unknown"] + "fedora", "freebsd", "netbsd", "openbsd", "rhel", "suse", "rocky", + "ubuntu", "unknown"] if "avoid-pep8-E402-import-not-top-of-file": -- cgit v1.2.3 From c8d3f99be84a4a04074a94c451387932bd086b26 Mon Sep 17 00:00:00 2001 From: Mike Russell Date: Wed, 16 Jun 2021 05:51:00 -0700 Subject: Small Doc Update for ReportEventStack and Test (#920) - small document update for ReportEventStack explaining post_files parameter - small unit test for test_reporting demonstrating the close of an event with optional post_files list --- cloudinit/reporting/events.py | 8 ++++++++ tests/unittests/test_reporting.py | 18 ++++++++++++++++++ tools/.github-cla-signers | 1 + 3 files changed, 27 insertions(+) (limited to 'tools/.github-cla-signers') diff --git a/cloudinit/reporting/events.py b/cloudinit/reporting/events.py index b8677c8b..9afad747 100644 --- a/cloudinit/reporting/events.py +++ b/cloudinit/reporting/events.py @@ -165,7 +165,15 @@ class ReportEventStack(object): :param result_on_exception: The result value to set if an exception is caught. default value is FAIL. + + :param post_files: + Can hold filepaths of files that are to get posted/created + regarding a given event. Something like success or failure information + in a given log file. For each filepath, if it's a valid regular file + it will get: read & encoded as base64 at the close of the event. + Default value, if None, is an empty list. """ + def __init__(self, name, description, message=None, parent=None, reporting_enabled=None, result_on_exception=status.FAIL, post_files=None): diff --git a/tests/unittests/test_reporting.py b/tests/unittests/test_reporting.py index 9f11fd5c..b78a6939 100644 --- a/tests/unittests/test_reporting.py +++ b/tests/unittests/test_reporting.py @@ -113,6 +113,7 @@ class TestReportingEvent(TestCase): class TestFinishReportingEvent(TestCase): + def test_as_has_result(self): result = events.status.SUCCESS name, desc = 'test_name', 'test_desc' @@ -121,6 +122,23 @@ class TestFinishReportingEvent(TestCase): self.assertTrue('result' in ret) self.assertEqual(ret['result'], result) + def test_has_result_with_optional_post_files(self): + result = events.status.SUCCESS + name, desc, files = 'test_name', 'test_desc', [ + '/really/fake/path/install.log'] + event = events.FinishReportingEvent( + name, desc, result, post_files=files) + ret = event.as_dict() + self.assertTrue('result' in ret) + self.assertTrue('files' in ret) + self.assertEqual(ret['result'], result) + posted_install_log = ret['files'][0] + self.assertTrue('path' in posted_install_log) + self.assertTrue('content' in posted_install_log) + self.assertTrue('encoding' in posted_install_log) + self.assertEqual(posted_install_log['path'], files[0]) + self.assertEqual(posted_install_log['encoding'], 'base64') + class TestBaseReportingHandler(TestCase): diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 14916d31..0b9fbe9e 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -22,6 +22,7 @@ emmanuelthome esposem giggsoff hamalq +irishgordo izzyleung johnsonshi jordimassaguerpla -- cgit v1.2.3 From 1243c5a1fc1f3546b40e48a2033a9abab546e40f Mon Sep 17 00:00:00 2001 From: Mark Mercado Date: Thu, 17 Jun 2021 10:37:23 -0400 Subject: Fix the spelling of "DigitalOcean" (#924) The name "DigitalOcean" doesn't have a space in it; it's a single compound word written in Pascal case (upper camel case). --- README.md | 2 +- cloudinit/sources/DataSourceDigitalOcean.py | 2 +- doc/rtd/topics/availability.rst | 2 +- doc/rtd/topics/datasources/digitalocean.rst | 4 ++-- tools/.github-cla-signers | 1 + 5 files changed, 6 insertions(+), 5 deletions(-) (limited to 'tools/.github-cla-signers') diff --git a/README.md b/README.md index 02b2f666..73d9e780 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ get in contact with that distribution and send them our way! | Supported OSes | Supported Public Clouds | Supported Private Clouds | | --- | --- | --- | -| Alpine Linux
ArchLinux
Debian
DragonFlyBSD
Fedora
FreeBSD
Gentoo Linux
NetBSD
OpenBSD
RHEL/CentOS/AlmaLinux/Rocky
SLES/openSUSE
Ubuntu










| Amazon Web Services
Microsoft Azure
Google Cloud Platform
Oracle Cloud Infrastructure
Softlayer
Rackspace Public Cloud
IBM Cloud
Digital Ocean
Bigstep
Hetzner
Joyent
CloudSigma
Alibaba Cloud
OVH
OpenNebula
Exoscale
Scaleway
CloudStack
AltCloud
SmartOS
HyperOne
Vultr
Rootbox
| Bare metal installs
OpenStack
LXD
KVM
Metal-as-a-Service (MAAS)















| +| Alpine Linux
ArchLinux
Debian
DragonFlyBSD
Fedora
FreeBSD
Gentoo Linux
NetBSD
OpenBSD
RHEL/CentOS/AlmaLinux/Rocky
SLES/openSUSE
Ubuntu










| Amazon Web Services
Microsoft Azure
Google Cloud Platform
Oracle Cloud Infrastructure
Softlayer
Rackspace Public Cloud
IBM Cloud
DigitalOcean
Bigstep
Hetzner
Joyent
CloudSigma
Alibaba Cloud
OVH
OpenNebula
Exoscale
Scaleway
CloudStack
AltCloud
SmartOS
HyperOne
Vultr
Rootbox
| Bare metal installs
OpenStack
LXD
KVM
Metal-as-a-Service (MAAS)















| ## To start developing cloud-init diff --git a/cloudinit/sources/DataSourceDigitalOcean.py b/cloudinit/sources/DataSourceDigitalOcean.py index 5040ce5b..08805d99 100644 --- a/cloudinit/sources/DataSourceDigitalOcean.py +++ b/cloudinit/sources/DataSourceDigitalOcean.py @@ -54,7 +54,7 @@ class DataSourceDigitalOcean(sources.DataSource): if not is_do: return False - LOG.info("Running on digital ocean. droplet_id=%s", droplet_id) + LOG.info("Running on DigitalOcean. droplet_id=%s", droplet_id) ipv4LL_nic = None if self.use_ip4LL: diff --git a/doc/rtd/topics/availability.rst b/doc/rtd/topics/availability.rst index e4480754..a45a49d6 100644 --- a/doc/rtd/topics/availability.rst +++ b/doc/rtd/topics/availability.rst @@ -43,7 +43,7 @@ environments in the public cloud: - Softlayer - Rackspace Public Cloud - IBM Cloud -- Digital Ocean +- DigitalOcean - Bigstep - Hetzner - Joyent diff --git a/doc/rtd/topics/datasources/digitalocean.rst b/doc/rtd/topics/datasources/digitalocean.rst index 88f1e5f5..a4910408 100644 --- a/doc/rtd/topics/datasources/digitalocean.rst +++ b/doc/rtd/topics/datasources/digitalocean.rst @@ -1,7 +1,7 @@ .. _datasource_digital_ocean: -Digital Ocean -============= +DigitalOcean +============ The `DigitalOcean`_ datasource consumes the content served from DigitalOcean's `metadata service`_. This metadata service serves information about the diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 0b9fbe9e..a7c36a8c 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -32,6 +32,7 @@ klausenbusk landon912 lucasmoura lungj +mamercad manuelisimo marlluslustosa matthewruffell -- cgit v1.2.3 From 894dd7c4fbca019ab2703de860b3d28e2722534b Mon Sep 17 00:00:00 2001 From: Renan Rodrigo Date: Mon, 5 Jul 2021 18:01:10 -0300 Subject: add renanrodrigo as a contributor (#938) --- tools/.github-cla-signers | 1 + 1 file changed, 1 insertion(+) (limited to 'tools/.github-cla-signers') diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index a7c36a8c..e2979ed4 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -44,6 +44,7 @@ olivierlemasle omBratteng onitake qubidt +renanrodrigo riedel slyon smoser -- cgit v1.2.3 From 758acf976f2cb67a85411467fa5fca2ea17a2283 Mon Sep 17 00:00:00 2001 From: Mal Graty Date: Tue, 3 Aug 2021 17:08:26 +0100 Subject: Implementing device_aliases as described in docs (#945) Implement missing device_aliases feature The device_aliases key has been documented as part of disk_setup for years, however the feature was never implemented. This implements the feature as documented allowing usercfg (rather than dsconfig) to create a mapping of device names. This is not to be confused with disk_aliases, a very similar map but existing solely for use by datasources. LP: #1867532 --- cloudinit/config/cc_disk_setup.py | 13 +- cloudinit/config/cc_mounts.py | 17 +- tests/integration_tests/bugs/test_lp1920939.py | 140 --------------- tests/integration_tests/modules/test_disk_setup.py | 192 +++++++++++++++++++++ .../unittests/test_handler/test_handler_mounts.py | 9 + tools/.github-cla-signers | 1 + 6 files changed, 225 insertions(+), 147 deletions(-) delete mode 100644 tests/integration_tests/bugs/test_lp1920939.py create mode 100644 tests/integration_tests/modules/test_disk_setup.py (limited to 'tools/.github-cla-signers') diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index 22af3813..3ec49ca5 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -125,9 +125,15 @@ def handle(_name, cfg, cloud, log, _args): See doc/examples/cloud-config-disk-setup.txt for documentation on the format. """ + device_aliases = cfg.get("device_aliases", {}) + + def alias_to_device(cand): + name = device_aliases.get(cand) + return cloud.device_name_to_device(name or cand) or name + disk_setup = cfg.get("disk_setup") if isinstance(disk_setup, dict): - update_disk_setup_devices(disk_setup, cloud.device_name_to_device) + update_disk_setup_devices(disk_setup, alias_to_device) log.debug("Partitioning disks: %s", str(disk_setup)) for disk, definition in disk_setup.items(): if not isinstance(definition, dict): @@ -145,7 +151,7 @@ def handle(_name, cfg, cloud, log, _args): fs_setup = cfg.get("fs_setup") if isinstance(fs_setup, list): log.debug("setting up filesystems: %s", str(fs_setup)) - update_fs_setup_devices(fs_setup, cloud.device_name_to_device) + update_fs_setup_devices(fs_setup, alias_to_device) for definition in fs_setup: if not isinstance(definition, dict): log.warning("Invalid file system definition: %s" % definition) @@ -174,7 +180,8 @@ def update_disk_setup_devices(disk_setup, tformer): del disk_setup[transformed] disk_setup[transformed] = disk_setup[origname] - disk_setup[transformed]['_origname'] = origname + if isinstance(disk_setup[transformed], dict): + disk_setup[transformed]['_origname'] = origname del disk_setup[origname] LOG.debug("updated disk_setup device entry '%s' to '%s'", origname, transformed) diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index c22d1698..eeb008d2 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -123,7 +123,7 @@ def _is_block_device(device_path, partition_path=None): return os.path.exists(sys_path) -def sanitize_devname(startname, transformer, log): +def sanitize_devname(startname, transformer, log, aliases=None): log.debug("Attempting to determine the real name of %s", startname) # workaround, allow user to specify 'ephemeral' @@ -137,9 +137,14 @@ def sanitize_devname(startname, transformer, log): return startname device_path, partition_number = util.expand_dotted_devname(devname) + orig = device_path + + if aliases: + device_path = aliases.get(device_path, device_path) + if orig != device_path: + log.debug("Mapped device alias %s to %s", orig, device_path) if is_meta_device_name(device_path): - orig = device_path device_path = transformer(device_path) if not device_path: return None @@ -394,6 +399,8 @@ def handle(_name, cfg, cloud, log, _args): fstab_devs[toks[0]] = line fstab_lines.append(line) + device_aliases = cfg.get("device_aliases", {}) + for i in range(len(cfgmnt)): # skip something that wasn't a list if not isinstance(cfgmnt[i], list): @@ -402,7 +409,8 @@ def handle(_name, cfg, cloud, log, _args): continue start = str(cfgmnt[i][0]) - sanitized = sanitize_devname(start, cloud.device_name_to_device, log) + sanitized = sanitize_devname(start, cloud.device_name_to_device, log, + aliases=device_aliases) if sanitized != start: log.debug("changed %s => %s" % (start, sanitized)) @@ -444,7 +452,8 @@ def handle(_name, cfg, cloud, log, _args): # entry has the same device name for defmnt in defmnts: start = defmnt[0] - sanitized = sanitize_devname(start, cloud.device_name_to_device, log) + sanitized = sanitize_devname(start, cloud.device_name_to_device, log, + aliases=device_aliases) if sanitized != start: log.debug("changed default device %s => %s" % (start, sanitized)) diff --git a/tests/integration_tests/bugs/test_lp1920939.py b/tests/integration_tests/bugs/test_lp1920939.py deleted file mode 100644 index 408792a6..00000000 --- a/tests/integration_tests/bugs/test_lp1920939.py +++ /dev/null @@ -1,140 +0,0 @@ -""" -Test that disk setup can run successfully on a mounted partition when -partprobe is being used. - -lp-1920939 -""" -import json -import os -import pytest -from uuid import uuid4 -from pycloudlib.lxd.instance import LXDInstance - -from cloudinit.subp import subp -from tests.integration_tests.instances import IntegrationInstance - -DISK_PATH = '/tmp/test_disk_setup_{}'.format(uuid4()) - - -def setup_and_mount_lxd_disk(instance: LXDInstance): - subp('lxc config device add {} test-disk-setup-disk disk source={}'.format( - instance.name, DISK_PATH).split()) - - -@pytest.yield_fixture -def create_disk(): - # 640k should be enough for anybody - subp('dd if=/dev/zero of={} bs=1k count=640'.format(DISK_PATH).split()) - yield - os.remove(DISK_PATH) - - -USERDATA = """\ -#cloud-config -disk_setup: - /dev/sdb: - table_type: mbr - layout: [50, 50] - overwrite: True -fs_setup: - - label: test - device: /dev/sdb1 - filesystem: ext4 - - label: test2 - device: /dev/sdb2 - filesystem: ext4 -mounts: -- ["/dev/sdb1", "/mnt1"] -- ["/dev/sdb2", "/mnt2"] -""" - -UPDATED_USERDATA = """\ -#cloud-config -disk_setup: - /dev/sdb: - table_type: mbr - layout: [100] - overwrite: True -fs_setup: - - label: test3 - device: /dev/sdb1 - filesystem: ext4 -mounts: -- ["/dev/sdb1", "/mnt3"] -""" - - -def _verify_first_disk_setup(client, log): - assert 'Traceback' not in log - assert 'WARN' not in log - lsblk = json.loads(client.execute('lsblk --json')) - sdb = [x for x in lsblk['blockdevices'] if x['name'] == 'sdb'][0] - assert len(sdb['children']) == 2 - assert sdb['children'][0]['name'] == 'sdb1' - assert sdb['children'][0]['mountpoint'] == '/mnt1' - assert sdb['children'][1]['name'] == 'sdb2' - assert sdb['children'][1]['mountpoint'] == '/mnt2' - - -@pytest.mark.user_data(USERDATA) -@pytest.mark.lxd_setup.with_args(setup_and_mount_lxd_disk) -@pytest.mark.ubuntu -@pytest.mark.lxd_vm -# Not bionic or xenial because the LXD agent gets in the way of us -# changing the userdata -@pytest.mark.not_bionic -@pytest.mark.not_xenial -def test_disk_setup_when_mounted(create_disk, client: IntegrationInstance): - """Test lp-1920939. - - We insert an extra disk into our VM, format it to have two partitions, - modify our cloud config to mount devices before disk setup, and modify - our userdata to setup a single partition on the disk. - - This allows cloud-init to attempt disk setup on a mounted partition. - When blockdev is in use, it will fail with - "blockdev: ioctl error on BLKRRPART: Device or resource busy" along - with a warning and a traceback. When partprobe is in use, everything - should work successfully. - """ - log = client.read_from_file('/var/log/cloud-init.log') - _verify_first_disk_setup(client, log) - - # Update our userdata and cloud.cfg to mount then perform new disk setup - client.write_to_file( - '/var/lib/cloud/seed/nocloud-net/user-data', - UPDATED_USERDATA - ) - client.execute("sed -i 's/write-files/write-files\\n - mounts/' " - "/etc/cloud/cloud.cfg") - - client.execute('cloud-init clean --logs') - client.restart() - - # Assert new setup works as expected - assert 'Traceback' not in log - assert 'WARN' not in log - - lsblk = json.loads(client.execute('lsblk --json')) - sdb = [x for x in lsblk['blockdevices'] if x['name'] == 'sdb'][0] - assert len(sdb['children']) == 1 - assert sdb['children'][0]['name'] == 'sdb1' - assert sdb['children'][0]['mountpoint'] == '/mnt3' - - -@pytest.mark.user_data(USERDATA) -@pytest.mark.lxd_setup.with_args(setup_and_mount_lxd_disk) -@pytest.mark.ubuntu -@pytest.mark.lxd_vm -def test_disk_setup_no_partprobe(create_disk, client: IntegrationInstance): - """Ensure disk setup still works as expected without partprobe.""" - # We can't do this part in a bootcmd because the path has already - # been found by the time we get to the bootcmd - client.execute('rm $(which partprobe)') - client.execute('cloud-init clean --logs') - client.restart() - - log = client.read_from_file('/var/log/cloud-init.log') - _verify_first_disk_setup(client, log) - - assert 'partprobe' not in log diff --git a/tests/integration_tests/modules/test_disk_setup.py b/tests/integration_tests/modules/test_disk_setup.py new file mode 100644 index 00000000..1fc96c52 --- /dev/null +++ b/tests/integration_tests/modules/test_disk_setup.py @@ -0,0 +1,192 @@ +import json +import os +import pytest +from uuid import uuid4 +from pycloudlib.lxd.instance import LXDInstance + +from cloudinit.subp import subp +from tests.integration_tests.instances import IntegrationInstance + +DISK_PATH = '/tmp/test_disk_setup_{}'.format(uuid4()) + + +def setup_and_mount_lxd_disk(instance: LXDInstance): + subp('lxc config device add {} test-disk-setup-disk disk source={}'.format( + instance.name, DISK_PATH).split()) + + +@pytest.yield_fixture +def create_disk(): + # 640k should be enough for anybody + subp('dd if=/dev/zero of={} bs=1k count=640'.format(DISK_PATH).split()) + yield + os.remove(DISK_PATH) + + +ALIAS_USERDATA = """\ +#cloud-config +device_aliases: + my_alias: /dev/sdb +disk_setup: + my_alias: + table_type: mbr + layout: [50, 50] + overwrite: True +fs_setup: +- label: fs1 + device: my_alias.1 + filesystem: ext4 +- label: fs2 + device: my_alias.2 + filesystem: ext4 +mounts: +- ["my_alias.1", "/mnt1"] +- ["my_alias.2", "/mnt2"] +""" + + +@pytest.mark.user_data(ALIAS_USERDATA) +@pytest.mark.lxd_setup.with_args(setup_and_mount_lxd_disk) +@pytest.mark.ubuntu +@pytest.mark.lxd_vm +class TestDeviceAliases: + """Test devices aliases work on disk setup/mount""" + + def test_device_alias(self, create_disk, client: IntegrationInstance): + log = client.read_from_file('/var/log/cloud-init.log') + assert ( + "updated disk_setup device entry 'my_alias' to '/dev/sdb'" + ) in log + assert 'changed my_alias.1 => /dev/sdb1' in log + assert 'changed my_alias.2 => /dev/sdb2' in log + assert 'WARN' not in log + assert 'Traceback' not in log + + lsblk = json.loads(client.execute('lsblk --json')) + sdb = [x for x in lsblk['blockdevices'] if x['name'] == 'sdb'][0] + assert len(sdb['children']) == 2 + assert sdb['children'][0]['name'] == 'sdb1' + assert sdb['children'][0]['mountpoint'] == '/mnt1' + assert sdb['children'][1]['name'] == 'sdb2' + assert sdb['children'][1]['mountpoint'] == '/mnt2' + + +PARTPROBE_USERDATA = """\ +#cloud-config +disk_setup: + /dev/sdb: + table_type: mbr + layout: [50, 50] + overwrite: True +fs_setup: + - label: test + device: /dev/sdb1 + filesystem: ext4 + - label: test2 + device: /dev/sdb2 + filesystem: ext4 +mounts: +- ["/dev/sdb1", "/mnt1"] +- ["/dev/sdb2", "/mnt2"] +""" + +UPDATED_PARTPROBE_USERDATA = """\ +#cloud-config +disk_setup: + /dev/sdb: + table_type: mbr + layout: [100] + overwrite: True +fs_setup: + - label: test3 + device: /dev/sdb1 + filesystem: ext4 +mounts: +- ["/dev/sdb1", "/mnt3"] +""" + + +@pytest.mark.user_data(PARTPROBE_USERDATA) +@pytest.mark.lxd_setup.with_args(setup_and_mount_lxd_disk) +@pytest.mark.ubuntu +@pytest.mark.lxd_vm +class TestPartProbeAvailability: + """Test disk setup works with partprobe + + Disk setup can run successfully on a mounted partition when + partprobe is being used. + + lp-1920939 + """ + + def _verify_first_disk_setup(self, client, log): + assert 'Traceback' not in log + assert 'WARN' not in log + lsblk = json.loads(client.execute('lsblk --json')) + sdb = [x for x in lsblk['blockdevices'] if x['name'] == 'sdb'][0] + assert len(sdb['children']) == 2 + assert sdb['children'][0]['name'] == 'sdb1' + assert sdb['children'][0]['mountpoint'] == '/mnt1' + assert sdb['children'][1]['name'] == 'sdb2' + assert sdb['children'][1]['mountpoint'] == '/mnt2' + + # Not bionic or xenial because the LXD agent gets in the way of us + # changing the userdata + @pytest.mark.not_bionic + @pytest.mark.not_xenial + def test_disk_setup_when_mounted( + self, create_disk, client: IntegrationInstance + ): + """Test lp-1920939. + + We insert an extra disk into our VM, format it to have two partitions, + modify our cloud config to mount devices before disk setup, and modify + our userdata to setup a single partition on the disk. + + This allows cloud-init to attempt disk setup on a mounted partition. + When blockdev is in use, it will fail with + "blockdev: ioctl error on BLKRRPART: Device or resource busy" along + with a warning and a traceback. When partprobe is in use, everything + should work successfully. + """ + log = client.read_from_file('/var/log/cloud-init.log') + self._verify_first_disk_setup(client, log) + + # Update our userdata and cloud.cfg to mount then perform new disk + # setup + client.write_to_file( + '/var/lib/cloud/seed/nocloud-net/user-data', + UPDATED_PARTPROBE_USERDATA, + ) + client.execute( + "sed -i 's/write-files/write-files\\n - mounts/' " + "/etc/cloud/cloud.cfg" + ) + + client.execute('cloud-init clean --logs') + client.restart() + + # Assert new setup works as expected + assert 'Traceback' not in log + assert 'WARN' not in log + + lsblk = json.loads(client.execute('lsblk --json')) + sdb = [x for x in lsblk['blockdevices'] if x['name'] == 'sdb'][0] + assert len(sdb['children']) == 1 + assert sdb['children'][0]['name'] == 'sdb1' + assert sdb['children'][0]['mountpoint'] == '/mnt3' + + def test_disk_setup_no_partprobe( + self, create_disk, client: IntegrationInstance + ): + """Ensure disk setup still works as expected without partprobe.""" + # We can't do this part in a bootcmd because the path has already + # been found by the time we get to the bootcmd + client.execute('rm $(which partprobe)') + client.execute('cloud-init clean --logs') + client.restart() + + log = client.read_from_file('/var/log/cloud-init.log') + self._verify_first_disk_setup(client, log) + + assert 'partprobe' not in log diff --git a/tests/unittests/test_handler/test_handler_mounts.py b/tests/unittests/test_handler/test_handler_mounts.py index e87069f6..69e8b30d 100644 --- a/tests/unittests/test_handler/test_handler_mounts.py +++ b/tests/unittests/test_handler/test_handler_mounts.py @@ -133,6 +133,15 @@ class TestSanitizeDevname(test_helpers.FilesystemMockingTestCase): disk_path, cc_mounts.sanitize_devname(disk_path, None, mock.Mock())) + def test_device_aliases_remapping(self): + disk_path = '/dev/sda' + self.mock_existence_of_disk(disk_path) + self.assertEqual(disk_path, + cc_mounts.sanitize_devname('mydata', + lambda x: None, + mock.Mock(), + {'mydata': disk_path})) + class TestSwapFileCreation(test_helpers.FilesystemMockingTestCase): diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index e2979ed4..3c2c6d14 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -32,6 +32,7 @@ klausenbusk landon912 lucasmoura lungj +mal mamercad manuelisimo marlluslustosa -- cgit v1.2.3 From 8b4a9bc7b81e61943af873bad92e2133f8275b0b Mon Sep 17 00:00:00 2001 From: Andrew Kutz <101085+akutz@users.noreply.github.com> Date: Mon, 9 Aug 2021 21:24:07 -0500 Subject: Datasource for VMware (#953) This patch finally introduces the Cloud-Init Datasource for VMware GuestInfo as a part of cloud-init proper. This datasource has existed since 2018, and rapidly became the de facto datasource for developers working with Packer, Terraform, for projects like kube-image-builder, and the de jure datasource for Photon OS. The major change to the datasource from its previous incarnation is the name. Now named DatasourceVMware, this new version of the datasource will allow multiple transport types in addition to GuestInfo keys. This datasource includes several unique features developed to address real-world situations: * Support for reading any key (metadata, userdata, vendordata) both from the guestinfo table when running on a VM in vSphere as well as from an environment variable when running inside of a container, useful for rapid dev/test. * Allows booting with DHCP while still providing full participation in Cloud-Init instance data and Jinja queries. The netifaces library provides the ability to inspect the network after it is online, and the runtime network configuration is then merged into the existing metadata and persisted to disk. * Advertises the local_ipv4 and local_ipv6 addresses via guestinfo as well. This is useful as Guest Tools is not always able to identify what would be considered the local address. The primary author and current steward of this datasource spoke at Cloud-Init Con 2020 where there was interest in contributing this datasource to the Cloud-Init codebase. The datasource currently lives in its own GitHub repository at https://github.com/vmware/cloud-init-vmware-guestinfo. Once the datasource is merged into Cloud-Init, the old repository will be deprecated. --- README.md | 2 +- cloudinit/settings.py | 1 + cloudinit/sources/DataSourceVMware.py | 871 +++++++++++++++++++++++++ doc/rtd/topics/availability.rst | 1 + doc/rtd/topics/datasources.rst | 1 + doc/rtd/topics/datasources/vmware.rst | 359 ++++++++++ requirements.txt | 9 + tests/unittests/test_datasource/test_common.py | 3 + tests/unittests/test_datasource/test_vmware.py | 377 +++++++++++ tests/unittests/test_ds_identify.py | 279 +++++++- tools/.github-cla-signers | 1 + tools/ds-identify | 76 ++- 12 files changed, 1977 insertions(+), 3 deletions(-) create mode 100644 cloudinit/sources/DataSourceVMware.py create mode 100644 doc/rtd/topics/datasources/vmware.rst create mode 100644 tests/unittests/test_datasource/test_vmware.py (limited to 'tools/.github-cla-signers') diff --git a/README.md b/README.md index caf9a6e9..5828c2fa 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ get in contact with that distribution and send them our way! | Supported OSes | Supported Public Clouds | Supported Private Clouds | | --- | --- | --- | -| Alpine Linux
ArchLinux
Debian
DragonFlyBSD
Fedora
FreeBSD
Gentoo Linux
NetBSD
OpenBSD
RHEL/CentOS/AlmaLinux/Rocky/PhotonOS/Virtuozzo/EuroLinux
SLES/openSUSE
Ubuntu










| Amazon Web Services
Microsoft Azure
Google Cloud Platform
Oracle Cloud Infrastructure
Softlayer
Rackspace Public Cloud
IBM Cloud
DigitalOcean
Bigstep
Hetzner
Joyent
CloudSigma
Alibaba Cloud
OVH
OpenNebula
Exoscale
Scaleway
CloudStack
AltCloud
SmartOS
HyperOne
Vultr
Rootbox
| Bare metal installs
OpenStack
LXD
KVM
Metal-as-a-Service (MAAS)















| +| Alpine Linux
ArchLinux
Debian
DragonFlyBSD
Fedora
FreeBSD
Gentoo Linux
NetBSD
OpenBSD
RHEL/CentOS/AlmaLinux/Rocky/PhotonOS/Virtuozzo/EuroLinux
SLES/openSUSE
Ubuntu










| Amazon Web Services
Microsoft Azure
Google Cloud Platform
Oracle Cloud Infrastructure
Softlayer
Rackspace Public Cloud
IBM Cloud
DigitalOcean
Bigstep
Hetzner
Joyent
CloudSigma
Alibaba Cloud
OVH
OpenNebula
Exoscale
Scaleway
CloudStack
AltCloud
SmartOS
HyperOne
Vultr
Rootbox
| Bare metal installs
OpenStack
LXD
KVM
Metal-as-a-Service (MAAS)
VMware















| ## To start developing cloud-init diff --git a/cloudinit/settings.py b/cloudinit/settings.py index 23e4c0ad..f69005ea 100644 --- a/cloudinit/settings.py +++ b/cloudinit/settings.py @@ -43,6 +43,7 @@ CFG_BUILTIN = { 'Exoscale', 'RbxCloud', 'UpCloud', + 'VMware', # At the end to act as a 'catch' when none of the above work... 'None', ], diff --git a/cloudinit/sources/DataSourceVMware.py b/cloudinit/sources/DataSourceVMware.py new file mode 100644 index 00000000..22ca63de --- /dev/null +++ b/cloudinit/sources/DataSourceVMware.py @@ -0,0 +1,871 @@ +# Cloud-Init DataSource for VMware +# +# Copyright (c) 2018-2021 VMware, Inc. All Rights Reserved. +# +# Authors: Anish Swaminathan +# Andrew Kutz +# +# This file is part of cloud-init. See LICENSE file for license information. + +"""Cloud-Init DataSource for VMware + +This module provides a cloud-init datasource for VMware systems and supports +multiple transports types, including: + + * EnvVars + * GuestInfo + +Netifaces (https://github.com/al45tair/netifaces) + + Please note this module relies on the netifaces project to introspect the + runtime, network configuration of the host on which this datasource is + running. This is in contrast to the rest of cloud-init which uses the + cloudinit/netinfo module. + + The reasons for using netifaces include: + + * Netifaces is built in C and is more portable across multiple systems + and more deterministic than shell exec'ing local network commands and + parsing their output. + + * Netifaces provides a stable way to determine the view of the host's + network after DHCP has brought the network online. Unlike most other + datasources, this datasource still provides support for JINJA queries + based on networking information even when the network is based on a + DHCP lease. While this does not tie this datasource directly to + netifaces, it does mean the ability to consistently obtain the + correct information is paramount. + + * It is currently possible to execute this datasource on macOS + (which many developers use today) to print the output of the + get_host_info function. This function calls netifaces to obtain + the same runtime network configuration that the datasource would + persist to the local system's instance data. + + However, the netinfo module fails on macOS. The result is either a + hung operation that requires a SIGINT to return control to the user, + or, if brew is used to install iproute2mac, the ip commands are used + but produce output the netinfo module is unable to parse. + + While macOS is not a target of cloud-init, this feature is quite + useful when working on this datasource. + + For more information about this behavior, please see the following + PR comment, https://bit.ly/3fG7OVh. + + The authors of this datasource are not opposed to moving away from + netifaces. The goal may be to eventually do just that. This proviso was + added to the top of this module as a way to remind future-us and others + why netifaces was used in the first place in order to either smooth the + transition away from netifaces or embrace it further up the cloud-init + stack. +""" + +import collections +import copy +from distutils.spawn import find_executable +import ipaddress +import json +import os +import socket +import time + +from cloudinit import dmi, log as logging +from cloudinit import sources +from cloudinit import util +from cloudinit.subp import subp, ProcessExecutionError + +import netifaces + + +PRODUCT_UUID_FILE_PATH = "/sys/class/dmi/id/product_uuid" + +LOG = logging.getLogger(__name__) +NOVAL = "No value found" + +DATA_ACCESS_METHOD_ENVVAR = "envvar" +DATA_ACCESS_METHOD_GUESTINFO = "guestinfo" + +VMWARE_RPCTOOL = find_executable("vmware-rpctool") +REDACT = "redact" +CLEANUP_GUESTINFO = "cleanup-guestinfo" +VMX_GUESTINFO = "VMX_GUESTINFO" +GUESTINFO_EMPTY_YAML_VAL = "---" + +LOCAL_IPV4 = "local-ipv4" +LOCAL_IPV6 = "local-ipv6" +WAIT_ON_NETWORK = "wait-on-network" +WAIT_ON_NETWORK_IPV4 = "ipv4" +WAIT_ON_NETWORK_IPV6 = "ipv6" + + +class DataSourceVMware(sources.DataSource): + """ + Setting the hostname: + The hostname is set by way of the metadata key "local-hostname". + + Setting the instance ID: + The instance ID may be set by way of the metadata key "instance-id". + However, if this value is absent then the instance ID is read + from the file /sys/class/dmi/id/product_uuid. + + Configuring the network: + The network is configured by setting the metadata key "network" + with a value consistent with Network Config Versions 1 or 2, + depending on the Linux distro's version of cloud-init: + + Network Config Version 1 - http://bit.ly/cloudinit-net-conf-v1 + Network Config Version 2 - http://bit.ly/cloudinit-net-conf-v2 + + For example, CentOS 7's official cloud-init package is version + 0.7.9 and does not support Network Config Version 2. However, + this datasource still supports supplying Network Config Version 2 + data as long as the Linux distro's cloud-init package is new + enough to parse the data. + + The metadata key "network.encoding" may be used to indicate the + format of the metadata key "network". Valid encodings are base64 + and gzip+base64. + """ + + dsname = "VMware" + + def __init__(self, sys_cfg, distro, paths, ud_proc=None): + sources.DataSource.__init__(self, sys_cfg, distro, paths, ud_proc) + + self.data_access_method = None + self.vmware_rpctool = VMWARE_RPCTOOL + + def _get_data(self): + """ + _get_data loads the metadata, userdata, and vendordata from one of + the following locations in the given order: + + * envvars + * guestinfo + + Please note when updating this function with support for new data + transports, the order should match the order in the dscheck_VMware + function from the file ds-identify. + """ + + # Initialize the locally scoped metadata, userdata, and vendordata + # variables. They are assigned below depending on the detected data + # access method. + md, ud, vd = None, None, None + + # First check to see if there is data via env vars. + if os.environ.get(VMX_GUESTINFO, ""): + md = guestinfo_envvar("metadata") + ud = guestinfo_envvar("userdata") + vd = guestinfo_envvar("vendordata") + + if md or ud or vd: + self.data_access_method = DATA_ACCESS_METHOD_ENVVAR + + # At this point, all additional data transports are valid only on + # a VMware platform. + if not self.data_access_method: + system_type = dmi.read_dmi_data("system-product-name") + if system_type is None: + LOG.debug("No system-product-name found") + return False + if "vmware" not in system_type.lower(): + LOG.debug("Not a VMware platform") + return False + + # If no data was detected, check the guestinfo transport next. + if not self.data_access_method: + if self.vmware_rpctool: + md = guestinfo("metadata", self.vmware_rpctool) + ud = guestinfo("userdata", self.vmware_rpctool) + vd = guestinfo("vendordata", self.vmware_rpctool) + + if md or ud or vd: + self.data_access_method = DATA_ACCESS_METHOD_GUESTINFO + + if not self.data_access_method: + LOG.error("failed to find a valid data access method") + return False + + LOG.info("using data access method %s", self._get_subplatform()) + + # Get the metadata. + self.metadata = process_metadata(load_json_or_yaml(md)) + + # Get the user data. + self.userdata_raw = ud + + # Get the vendor data. + self.vendordata_raw = vd + + # Redact any sensitive information. + self.redact_keys() + + # get_data returns true if there is any available metadata, + # userdata, or vendordata. + if self.metadata or self.userdata_raw or self.vendordata_raw: + return True + else: + return False + + def setup(self, is_new_instance): + """setup(is_new_instance) + + This is called before user-data and vendor-data have been processed. + + Unless the datasource has set mode to 'local', then networking + per 'fallback' or per 'network_config' will have been written and + brought up the OS at this point. + """ + + host_info = wait_on_network(self.metadata) + LOG.info("got host-info: %s", host_info) + + # Reflect any possible local IPv4 or IPv6 addresses in the guest + # info. + advertise_local_ip_addrs(host_info) + + # Ensure the metadata gets updated with information about the + # host, including the network interfaces, default IP addresses, + # etc. + self.metadata = util.mergemanydict([self.metadata, host_info]) + + # Persist the instance data for versions of cloud-init that support + # doing so. This occurs here rather than in the get_data call in + # order to ensure that the network interfaces are up and can be + # persisted with the metadata. + self.persist_instance_data() + + def _get_subplatform(self): + get_key_name_fn = None + if self.data_access_method == DATA_ACCESS_METHOD_ENVVAR: + get_key_name_fn = get_guestinfo_envvar_key_name + elif self.data_access_method == DATA_ACCESS_METHOD_GUESTINFO: + get_key_name_fn = get_guestinfo_key_name + else: + return sources.METADATA_UNKNOWN + + return "%s (%s)" % ( + self.data_access_method, + get_key_name_fn("metadata"), + ) + + @property + def network_config(self): + if "network" in self.metadata: + LOG.debug("using metadata network config") + else: + LOG.debug("using fallback network config") + self.metadata["network"] = { + "config": self.distro.generate_fallback_config(), + } + return self.metadata["network"]["config"] + + def get_instance_id(self): + # Pull the instance ID out of the metadata if present. Otherwise + # read the file /sys/class/dmi/id/product_uuid for the instance ID. + if self.metadata and "instance-id" in self.metadata: + return self.metadata["instance-id"] + with open(PRODUCT_UUID_FILE_PATH, "r") as id_file: + self.metadata["instance-id"] = str(id_file.read()).rstrip().lower() + return self.metadata["instance-id"] + + def get_public_ssh_keys(self): + for key_name in ( + "public-keys-data", + "public_keys_data", + "public-keys", + "public_keys", + ): + if key_name in self.metadata: + return sources.normalize_pubkey_data(self.metadata[key_name]) + return [] + + def redact_keys(self): + # Determine if there are any keys to redact. + keys_to_redact = None + if REDACT in self.metadata: + keys_to_redact = self.metadata[REDACT] + elif CLEANUP_GUESTINFO in self.metadata: + # This is for backwards compatibility. + keys_to_redact = self.metadata[CLEANUP_GUESTINFO] + + if self.data_access_method == DATA_ACCESS_METHOD_GUESTINFO: + guestinfo_redact_keys(keys_to_redact, self.vmware_rpctool) + + +def decode(key, enc_type, data): + """ + decode returns the decoded string value of data + key is a string used to identify the data being decoded in log messages + """ + LOG.debug("Getting encoded data for key=%s, enc=%s", key, enc_type) + + raw_data = None + if enc_type in ["gzip+base64", "gz+b64"]: + LOG.debug("Decoding %s format %s", enc_type, key) + raw_data = util.decomp_gzip(util.b64d(data)) + elif enc_type in ["base64", "b64"]: + LOG.debug("Decoding %s format %s", enc_type, key) + raw_data = util.b64d(data) + else: + LOG.debug("Plain-text data %s", key) + raw_data = data + + return util.decode_binary(raw_data) + + +def get_none_if_empty_val(val): + """ + get_none_if_empty_val returns None if the provided value, once stripped + of its trailing whitespace, is empty or equal to GUESTINFO_EMPTY_YAML_VAL. + + The return value is always a string, regardless of whether the input is + a bytes class or a string. + """ + + # If the provided value is a bytes class, convert it to a string to + # simplify the rest of this function's logic. + val = util.decode_binary(val) + val = val.rstrip() + if len(val) == 0 or val == GUESTINFO_EMPTY_YAML_VAL: + return None + return val + + +def advertise_local_ip_addrs(host_info): + """ + advertise_local_ip_addrs gets the local IP address information from + the provided host_info map and sets the addresses in the guestinfo + namespace + """ + if not host_info: + return + + # Reflect any possible local IPv4 or IPv6 addresses in the guest + # info. + local_ipv4 = host_info.get(LOCAL_IPV4) + if local_ipv4: + guestinfo_set_value(LOCAL_IPV4, local_ipv4) + LOG.info("advertised local ipv4 address %s in guestinfo", local_ipv4) + + local_ipv6 = host_info.get(LOCAL_IPV6) + if local_ipv6: + guestinfo_set_value(LOCAL_IPV6, local_ipv6) + LOG.info("advertised local ipv6 address %s in guestinfo", local_ipv6) + + +def handle_returned_guestinfo_val(key, val): + """ + handle_returned_guestinfo_val returns the provided value if it is + not empty or set to GUESTINFO_EMPTY_YAML_VAL, otherwise None is + returned + """ + val = get_none_if_empty_val(val) + if val: + return val + LOG.debug("No value found for key %s", key) + return None + + +def get_guestinfo_key_name(key): + return "guestinfo." + key + + +def get_guestinfo_envvar_key_name(key): + return ("vmx." + get_guestinfo_key_name(key)).upper().replace(".", "_", -1) + + +def guestinfo_envvar(key): + val = guestinfo_envvar_get_value(key) + if not val: + return None + enc_type = guestinfo_envvar_get_value(key + ".encoding") + return decode(get_guestinfo_envvar_key_name(key), enc_type, val) + + +def guestinfo_envvar_get_value(key): + env_key = get_guestinfo_envvar_key_name(key) + return handle_returned_guestinfo_val(key, os.environ.get(env_key, "")) + + +def guestinfo(key, vmware_rpctool=VMWARE_RPCTOOL): + """ + guestinfo returns the guestinfo value for the provided key, decoding + the value when required + """ + val = guestinfo_get_value(key, vmware_rpctool) + if not val: + return None + enc_type = guestinfo_get_value(key + ".encoding", vmware_rpctool) + return decode(get_guestinfo_key_name(key), enc_type, val) + + +def guestinfo_get_value(key, vmware_rpctool=VMWARE_RPCTOOL): + """ + Returns a guestinfo value for the specified key. + """ + LOG.debug("Getting guestinfo value for key %s", key) + + try: + (stdout, stderr) = subp( + [ + vmware_rpctool, + "info-get " + get_guestinfo_key_name(key), + ] + ) + if stderr == NOVAL: + LOG.debug("No value found for key %s", key) + elif not stdout: + LOG.error("Failed to get guestinfo value for key %s", key) + return handle_returned_guestinfo_val(key, stdout) + except ProcessExecutionError as error: + if error.stderr == NOVAL: + LOG.debug("No value found for key %s", key) + else: + util.logexc( + LOG, + "Failed to get guestinfo value for key %s: %s", + key, + error, + ) + except Exception: + util.logexc( + LOG, + "Unexpected error while trying to get " + + "guestinfo value for key %s", + key, + ) + + return None + + +def guestinfo_set_value(key, value, vmware_rpctool=VMWARE_RPCTOOL): + """ + Sets a guestinfo value for the specified key. Set value to an empty string + to clear an existing guestinfo key. + """ + + # If value is an empty string then set it to a single space as it is not + # possible to set a guestinfo key to an empty string. Setting a guestinfo + # key to a single space is as close as it gets to clearing an existing + # guestinfo key. + if value == "": + value = " " + + LOG.debug("Setting guestinfo key=%s to value=%s", key, value) + + try: + subp( + [ + vmware_rpctool, + ("info-set %s %s" % (get_guestinfo_key_name(key), value)), + ] + ) + return True + except ProcessExecutionError as error: + util.logexc( + LOG, + "Failed to set guestinfo key=%s to value=%s: %s", + key, + value, + error, + ) + except Exception: + util.logexc( + LOG, + "Unexpected error while trying to set " + + "guestinfo key=%s to value=%s", + key, + value, + ) + + return None + + +def guestinfo_redact_keys(keys, vmware_rpctool=VMWARE_RPCTOOL): + """ + guestinfo_redact_keys redacts guestinfo of all of the keys in the given + list. each key will have its value set to "---". Since the value is valid + YAML, cloud-init can still read it if it tries. + """ + if not keys: + return + if not type(keys) in (list, tuple): + keys = [keys] + for key in keys: + key_name = get_guestinfo_key_name(key) + LOG.info("clearing %s", key_name) + if not guestinfo_set_value( + key, GUESTINFO_EMPTY_YAML_VAL, vmware_rpctool + ): + LOG.error("failed to clear %s", key_name) + LOG.info("clearing %s.encoding", key_name) + if not guestinfo_set_value(key + ".encoding", "", vmware_rpctool): + LOG.error("failed to clear %s.encoding", key_name) + + +def load_json_or_yaml(data): + """ + load first attempts to unmarshal the provided data as JSON, and if + that fails then attempts to unmarshal the data as YAML. If data is + None then a new dictionary is returned. + """ + if not data: + return {} + try: + return util.load_json(data) + except (json.JSONDecodeError, TypeError): + return util.load_yaml(data) + + +def process_metadata(data): + """ + process_metadata processes metadata and loads the optional network + configuration. + """ + network = None + if "network" in data: + network = data["network"] + del data["network"] + + network_enc = None + if "network.encoding" in data: + network_enc = data["network.encoding"] + del data["network.encoding"] + + if network: + if isinstance(network, collections.abc.Mapping): + LOG.debug("network data copied to 'config' key") + network = {"config": copy.deepcopy(network)} + else: + LOG.debug("network data to be decoded %s", network) + dec_net = decode("metadata.network", network_enc, network) + network = { + "config": load_json_or_yaml(dec_net), + } + + LOG.debug("network data %s", network) + data["network"] = network + + return data + + +# Used to match classes to dependencies +datasources = [ + (DataSourceVMware, (sources.DEP_FILESYSTEM,)), # Run at init-local + (DataSourceVMware, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), +] + + +def get_datasource_list(depends): + """ + Return a list of data sources that match this set of dependencies + """ + return sources.list_from_depends(depends, datasources) + + +def get_default_ip_addrs(): + """ + Returns the default IPv4 and IPv6 addresses based on the device(s) used for + the default route. Please note that None may be returned for either address + family if that family has no default route or if there are multiple + addresses associated with the device used by the default route for a given + address. + """ + # TODO(promote and use netifaces in cloudinit.net* modules) + gateways = netifaces.gateways() + if "default" not in gateways: + return None, None + + default_gw = gateways["default"] + if ( + netifaces.AF_INET not in default_gw + and netifaces.AF_INET6 not in default_gw + ): + return None, None + + ipv4 = None + ipv6 = None + + gw4 = default_gw.get(netifaces.AF_INET) + if gw4: + _, dev4 = gw4 + addr4_fams = netifaces.ifaddresses(dev4) + if addr4_fams: + af_inet4 = addr4_fams.get(netifaces.AF_INET) + if af_inet4: + if len(af_inet4) > 1: + LOG.warning( + "device %s has more than one ipv4 address: %s", + dev4, + af_inet4, + ) + elif "addr" in af_inet4[0]: + ipv4 = af_inet4[0]["addr"] + + # Try to get the default IPv6 address by first seeing if there is a default + # IPv6 route. + gw6 = default_gw.get(netifaces.AF_INET6) + if gw6: + _, dev6 = gw6 + addr6_fams = netifaces.ifaddresses(dev6) + if addr6_fams: + af_inet6 = addr6_fams.get(netifaces.AF_INET6) + if af_inet6: + if len(af_inet6) > 1: + LOG.warning( + "device %s has more than one ipv6 address: %s", + dev6, + af_inet6, + ) + elif "addr" in af_inet6[0]: + ipv6 = af_inet6[0]["addr"] + + # If there is a default IPv4 address but not IPv6, then see if there is a + # single IPv6 address associated with the same device associated with the + # default IPv4 address. + if ipv4 and not ipv6: + af_inet6 = addr4_fams.get(netifaces.AF_INET6) + if af_inet6: + if len(af_inet6) > 1: + LOG.warning( + "device %s has more than one ipv6 address: %s", + dev4, + af_inet6, + ) + elif "addr" in af_inet6[0]: + ipv6 = af_inet6[0]["addr"] + + # If there is a default IPv6 address but not IPv4, then see if there is a + # single IPv4 address associated with the same device associated with the + # default IPv6 address. + if not ipv4 and ipv6: + af_inet4 = addr6_fams.get(netifaces.AF_INET) + if af_inet4: + if len(af_inet4) > 1: + LOG.warning( + "device %s has more than one ipv4 address: %s", + dev6, + af_inet4, + ) + elif "addr" in af_inet4[0]: + ipv4 = af_inet4[0]["addr"] + + return ipv4, ipv6 + + +# patched socket.getfqdn() - see https://bugs.python.org/issue5004 + + +def getfqdn(name=""): + """Get fully qualified domain name from name. + An empty argument is interpreted as meaning the local host. + """ + # TODO(may want to promote this function to util.getfqdn) + # TODO(may want to extend util.get_hostname to accept fqdn=True param) + name = name.strip() + if not name or name == "0.0.0.0": + name = util.get_hostname() + try: + addrs = socket.getaddrinfo( + name, None, 0, socket.SOCK_DGRAM, 0, socket.AI_CANONNAME + ) + except socket.error: + pass + else: + for addr in addrs: + if addr[3]: + name = addr[3] + break + return name + + +def is_valid_ip_addr(val): + """ + Returns false if the address is loopback, link local or unspecified; + otherwise true is returned. + """ + # TODO(extend cloudinit.net.is_ip_addr exclude link_local/loopback etc) + # TODO(migrate to use cloudinit.net.is_ip_addr)# + + addr = None + try: + addr = ipaddress.ip_address(val) + except ipaddress.AddressValueError: + addr = ipaddress.ip_address(str(val)) + except Exception: + return None + + if addr.is_link_local or addr.is_loopback or addr.is_unspecified: + return False + return True + + +def get_host_info(): + """ + Returns host information such as the host name and network interfaces. + """ + # TODO(look to promote netifices use up in cloud-init netinfo funcs) + host_info = { + "network": { + "interfaces": { + "by-mac": collections.OrderedDict(), + "by-ipv4": collections.OrderedDict(), + "by-ipv6": collections.OrderedDict(), + }, + }, + } + hostname = getfqdn(util.get_hostname()) + if hostname: + host_info["hostname"] = hostname + host_info["local-hostname"] = hostname + host_info["local_hostname"] = hostname + + default_ipv4, default_ipv6 = get_default_ip_addrs() + if default_ipv4: + host_info[LOCAL_IPV4] = default_ipv4 + if default_ipv6: + host_info[LOCAL_IPV6] = default_ipv6 + + by_mac = host_info["network"]["interfaces"]["by-mac"] + by_ipv4 = host_info["network"]["interfaces"]["by-ipv4"] + by_ipv6 = host_info["network"]["interfaces"]["by-ipv6"] + + ifaces = netifaces.interfaces() + for dev_name in ifaces: + addr_fams = netifaces.ifaddresses(dev_name) + af_link = addr_fams.get(netifaces.AF_LINK) + af_inet4 = addr_fams.get(netifaces.AF_INET) + af_inet6 = addr_fams.get(netifaces.AF_INET6) + + mac = None + if af_link and "addr" in af_link[0]: + mac = af_link[0]["addr"] + + # Do not bother recording localhost + if mac == "00:00:00:00:00:00": + continue + + if mac and (af_inet4 or af_inet6): + key = mac + val = {} + if af_inet4: + af_inet4_vals = [] + for ip_info in af_inet4: + if not is_valid_ip_addr(ip_info["addr"]): + continue + af_inet4_vals.append(ip_info) + val["ipv4"] = af_inet4_vals + if af_inet6: + af_inet6_vals = [] + for ip_info in af_inet6: + if not is_valid_ip_addr(ip_info["addr"]): + continue + af_inet6_vals.append(ip_info) + val["ipv6"] = af_inet6_vals + by_mac[key] = val + + if af_inet4: + for ip_info in af_inet4: + key = ip_info["addr"] + if not is_valid_ip_addr(key): + continue + val = copy.deepcopy(ip_info) + del val["addr"] + if mac: + val["mac"] = mac + by_ipv4[key] = val + + if af_inet6: + for ip_info in af_inet6: + key = ip_info["addr"] + if not is_valid_ip_addr(key): + continue + val = copy.deepcopy(ip_info) + del val["addr"] + if mac: + val["mac"] = mac + by_ipv6[key] = val + + return host_info + + +def wait_on_network(metadata): + # Determine whether we need to wait on the network coming online. + wait_on_ipv4 = False + wait_on_ipv6 = False + if WAIT_ON_NETWORK in metadata: + wait_on_network = metadata[WAIT_ON_NETWORK] + if WAIT_ON_NETWORK_IPV4 in wait_on_network: + wait_on_ipv4_val = wait_on_network[WAIT_ON_NETWORK_IPV4] + if isinstance(wait_on_ipv4_val, bool): + wait_on_ipv4 = wait_on_ipv4_val + else: + wait_on_ipv4 = util.translate_bool(wait_on_ipv4_val) + if WAIT_ON_NETWORK_IPV6 in wait_on_network: + wait_on_ipv6_val = wait_on_network[WAIT_ON_NETWORK_IPV6] + if isinstance(wait_on_ipv6_val, bool): + wait_on_ipv6 = wait_on_ipv6_val + else: + wait_on_ipv6 = util.translate_bool(wait_on_ipv6_val) + + # Get information about the host. + host_info = None + while host_info is None: + # This loop + sleep results in two logs every second while waiting + # for either ipv4 or ipv6 up. Do we really need to log each iteration + # or can we log once and log on successful exit? + host_info = get_host_info() + + network = host_info.get("network") or {} + interfaces = network.get("interfaces") or {} + by_ipv4 = interfaces.get("by-ipv4") or {} + by_ipv6 = interfaces.get("by-ipv6") or {} + + if wait_on_ipv4: + ipv4_ready = len(by_ipv4) > 0 if by_ipv4 else False + if not ipv4_ready: + host_info = None + + if wait_on_ipv6: + ipv6_ready = len(by_ipv6) > 0 if by_ipv6 else False + if not ipv6_ready: + host_info = None + + if host_info is None: + LOG.debug( + "waiting on network: wait4=%s, ready4=%s, wait6=%s, ready6=%s", + wait_on_ipv4, + ipv4_ready, + wait_on_ipv6, + ipv6_ready, + ) + time.sleep(1) + + LOG.debug("waiting on network complete") + return host_info + + +def main(): + """ + Executed when this file is used as a program. + """ + try: + logging.setupBasicLogging() + except Exception: + pass + metadata = { + "wait-on-network": {"ipv4": True, "ipv6": "false"}, + "network": {"config": {"dhcp": True}}, + } + host_info = wait_on_network(metadata) + metadata = util.mergemanydict([metadata, host_info]) + print(util.json_dumps(metadata)) + + +if __name__ == "__main__": + main() + +# vi: ts=4 expandtab diff --git a/doc/rtd/topics/availability.rst b/doc/rtd/topics/availability.rst index e0644534..71827177 100644 --- a/doc/rtd/topics/availability.rst +++ b/doc/rtd/topics/availability.rst @@ -67,5 +67,6 @@ Additionally, cloud-init is supported on these private clouds: - LXD - KVM - Metal-as-a-Service (MAAS) +- VMware .. vi: textwidth=79 diff --git a/doc/rtd/topics/datasources.rst b/doc/rtd/topics/datasources.rst index 497b1467..f5aee1c2 100644 --- a/doc/rtd/topics/datasources.rst +++ b/doc/rtd/topics/datasources.rst @@ -50,6 +50,7 @@ The following is a list of documents for each supported datasource: datasources/upcloud.rst datasources/zstack.rst datasources/vultr.rst + datasources/vmware.rst Creation ======== diff --git a/doc/rtd/topics/datasources/vmware.rst b/doc/rtd/topics/datasources/vmware.rst new file mode 100644 index 00000000..996eb61f --- /dev/null +++ b/doc/rtd/topics/datasources/vmware.rst @@ -0,0 +1,359 @@ +.. _datasource_vmware: + +VMware +====== + +This datasource is for use with systems running on a VMware platform such as +vSphere and currently supports the following data transports: + + +* `GuestInfo `_ keys + +Configuration +------------- + +The configuration method is dependent upon the transport: + +GuestInfo Keys +^^^^^^^^^^^^^^ + +One method of providing meta, user, and vendor data is by setting the following +key/value pairs on a VM's ``extraConfig`` `property `_ : + +.. list-table:: + :header-rows: 1 + + * - Property + - Description + * - ``guestinfo.metadata`` + - A YAML or JSON document containing the cloud-init metadata. + * - ``guestinfo.metadata.encoding`` + - The encoding type for ``guestinfo.metadata``. + * - ``guestinfo.userdata`` + - A YAML document containing the cloud-init user data. + * - ``guestinfo.userdata.encoding`` + - The encoding type for ``guestinfo.userdata``. + * - ``guestinfo.vendordata`` + - A YAML document containing the cloud-init vendor data. + * - ``guestinfo.vendordata.encoding`` + - The encoding type for ``guestinfo.vendordata``. + + +All ``guestinfo.*.encoding`` values may be set to ``base64`` or +``gzip+base64``. + +Features +-------- + +This section reviews several features available in this datasource, regardless +of how the meta, user, and vendor data was discovered. + +Instance data and lazy networks +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +One of the hallmarks of cloud-init is `its use of instance-data and JINJA +queries <../instancedata.html#using-instance-data>`_ +-- the ability to write queries in user and vendor data that reference runtime +information present in ``/run/cloud-init/instance-data.json``. This works well +when the metadata provides all of the information up front, such as the network +configuration. For systems that rely on DHCP, however, this information may not +be available when the metadata is persisted to disk. + +This datasource ensures that even if the instance is using DHCP to configure +networking, the same details about the configured network are available in +``/run/cloud-init/instance-data.json`` as if static networking was used. This +information collected at runtime is easy to demonstrate by executing the +datasource on the command line. From the root of this repository, run the +following command: + +.. code-block:: bash + + PYTHONPATH="$(pwd)" python3 cloudinit/sources/DataSourceVMware.py + +The above command will result in output similar to the below JSON: + +.. code-block:: json + + { + "hostname": "akutz.localhost", + "local-hostname": "akutz.localhost", + "local-ipv4": "192.168.0.188", + "local_hostname": "akutz.localhost", + "network": { + "config": { + "dhcp": true + }, + "interfaces": { + "by-ipv4": { + "172.0.0.2": { + "netmask": "255.255.255.255", + "peer": "172.0.0.2" + }, + "192.168.0.188": { + "broadcast": "192.168.0.255", + "mac": "64:4b:f0:18:9a:21", + "netmask": "255.255.255.0" + } + }, + "by-ipv6": { + "fd8e:d25e:c5b6:1:1f5:b2fd:8973:22f2": { + "flags": 208, + "mac": "64:4b:f0:18:9a:21", + "netmask": "ffff:ffff:ffff:ffff::/64" + } + }, + "by-mac": { + "64:4b:f0:18:9a:21": { + "ipv4": [ + { + "addr": "192.168.0.188", + "broadcast": "192.168.0.255", + "netmask": "255.255.255.0" + } + ], + "ipv6": [ + { + "addr": "fd8e:d25e:c5b6:1:1f5:b2fd:8973:22f2", + "flags": 208, + "netmask": "ffff:ffff:ffff:ffff::/64" + } + ] + }, + "ac:de:48:00:11:22": { + "ipv6": [] + } + } + } + }, + "wait-on-network": { + "ipv4": true, + "ipv6": "false" + } + } + + +Redacting sensitive information +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Sometimes the cloud-init userdata might contain sensitive information, and it +may be desirable to have the ``guestinfo.userdata`` key (or other guestinfo +keys) redacted as soon as its data is read by the datasource. This is possible +by adding the following to the metadata: + +.. code-block:: yaml + + redact: # formerly named cleanup-guestinfo, which will also work + - userdata + - vendordata + +When the above snippet is added to the metadata, the datasource will iterate +over the elements in the ``redact`` array and clear each of the keys. For +example, when the guestinfo transport is used, the above snippet will cause +the following commands to be executed: + +.. code-block:: shell + + vmware-rpctool "info-set guestinfo.userdata ---" + vmware-rpctool "info-set guestinfo.userdata.encoding " + vmware-rpctool "info-set guestinfo.vendordata ---" + vmware-rpctool "info-set guestinfo.vendordata.encoding " + +Please note that keys are set to the valid YAML string ``---`` as it is not +possible remove an existing key from the guestinfo key-space. A key's analogous +encoding property will be set to a single white-space character, causing the +datasource to treat the actual key value as plain-text, thereby loading it as +an empty YAML doc (hence the aforementioned ``---``\ ). + +Reading the local IP addresses +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This datasource automatically discovers the local IPv4 and IPv6 addresses for +a guest operating system based on the default routes. However, when inspecting +a VM externally, it's not possible to know what the *default* IP address is for +the guest OS. That's why this datasource sets the discovered, local IPv4 and +IPv6 addresses back in the guestinfo namespace as the following keys: + + +* ``guestinfo.local-ipv4`` +* ``guestinfo.local-ipv6`` + +It is possible that a host may not have any default, local IP addresses. It's +also possible the reported, local addresses are link-local addresses. But these +two keys may be used to discover what this datasource determined were the local +IPv4 and IPv6 addresses for a host. + +Waiting on the network +^^^^^^^^^^^^^^^^^^^^^^ + +Sometimes cloud-init may bring up the network, but it will not finish coming +online before the datasource's ``setup`` function is called, resulting in an +``/var/run/cloud-init/instance-data.json`` file that does not have the correct +network information. It is possible to instruct the datasource to wait until an +IPv4 or IPv6 address is available before writing the instance data with the +following metadata properties: + +.. code-block:: yaml + + wait-on-network: + ipv4: true + ipv6: true + +If either of the above values are true, then the datasource will sleep for a +second, check the network status, and repeat until one or both addresses from +the specified families are available. + +Walkthrough +----------- + +The following series of steps is a demonstration on how to configure a VM with +this datasource: + + +#. Create the metadata file for the VM. Save the following YAML to a file named + ``metadata.yaml``\ : + + .. code-block:: yaml + + instance-id: cloud-vm + local-hostname: cloud-vm + network: + version: 2 + ethernets: + nics: + match: + name: ens* + dhcp4: yes + +#. Create the userdata file ``userdata.yaml``\ : + + .. code-block:: yaml + + #cloud-config + + users: + - default + - name: akutz + primary_group: akutz + sudo: ALL=(ALL) NOPASSWD:ALL + groups: sudo, wheel + ssh_import_id: None + lock_passwd: true + ssh_authorized_keys: + - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDE0c5FczvcGSh/tG4iw+Fhfi/O5/EvUM/96js65tly4++YTXK1d9jcznPS5ruDlbIZ30oveCBd3kT8LLVFwzh6hepYTf0YmCTpF4eDunyqmpCXDvVscQYRXyasEm5olGmVe05RrCJSeSShAeptv4ueIn40kZKOghinGWLDSZG4+FFfgrmcMCpx5YSCtX2gvnEYZJr0czt4rxOZuuP7PkJKgC/mt2PcPjooeX00vAj81jjU2f3XKrjjz2u2+KIt9eba+vOQ6HiC8c2IzRkUAJ5i1atLy8RIbejo23+0P4N2jjk17QySFOVHwPBDTYb0/0M/4ideeU74EN/CgVsvO6JrLsPBR4dojkV5qNbMNxIVv5cUwIy2ThlLgqpNCeFIDLCWNZEFKlEuNeSQ2mPtIO7ETxEL2Cz5y/7AIuildzYMc6wi2bofRC8HmQ7rMXRWdwLKWsR0L7SKjHblIwarxOGqLnUI+k2E71YoP7SZSlxaKi17pqkr0OMCF+kKqvcvHAQuwGqyumTEWOlH6TCx1dSPrW+pVCZSHSJtSTfDW2uzL6y8k10MT06+pVunSrWo5LHAXcS91htHV1M1UrH/tZKSpjYtjMb5+RonfhaFRNzvj7cCE1f3Kp8UVqAdcGBTtReoE8eRUT63qIxjw03a7VwAyB2w+9cu1R9/vAo8SBeRqw== sakutz@gmail.com + +#. Please note this step requires that the VM be powered off. All of the + commands below use the VMware CLI tool, `govc `_. + + Go ahead and assign the path to the VM to the environment variable ``VM``\ : + + .. code-block:: shell + + export VM="/inventory/path/to/the/vm" + +#. Power off the VM: + + .. raw:: html + +
+ + ⚠️ First Boot Mode + + To ensure the next power-on operation results in a first-boot scenario for + cloud-init, it may be necessary to run the following command just before + powering off the VM: + + .. code-block:: bash + + cloud-init clean + + Otherwise cloud-init may not run in first-boot mode. For more information + on how the boot mode is determined, please see the + `First Boot Documentation <../boot.html#first-boot-determination>`_. + + .. raw:: html + +
+ + .. code-block:: shell + + govc vm.power -off "${VM}" + +#. + Export the environment variables that contain the cloud-init metadata and + userdata: + + .. code-block:: shell + + export METADATA=$(gzip -c9 /dev/null || base64; }) \ + USERDATA=$(gzip -c9 /dev/null || base64; }) + +#. + Assign the metadata and userdata to the VM: + + .. code-block:: shell + + govc vm.change -vm "${VM}" \ + -e guestinfo.metadata="${METADATA}" \ + -e guestinfo.metadata.encoding="gzip+base64" \ + -e guestinfo.userdata="${USERDATA}" \ + -e guestinfo.userdata.encoding="gzip+base64" + + Please note the above commands include specifying the encoding for the + properties. This is important as it informs the datasource how to decode + the data for cloud-init. Valid values for ``metadata.encoding`` and + ``userdata.encoding`` include: + + + * ``base64`` + * ``gzip+base64`` + +#. + Power on the VM: + + .. code-block:: shell + + govc vm.power -vm "${VM}" -on + +If all went according to plan, the CentOS box is: + +* Locked down, allowing SSH access only for the user in the userdata +* Configured for a dynamic IP address via DHCP +* Has a hostname of ``cloud-vm`` + +Examples +-------- + +This section reviews common configurations: + +Setting the hostname +^^^^^^^^^^^^^^^^^^^^ + +The hostname is set by way of the metadata key ``local-hostname``. + +Setting the instance ID +^^^^^^^^^^^^^^^^^^^^^^^ + +The instance ID may be set by way of the metadata key ``instance-id``. However, +if this value is absent then then the instance ID is read from the file +``/sys/class/dmi/id/product_uuid``. + +Providing public SSH keys +^^^^^^^^^^^^^^^^^^^^^^^^^ + +The public SSH keys may be set by way of the metadata key ``public-keys-data``. +Each newline-terminated string will be interpreted as a separate SSH public +key, which will be placed in distro's default user's +``~/.ssh/authorized_keys``. If the value is empty or absent, then nothing will +be written to ``~/.ssh/authorized_keys``. + +Configuring the network +^^^^^^^^^^^^^^^^^^^^^^^ + +The network is configured by setting the metadata key ``network`` with a value +consistent with Network Config Versions +`1 <../network-config-format-v1.html>`_ or +`2 <../network-config-format-v2.html>`_\ , depending on the Linux +distro's version of cloud-init. + +The metadata key ``network.encoding`` may be used to indicate the format of +the metadata key "network". Valid encodings are ``base64`` and ``gzip+base64``. diff --git a/requirements.txt b/requirements.txt index 5817da3b..41d01d62 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,3 +32,12 @@ jsonpatch # For validating cloud-config sections per schema definitions jsonschema + +# Used by DataSourceVMware to inspect the host's network configuration during +# the "setup()" function. +# +# This allows a host that uses DHCP to bring up the network during BootLocal +# and still participate in instance-data by gathering the network in detail at +# runtime and merge that information into the metadata and repersist that to +# disk. +netifaces>=0.10.9 diff --git a/tests/unittests/test_datasource/test_common.py b/tests/unittests/test_datasource/test_common.py index 5e9c547a..00f0a78c 100644 --- a/tests/unittests/test_datasource/test_common.py +++ b/tests/unittests/test_datasource/test_common.py @@ -29,6 +29,7 @@ from cloudinit.sources import ( DataSourceSmartOS as SmartOS, DataSourceUpCloud as UpCloud, DataSourceVultr as Vultr, + DataSourceVMware as VMware, ) from cloudinit.sources import DataSourceNone as DSNone @@ -52,6 +53,7 @@ DEFAULT_LOCAL = [ RbxCloud.DataSourceRbxCloud, Scaleway.DataSourceScaleway, UpCloud.DataSourceUpCloudLocal, + VMware.DataSourceVMware, ] DEFAULT_NETWORK = [ @@ -68,6 +70,7 @@ DEFAULT_NETWORK = [ OpenStack.DataSourceOpenStack, OVF.DataSourceOVFNet, UpCloud.DataSourceUpCloud, + VMware.DataSourceVMware, ] diff --git a/tests/unittests/test_datasource/test_vmware.py b/tests/unittests/test_datasource/test_vmware.py new file mode 100644 index 00000000..597db7c8 --- /dev/null +++ b/tests/unittests/test_datasource/test_vmware.py @@ -0,0 +1,377 @@ +# Copyright (c) 2021 VMware, Inc. All Rights Reserved. +# +# Authors: Andrew Kutz +# +# This file is part of cloud-init. See LICENSE file for license information. + +import base64 +import gzip +from cloudinit import dmi, helpers, safeyaml +from cloudinit import settings +from cloudinit.sources import DataSourceVMware +from cloudinit.tests.helpers import ( + mock, + CiTestCase, + FilesystemMockingTestCase, + populate_dir, +) + +import os + +PRODUCT_NAME_FILE_PATH = "/sys/class/dmi/id/product_name" +PRODUCT_NAME = "VMware7,1" +PRODUCT_UUID = "82343CED-E4C7-423B-8F6B-0D34D19067AB" +REROOT_FILES = { + DataSourceVMware.PRODUCT_UUID_FILE_PATH: PRODUCT_UUID, + PRODUCT_NAME_FILE_PATH: PRODUCT_NAME, +} + +VMW_MULTIPLE_KEYS = [ + "ssh-rsa AAAAB3NzaC1yc2EAAAA... test1@vmw.com", + "ssh-rsa AAAAB3NzaC1yc2EAAAA... test2@vmw.com", +] +VMW_SINGLE_KEY = "ssh-rsa AAAAB3NzaC1yc2EAAAA... test@vmw.com" + +VMW_METADATA_YAML = """instance-id: cloud-vm +local-hostname: cloud-vm +network: + version: 2 + ethernets: + nics: + match: + name: ens* + dhcp4: yes +""" + +VMW_USERDATA_YAML = """## template: jinja +#cloud-config +users: +- default +""" + +VMW_VENDORDATA_YAML = """## template: jinja +#cloud-config +runcmd: +- echo "Hello, world." +""" + + +class TestDataSourceVMware(CiTestCase): + """ + Test common functionality that is not transport specific. + """ + + def setUp(self): + super(TestDataSourceVMware, self).setUp() + self.tmp = self.tmp_dir() + + def test_no_data_access_method(self): + ds = get_ds(self.tmp) + ds.vmware_rpctool = None + ret = ds.get_data() + self.assertFalse(ret) + + def test_get_host_info(self): + host_info = DataSourceVMware.get_host_info() + self.assertTrue(host_info) + self.assertTrue(host_info["hostname"]) + self.assertTrue(host_info["local-hostname"]) + self.assertTrue(host_info["local_hostname"]) + self.assertTrue(host_info[DataSourceVMware.LOCAL_IPV4]) + + +class TestDataSourceVMwareEnvVars(FilesystemMockingTestCase): + """ + Test the envvar transport. + """ + + def setUp(self): + super(TestDataSourceVMwareEnvVars, self).setUp() + self.tmp = self.tmp_dir() + os.environ[DataSourceVMware.VMX_GUESTINFO] = "1" + self.create_system_files() + + def tearDown(self): + del os.environ[DataSourceVMware.VMX_GUESTINFO] + return super(TestDataSourceVMwareEnvVars, self).tearDown() + + def create_system_files(self): + rootd = self.tmp_dir() + populate_dir( + rootd, + { + DataSourceVMware.PRODUCT_UUID_FILE_PATH: PRODUCT_UUID, + }, + ) + self.assertTrue(self.reRoot(rootd)) + + def assert_get_data_ok(self, m_fn, m_fn_call_count=6): + ds = get_ds(self.tmp) + ds.vmware_rpctool = None + ret = ds.get_data() + self.assertTrue(ret) + self.assertEqual(m_fn_call_count, m_fn.call_count) + self.assertEqual( + ds.data_access_method, DataSourceVMware.DATA_ACCESS_METHOD_ENVVAR + ) + return ds + + def assert_metadata(self, metadata, m_fn, m_fn_call_count=6): + ds = self.assert_get_data_ok(m_fn, m_fn_call_count) + assert_metadata(self, ds, metadata) + + @mock.patch( + "cloudinit.sources.DataSourceVMware.guestinfo_envvar_get_value" + ) + def test_get_subplatform(self, m_fn): + m_fn.side_effect = [VMW_METADATA_YAML, "", "", "", "", ""] + ds = self.assert_get_data_ok(m_fn, m_fn_call_count=4) + self.assertEqual( + ds.subplatform, + "%s (%s)" + % ( + DataSourceVMware.DATA_ACCESS_METHOD_ENVVAR, + DataSourceVMware.get_guestinfo_envvar_key_name("metadata"), + ), + ) + + @mock.patch( + "cloudinit.sources.DataSourceVMware.guestinfo_envvar_get_value" + ) + def test_get_data_metadata_only(self, m_fn): + m_fn.side_effect = [VMW_METADATA_YAML, "", "", "", "", ""] + self.assert_get_data_ok(m_fn, m_fn_call_count=4) + + @mock.patch( + "cloudinit.sources.DataSourceVMware.guestinfo_envvar_get_value" + ) + def test_get_data_userdata_only(self, m_fn): + m_fn.side_effect = ["", VMW_USERDATA_YAML, "", ""] + self.assert_get_data_ok(m_fn, m_fn_call_count=4) + + @mock.patch( + "cloudinit.sources.DataSourceVMware.guestinfo_envvar_get_value" + ) + def test_get_data_vendordata_only(self, m_fn): + m_fn.side_effect = ["", "", VMW_VENDORDATA_YAML, ""] + self.assert_get_data_ok(m_fn, m_fn_call_count=4) + + @mock.patch( + "cloudinit.sources.DataSourceVMware.guestinfo_envvar_get_value" + ) + def test_get_data_metadata_base64(self, m_fn): + data = base64.b64encode(VMW_METADATA_YAML.encode("utf-8")) + m_fn.side_effect = [data, "base64", "", ""] + self.assert_get_data_ok(m_fn, m_fn_call_count=4) + + @mock.patch( + "cloudinit.sources.DataSourceVMware.guestinfo_envvar_get_value" + ) + def test_get_data_metadata_b64(self, m_fn): + data = base64.b64encode(VMW_METADATA_YAML.encode("utf-8")) + m_fn.side_effect = [data, "b64", "", ""] + self.assert_get_data_ok(m_fn, m_fn_call_count=4) + + @mock.patch( + "cloudinit.sources.DataSourceVMware.guestinfo_envvar_get_value" + ) + def test_get_data_metadata_gzip_base64(self, m_fn): + data = VMW_METADATA_YAML.encode("utf-8") + data = gzip.compress(data) + data = base64.b64encode(data) + m_fn.side_effect = [data, "gzip+base64", "", ""] + self.assert_get_data_ok(m_fn, m_fn_call_count=4) + + @mock.patch( + "cloudinit.sources.DataSourceVMware.guestinfo_envvar_get_value" + ) + def test_get_data_metadata_gz_b64(self, m_fn): + data = VMW_METADATA_YAML.encode("utf-8") + data = gzip.compress(data) + data = base64.b64encode(data) + m_fn.side_effect = [data, "gz+b64", "", ""] + self.assert_get_data_ok(m_fn, m_fn_call_count=4) + + @mock.patch( + "cloudinit.sources.DataSourceVMware.guestinfo_envvar_get_value" + ) + def test_metadata_single_ssh_key(self, m_fn): + metadata = DataSourceVMware.load_json_or_yaml(VMW_METADATA_YAML) + metadata["public_keys"] = VMW_SINGLE_KEY + metadata_yaml = safeyaml.dumps(metadata) + m_fn.side_effect = [metadata_yaml, "", "", ""] + self.assert_metadata(metadata, m_fn, m_fn_call_count=4) + + @mock.patch( + "cloudinit.sources.DataSourceVMware.guestinfo_envvar_get_value" + ) + def test_metadata_multiple_ssh_keys(self, m_fn): + metadata = DataSourceVMware.load_json_or_yaml(VMW_METADATA_YAML) + metadata["public_keys"] = VMW_MULTIPLE_KEYS + metadata_yaml = safeyaml.dumps(metadata) + m_fn.side_effect = [metadata_yaml, "", "", ""] + self.assert_metadata(metadata, m_fn, m_fn_call_count=4) + + +class TestDataSourceVMwareGuestInfo(FilesystemMockingTestCase): + """ + Test the guestinfo transport on a VMware platform. + """ + + def setUp(self): + super(TestDataSourceVMwareGuestInfo, self).setUp() + self.tmp = self.tmp_dir() + self.create_system_files() + + def create_system_files(self): + rootd = self.tmp_dir() + populate_dir( + rootd, + { + DataSourceVMware.PRODUCT_UUID_FILE_PATH: PRODUCT_UUID, + PRODUCT_NAME_FILE_PATH: PRODUCT_NAME, + }, + ) + self.assertTrue(self.reRoot(rootd)) + + def assert_get_data_ok(self, m_fn, m_fn_call_count=6): + ds = get_ds(self.tmp) + ds.vmware_rpctool = "vmware-rpctool" + ret = ds.get_data() + self.assertTrue(ret) + self.assertEqual(m_fn_call_count, m_fn.call_count) + self.assertEqual( + ds.data_access_method, + DataSourceVMware.DATA_ACCESS_METHOD_GUESTINFO, + ) + return ds + + def assert_metadata(self, metadata, m_fn, m_fn_call_count=6): + ds = self.assert_get_data_ok(m_fn, m_fn_call_count) + assert_metadata(self, ds, metadata) + + def test_ds_valid_on_vmware_platform(self): + system_type = dmi.read_dmi_data("system-product-name") + self.assertEqual(system_type, PRODUCT_NAME) + + @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_get_value") + def test_get_subplatform(self, m_fn): + m_fn.side_effect = [VMW_METADATA_YAML, "", "", "", "", ""] + ds = self.assert_get_data_ok(m_fn, m_fn_call_count=4) + self.assertEqual( + ds.subplatform, + "%s (%s)" + % ( + DataSourceVMware.DATA_ACCESS_METHOD_GUESTINFO, + DataSourceVMware.get_guestinfo_key_name("metadata"), + ), + ) + + @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_get_value") + def test_get_data_userdata_only(self, m_fn): + m_fn.side_effect = ["", VMW_USERDATA_YAML, "", ""] + self.assert_get_data_ok(m_fn, m_fn_call_count=4) + + @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_get_value") + def test_get_data_vendordata_only(self, m_fn): + m_fn.side_effect = ["", "", VMW_VENDORDATA_YAML, ""] + self.assert_get_data_ok(m_fn, m_fn_call_count=4) + + @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_get_value") + def test_metadata_single_ssh_key(self, m_fn): + metadata = DataSourceVMware.load_json_or_yaml(VMW_METADATA_YAML) + metadata["public_keys"] = VMW_SINGLE_KEY + metadata_yaml = safeyaml.dumps(metadata) + m_fn.side_effect = [metadata_yaml, "", "", ""] + self.assert_metadata(metadata, m_fn, m_fn_call_count=4) + + @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_get_value") + def test_metadata_multiple_ssh_keys(self, m_fn): + metadata = DataSourceVMware.load_json_or_yaml(VMW_METADATA_YAML) + metadata["public_keys"] = VMW_MULTIPLE_KEYS + metadata_yaml = safeyaml.dumps(metadata) + m_fn.side_effect = [metadata_yaml, "", "", ""] + self.assert_metadata(metadata, m_fn, m_fn_call_count=4) + + @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_get_value") + def test_get_data_metadata_base64(self, m_fn): + data = base64.b64encode(VMW_METADATA_YAML.encode("utf-8")) + m_fn.side_effect = [data, "base64", "", ""] + self.assert_get_data_ok(m_fn, m_fn_call_count=4) + + @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_get_value") + def test_get_data_metadata_b64(self, m_fn): + data = base64.b64encode(VMW_METADATA_YAML.encode("utf-8")) + m_fn.side_effect = [data, "b64", "", ""] + self.assert_get_data_ok(m_fn, m_fn_call_count=4) + + @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_get_value") + def test_get_data_metadata_gzip_base64(self, m_fn): + data = VMW_METADATA_YAML.encode("utf-8") + data = gzip.compress(data) + data = base64.b64encode(data) + m_fn.side_effect = [data, "gzip+base64", "", ""] + self.assert_get_data_ok(m_fn, m_fn_call_count=4) + + @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_get_value") + def test_get_data_metadata_gz_b64(self, m_fn): + data = VMW_METADATA_YAML.encode("utf-8") + data = gzip.compress(data) + data = base64.b64encode(data) + m_fn.side_effect = [data, "gz+b64", "", ""] + self.assert_get_data_ok(m_fn, m_fn_call_count=4) + + +class TestDataSourceVMwareGuestInfo_InvalidPlatform(FilesystemMockingTestCase): + """ + Test the guestinfo transport on a non-VMware platform. + """ + + def setUp(self): + super(TestDataSourceVMwareGuestInfo_InvalidPlatform, self).setUp() + self.tmp = self.tmp_dir() + self.create_system_files() + + def create_system_files(self): + rootd = self.tmp_dir() + populate_dir( + rootd, + { + DataSourceVMware.PRODUCT_UUID_FILE_PATH: PRODUCT_UUID, + }, + ) + self.assertTrue(self.reRoot(rootd)) + + @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_get_value") + def test_ds_invalid_on_non_vmware_platform(self, m_fn): + system_type = dmi.read_dmi_data("system-product-name") + self.assertEqual(system_type, None) + + m_fn.side_effect = [VMW_METADATA_YAML, "", "", "", "", ""] + ds = get_ds(self.tmp) + ds.vmware_rpctool = "vmware-rpctool" + ret = ds.get_data() + self.assertFalse(ret) + + +def assert_metadata(test_obj, ds, metadata): + test_obj.assertEqual(metadata.get("instance-id"), ds.get_instance_id()) + test_obj.assertEqual(metadata.get("local-hostname"), ds.get_hostname()) + + expected_public_keys = metadata.get("public_keys") + if not isinstance(expected_public_keys, list): + expected_public_keys = [expected_public_keys] + + test_obj.assertEqual(expected_public_keys, ds.get_public_ssh_keys()) + test_obj.assertIsInstance(ds.get_public_ssh_keys(), list) + + +def get_ds(temp_dir): + ds = DataSourceVMware.DataSourceVMware( + settings.CFG_BUILTIN, None, helpers.Paths({"run_dir": temp_dir}) + ) + ds.vmware_rpctool = "vmware-rpctool" + return ds + + +# vi: ts=4 expandtab diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py index 1d8aaf18..8617d7bd 100644 --- a/tests/unittests/test_ds_identify.py +++ b/tests/unittests/test_ds_identify.py @@ -649,6 +649,50 @@ class TestDsIdentify(DsIdentifyBase): """EC2: bobrightbox.com in product_serial is not brightbox'""" self._test_ds_not_found('Ec2-E24Cloud-negative') + def test_vmware_no_valid_transports(self): + """VMware: no valid transports""" + self._test_ds_not_found('VMware-NoValidTransports') + + def test_vmware_envvar_no_data(self): + """VMware: envvar transport no data""" + self._test_ds_not_found('VMware-EnvVar-NoData') + + def test_vmware_envvar_no_virt_id(self): + """VMware: envvar transport success if no virt id""" + self._test_ds_found('VMware-EnvVar-NoVirtID') + + def test_vmware_envvar_activated_by_metadata(self): + """VMware: envvar transport activated by metadata""" + self._test_ds_found('VMware-EnvVar-Metadata') + + def test_vmware_envvar_activated_by_userdata(self): + """VMware: envvar transport activated by userdata""" + self._test_ds_found('VMware-EnvVar-Userdata') + + def test_vmware_envvar_activated_by_vendordata(self): + """VMware: envvar transport activated by vendordata""" + self._test_ds_found('VMware-EnvVar-Vendordata') + + def test_vmware_guestinfo_no_data(self): + """VMware: guestinfo transport no data""" + self._test_ds_not_found('VMware-GuestInfo-NoData') + + def test_vmware_guestinfo_no_virt_id(self): + """VMware: guestinfo transport fails if no virt id""" + self._test_ds_not_found('VMware-GuestInfo-NoVirtID') + + def test_vmware_guestinfo_activated_by_metadata(self): + """VMware: guestinfo transport activated by metadata""" + self._test_ds_found('VMware-GuestInfo-Metadata') + + def test_vmware_guestinfo_activated_by_userdata(self): + """VMware: guestinfo transport activated by userdata""" + self._test_ds_found('VMware-GuestInfo-Userdata') + + def test_vmware_guestinfo_activated_by_vendordata(self): + """VMware: guestinfo transport activated by vendordata""" + self._test_ds_found('VMware-GuestInfo-Vendordata') + class TestBSDNoSys(DsIdentifyBase): """Test *BSD code paths @@ -1136,7 +1180,240 @@ VALID_CFG = { 'Ec2-E24Cloud-negative': { 'ds': 'Ec2', 'files': {P_SYS_VENDOR: 'e24cloudyday\n'}, - } + }, + 'VMware-NoValidTransports': { + 'ds': 'VMware', + 'mocks': [ + MOCK_VIRT_IS_VMWARE, + ], + }, + 'VMware-EnvVar-NoData': { + 'ds': 'VMware', + 'mocks': [ + { + 'name': 'vmware_has_envvar_vmx_guestinfo', + 'ret': 0, + }, + { + 'name': 'vmware_has_envvar_vmx_guestinfo_metadata', + 'ret': 1, + }, + { + 'name': 'vmware_has_envvar_vmx_guestinfo_userdata', + 'ret': 1, + }, + { + 'name': 'vmware_has_envvar_vmx_guestinfo_vendordata', + 'ret': 1, + }, + MOCK_VIRT_IS_VMWARE, + ], + }, + 'VMware-EnvVar-NoVirtID': { + 'ds': 'VMware', + 'mocks': [ + { + 'name': 'vmware_has_envvar_vmx_guestinfo', + 'ret': 0, + }, + { + 'name': 'vmware_has_envvar_vmx_guestinfo_metadata', + 'ret': 0, + }, + { + 'name': 'vmware_has_envvar_vmx_guestinfo_userdata', + 'ret': 1, + }, + { + 'name': 'vmware_has_envvar_vmx_guestinfo_vendordata', + 'ret': 1, + }, + ], + }, + 'VMware-EnvVar-Metadata': { + 'ds': 'VMware', + 'mocks': [ + { + 'name': 'vmware_has_envvar_vmx_guestinfo', + 'ret': 0, + }, + { + 'name': 'vmware_has_envvar_vmx_guestinfo_metadata', + 'ret': 0, + }, + { + 'name': 'vmware_has_envvar_vmx_guestinfo_userdata', + 'ret': 1, + }, + { + 'name': 'vmware_has_envvar_vmx_guestinfo_vendordata', + 'ret': 1, + }, + MOCK_VIRT_IS_VMWARE, + ], + }, + 'VMware-EnvVar-Userdata': { + 'ds': 'VMware', + 'mocks': [ + { + 'name': 'vmware_has_envvar_vmx_guestinfo', + 'ret': 0, + }, + { + 'name': 'vmware_has_envvar_vmx_guestinfo_metadata', + 'ret': 1, + }, + { + 'name': 'vmware_has_envvar_vmx_guestinfo_userdata', + 'ret': 0, + }, + { + 'name': 'vmware_has_envvar_vmx_guestinfo_vendordata', + 'ret': 1, + }, + MOCK_VIRT_IS_VMWARE, + ], + }, + 'VMware-EnvVar-Vendordata': { + 'ds': 'VMware', + 'mocks': [ + { + 'name': 'vmware_has_envvar_vmx_guestinfo', + 'ret': 0, + }, + { + 'name': 'vmware_has_envvar_vmx_guestinfo_metadata', + 'ret': 1, + }, + { + 'name': 'vmware_has_envvar_vmx_guestinfo_userdata', + 'ret': 1, + }, + { + 'name': 'vmware_has_envvar_vmx_guestinfo_vendordata', + 'ret': 0, + }, + MOCK_VIRT_IS_VMWARE, + ], + }, + 'VMware-GuestInfo-NoData': { + 'ds': 'VMware', + 'mocks': [ + { + 'name': 'vmware_has_rpctool', + 'ret': 0, + 'out': '/usr/bin/vmware-rpctool', + }, + { + 'name': 'vmware_rpctool_guestinfo_metadata', + 'ret': 1, + }, + { + 'name': 'vmware_rpctool_guestinfo_userdata', + 'ret': 1, + }, + { + 'name': 'vmware_rpctool_guestinfo_vendordata', + 'ret': 1, + }, + MOCK_VIRT_IS_VMWARE, + ], + }, + 'VMware-GuestInfo-NoVirtID': { + 'ds': 'VMware', + 'mocks': [ + { + 'name': 'vmware_has_rpctool', + 'ret': 0, + 'out': '/usr/bin/vmware-rpctool', + }, + { + 'name': 'vmware_rpctool_guestinfo_metadata', + 'ret': 0, + 'out': '---', + }, + { + 'name': 'vmware_rpctool_guestinfo_userdata', + 'ret': 1, + }, + { + 'name': 'vmware_rpctool_guestinfo_vendordata', + 'ret': 1, + }, + ], + }, + 'VMware-GuestInfo-Metadata': { + 'ds': 'VMware', + 'mocks': [ + { + 'name': 'vmware_has_rpctool', + 'ret': 0, + 'out': '/usr/bin/vmware-rpctool', + }, + { + 'name': 'vmware_rpctool_guestinfo_metadata', + 'ret': 0, + 'out': '---', + }, + { + 'name': 'vmware_rpctool_guestinfo_userdata', + 'ret': 1, + }, + { + 'name': 'vmware_rpctool_guestinfo_vendordata', + 'ret': 1, + }, + MOCK_VIRT_IS_VMWARE, + ], + }, + 'VMware-GuestInfo-Userdata': { + 'ds': 'VMware', + 'mocks': [ + { + 'name': 'vmware_has_rpctool', + 'ret': 0, + 'out': '/usr/bin/vmware-rpctool', + }, + { + 'name': 'vmware_rpctool_guestinfo_metadata', + 'ret': 1, + }, + { + 'name': 'vmware_rpctool_guestinfo_userdata', + 'ret': 0, + 'out': '---', + }, + { + 'name': 'vmware_rpctool_guestinfo_vendordata', + 'ret': 1, + }, + MOCK_VIRT_IS_VMWARE, + ], + }, + 'VMware-GuestInfo-Vendordata': { + 'ds': 'VMware', + 'mocks': [ + { + 'name': 'vmware_has_rpctool', + 'ret': 0, + 'out': '/usr/bin/vmware-rpctool', + }, + { + 'name': 'vmware_rpctool_guestinfo_metadata', + 'ret': 1, + }, + { + 'name': 'vmware_rpctool_guestinfo_userdata', + 'ret': 1, + }, + { + 'name': 'vmware_rpctool_guestinfo_vendordata', + 'ret': 0, + 'out': '---', + }, + MOCK_VIRT_IS_VMWARE, + ], + }, } # vi: ts=4 expandtab diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 3c2c6d14..5089dd70 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -1,5 +1,6 @@ ader1990 ajmyyra +akutz AlexBaranowski Aman306 andrewbogott diff --git a/tools/ds-identify b/tools/ds-identify index 73e27c71..234ffa81 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -125,7 +125,7 @@ DI_DSNAME="" # be searched if there is no setting found in config. DI_DSLIST_DEFAULT="MAAS ConfigDrive NoCloud AltCloud Azure Bigstep \ CloudSigma CloudStack DigitalOcean Vultr AliYun Ec2 GCE OpenNebula OpenStack \ -OVF SmartOS Scaleway Hetzner IBMCloud Oracle Exoscale RbxCloud UpCloud" +OVF SmartOS Scaleway Hetzner IBMCloud Oracle Exoscale RbxCloud UpCloud VMware" DI_DSLIST="" DI_MODE="" DI_ON_FOUND="" @@ -1364,6 +1364,80 @@ dscheck_Vultr() { return $DS_NOT_FOUND } +vmware_has_envvar_vmx_guestinfo() { + [ -n "${VMX_GUESTINFO:-}" ] +} + +vmware_has_envvar_vmx_guestinfo_metadata() { + [ -n "${VMX_GUESTINFO_METADATA:-}" ] +} + +vmware_has_envvar_vmx_guestinfo_userdata() { + [ -n "${VMX_GUESTINFO_USERDATA:-}" ] +} + +vmware_has_envvar_vmx_guestinfo_vendordata() { + [ -n "${VMX_GUESTINFO_VENDORDATA:-}" ] +} + +vmware_has_rpctool() { + command -v vmware-rpctool >/dev/null 2>&1 +} + +vmware_rpctool_guestinfo_metadata() { + vmware-rpctool "info-get guestinfo.metadata" +} + +vmware_rpctool_guestinfo_userdata() { + vmware-rpctool "info-get guestinfo.userdata" +} + +vmware_rpctool_guestinfo_vendordata() { + vmware-rpctool "info-get guestinfo.vendordata" +} + +dscheck_VMware() { + # Checks to see if there is valid data for the VMware datasource. + # The data transports are checked in the following order: + # + # * envvars + # * guestinfo + # + # Please note when updating this function with support for new data + # transports, the order should match the order in the _get_data + # function from the file DataSourceVMware.py. + + # Check to see if running in a container and the VMware + # datasource is configured via environment variables. + if vmware_has_envvar_vmx_guestinfo; then + if vmware_has_envvar_vmx_guestinfo_metadata || \ + vmware_has_envvar_vmx_guestinfo_userdata || \ + vmware_has_envvar_vmx_guestinfo_vendordata; then + return "${DS_FOUND}" + fi + fi + + # Do not proceed unless the detected platform is VMware. + if [ ! "${DI_VIRT}" = "vmware" ]; then + return "${DS_NOT_FOUND}" + fi + + # Do not proceed if the vmware-rpctool command is not present. + if ! vmware_has_rpctool; then + return "${DS_NOT_FOUND}" + fi + + # Activate the VMware datasource only if any of the fields used + # by the datasource are present in the guestinfo table. + if { vmware_rpctool_guestinfo_metadata || \ + vmware_rpctool_guestinfo_userdata || \ + vmware_rpctool_guestinfo_vendordata; } >/dev/null 2>&1; then + return "${DS_FOUND}" + fi + + return "${DS_NOT_FOUND}" +} + collect_info() { read_uname_info read_virt -- cgit v1.2.3 From 3c8585919b07bbe6e7480c70167bc0a90b6303c7 Mon Sep 17 00:00:00 2001 From: Noah Fontes Date: Tue, 10 Aug 2021 07:27:31 -0700 Subject: Add Puppet contributors to CLA signers (#964) --- tools/.github-cla-signers | 3 +++ 1 file changed, 3 insertions(+) (limited to 'tools/.github-cla-signers') diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 5089dd70..cf06ca3d 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -13,6 +13,7 @@ BirknerAlex bmhughes candlerb cawamata +ciprianbadescu dankenigsberg ddymko dermotbradley @@ -21,8 +22,10 @@ eandersson eb3095 emmanuelthome esposem +GabrielNagy giggsoff hamalq +impl irishgordo izzyleung johnsonshi -- cgit v1.2.3 From 0bf8d575dc91b68084ef4d88869ac719b23924d0 Mon Sep 17 00:00:00 2001 From: sarahwzadara <83585923+sarahwzadara@users.noreply.github.com> Date: Mon, 16 Aug 2021 17:55:10 +0300 Subject: add Zadara Edge Cloud Platform to the supported clouds list (#963) --- doc/rtd/topics/availability.rst | 1 + tools/.github-cla-signers | 1 + 2 files changed, 2 insertions(+) (limited to 'tools/.github-cla-signers') diff --git a/doc/rtd/topics/availability.rst b/doc/rtd/topics/availability.rst index 71827177..d8ca9d16 100644 --- a/doc/rtd/topics/availability.rst +++ b/doc/rtd/topics/availability.rst @@ -59,6 +59,7 @@ environments in the public cloud: - SmartOS - UpCloud - Vultr +- Zadara Edge Cloud Platform Additionally, cloud-init is supported on these private clouds: diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index cf06ca3d..4fa108aa 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -51,6 +51,7 @@ onitake qubidt renanrodrigo riedel +sarahwzadara slyon smoser sshedi -- cgit v1.2.3 From f6c71fd758a3589ac5f62fd251d60b00edf5bb1c Mon Sep 17 00:00:00 2001 From: Alexandr Kravchenko Date: Wed, 1 Sep 2021 22:01:09 +0300 Subject: Add CloudLinux OS support (#1003) https://www.cloudlinux.com/ --- README.md | 2 +- cloudinit/config/cc_ntp.py | 6 +++--- cloudinit/config/cc_yum_add_repo.py | 8 ++++---- cloudinit/distros/__init__.py | 4 ++-- cloudinit/distros/cloudlinux.py | 9 +++++++++ cloudinit/net/sysconfig.py | 4 ++-- cloudinit/tests/test_util.py | 33 ++++++++++++++++++++++++++++++++- cloudinit/util.py | 5 +++-- config/cloud.cfg.tmpl | 6 +++--- systemd/cloud-init-generator.tmpl | 2 +- systemd/cloud-init.service.tmpl | 2 +- tests/unittests/test_cli.py | 6 +++--- tools/.github-cla-signers | 1 + tools/render-cloudcfg | 2 +- 14 files changed, 66 insertions(+), 24 deletions(-) create mode 100644 cloudinit/distros/cloudlinux.py (limited to 'tools/.github-cla-signers') diff --git a/README.md b/README.md index 5828c2fa..b705a065 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ get in contact with that distribution and send them our way! | Supported OSes | Supported Public Clouds | Supported Private Clouds | | --- | --- | --- | -| Alpine Linux
ArchLinux
Debian
DragonFlyBSD
Fedora
FreeBSD
Gentoo Linux
NetBSD
OpenBSD
RHEL/CentOS/AlmaLinux/Rocky/PhotonOS/Virtuozzo/EuroLinux
SLES/openSUSE
Ubuntu










| Amazon Web Services
Microsoft Azure
Google Cloud Platform
Oracle Cloud Infrastructure
Softlayer
Rackspace Public Cloud
IBM Cloud
DigitalOcean
Bigstep
Hetzner
Joyent
CloudSigma
Alibaba Cloud
OVH
OpenNebula
Exoscale
Scaleway
CloudStack
AltCloud
SmartOS
HyperOne
Vultr
Rootbox
| Bare metal installs
OpenStack
LXD
KVM
Metal-as-a-Service (MAAS)
VMware















| +| Alpine Linux
ArchLinux
Debian
DragonFlyBSD
Fedora
FreeBSD
Gentoo Linux
NetBSD
OpenBSD
RHEL/CentOS/AlmaLinux/Rocky/PhotonOS/Virtuozzo/EuroLinux/CloudLinux
SLES/openSUSE
Ubuntu










| Amazon Web Services
Microsoft Azure
Google Cloud Platform
Oracle Cloud Infrastructure
Softlayer
Rackspace Public Cloud
IBM Cloud
DigitalOcean
Bigstep
Hetzner
Joyent
CloudSigma
Alibaba Cloud
OVH
OpenNebula
Exoscale
Scaleway
CloudStack
AltCloud
SmartOS
HyperOne
Vultr
Rootbox
| Bare metal installs
OpenStack
LXD
KVM
Metal-as-a-Service (MAAS)
VMware















| ## To start developing cloud-init diff --git a/cloudinit/config/cc_ntp.py b/cloudinit/config/cc_ntp.py index 7c371a49..e2231cbb 100644 --- a/cloudinit/config/cc_ntp.py +++ b/cloudinit/config/cc_ntp.py @@ -24,9 +24,9 @@ LOG = logging.getLogger(__name__) frequency = PER_INSTANCE NTP_CONF = '/etc/ntp.conf' NR_POOL_SERVERS = 4 -distros = ['almalinux', 'alpine', 'centos', 'debian', 'eurolinux', 'fedora', - 'opensuse', 'photon', 'rhel', 'rocky', 'sles', 'ubuntu', - 'virtuozzo'] +distros = ['almalinux', 'alpine', 'centos', 'cloudlinux', 'debian', + 'eurolinux', 'fedora', 'opensuse', 'photon', 'rhel', 'rocky', + 'sles', 'ubuntu', 'virtuozzo'] NTP_CLIENT_CONFIG = { 'chrony': { diff --git a/cloudinit/config/cc_yum_add_repo.py b/cloudinit/config/cc_yum_add_repo.py index b7a48dcc..899cb082 100644 --- a/cloudinit/config/cc_yum_add_repo.py +++ b/cloudinit/config/cc_yum_add_repo.py @@ -18,8 +18,8 @@ entry, the config entry will be skipped. **Module frequency:** per always -**Supported distros:** almalinux, centos, eurolinux, fedora, photon, rhel, - rocky, virtuozzo +**Supported distros:** almalinux, centos, cloudlinux, eurolinux, fedora, + photon, rhel, rocky, virtuozzo **Config keys**:: @@ -37,8 +37,8 @@ from configparser import ConfigParser from cloudinit import util -distros = ['almalinux', 'centos', 'eurolinux', 'fedora', 'photon', 'rhel', - 'rocky', 'virtuozzo'] +distros = ['almalinux', 'centos', 'cloudlinux', 'eurolinux', 'fedora', + 'photon', 'rhel', 'rocky', 'virtuozzo'] def _canonicalize_id(repo_id): diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 2e629143..a0526948 100755 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -49,8 +49,8 @@ OSFAMILIES = { 'debian': ['debian', 'ubuntu'], 'freebsd': ['freebsd'], 'gentoo': ['gentoo'], - 'redhat': ['almalinux', 'amazon', 'centos', 'eurolinux', 'fedora', - 'photon', 'rhel', 'rocky', 'virtuozzo'], + 'redhat': ['almalinux', 'amazon', 'centos', 'cloudlinux', 'eurolinux', + 'fedora', 'photon', 'rhel', 'rocky', 'virtuozzo'], 'suse': ['opensuse', 'sles'], } diff --git a/cloudinit/distros/cloudlinux.py b/cloudinit/distros/cloudlinux.py new file mode 100644 index 00000000..edb3165d --- /dev/null +++ b/cloudinit/distros/cloudlinux.py @@ -0,0 +1,9 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit.distros import rhel + + +class Distro(rhel.Distro): + pass + +# vi: ts=4 expandtab diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index 06f7255e..7b8e4da7 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -18,8 +18,8 @@ from .network_state import ( is_ipv6_addr, net_prefix_to_ipv4_mask, subnet_is_ipv6, IPV6_DYNAMIC_TYPES) LOG = logging.getLogger(__name__) -KNOWN_DISTROS = ['almalinux', 'centos', 'eurolinux', 'fedora', 'rhel', 'rocky', - 'suse', 'virtuozzo'] +KNOWN_DISTROS = ['almalinux', 'centos', 'cloudlinux', 'eurolinux', 'fedora', + 'rhel', 'rocky', 'suse', 'virtuozzo'] NM_CFG_FILE = "/etc/NetworkManager/NetworkManager.conf" diff --git a/cloudinit/tests/test_util.py b/cloudinit/tests/test_util.py index 9dd01158..f11cfb27 100644 --- a/cloudinit/tests/test_util.py +++ b/cloudinit/tests/test_util.py @@ -186,6 +186,20 @@ OS_RELEASE_VIRTUOZZO_8 = dedent("""\ BUG_REPORT_URL="https://bugs.openvz.org" """) +OS_RELEASE_CLOUDLINUX_8 = dedent("""\ + NAME="CloudLinux" + VERSION="8.4 (Valery Rozhdestvensky)" + ID="cloudlinux" + ID_LIKE="rhel fedora centos" + VERSION_ID="8.4" + PLATFORM_ID="platform:el8" + PRETTY_NAME="CloudLinux 8.4 (Valery Rozhdestvensky)" + ANSI_COLOR="0;31" + CPE_NAME="cpe:/o:cloudlinux:cloudlinux:8.4:GA:server" + HOME_URL="https://www.cloudlinux.com/" + BUG_REPORT_URL="https://www.cloudlinux.com/support" +""") + REDHAT_RELEASE_CENTOS_6 = "CentOS release 6.10 (Final)" REDHAT_RELEASE_CENTOS_7 = "CentOS Linux release 7.5.1804 (Core)" REDHAT_RELEASE_REDHAT_6 = ( @@ -200,7 +214,8 @@ REDHAT_RELEASE_ROCKY_8 = ( "Rocky Linux release 8.3 (Green Obsidian)") REDHAT_RELEASE_VIRTUOZZO_8 = ( "Virtuozzo Linux release 8") - +REDHAT_RELEASE_CLOUDLINUX_8 = ( + "CloudLinux release 8.4 (Valery Rozhdestvensky)") OS_RELEASE_DEBIAN = dedent("""\ PRETTY_NAME="Debian GNU/Linux 9 (stretch)" NAME="Debian GNU/Linux" @@ -679,6 +694,22 @@ class TestGetLinuxDistro(CiTestCase): dist = util.get_linux_distro() self.assertEqual(('virtuozzo', '8', 'Virtuozzo Linux'), dist) + @mock.patch('cloudinit.util.load_file') + def test_get_linux_cloud8_rhrelease(self, m_os_release, m_path_exists): + """Verify cloudlinux 8 read from redhat-release.""" + m_os_release.return_value = REDHAT_RELEASE_CLOUDLINUX_8 + m_path_exists.side_effect = TestGetLinuxDistro.redhat_release_exists + dist = util.get_linux_distro() + self.assertEqual(('cloudlinux', '8.4', 'Valery Rozhdestvensky'), dist) + + @mock.patch('cloudinit.util.load_file') + def test_get_linux_cloud8_osrelease(self, m_os_release, m_path_exists): + """Verify cloudlinux 8 read from os-release.""" + m_os_release.return_value = OS_RELEASE_CLOUDLINUX_8 + m_path_exists.side_effect = TestGetLinuxDistro.os_release_exists + dist = util.get_linux_distro() + self.assertEqual(('cloudlinux', '8.4', 'Valery Rozhdestvensky'), dist) + @mock.patch('cloudinit.util.load_file') def test_get_linux_debian(self, m_os_release, m_path_exists): """Verify we get the correct name and release name on Debian.""" diff --git a/cloudinit/util.py b/cloudinit/util.py index c53f6453..894245bf 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -548,8 +548,9 @@ def system_info(): if system == "linux": linux_dist = info['dist'][0].lower() if linux_dist in ( - 'almalinux', 'alpine', 'arch', 'centos', 'debian', 'eurolinux', - 'fedora', 'photon', 'rhel', 'rocky', 'suse', 'virtuozzo'): + 'almalinux', 'alpine', 'arch', 'centos', 'cloudlinux', + 'debian', 'eurolinux', 'fedora', 'photon', 'rhel', 'rocky', + 'suse', 'virtuozzo'): var = linux_dist elif linux_dist in ('ubuntu', 'linuxmint', 'mint'): var = 'ubuntu' diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl index 825deff4..e5fbc10b 100644 --- a/config/cloud.cfg.tmpl +++ b/config/cloud.cfg.tmpl @@ -32,7 +32,7 @@ disable_root: false disable_root: true {% endif %} -{% if variant in ["almalinux", "alpine", "amazon", "centos", "eurolinux", +{% if variant in ["almalinux", "alpine", "amazon", "centos", "cloudlinux", "eurolinux", "fedora", "rhel", "rocky", "virtuozzo"] %} mount_default_fields: [~, ~, 'auto', 'defaults,nofail', '0', '2'] {% if variant == "amazon" %} @@ -173,7 +173,7 @@ cloud_final_modules: # (not accessible to handlers/transforms) system_info: # This will affect which distro class gets used -{% if variant in ["almalinux", "alpine", "amazon", "arch", "centos", "debian", +{% if variant in ["almalinux", "alpine", "amazon", "arch", "centos", "cloudlinux", "debian", "eurolinux", "fedora", "freebsd", "netbsd", "openbsd", "photon", "rhel", "rocky", "suse", "ubuntu", "virtuozzo"] %} distro: {{ variant }} @@ -228,7 +228,7 @@ system_info: primary: http://ports.ubuntu.com/ubuntu-ports security: http://ports.ubuntu.com/ubuntu-ports ssh_svcname: ssh -{% elif variant in ["almalinux", "alpine", "amazon", "arch", "centos", "eurolinux", +{% elif variant in ["almalinux", "alpine", "amazon", "arch", "centos", "cloudlinux", "eurolinux", "fedora", "rhel", "rocky", "suse", "virtuozzo"] %} # Default user name + that default users groups (if added/used) default_user: diff --git a/systemd/cloud-init-generator.tmpl b/systemd/cloud-init-generator.tmpl index 3dbe5947..7fe009ec 100644 --- a/systemd/cloud-init-generator.tmpl +++ b/systemd/cloud-init-generator.tmpl @@ -83,7 +83,7 @@ default() { check_for_datasource() { local ds_rc="" -{% if variant in ["almalinux", "centos", "eurolinux", "fedora", "rhel", +{% if variant in ["almalinux", "centos", "cloudlinux", "eurolinux", "fedora", "rhel", "rocky", "virtuozzo"] %} local dsidentify="/usr/libexec/cloud-init/ds-identify" {% else %} diff --git a/systemd/cloud-init.service.tmpl b/systemd/cloud-init.service.tmpl index 636f59be..6c3a8f20 100644 --- a/systemd/cloud-init.service.tmpl +++ b/systemd/cloud-init.service.tmpl @@ -12,7 +12,7 @@ After=systemd-networkd-wait-online.service {% if variant in ["ubuntu", "unknown", "debian"] %} After=networking.service {% endif %} -{% if variant in ["almalinux", "centos", "eurolinux", "fedora", "rhel", +{% if variant in ["almalinux", "centos", "cloudlinux", "eurolinux", "fedora", "rhel", "rocky", "virtuozzo"] %} After=network.service After=NetworkManager.service diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py index a39e1d0c..b8ae9e47 100644 --- a/tests/unittests/test_cli.py +++ b/tests/unittests/test_cli.py @@ -224,9 +224,9 @@ class TestCLI(test_helpers.FilesystemMockingTestCase): self._call_main(['cloud-init', 'devel', 'schema', '--docs', 'all']) expected_doc_sections = [ '**Supported distros:** all', - ('**Supported distros:** almalinux, alpine, centos, debian, ' - 'eurolinux, fedora, opensuse, photon, rhel, rocky, sles, ubuntu, ' - 'virtuozzo'), + ('**Supported distros:** almalinux, alpine, centos, cloudlinux, ' + 'debian, eurolinux, fedora, opensuse, photon, rhel, rocky, ' + 'sles, ubuntu, virtuozzo'), '**Config schema**:\n **resize_rootfs:** (true/false/noblock)', '**Examples**::\n\n runcmd:\n - [ ls, -l, / ]\n' ] diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 4fa108aa..ba0c6d79 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -28,6 +28,7 @@ hamalq impl irishgordo izzyleung +JohnKepplers johnsonshi jordimassaguerpla jqueuniet diff --git a/tools/render-cloudcfg b/tools/render-cloudcfg index 30f82521..78a48c30 100755 --- a/tools/render-cloudcfg +++ b/tools/render-cloudcfg @@ -4,7 +4,7 @@ import argparse import os import sys -VARIANTS = ["almalinux", "alpine", "amazon", "arch", "centos", "debian", +VARIANTS = ["almalinux", "alpine", "amazon", "arch", "centos", "cloudlinux", "debian", "eurolinux", "fedora", "freebsd", "netbsd", "openbsd", "photon", "rhel", "suse","rocky", "ubuntu", "unknown", "virtuozzo"] -- cgit v1.2.3 From 7fe0f900adcd45a74a24b7f9b251e2ee35b53b54 Mon Sep 17 00:00:00 2001 From: Andy Fiddaman Date: Thu, 2 Sep 2021 14:26:28 +0000 Subject: cc_update_etc_hosts: Use the distribution-defined path for the hosts file (#983) The distribution class has a field that specifies the location of the system hosts file and this can be overridden in subclasses. While the field is correctly used in distro.update_etc_hosts(), the update_etc_hosts module does not use it and just assumes '/etc/hosts' This fixes the module to use the distribution-specific variable. --- cloudinit/config/cc_update_etc_hosts.py | 32 ++++++++++++++++++-------------- tools/.github-cla-signers | 1 + 2 files changed, 19 insertions(+), 14 deletions(-) (limited to 'tools/.github-cla-signers') diff --git a/cloudinit/config/cc_update_etc_hosts.py b/cloudinit/config/cc_update_etc_hosts.py index 03fffb96..3a78fccc 100644 --- a/cloudinit/config/cc_update_etc_hosts.py +++ b/cloudinit/config/cc_update_etc_hosts.py @@ -9,27 +9,28 @@ """ Update Etc Hosts ---------------- -**Summary:** update ``/etc/hosts`` +**Summary:** update the hosts file (usually ``/etc/hosts``) -This module will update the contents of ``/etc/hosts`` based on the -hostname/fqdn specified in config. Management of ``/etc/hosts`` is controlled -using ``manage_etc_hosts``. If this is set to false, cloud-init will not manage -``/etc/hosts`` at all. This is the default behavior. +This module will update the contents of the local hosts database (hosts file; +usually ``/etc/hosts``) based on the hostname/fqdn specified in config. +Management of the hosts file is controlled using ``manage_etc_hosts``. If this +is set to false, cloud-init will not manage the hosts file at all. This is the +default behavior. -If set to ``true`` or ``template``, cloud-init will generate ``/etc/hosts`` +If set to ``true`` or ``template``, cloud-init will generate the hosts file using the template located in ``/etc/cloud/templates/hosts.tmpl``. In the ``/etc/cloud/templates/hosts.tmpl`` template, the strings ``$hostname`` and ``$fqdn`` will be replaced with the hostname and fqdn respectively. If ``manage_etc_hosts`` is set to ``localhost``, then cloud-init will not -rewrite ``/etc/hosts`` entirely, but rather will ensure that a entry for the -fqdn with a distribution dependent ip is present in ``/etc/hosts`` (i.e. -``ping `` will ping ``127.0.0.1`` or ``127.0.1.1`` or other ip). +rewrite the hosts file entirely, but rather will ensure that a entry for the +fqdn with a distribution dependent ip is present (i.e. ``ping `` will +ping ``127.0.0.1`` or ``127.0.1.1`` or other ip). .. note:: if ``manage_etc_hosts`` is set ``true`` or ``template``, the contents - of ``/etc/hosts`` will be updated every boot. to make any changes to - ``/etc/hosts`` persistant they must be made in + of the hosts file will be updated every boot. To make any changes to + the hosts file persistent they must be made in ``/etc/cloud/templates/hosts.tmpl`` .. note:: @@ -59,6 +60,9 @@ frequency = PER_ALWAYS def handle(name, cfg, cloud, log, _args): manage_hosts = util.get_cfg_option_str(cfg, "manage_etc_hosts", False) + + hosts_fn = cloud.distro.hosts_fn + if util.translate_bool(manage_hosts, addons=['template']): (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) if not hostname: @@ -74,7 +78,7 @@ def handle(name, cfg, cloud, log, _args): " found for distro %s") % (cloud.distro.osfamily)) - templater.render_to_file(tpl_fn_name, '/etc/hosts', + templater.render_to_file(tpl_fn_name, hosts_fn, {'hostname': hostname, 'fqdn': fqdn}) elif manage_hosts == "localhost": @@ -84,10 +88,10 @@ def handle(name, cfg, cloud, log, _args): " but no hostname was found")) return - log.debug("Managing localhost in /etc/hosts") + log.debug("Managing localhost in %s", hosts_fn) cloud.distro.update_etc_hosts(hostname, fqdn) else: log.debug(("Configuration option 'manage_etc_hosts' is not set," - " not managing /etc/hosts in module %s"), name) + " not managing %s in module %s"), hosts_fn, name) # vi: ts=4 expandtab diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index ba0c6d79..d1939718 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -14,6 +14,7 @@ bmhughes candlerb cawamata ciprianbadescu +citrus-it dankenigsberg ddymko dermotbradley -- cgit v1.2.3 From f4c47e3e25d1fb79e2673e37f8fc67750d025be2 Mon Sep 17 00:00:00 2001 From: zhuzaifangxuele <52022596+zhuzaifangxuele@users.noreply.github.com> Date: Tue, 14 Sep 2021 02:21:28 +0800 Subject: Support openEuler OS (#1012) openEuler Homepage: https://www.openeuler.org/en/ --- README.md | 2 +- cloudinit/config/cc_ntp.py | 4 ++-- cloudinit/config/cc_yum_add_repo.py | 4 ++-- cloudinit/distros/__init__.py | 2 +- cloudinit/distros/openEuler.py | 9 +++++++++ cloudinit/net/sysconfig.py | 2 +- cloudinit/tests/test_util.py | 17 +++++++++++++++++ cloudinit/util.py | 4 ++-- config/cloud.cfg.tmpl | 6 +++--- systemd/cloud-init-generator.tmpl | 4 ++-- systemd/cloud-init.service.tmpl | 4 ++-- tests/unittests/test_cli.py | 4 ++-- tools/.github-cla-signers | 1 + tools/render-cloudcfg | 2 +- 14 files changed, 46 insertions(+), 19 deletions(-) create mode 100644 cloudinit/distros/openEuler.py (limited to 'tools/.github-cla-signers') diff --git a/README.md b/README.md index b705a065..e96541ef 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ get in contact with that distribution and send them our way! | Supported OSes | Supported Public Clouds | Supported Private Clouds | | --- | --- | --- | -| Alpine Linux
ArchLinux
Debian
DragonFlyBSD
Fedora
FreeBSD
Gentoo Linux
NetBSD
OpenBSD
RHEL/CentOS/AlmaLinux/Rocky/PhotonOS/Virtuozzo/EuroLinux/CloudLinux
SLES/openSUSE
Ubuntu










| Amazon Web Services
Microsoft Azure
Google Cloud Platform
Oracle Cloud Infrastructure
Softlayer
Rackspace Public Cloud
IBM Cloud
DigitalOcean
Bigstep
Hetzner
Joyent
CloudSigma
Alibaba Cloud
OVH
OpenNebula
Exoscale
Scaleway
CloudStack
AltCloud
SmartOS
HyperOne
Vultr
Rootbox
| Bare metal installs
OpenStack
LXD
KVM
Metal-as-a-Service (MAAS)
VMware















| +| Alpine Linux
ArchLinux
Debian
DragonFlyBSD
Fedora
FreeBSD
Gentoo Linux
NetBSD
OpenBSD
openEuler
RHEL/CentOS/AlmaLinux/Rocky/PhotonOS/Virtuozzo/EuroLinux/CloudLinux
SLES/openSUSE
Ubuntu










| Amazon Web Services
Microsoft Azure
Google Cloud Platform
Oracle Cloud Infrastructure
Softlayer
Rackspace Public Cloud
IBM Cloud
DigitalOcean
Bigstep
Hetzner
Joyent
CloudSigma
Alibaba Cloud
OVH
OpenNebula
Exoscale
Scaleway
CloudStack
AltCloud
SmartOS
HyperOne
Vultr
Rootbox
| Bare metal installs
OpenStack
LXD
KVM
Metal-as-a-Service (MAAS)
VMware















| ## To start developing cloud-init diff --git a/cloudinit/config/cc_ntp.py b/cloudinit/config/cc_ntp.py index e2231cbb..f4468c9d 100644 --- a/cloudinit/config/cc_ntp.py +++ b/cloudinit/config/cc_ntp.py @@ -25,8 +25,8 @@ frequency = PER_INSTANCE NTP_CONF = '/etc/ntp.conf' NR_POOL_SERVERS = 4 distros = ['almalinux', 'alpine', 'centos', 'cloudlinux', 'debian', - 'eurolinux', 'fedora', 'opensuse', 'photon', 'rhel', 'rocky', - 'sles', 'ubuntu', 'virtuozzo'] + 'eurolinux', 'fedora', 'openEuler', 'opensuse', 'photon', + 'rhel', 'rocky', 'sles', 'ubuntu', 'virtuozzo'] NTP_CLIENT_CONFIG = { 'chrony': { diff --git a/cloudinit/config/cc_yum_add_repo.py b/cloudinit/config/cc_yum_add_repo.py index 899cb082..bcca86cb 100644 --- a/cloudinit/config/cc_yum_add_repo.py +++ b/cloudinit/config/cc_yum_add_repo.py @@ -19,7 +19,7 @@ entry, the config entry will be skipped. **Module frequency:** per always **Supported distros:** almalinux, centos, cloudlinux, eurolinux, fedora, - photon, rhel, rocky, virtuozzo + openEuler, photon, rhel, rocky, virtuozzo **Config keys**:: @@ -38,7 +38,7 @@ from configparser import ConfigParser from cloudinit import util distros = ['almalinux', 'centos', 'cloudlinux', 'eurolinux', 'fedora', - 'photon', 'rhel', 'rocky', 'virtuozzo'] + 'openEuler', 'photon', 'rhel', 'rocky', 'virtuozzo'] def _canonicalize_id(repo_id): diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index a0526948..63e78591 100755 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -50,7 +50,7 @@ OSFAMILIES = { 'freebsd': ['freebsd'], 'gentoo': ['gentoo'], 'redhat': ['almalinux', 'amazon', 'centos', 'cloudlinux', 'eurolinux', - 'fedora', 'photon', 'rhel', 'rocky', 'virtuozzo'], + 'fedora', 'openEuler', 'photon', 'rhel', 'rocky', 'virtuozzo'], 'suse': ['opensuse', 'sles'], } diff --git a/cloudinit/distros/openEuler.py b/cloudinit/distros/openEuler.py new file mode 100644 index 00000000..edb3165d --- /dev/null +++ b/cloudinit/distros/openEuler.py @@ -0,0 +1,9 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit.distros import rhel + + +class Distro(rhel.Distro): + pass + +# vi: ts=4 expandtab diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index 7b8e4da7..ef4543b4 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -19,7 +19,7 @@ from .network_state import ( LOG = logging.getLogger(__name__) KNOWN_DISTROS = ['almalinux', 'centos', 'cloudlinux', 'eurolinux', 'fedora', - 'rhel', 'rocky', 'suse', 'virtuozzo'] + 'openEuler', 'rhel', 'rocky', 'suse', 'virtuozzo'] NM_CFG_FILE = "/etc/NetworkManager/NetworkManager.conf" diff --git a/cloudinit/tests/test_util.py b/cloudinit/tests/test_util.py index f11cfb27..977ad8e0 100644 --- a/cloudinit/tests/test_util.py +++ b/cloudinit/tests/test_util.py @@ -200,6 +200,15 @@ OS_RELEASE_CLOUDLINUX_8 = dedent("""\ BUG_REPORT_URL="https://www.cloudlinux.com/support" """) +OS_RELEASE_OPENEULER_20 = dedent("""\ + NAME="openEuler" + VERSION="20.03 (LTS-SP2)" + ID="openEuler" + VERSION_ID="20.03" + PRETTY_NAME="openEuler 20.03 (LTS-SP2)" + ANSI_COLOR="0;31" +""") + REDHAT_RELEASE_CENTOS_6 = "CentOS release 6.10 (Final)" REDHAT_RELEASE_CENTOS_7 = "CentOS Linux release 7.5.1804 (Core)" REDHAT_RELEASE_REDHAT_6 = ( @@ -718,6 +727,14 @@ class TestGetLinuxDistro(CiTestCase): dist = util.get_linux_distro() self.assertEqual(('debian', '9', 'stretch'), dist) + @mock.patch('cloudinit.util.load_file') + def test_get_linux_openeuler(self, m_os_release, m_path_exists): + """Verify get the correct name and release name on Openeuler.""" + m_os_release.return_value = OS_RELEASE_OPENEULER_20 + m_path_exists.side_effect = TestGetLinuxDistro.os_release_exists + dist = util.get_linux_distro() + self.assertEqual(('openEuler', '20.03', 'LTS-SP2'), dist) + @mock.patch('cloudinit.util.load_file') def test_get_linux_opensuse(self, m_os_release, m_path_exists): """Verify we get the correct name and machine arch on openSUSE diff --git a/cloudinit/util.py b/cloudinit/util.py index 894245bf..9662b30b 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -549,8 +549,8 @@ def system_info(): linux_dist = info['dist'][0].lower() if linux_dist in ( 'almalinux', 'alpine', 'arch', 'centos', 'cloudlinux', - 'debian', 'eurolinux', 'fedora', 'photon', 'rhel', 'rocky', - 'suse', 'virtuozzo'): + 'debian', 'eurolinux', 'fedora', 'openEuler', 'photon', + 'rhel', 'rocky', 'suse', 'virtuozzo'): var = linux_dist elif linux_dist in ('ubuntu', 'linuxmint', 'mint'): var = 'ubuntu' diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl index e5fbc10b..de1d75e5 100644 --- a/config/cloud.cfg.tmpl +++ b/config/cloud.cfg.tmpl @@ -33,7 +33,7 @@ disable_root: true {% endif %} {% if variant in ["almalinux", "alpine", "amazon", "centos", "cloudlinux", "eurolinux", - "fedora", "rhel", "rocky", "virtuozzo"] %} + "fedora", "openEuler", "rhel", "rocky", "virtuozzo"] %} mount_default_fields: [~, ~, 'auto', 'defaults,nofail', '0', '2'] {% if variant == "amazon" %} resize_rootfs: noblock @@ -174,7 +174,7 @@ cloud_final_modules: system_info: # This will affect which distro class gets used {% if variant in ["almalinux", "alpine", "amazon", "arch", "centos", "cloudlinux", "debian", - "eurolinux", "fedora", "freebsd", "netbsd", "openbsd", + "eurolinux", "fedora", "freebsd", "netbsd", "openbsd", "openEuler", "photon", "rhel", "rocky", "suse", "ubuntu", "virtuozzo"] %} distro: {{ variant }} {% elif variant in ["dragonfly"] %} @@ -229,7 +229,7 @@ system_info: security: http://ports.ubuntu.com/ubuntu-ports ssh_svcname: ssh {% elif variant in ["almalinux", "alpine", "amazon", "arch", "centos", "cloudlinux", "eurolinux", - "fedora", "rhel", "rocky", "suse", "virtuozzo"] %} + "fedora", "openEuler", "rhel", "rocky", "suse", "virtuozzo"] %} # Default user name + that default users groups (if added/used) default_user: {% if variant == "amazon" %} diff --git a/systemd/cloud-init-generator.tmpl b/systemd/cloud-init-generator.tmpl index 7fe009ec..7d1e7256 100644 --- a/systemd/cloud-init-generator.tmpl +++ b/systemd/cloud-init-generator.tmpl @@ -83,8 +83,8 @@ default() { check_for_datasource() { local ds_rc="" -{% if variant in ["almalinux", "centos", "cloudlinux", "eurolinux", "fedora", "rhel", - "rocky", "virtuozzo"] %} +{% if variant in ["almalinux", "centos", "cloudlinux", "eurolinux", "fedora", + "openEuler", "rhel", "rocky", "virtuozzo"] %} local dsidentify="/usr/libexec/cloud-init/ds-identify" {% else %} local dsidentify="/usr/lib/cloud-init/ds-identify" diff --git a/systemd/cloud-init.service.tmpl b/systemd/cloud-init.service.tmpl index 6c3a8f20..de3f3d91 100644 --- a/systemd/cloud-init.service.tmpl +++ b/systemd/cloud-init.service.tmpl @@ -12,8 +12,8 @@ After=systemd-networkd-wait-online.service {% if variant in ["ubuntu", "unknown", "debian"] %} After=networking.service {% endif %} -{% if variant in ["almalinux", "centos", "cloudlinux", "eurolinux", "fedora", "rhel", - "rocky", "virtuozzo"] %} +{% if variant in ["almalinux", "centos", "cloudlinux", "eurolinux", "fedora", + "openEuler", "rhel", "rocky", "virtuozzo"] %} After=network.service After=NetworkManager.service {% endif %} diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py index b8ae9e47..1459fd9c 100644 --- a/tests/unittests/test_cli.py +++ b/tests/unittests/test_cli.py @@ -225,8 +225,8 @@ class TestCLI(test_helpers.FilesystemMockingTestCase): expected_doc_sections = [ '**Supported distros:** all', ('**Supported distros:** almalinux, alpine, centos, cloudlinux, ' - 'debian, eurolinux, fedora, opensuse, photon, rhel, rocky, ' - 'sles, ubuntu, virtuozzo'), + 'debian, eurolinux, fedora, openEuler, opensuse, photon, rhel, ' + 'rocky, sles, ubuntu, virtuozzo'), '**Config schema**:\n **resize_rootfs:** (true/false/noblock)', '**Examples**::\n\n runcmd:\n - [ ls, -l, / ]\n' ] diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index d1939718..fa395553 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -68,3 +68,4 @@ Vultaire WebSpider xiachen-rh xnox +zhuzaifangxuele diff --git a/tools/render-cloudcfg b/tools/render-cloudcfg index 78a48c30..186d61b7 100755 --- a/tools/render-cloudcfg +++ b/tools/render-cloudcfg @@ -5,7 +5,7 @@ import os import sys VARIANTS = ["almalinux", "alpine", "amazon", "arch", "centos", "cloudlinux", "debian", - "eurolinux", "fedora", "freebsd", "netbsd", "openbsd", "photon", + "eurolinux", "fedora", "freebsd", "netbsd", "openbsd", "openEuler", "photon", "rhel", "suse","rocky", "ubuntu", "unknown", "virtuozzo"] -- cgit v1.2.3 From 26a92b0d883492beefacee80a7e7a2ab2a2c648f Mon Sep 17 00:00:00 2001 From: Jille Timmermans Date: Tue, 14 Sep 2021 21:06:43 +0200 Subject: Add Jille to tools/.github-cla-signers (#1016) --- tools/.github-cla-signers | 1 + 1 file changed, 1 insertion(+) (limited to 'tools/.github-cla-signers') diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index fa395553..42a7e4c1 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -29,6 +29,7 @@ hamalq impl irishgordo izzyleung +Jille JohnKepplers johnsonshi jordimassaguerpla -- cgit v1.2.3 From d4fe7bc4c320023263075258027eae210e7497e1 Mon Sep 17 00:00:00 2001 From: Richard Hansen Date: Thu, 16 Sep 2021 10:12:41 -0400 Subject: Update IPv6 entries in /etc/hosts (#1021) Add IPv6 localhost (::1) entry. See: https://salsa.debian.org/installer-team/netcfg/-/blob/1767c9264e04652b9150ffc7b25568e4ea6b2bdd/netcfg.h#L42 https://salsa.debian.org/md/netbase/-/blob/9de8afcad482418cc4956dc09bbf6a2e8624d574/debian/netbase.postinst#L8 Also remove ip6-localnet, ip6-mcastprefix, and ip6-allhosts. See: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=499800 https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=688090 LP: #1943798 --- templates/hosts.debian.tmpl | 5 +---- tools/.github-cla-signers | 1 + 2 files changed, 2 insertions(+), 4 deletions(-) (limited to 'tools/.github-cla-signers') diff --git a/templates/hosts.debian.tmpl b/templates/hosts.debian.tmpl index 7e29907a..afeccf9a 100644 --- a/templates/hosts.debian.tmpl +++ b/templates/hosts.debian.tmpl @@ -17,10 +17,7 @@ you need to add the following to config: 127.0.0.1 localhost # The following lines are desirable for IPv6 capable hosts -::1 ip6-localhost ip6-loopback -fe00::0 ip6-localnet -ff00::0 ip6-mcastprefix +::1 localhost ip6-localhost ip6-loopback ff02::1 ip6-allnodes ff02::2 ip6-allrouters -ff02::3 ip6-allhosts diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 42a7e4c1..d9d43ba9 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -53,6 +53,7 @@ omBratteng onitake qubidt renanrodrigo +rhansen riedel sarahwzadara slyon -- cgit v1.2.3 From e27c30748e88409b1646a552f994edf9ed9d017e Mon Sep 17 00:00:00 2001 From: vteratipally <67723486+vteratipally@users.noreply.github.com> Date: Mon, 20 Sep 2021 21:53:05 -0700 Subject: Add retries to DataSourceGCE.py when connecting to GCE (#1005) Add retries to DatasourceGCE when connecting to GCE. Sometimes when the trying to fetch the metadata, cloud-init fails and the fallback datasource NoCloud is used which is not expected. Add retries to ensure loading of the data source. --- cloudinit/sources/DataSourceGCE.py | 19 +++++++++++++------ cloudinit/sources/__init__.py | 23 ++++++++++++++++++----- cloudinit/sources/tests/test_init.py | 17 +++++++++++------ doc/rtd/topics/datasources/gce.rst | 22 ++++++++++++++++++++++ tools/.github-cla-signers | 1 + 5 files changed, 65 insertions(+), 17 deletions(-) (limited to 'tools/.github-cla-signers') diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 746caddb..ecdc458b 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -27,8 +27,10 @@ HEADERS = {'Metadata-Flavor': 'Google'} class GoogleMetadataFetcher(object): - def __init__(self, metadata_address): + def __init__(self, metadata_address, num_retries, sec_between_retries): self.metadata_address = metadata_address + self.num_retries = num_retries + self.sec_between_retries = sec_between_retries def get_value(self, path, is_text, is_recursive=False): value = None @@ -36,7 +38,9 @@ class GoogleMetadataFetcher(object): url = self.metadata_address + path if is_recursive: url += '/?recursive=True' - resp = url_helper.readurl(url=url, headers=HEADERS) + resp = url_helper.readurl(url=url, headers=HEADERS, + retries=self.num_retries, + sec_between=self.sec_between_retries) except url_helper.UrlError as exc: msg = "url %s raised exception %s" LOG.debug(msg, path, exc) @@ -68,9 +72,11 @@ class DataSourceGCE(sources.DataSource): self.metadata_address = self.ds_cfg['metadata_url'] def _get_data(self): + url_params = self.get_url_params() ret = util.log_time( LOG.debug, 'Crawl of GCE metadata service', - read_md, kwargs={'address': self.metadata_address}) + read_md, kwargs={'address': self.metadata_address, + 'url_params': url_params}) if not ret['success']: if ret['platform_reports_gce']: @@ -176,7 +182,7 @@ def _parse_public_keys(public_keys_data, default_user=None): return public_keys -def read_md(address=None, platform_check=True): +def read_md(address=None, url_params=None, platform_check=True): if address is None: address = MD_V1_URL @@ -203,8 +209,9 @@ def read_md(address=None, platform_check=True): ('instance-data', ('instance/attributes',), False, False, True), ('project-data', ('project/attributes',), False, False, True), ] - - metadata_fetcher = GoogleMetadataFetcher(address) + metadata_fetcher = GoogleMetadataFetcher(address, + url_params.num_retries, + url_params.sec_between_retries) md = {} # Iterate over url_map keys to get metadata items. for (mkey, paths, required, is_text, is_recursive) in url_map: diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 54b8240a..d61d280d 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -138,7 +138,8 @@ def redact_sensitive_keys(metadata, redact_value=REDACT_SENSITIVE_VALUE): URLParams = namedtuple( - 'URLParms', ['max_wait_seconds', 'timeout_seconds', 'num_retries']) + 'URLParms', ['max_wait_seconds', 'timeout_seconds', + 'num_retries', 'sec_between_retries']) class DataSource(CloudInitPickleMixin, metaclass=abc.ABCMeta): @@ -175,9 +176,10 @@ class DataSource(CloudInitPickleMixin, metaclass=abc.ABCMeta): NetworkConfigSource.ds) # read_url_params - url_max_wait = -1 # max_wait < 0 means do not wait - url_timeout = 10 # timeout for each metadata url read attempt - url_retries = 5 # number of times to retry url upon 404 + url_max_wait = -1 # max_wait < 0 means do not wait + url_timeout = 10 # timeout for each metadata url read attempt + url_retries = 5 # number of times to retry url upon 404 + url_sec_between_retries = 1 # amount of seconds to wait between retries # The datasource defines a set of supported EventTypes during which # the datasource can react to changes in metadata and regenerate @@ -422,7 +424,18 @@ class DataSource(CloudInitPickleMixin, metaclass=abc.ABCMeta): LOG, "Config retries '%s' is not an int, using default '%s'", self.ds_cfg.get('retries'), retries) - return URLParams(max_wait, timeout, retries) + sec_between_retries = self.url_sec_between_retries + try: + sec_between_retries = int(self.ds_cfg.get( + "sec_between_retries", + self.url_sec_between_retries)) + except Exception: + util.logexc( + LOG, "Config sec_between_retries '%s' is not an int," + " using default '%s'", + self.ds_cfg.get("sec_between_retries"), sec_between_retries) + + return URLParams(max_wait, timeout, retries, sec_between_retries) def get_userdata(self, apply_filter=False): if self.userdata is None: diff --git a/cloudinit/sources/tests/test_init.py b/cloudinit/sources/tests/test_init.py index a2b052a6..ae09cb17 100644 --- a/cloudinit/sources/tests/test_init.py +++ b/cloudinit/sources/tests/test_init.py @@ -97,6 +97,8 @@ class TestDataSource(CiTestCase): self.assertEqual(params.max_wait_seconds, self.datasource.url_max_wait) self.assertEqual(params.timeout_seconds, self.datasource.url_timeout) self.assertEqual(params.num_retries, self.datasource.url_retries) + self.assertEqual(params.sec_between_retries, + self.datasource.url_sec_between_retries) def test_datasource_get_url_params_subclassed(self): """Subclasses can override get_url_params defaults.""" @@ -104,7 +106,7 @@ class TestDataSource(CiTestCase): distro = 'distrotest' # generally should be a Distro object datasource = DataSourceTestSubclassNet(sys_cfg, distro, self.paths) expected = (datasource.url_max_wait, datasource.url_timeout, - datasource.url_retries) + datasource.url_retries, datasource.url_sec_between_retries) url_params = datasource.get_url_params() self.assertNotEqual(self.datasource.get_url_params(), url_params) self.assertEqual(expected, url_params) @@ -114,14 +116,16 @@ class TestDataSource(CiTestCase): sys_cfg = { 'datasource': { 'MyTestSubclass': { - 'max_wait': '1', 'timeout': '2', 'retries': '3'}}} + 'max_wait': '1', 'timeout': '2', + 'retries': '3', 'sec_between_retries': 4 + }}} datasource = DataSourceTestSubclassNet( sys_cfg, self.distro, self.paths) - expected = (1, 2, 3) + expected = (1, 2, 3, 4) url_params = datasource.get_url_params() self.assertNotEqual( (datasource.url_max_wait, datasource.url_timeout, - datasource.url_retries), + datasource.url_retries, datasource.url_sec_between_retries), url_params) self.assertEqual(expected, url_params) @@ -130,7 +134,8 @@ class TestDataSource(CiTestCase): # Set an override that is below 0 which gets ignored. sys_cfg = {'datasource': {'_undef': {'timeout': '-1'}}} datasource = DataSource(sys_cfg, self.distro, self.paths) - (_max_wait, timeout, _retries) = datasource.get_url_params() + (_max_wait, timeout, _retries, + _sec_between_retries) = datasource.get_url_params() self.assertEqual(0, timeout) def test_datasource_get_url_uses_defaults_on_errors(self): @@ -142,7 +147,7 @@ class TestDataSource(CiTestCase): datasource = DataSource(sys_cfg, self.distro, self.paths) url_params = datasource.get_url_params() expected = (datasource.url_max_wait, datasource.url_timeout, - datasource.url_retries) + datasource.url_retries, datasource.url_sec_between_retries) self.assertEqual(expected, url_params) logs = self.logs.getvalue() expected_logs = [ diff --git a/doc/rtd/topics/datasources/gce.rst b/doc/rtd/topics/datasources/gce.rst index 8406695c..f3590282 100644 --- a/doc/rtd/topics/datasources/gce.rst +++ b/doc/rtd/topics/datasources/gce.rst @@ -15,6 +15,28 @@ to provide ``public-keys``. ``user-data`` and ``user-data-encoding`` can be provided to cloud-init by setting those custom metadata keys for an *instance*. +Configuration +------------- +The following configuration can be set for the datasource in system +configuration (in `/etc/cloud/cloud.cfg` or `/etc/cloud/cloud.cfg.d/`). + +The settings that may be configured are: + + * **retries**: The number of retries that should be done for an http request. + This value is used only after metadata_url is selected. (default: 5) + * **sec_between_retries**: The amount of wait time between the retries when + crawling the metadata service. (default: 1) + + +An example configuration with the default values is provided below: + +.. sourcecode:: yaml + + datasource: + GCE: + retries: 5 + sec_between_retries: 1 + .. _GCE metadata docs: https://cloud.google.com/compute/docs/storing-retrieving-metadata#querying .. vi: textwidth=78 diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index d9d43ba9..4f668112 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -66,6 +66,7 @@ timothegenzmer tnt-dev tomponline tsanghan +vteratipally Vultaire WebSpider xiachen-rh -- cgit v1.2.3 From 21aec00f77dc51cee2612978ed4ebcca9d91a697 Mon Sep 17 00:00:00 2001 From: jshen28 Date: Thu, 30 Sep 2021 00:38:12 +0800 Subject: Add jshen28 as contributor (#1035) --- tools/.github-cla-signers | 1 + 1 file changed, 1 insertion(+) (limited to 'tools/.github-cla-signers') diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 4f668112..051aff04 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -35,6 +35,7 @@ johnsonshi jordimassaguerpla jqueuniet jsf9k +jshen28 klausenbusk landon912 lucasmoura -- cgit v1.2.3 From 934c8d71fd99b2ca24db01ca853ff42faa88fb5e Mon Sep 17 00:00:00 2001 From: Andrew Gein Date: Thu, 30 Sep 2021 01:10:55 +0500 Subject: Add andgein as contributor (#1042) --- tools/.github-cla-signers | 1 + 1 file changed, 1 insertion(+) (limited to 'tools/.github-cla-signers') diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 051aff04..3e16ddf3 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -3,6 +3,7 @@ ajmyyra akutz AlexBaranowski Aman306 +andgein andrewbogott andrewlukoshko antonyc -- cgit v1.2.3 From 725a7f7f19eb39b472e1f24b447fc9a596bf1748 Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Thu, 7 Oct 2021 14:08:13 -0600 Subject: Allow comments in runcmd and report failed commands correctly (#1049) Allow comments in runcmd and report failed commands correctly A `runcmd` script may fail to parse properly, but does not mark `runcmd` as failed when that occurs. Additionally `shellify()` fails to correctly parse scripts that contain a comment line. Rectify both issues and add unit tests to verify correct behavior. LP: #1853146 --- cloudinit/config/cc_runcmd.py | 7 +++--- cloudinit/tests/test_util.py | 5 ++++ cloudinit/util.py | 3 +++ .../unittests/test_handler/test_handler_runcmd.py | 27 +++++++++++++++++----- tools/.github-cla-signers | 1 + 5 files changed, 34 insertions(+), 9 deletions(-) (limited to 'tools/.github-cla-signers') diff --git a/cloudinit/config/cc_runcmd.py b/cloudinit/config/cc_runcmd.py index 1f75d6c5..15960c7d 100644 --- a/cloudinit/config/cc_runcmd.py +++ b/cloudinit/config/cc_runcmd.py @@ -65,7 +65,8 @@ schema = { 'items': { 'oneOf': [ {'type': 'array', 'items': {'type': 'string'}}, - {'type': 'string'}] + {'type': 'string'}, + {'type': 'null'}] }, 'additionalItems': False, # Reject items of non-string non-list 'additionalProperties': False, @@ -90,7 +91,7 @@ def handle(name, cfg, cloud, log, _args): try: content = util.shellify(cmd) util.write_file(out_fn, content, 0o700) - except Exception: - util.logexc(log, "Failed to shellify %s into file %s", cmd, out_fn) + except Exception as e: + raise type(e)('Failed to shellify {} into file {}'.format(cmd, out_fn)) # vi: ts=4 expandtab diff --git a/cloudinit/tests/test_util.py b/cloudinit/tests/test_util.py index 977ad8e0..ab5eb35c 100644 --- a/cloudinit/tests/test_util.py +++ b/cloudinit/tests/test_util.py @@ -349,6 +349,11 @@ class TestShellify(CiTestCase): util.shellify(["echo hi mom", ["echo", "hi dad"], ('echo', 'hi', 'sis')])) + def test_supports_comments(self): + self.assertEqual( + '\n'.join(["#!/bin/sh", "echo start", "echo end", ""]), + util.shellify(["echo start", None, "echo end"])) + class TestGetHostnameFqdn(CiTestCase): diff --git a/cloudinit/util.py b/cloudinit/util.py index 22d8917e..1b4384e1 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2031,6 +2031,9 @@ def shellify(cmdlist, add_header=True): elif isinstance(args, str): content = "%s%s\n" % (content, args) cmds_made += 1 + # Yaml parsing of a comment results in None + elif args is None: + pass else: raise TypeError( "Unable to shellify type '%s'. Expected list, string, tuple. " diff --git a/tests/unittests/test_handler/test_handler_runcmd.py b/tests/unittests/test_handler/test_handler_runcmd.py index 73237d68..c03efa67 100644 --- a/tests/unittests/test_handler/test_handler_runcmd.py +++ b/tests/unittests/test_handler/test_handler_runcmd.py @@ -7,6 +7,7 @@ from cloudinit.tests.helpers import ( CiTestCase, FilesystemMockingTestCase, SchemaTestCaseMixin, skipUnlessJsonSchema) +from unittest.mock import patch import logging import os import stat @@ -41,15 +42,27 @@ class TestRuncmd(FilesystemMockingTestCase): "Skipping module named notimportant, no 'runcmd' key", self.logs.getvalue()) + @patch('cloudinit.util.shellify') + def test_runcmd_shellify_fails(self, cls): + """When shellify fails throw exception""" + cls.side_effect = TypeError("patched shellify") + valid_config = {'runcmd': ['echo 42']} + cc = self._get_cloud('ubuntu') + with self.assertRaises(TypeError) as cm: + with self.allow_subp(['/bin/sh']): + handle('cc_runcmd', valid_config, cc, LOG, None) + self.assertIn("Failed to shellify", str(cm.exception)) + def test_handler_invalid_command_set(self): """Commands which can't be converted to shell will raise errors.""" invalid_config = {'runcmd': 1} cc = self._get_cloud('ubuntu') - handle('cc_runcmd', invalid_config, cc, LOG, []) + with self.assertRaises(TypeError) as cm: + handle('cc_runcmd', invalid_config, cc, LOG, []) self.assertIn( 'Failed to shellify 1 into file' ' /var/lib/cloud/instances/iid-datasource-none/scripts/runcmd', - self.logs.getvalue()) + str(cm.exception)) @skipUnlessJsonSchema() def test_handler_schema_validation_warns_non_array_type(self): @@ -60,11 +73,12 @@ class TestRuncmd(FilesystemMockingTestCase): """ invalid_config = {'runcmd': 1} cc = self._get_cloud('ubuntu') - handle('cc_runcmd', invalid_config, cc, LOG, []) + with self.assertRaises(TypeError) as cm: + handle('cc_runcmd', invalid_config, cc, LOG, []) self.assertIn( 'Invalid config:\nruncmd: 1 is not of type \'array\'', self.logs.getvalue()) - self.assertIn('Failed to shellify', self.logs.getvalue()) + self.assertIn('Failed to shellify', str(cm.exception)) @skipUnlessJsonSchema() def test_handler_schema_validation_warns_non_array_item_type(self): @@ -76,7 +90,8 @@ class TestRuncmd(FilesystemMockingTestCase): invalid_config = { 'runcmd': ['ls /', 20, ['wget', 'http://stuff/blah'], {'a': 'n'}]} cc = self._get_cloud('ubuntu') - handle('cc_runcmd', invalid_config, cc, LOG, []) + with self.assertRaises(TypeError) as cm: + handle('cc_runcmd', invalid_config, cc, LOG, []) expected_warnings = [ 'runcmd.1: 20 is not valid under any of the given schemas', 'runcmd.3: {\'a\': \'n\'} is not valid under any of the given' @@ -85,7 +100,7 @@ class TestRuncmd(FilesystemMockingTestCase): logs = self.logs.getvalue() for warning in expected_warnings: self.assertIn(warning, logs) - self.assertIn('Failed to shellify', logs) + self.assertIn('Failed to shellify', str(cm.exception)) def test_handler_write_valid_runcmd_schema_to_file(self): """Valid runcmd schema is written to a runcmd shell script.""" diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 3e16ddf3..0aa168d6 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -27,6 +27,7 @@ esposem GabrielNagy giggsoff hamalq +holmanb impl irishgordo izzyleung -- cgit v1.2.3 From a0a68a24c34ee268962e7a3c3844c59ab4036bf9 Mon Sep 17 00:00:00 2001 From: Thomas Weißschuh Date: Tue, 19 Oct 2021 02:09:40 +0200 Subject: VMware: read network-config from ISO (#1066) There is no reason for the ISO missing this functionality. As discussed in https://github.com/canonical/cloud-init/pull/947/files#r707338489 --- cloudinit/sources/DataSourceOVF.py | 3 +-- doc/sources/ovf/example/ovf-env.xml | 2 +- tests/unittests/test_datasource/test_ovf.py | 10 ++++++++-- tools/.github-cla-signers | 1 + 4 files changed, 11 insertions(+), 5 deletions(-) (limited to 'tools/.github-cla-signers') diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 08a205f1..5257a534 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -360,8 +360,7 @@ class DataSourceOVF(sources.DataSource): if contents: break if contents: - read_network = ('com.vmware.guestInfo' == name) - (md, ud, cfg) = read_ovf_environment(contents, read_network) + (md, ud, cfg) = read_ovf_environment(contents, True) self.environment = contents if 'network-config' in md and md['network-config']: self._network_config = md['network-config'] diff --git a/doc/sources/ovf/example/ovf-env.xml b/doc/sources/ovf/example/ovf-env.xml index 4ef4ee63..e5f4e262 100644 --- a/doc/sources/ovf/example/ovf-env.xml +++ b/doc/sources/ovf/example/ovf-env.xml @@ -42,7 +42,7 @@