From 76652f3e07b6f659b2fd166a6619cb427dc6bc7e Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 22 Jun 2020 14:44:37 -0400 Subject: Hetzner: support reading user-data that is base64 encoded. (#448) Hetzner cloud only supports user-data as a string (presumably utf-8). In order to allow users on Hetzner to provide binary data to cloud-init, we will attempt to base64decode the userdata. The change here adds a 'maybe_b64decode' function that will decode data if and only if is base64 encoded. The reason for not using util.b64d is that we do not want the return value decoded to a string, and util.b64d will do that if it can. Additionally we call decode with validate=True which oddly is not the default. LP: #1884071 --- cloudinit/sources/DataSourceHetzner.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'cloudinit/sources/DataSourceHetzner.py') diff --git a/cloudinit/sources/DataSourceHetzner.py b/cloudinit/sources/DataSourceHetzner.py index 50298330..70e4274c 100644 --- a/cloudinit/sources/DataSourceHetzner.py +++ b/cloudinit/sources/DataSourceHetzner.py @@ -59,7 +59,14 @@ class DataSourceHetzner(sources.DataSource): self.userdata_address, timeout=self.timeout, sec_between=self.wait_retry, retries=self.retries) - self.userdata_raw = ud + # Hetzner cloud does not support binary user-data. So here, do a + # base64 decode of the data if we can. The end result being that a + # user can provide base64 encoded (possibly gzipped) data as user-data. + # + # The fallout is that in the event of b64 encoded user-data, + # /var/lib/cloud-init/cloud-config.txt will not be identical to the + # user-data provided. It will be decoded. + self.userdata_raw = hc_helper.maybe_b64decode(ud) self.metadata_full = md """hostname is name provided by user at launch. The API enforces -- cgit v1.2.3 From 3cec3881062490727c5fff1b16b53f0176f976f0 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Mon, 13 Jul 2020 12:00:32 -0400 Subject: cloudinit: remove global disable of pylint W0105 and fix errors (#480) This includes a fix to a test that had a string concatenation issue, and so was only testing a prefix of what was intended. --- .pylintrc | 3 +- cloudinit/analyze/show.py | 41 ++++++++++------------ cloudinit/config/cc_resizefs.py | 14 ++++---- cloudinit/distros/opensuse.py | 2 +- cloudinit/net/eni.py | 6 ++-- cloudinit/sources/DataSourceAzure.py | 14 ++++---- cloudinit/sources/DataSourceHetzner.py | 6 ++-- .../unittests/test_datasource/test_azure_helper.py | 7 ++-- tests/unittests/test_reporting_hyperv.py | 6 ++-- 9 files changed, 45 insertions(+), 54 deletions(-) (limited to 'cloudinit/sources/DataSourceHetzner.py') diff --git a/.pylintrc b/.pylintrc index 4d5d066d..f04603b9 100644 --- a/.pylintrc +++ b/.pylintrc @@ -7,7 +7,6 @@ jobs=4 [MESSAGES CONTROL] # Errors and warings with some filtered: -# W0105(pointless-string-statement) # W0107(unnecessary-pass) # W0201(attribute-defined-outside-init) # W0212(protected-access) @@ -28,7 +27,7 @@ jobs=4 # W0703(broad-except) # W1401(anomalous-backslash-in-string) -disable=C, F, I, R, W0105, W0107, W0201, W0212, W0221, W0222, W0223, W0231, W0311, W0511, W0602, W0603, W0611, W0613, W0621, W0622, W0631, W0703, W1401 +disable=C, F, I, R, W0107, W0201, W0212, W0221, W0222, W0223, W0231, W0311, W0511, W0602, W0603, W0611, W0613, W0621, W0622, W0631, W0703, W1401 [REPORTS] diff --git a/cloudinit/analyze/show.py b/cloudinit/analyze/show.py index cca1fa7f..0c825b23 100644 --- a/cloudinit/analyze/show.py +++ b/cloudinit/analyze/show.py @@ -15,28 +15,25 @@ from cloudinit import subp from cloudinit import util from cloudinit.distros import uses_systemd -# An event: -''' -{ - "description": "executing late commands", - "event_type": "start", - "level": "INFO", - "name": "cmd-install/stage-late" - "origin": "cloudinit", - "timestamp": 1461164249.1590767, -}, - - { - "description": "executing late commands", - "event_type": "finish", - "level": "INFO", - "name": "cmd-install/stage-late", - "origin": "cloudinit", - "result": "SUCCESS", - "timestamp": 1461164249.1590767 - } - -''' +# Example events: +# { +# "description": "executing late commands", +# "event_type": "start", +# "level": "INFO", +# "name": "cmd-install/stage-late" +# "origin": "cloudinit", +# "timestamp": 1461164249.1590767, +# } +# { +# "description": "executing late commands", +# "event_type": "finish", +# "level": "INFO", +# "name": "cmd-install/stage-late", +# "origin": "cloudinit", +# "result": "SUCCESS", +# "timestamp": 1461164249.1590767 +# } + format_key = { '%d': 'delta', '%D': 'description', diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index 8de4db30..978d2ee0 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -118,14 +118,12 @@ def _can_skip_resize_ufs(mount_point, devpth): if o == "-f": frag_sz = int(a) # check the current partition size - """ - # gpart show /dev/da0 -=> 40 62914480 da0 GPT (30G) - 40 1024 1 freebsd-boot (512K) - 1064 58719232 2 freebsd-ufs (28G) - 58720296 3145728 3 freebsd-swap (1.5G) - 61866024 1048496 - free - (512M) - """ + # Example output from `gpart show /dev/da0`: + # => 40 62914480 da0 GPT (30G) + # 40 1024 1 freebsd-boot (512K) + # 1064 58719232 2 freebsd-ufs (28G) + # 58720296 3145728 3 freebsd-swap (1.5G) + # 61866024 1048496 - free - (512M) expect_sz = None m = re.search('^(/dev/.+)p([0-9])$', devpth) gpart_res = _get_gpart_output(m.group(1)) diff --git a/cloudinit/distros/opensuse.py b/cloudinit/distros/opensuse.py index ffb7d0e8..b8e557b8 100644 --- a/cloudinit/distros/opensuse.py +++ b/cloudinit/distros/opensuse.py @@ -185,7 +185,7 @@ class Distro(distros.Distro): def preferred_ntp_clients(self): """The preferred ntp client is dependent on the version.""" - """Allow distro to determine the preferred ntp client list""" + # Allow distro to determine the preferred ntp client list if not self._preferred_ntp_clients: distro_info = util.system_info()['dist'] name = distro_info[0] diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index b4c69457..13c041f3 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -483,10 +483,8 @@ class Renderer(renderer.Renderer): if searchdomains: lo['subnets'][0]["dns_search"] = (" ".join(searchdomains)) - ''' Apply a sort order to ensure that we write out - the physical interfaces first; this is critical for - bonding - ''' + # Apply a sort order to ensure that we write out the physical + # interfaces first; this is critical for bonding order = { 'loopback': 0, 'physical': 1, diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 41431a7a..5e25b956 100755 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -166,12 +166,11 @@ def get_resource_disk_on_freebsd(port_id): port_id = port_id - 2 g1 = "000" + str(port_id) g0g1 = "{0}-{1}".format(g0, g1) - """ - search 'X' from - 'dev.storvsc.X.%pnpinfo: - classid=32412632-86cb-44a2-9b5c-50d1417354f5 - deviceid=00000000-0001-8899-0000-000000000000' - """ + + # search 'X' from + # 'dev.storvsc.X.%pnpinfo: + # classid=32412632-86cb-44a2-9b5c-50d1417354f5 + # deviceid=00000000-0001-8899-0000-000000000000' sysctl_out = get_dev_storvsc_sysctl() storvscid = find_storvscid_from_sysctl_pnpinfo(sysctl_out, g0g1) @@ -1485,13 +1484,12 @@ def maybe_remove_ubuntu_network_config_scripts(paths=None): def _is_platform_viable(seed_dir): + """Check platform environment to report if this datasource may run.""" with events.ReportEventStack( name="check-platform-viability", description="found azure asset tag", parent=azure_ds_reporter ) as evt: - - """Check platform environment to report if this datasource may run.""" asset_tag = util.read_dmi_data('chassis-asset-tag') if asset_tag == AZURE_CHASSIS_ASSET_TAG: return True diff --git a/cloudinit/sources/DataSourceHetzner.py b/cloudinit/sources/DataSourceHetzner.py index 70e4274c..a86035e0 100644 --- a/cloudinit/sources/DataSourceHetzner.py +++ b/cloudinit/sources/DataSourceHetzner.py @@ -69,9 +69,9 @@ class DataSourceHetzner(sources.DataSource): self.userdata_raw = hc_helper.maybe_b64decode(ud) self.metadata_full = md - """hostname is name provided by user at launch. The API enforces - it is a valid hostname, but it is not guaranteed to be resolvable - in dns or fully qualified.""" + # hostname is name provided by user at launch. The API enforces it is + # a valid hostname, but it is not guaranteed to be resolvable in dns or + # fully qualified. self.metadata['instance-id'] = md['instance-id'] self.metadata['local-hostname'] = md['hostname'] self.metadata['network-config'] = md.get('network-config', None) diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 71ef57f0..f314cd4a 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -404,11 +404,10 @@ class TestWALinuxAgentShim(CiTestCase): self.GoalState.call_args_list) def test_certificates_used_to_determine_public_keys(self): + # if register_with_azure_and_fetch_data() isn't passed some info about + # the user's public keys, there's no point in even trying to parse the + # certificates shim = wa_shim() - """if register_with_azure_and_fetch_data() isn't passed some info about - the user's public keys, there's no point in even trying to parse - the certificates - """ mypk = [{'fingerprint': 'fp1', 'path': 'path1'}, {'fingerprint': 'fp3', 'path': 'path3', 'value': ''}] certs = {'fp1': 'expected-key', diff --git a/tests/unittests/test_reporting_hyperv.py b/tests/unittests/test_reporting_hyperv.py index b60a66ab..bacf5da9 100644 --- a/tests/unittests/test_reporting_hyperv.py +++ b/tests/unittests/test_reporting_hyperv.py @@ -139,8 +139,10 @@ class TextKvpReporter(CiTestCase): @mock.patch('cloudinit.subp.subp') def test_get_boot_telemetry(self, m_subp, m_sysd): reporter = HyperVKvpReportingHandler(kvp_file_path=self.tmp_file_path) - datetime_pattern = r"\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]" - r"\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z)" + datetime_pattern = ( + r"\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]" + r"\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z)" + ) # get_boot_telemetry makes two subp calls to systemctl. We provide # a list of values that the subp calls should return -- cgit v1.2.3