From bae9b11da9ed7dd0b16fe5adeaf4774b7cc628cf Mon Sep 17 00:00:00 2001 From: James Falcon Date: Wed, 15 Dec 2021 20:16:38 -0600 Subject: Adopt Black and isort (SC-700) (#1157) Applied Black and isort, fixed any linting issues, updated tox.ini and CI. --- cloudinit/url_helper.py | 273 +++++++++++++++++++++++++++++++----------------- 1 file changed, 179 insertions(+), 94 deletions(-) (limited to 'cloudinit/url_helper.py') diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index caa88435..847e5379 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -17,7 +17,7 @@ from errno import ENOENT from functools import partial from http.client import NOT_FOUND from itertools import count -from urllib.parse import urlparse, urlunparse, quote +from urllib.parse import quote, urlparse, urlunparse import requests from requests import exceptions @@ -32,32 +32,33 @@ LOG = logging.getLogger(__name__) SSL_ENABLED = False CONFIG_ENABLED = False # This was added in 0.7 (but taken out in >=1.0) _REQ_VER = None -REDACTED = 'REDACTED' +REDACTED = "REDACTED" try: from distutils.version import LooseVersion + import pkg_resources - _REQ = pkg_resources.get_distribution('requests') + + _REQ = pkg_resources.get_distribution("requests") _REQ_VER = LooseVersion(_REQ.version) # pylint: disable=no-member - if _REQ_VER >= LooseVersion('0.8.8'): + if _REQ_VER >= LooseVersion("0.8.8"): SSL_ENABLED = True - if LooseVersion('0.7.0') <= _REQ_VER < LooseVersion('1.0.0'): + if LooseVersion("0.7.0") <= _REQ_VER < LooseVersion("1.0.0"): CONFIG_ENABLED = True except ImportError: pass def _cleanurl(url): - parsed_url = list(urlparse(url, scheme='http')) + parsed_url = list(urlparse(url, scheme="http")) if not parsed_url[1] and parsed_url[2]: # Swap these since this seems to be a common # occurrence when given urls like 'www.google.com' parsed_url[1] = parsed_url[2] - parsed_url[2] = '' + parsed_url[2] = "" return urlunparse(parsed_url) def combine_url(base, *add_ons): - def combine_single(url, add_on): url_parsed = list(urlparse(url)) path = url_parsed[2] @@ -87,7 +88,7 @@ def read_file_or_url(url, **kwargs): if url.lower().startswith("file://"): if kwargs.get("data"): LOG.warning("Unable to post data to file resource %s", url) - file_path = url[len("file://"):] + file_path = url[len("file://") :] try: with open(file_path, "rb") as fp: contents = fp.read() @@ -117,7 +118,7 @@ class StringResponse(object): return True def __str__(self): - return self.contents.decode('utf-8') + return self.contents.decode("utf-8") class FileResponse(StringResponse): @@ -173,28 +174,46 @@ class UrlError(IOError): def _get_ssl_args(url, ssl_details): ssl_args = {} scheme = urlparse(url).scheme - if scheme == 'https' and ssl_details: + if scheme == "https" and ssl_details: if not SSL_ENABLED: - LOG.warning("SSL is not supported in requests v%s, " - "cert. verification can not occur!", _REQ_VER) + LOG.warning( + "SSL is not supported in requests v%s, " + "cert. verification can not occur!", + _REQ_VER, + ) else: - if 'ca_certs' in ssl_details and ssl_details['ca_certs']: - ssl_args['verify'] = ssl_details['ca_certs'] + if "ca_certs" in ssl_details and ssl_details["ca_certs"]: + ssl_args["verify"] = ssl_details["ca_certs"] else: - ssl_args['verify'] = True - if 'cert_file' in ssl_details and 'key_file' in ssl_details: - ssl_args['cert'] = [ssl_details['cert_file'], - ssl_details['key_file']] - elif 'cert_file' in ssl_details: - ssl_args['cert'] = str(ssl_details['cert_file']) + ssl_args["verify"] = True + if "cert_file" in ssl_details and "key_file" in ssl_details: + ssl_args["cert"] = [ + ssl_details["cert_file"], + ssl_details["key_file"], + ] + elif "cert_file" in ssl_details: + ssl_args["cert"] = str(ssl_details["cert_file"]) return ssl_args -def readurl(url, data=None, timeout=None, retries=0, sec_between=1, - headers=None, headers_cb=None, headers_redact=None, - ssl_details=None, check_status=True, allow_redirects=True, - exception_cb=None, session=None, infinite=False, log_req_resp=True, - request_method=None): +def readurl( + url, + data=None, + timeout=None, + retries=0, + sec_between=1, + headers=None, + headers_cb=None, + headers_redact=None, + ssl_details=None, + check_status=True, + allow_redirects=True, + exception_cb=None, + session=None, + infinite=False, + log_req_resp=True, + request_method=None, +): """Wrapper around requests.Session to read the url and retry if necessary :param url: Mandatory url to request. @@ -227,15 +246,15 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, """ url = _cleanurl(url) req_args = { - 'url': url, + "url": url, } req_args.update(_get_ssl_args(url, ssl_details)) - req_args['allow_redirects'] = allow_redirects + req_args["allow_redirects"] = allow_redirects if not request_method: - request_method = 'POST' if data else 'GET' - req_args['method'] = request_method + request_method = "POST" if data else "GET" + req_args["method"] = request_method if timeout is not None: - req_args['timeout'] = max(float(timeout), 0) + req_args["timeout"] = max(float(timeout), 0) if headers_redact is None: headers_redact = [] # It doesn't seem like config @@ -243,31 +262,33 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, # need to manually do the retries if it wasn't... if CONFIG_ENABLED: req_config = { - 'store_cookies': False, + "store_cookies": False, } # Don't use the retry support built-in # since it doesn't allow for 'sleep_times' # in between tries.... # if retries: # req_config['max_retries'] = max(int(retries), 0) - req_args['config'] = req_config + req_args["config"] = req_config manual_tries = 1 if retries: manual_tries = max(int(retries) + 1, 1) def_headers = { - 'User-Agent': 'Cloud-Init/%s' % (version.version_string()), + "User-Agent": "Cloud-Init/%s" % (version.version_string()), } if headers: def_headers.update(headers) headers = def_headers if not headers_cb: + def _cb(url): return headers + headers_cb = _cb if data: - req_args['data'] = data + req_args["data"] = data if sec_between is None: sec_between = -1 @@ -276,12 +297,12 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, # doesn't handle sleeping between tries... # Infinitely retry if infinite is True for i in count() if infinite else range(0, manual_tries): - req_args['headers'] = headers_cb(url) + req_args["headers"] = headers_cb(url) filtered_req_args = {} for (k, v) in req_args.items(): - if k == 'data': + if k == "data": continue - if k == 'headers' and headers_redact: + if k == "headers" and headers_redact: matched_headers = [k for k in headers_redact if v.get(k)] if matched_headers: filtered_req_args[k] = copy.deepcopy(v) @@ -292,9 +313,13 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, try: if log_req_resp: - LOG.debug("[%s/%s] open '%s' with %s configuration", i, - "infinite" if infinite else manual_tries, url, - filtered_req_args) + LOG.debug( + "[%s/%s] open '%s' with %s configuration", + i, + "infinite" if infinite else manual_tries, + url, + filtered_req_args, + ) if session is None: session = requests.Session() @@ -304,19 +329,33 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, if check_status: r.raise_for_status() - LOG.debug("Read from %s (%s, %sb) after %s attempts", url, - r.status_code, len(r.content), (i + 1)) + LOG.debug( + "Read from %s (%s, %sb) after %s attempts", + url, + r.status_code, + len(r.content), + (i + 1), + ) # Doesn't seem like we can make it use a different # subclass for responses, so add our own backward-compat # attrs return UrlResponse(r) except exceptions.RequestException as e: - if (isinstance(e, (exceptions.HTTPError)) and - hasattr(e, 'response') and # This appeared in v 0.10.8 - hasattr(e.response, 'status_code')): - excps.append(UrlError(e, code=e.response.status_code, - headers=e.response.headers, - url=url)) + if ( + isinstance(e, (exceptions.HTTPError)) + and hasattr(e, "response") + and hasattr( # This appeared in v 0.10.8 + e.response, "status_code" + ) + ): + excps.append( + UrlError( + e, + code=e.response.status_code, + headers=e.response.headers, + url=url, + ) + ) else: excps.append(UrlError(e, url=url)) if SSL_ENABLED and isinstance(e, exceptions.SSLError): @@ -328,22 +367,33 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, # to continue retrying and False to break and re-raise the # exception break - if (infinite and sec_between > 0) or \ - (i + 1 < manual_tries and sec_between > 0): + if (infinite and sec_between > 0) or ( + i + 1 < manual_tries and sec_between > 0 + ): if log_req_resp: LOG.debug( "Please wait %s seconds while we wait to try again", - sec_between) + sec_between, + ) time.sleep(sec_between) if excps: raise excps[-1] return None # Should throw before this... -def wait_for_url(urls, max_wait=None, timeout=None, status_cb=None, - headers_cb=None, headers_redact=None, sleep_time=1, - exception_cb=None, sleep_time_cb=None, request_method=None): +def wait_for_url( + urls, + max_wait=None, + timeout=None, + status_cb=None, + headers_cb=None, + headers_redact=None, + sleep_time=1, + exception_cb=None, + sleep_time_cb=None, + request_method=None, +): """ urls: a list of urls to try max_wait: roughly the maximum time to wait before giving up @@ -388,9 +438,9 @@ def wait_for_url(urls, max_wait=None, timeout=None, status_cb=None, status_cb = log_status_cb def timeup(max_wait, start_time): - if (max_wait is None): + if max_wait is None: return False - return ((max_wait <= 0) or (time.time() - start_time > max_wait)) + return (max_wait <= 0) or (time.time() - start_time > max_wait) loop_n = 0 response = None @@ -404,8 +454,11 @@ def wait_for_url(urls, max_wait=None, timeout=None, status_cb=None, if loop_n != 0: if timeup(max_wait, start_time): break - if (max_wait is not None and - timeout and (now + timeout > (start_time + max_wait))): + if ( + max_wait is not None + and timeout + and (now + timeout > (start_time + max_wait)) + ): # shorten timeout to not run way over max_time timeout = int((start_time + max_wait) - now) @@ -418,17 +471,29 @@ def wait_for_url(urls, max_wait=None, timeout=None, status_cb=None, headers = {} response = readurl( - url, headers=headers, headers_redact=headers_redact, - timeout=timeout, check_status=False, - request_method=request_method) + url, + headers=headers, + headers_redact=headers_redact, + timeout=timeout, + check_status=False, + request_method=request_method, + ) if not response.contents: reason = "empty response [%s]" % (response.code) - url_exc = UrlError(ValueError(reason), code=response.code, - headers=response.headers, url=url) + url_exc = UrlError( + ValueError(reason), + code=response.code, + headers=response.headers, + url=url, + ) elif not response.ok(): reason = "bad status code [%s]" % (response.code) - url_exc = UrlError(ValueError(reason), code=response.code, - headers=response.headers, url=url) + url_exc = UrlError( + ValueError(reason), + code=response.code, + headers=response.headers, + url=url, + ) else: return url, response.contents except UrlError as e: @@ -440,10 +505,12 @@ def wait_for_url(urls, max_wait=None, timeout=None, status_cb=None, time_taken = int(time.time() - start_time) max_wait_str = "%ss" % max_wait if max_wait else "unlimited" - status_msg = "Calling '%s' failed [%s/%s]: %s" % (url, - time_taken, - max_wait_str, - reason) + status_msg = "Calling '%s' failed [%s/%s]: %s" % ( + url, + time_taken, + max_wait_str, + reason, + ) status_cb(status_msg) if exception_cb: # This can be used to alter the headers that will be sent @@ -455,17 +522,23 @@ def wait_for_url(urls, max_wait=None, timeout=None, status_cb=None, break loop_n = loop_n + 1 - LOG.debug("Please wait %s seconds while we wait to try again", - sleep_time) + LOG.debug( + "Please wait %s seconds while we wait to try again", sleep_time + ) time.sleep(sleep_time) return False, None class OauthUrlHelper(object): - def __init__(self, consumer_key=None, token_key=None, - token_secret=None, consumer_secret=None, - skew_data_file="/run/oauth_skew.json"): + def __init__( + self, + consumer_key=None, + token_key=None, + token_secret=None, + consumer_secret=None, + skew_data_file="/run/oauth_skew.json", + ): self.consumer_key = consumer_key self.consumer_secret = consumer_secret or "" self.token_key = token_key @@ -477,8 +550,10 @@ class OauthUrlHelper(object): if not any(required): self._do_oauth = False elif not all(required): - raise ValueError("all or none of token_key, token_secret, or " - "consumer_key can be set") + raise ValueError( + "all or none of token_key, token_secret, or " + "consumer_key can be set" + ) old = self.read_skew_file() self.skew_data = old or {} @@ -501,16 +576,17 @@ class OauthUrlHelper(object): fp.write(json.dumps(cur)) def exception_cb(self, msg, exception): - if not (isinstance(exception, UrlError) and - (exception.code == 403 or exception.code == 401)): + if not ( + isinstance(exception, UrlError) + and (exception.code == 403 or exception.code == 401) + ): return - if 'date' not in exception.headers: - LOG.warning("Missing header 'date' in %s response", - exception.code) + if "date" not in exception.headers: + LOG.warning("Missing header 'date' in %s response", exception.code) return - date = exception.headers['date'] + date = exception.headers["date"] try: remote_time = time.mktime(parsedate(date)) except Exception as e: @@ -537,15 +613,21 @@ class OauthUrlHelper(object): timestamp = int(time.time()) + self.skew_data[host] return oauth_headers( - url=url, consumer_key=self.consumer_key, - token_key=self.token_key, token_secret=self.token_secret, - consumer_secret=self.consumer_secret, timestamp=timestamp) + url=url, + consumer_key=self.consumer_key, + token_key=self.token_key, + token_secret=self.token_secret, + consumer_secret=self.consumer_secret, + timestamp=timestamp, + ) def _wrapped(self, wrapped_func, args, kwargs): - kwargs['headers_cb'] = partial( - self._headers_cb, kwargs.get('headers_cb')) - kwargs['exception_cb'] = partial( - self._exception_cb, kwargs.get('exception_cb')) + kwargs["headers_cb"] = partial( + self._headers_cb, kwargs.get("headers_cb") + ) + kwargs["exception_cb"] = partial( + self._exception_cb, kwargs.get("exception_cb") + ) return wrapped_func(*args, **kwargs) def wait_for_url(self, *args, **kwargs): @@ -571,12 +653,13 @@ class OauthUrlHelper(object): return headers -def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret, - timestamp=None): +def oauth_headers( + url, consumer_key, token_key, token_secret, consumer_secret, timestamp=None +): try: import oauthlib.oauth1 as oauth1 except ImportError as e: - raise NotImplementedError('oauth support is not available') from e + raise NotImplementedError("oauth support is not available") from e if timestamp: timestamp = str(timestamp) @@ -589,7 +672,8 @@ def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret, resource_owner_key=token_key, resource_owner_secret=token_secret, signature_method=oauth1.SIGNATURE_PLAINTEXT, - timestamp=timestamp) + timestamp=timestamp, + ) _uri, signed_headers, _body = client.sign(url) return signed_headers @@ -607,4 +691,5 @@ def retry_on_url_exc(msg, exc): return True return False + # vi: ts=4 expandtab -- cgit v1.2.3 From e3f3485d875f021915654bf2b64678e151a8d6f6 Mon Sep 17 00:00:00 2001 From: Shreenidhi Shedi <53473811+sshedi@users.noreply.github.com> Date: Thu, 13 Jan 2022 01:22:32 +0530 Subject: Remove distutils usage (#1177) distutils is getting deprecated soon. Let's replace it with suggested alternatives as suggested in: https://www.python.org/dev/peps/pep-0632/ Remove `requests` version check and related code from url_helper.py as the versions specified are old enough to no longer be relevant. Signed-off-by: Shreenidhi Shedi --- cloudinit/sources/DataSourceVMware.py | 5 +-- cloudinit/url_helper.py | 60 +++++-------------------- setup.py | 10 +++-- tests/integration_tests/integration_settings.py | 12 ++--- 4 files changed, 26 insertions(+), 61 deletions(-) (limited to 'cloudinit/url_helper.py') diff --git a/cloudinit/sources/DataSourceVMware.py b/cloudinit/sources/DataSourceVMware.py index ed7f487a..6ef7c9d5 100644 --- a/cloudinit/sources/DataSourceVMware.py +++ b/cloudinit/sources/DataSourceVMware.py @@ -68,14 +68,13 @@ import json import os import socket import time -from distutils.spawn import find_executable import netifaces from cloudinit import dmi from cloudinit import log as logging from cloudinit import sources, util -from cloudinit.subp import ProcessExecutionError, subp +from cloudinit.subp import ProcessExecutionError, subp, which PRODUCT_UUID_FILE_PATH = "/sys/class/dmi/id/product_uuid" @@ -85,7 +84,7 @@ NOVAL = "No value found" DATA_ACCESS_METHOD_ENVVAR = "envvar" DATA_ACCESS_METHOD_GUESTINFO = "guestinfo" -VMWARE_RPCTOOL = find_executable("vmware-rpctool") +VMWARE_RPCTOOL = which("vmware-rpctool") REDACT = "redact" CLEANUP_GUESTINFO = "cleanup-guestinfo" VMX_GUESTINFO = "VMX_GUESTINFO" diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 847e5379..790e2fbf 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -27,25 +27,7 @@ from cloudinit import version LOG = logging.getLogger(__name__) - -# Check if requests has ssl support (added in requests >= 0.8.8) -SSL_ENABLED = False -CONFIG_ENABLED = False # This was added in 0.7 (but taken out in >=1.0) -_REQ_VER = None REDACTED = "REDACTED" -try: - from distutils.version import LooseVersion - - import pkg_resources - - _REQ = pkg_resources.get_distribution("requests") - _REQ_VER = LooseVersion(_REQ.version) # pylint: disable=no-member - if _REQ_VER >= LooseVersion("0.8.8"): - SSL_ENABLED = True - if LooseVersion("0.7.0") <= _REQ_VER < LooseVersion("1.0.0"): - CONFIG_ENABLED = True -except ImportError: - pass def _cleanurl(url): @@ -175,24 +157,17 @@ def _get_ssl_args(url, ssl_details): ssl_args = {} scheme = urlparse(url).scheme if scheme == "https" and ssl_details: - if not SSL_ENABLED: - LOG.warning( - "SSL is not supported in requests v%s, " - "cert. verification can not occur!", - _REQ_VER, - ) + if "ca_certs" in ssl_details and ssl_details["ca_certs"]: + ssl_args["verify"] = ssl_details["ca_certs"] else: - if "ca_certs" in ssl_details and ssl_details["ca_certs"]: - ssl_args["verify"] = ssl_details["ca_certs"] - else: - ssl_args["verify"] = True - if "cert_file" in ssl_details and "key_file" in ssl_details: - ssl_args["cert"] = [ - ssl_details["cert_file"], - ssl_details["key_file"], - ] - elif "cert_file" in ssl_details: - ssl_args["cert"] = str(ssl_details["cert_file"]) + ssl_args["verify"] = True + if "cert_file" in ssl_details and "key_file" in ssl_details: + ssl_args["cert"] = [ + ssl_details["cert_file"], + ssl_details["key_file"], + ] + elif "cert_file" in ssl_details: + ssl_args["cert"] = str(ssl_details["cert_file"]) return ssl_args @@ -257,19 +232,6 @@ def readurl( req_args["timeout"] = max(float(timeout), 0) if headers_redact is None: headers_redact = [] - # It doesn't seem like config - # was added in older library versions (or newer ones either), thus we - # need to manually do the retries if it wasn't... - if CONFIG_ENABLED: - req_config = { - "store_cookies": False, - } - # Don't use the retry support built-in - # since it doesn't allow for 'sleep_times' - # in between tries.... - # if retries: - # req_config['max_retries'] = max(int(retries), 0) - req_args["config"] = req_config manual_tries = 1 if retries: manual_tries = max(int(retries) + 1, 1) @@ -358,7 +320,7 @@ def readurl( ) else: excps.append(UrlError(e, url=url)) - if SSL_ENABLED and isinstance(e, exceptions.SSLError): + if isinstance(e, exceptions.SSLError): # ssl exceptions are not going to get fixed by waiting a # few seconds break diff --git a/setup.py b/setup.py index e6405ad9..c98a4703 100755 --- a/setup.py +++ b/setup.py @@ -15,13 +15,17 @@ import shutil import subprocess import sys import tempfile -from distutils.errors import DistutilsArgError from glob import glob import setuptools from setuptools.command.egg_info import egg_info from setuptools.command.install import install +try: + from setuptools.errors import DistutilsError +except ImportError: + from distutils.errors import DistutilsArgError as DistutilsError + RENDERED_TMPD_PREFIX = "RENDERED_TEMPD" VARIANT = None @@ -245,9 +249,7 @@ class InitsysInstallData(install): bad = [f for f in self.init_system if f not in INITSYS_TYPES] if len(bad) != 0: - raise DistutilsArgError( - "Invalid --init-system: %s" % ",".join(bad) - ) + raise DistutilsError("Invalid --init-system: %s" % ",".join(bad)) for system in self.init_system: # add data files for anything that starts with '.' diff --git a/tests/integration_tests/integration_settings.py b/tests/integration_tests/integration_settings.py index 641ce297..02037a88 100644 --- a/tests/integration_tests/integration_settings.py +++ b/tests/integration_tests/integration_settings.py @@ -1,6 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. import os -from distutils.util import strtobool + +from cloudinit.util import is_false, is_true ################################################################## # LAUNCH SETTINGS @@ -126,8 +127,9 @@ for setting in current_settings: "CLOUD_INIT_{}".format(setting), globals()[setting] ) if isinstance(env_setting, str): - try: - env_setting = bool(strtobool(env_setting.strip())) - except ValueError: - pass + env_setting = env_setting.strip() + if is_true(env_setting): + env_setting = True + elif is_false(env_setting): + env_setting = False globals()[setting] = env_setting -- cgit v1.2.3 From 0b41b359a70bbbf3a648862a9b849d60b9ff6c3b Mon Sep 17 00:00:00 2001 From: Chris Patterson Date: Fri, 11 Feb 2022 23:40:45 -0500 Subject: sources/azure: address mypy/pyright typing complaints (#1245) Raise runtime errors for unhandled cases which would cause other exceptions. Ignore types for a few cases where a non-trivial refactor would be required to prevent the warning. Signed-off-by: Chris Patterson --- cloudinit/distros/__init__.py | 10 +++--- cloudinit/distros/networking.py | 3 +- cloudinit/sources/DataSourceAzure.py | 62 +++++++++++++++++++++-------------- cloudinit/sources/helpers/azure.py | 18 +++++----- cloudinit/url_helper.py | 7 ++-- tests/unittests/sources/test_azure.py | 4 +-- 6 files changed, 59 insertions(+), 45 deletions(-) (limited to 'cloudinit/url_helper.py') diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index a261c16e..76acd6a3 100755 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -16,7 +16,7 @@ import stat import string import urllib.parse from io import StringIO -from typing import Any, Mapping # noqa: F401 +from typing import Any, Mapping, Type from cloudinit import importer from cloudinit import log as logging @@ -26,7 +26,7 @@ from cloudinit.features import ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES from cloudinit.net import activators, eni, network_state, renderers from cloudinit.net.network_state import parse_net_config_data -from .networking import LinuxNetworking +from .networking import LinuxNetworking, Networking # Used when a cloud-config module can be run on all cloud-init distibutions. # The value 'all' is surfaced in module documentation for distro support. @@ -76,9 +76,9 @@ class Distro(persistence.CloudInitPickleMixin, metaclass=abc.ABCMeta): hostname_conf_fn = "/etc/hostname" tz_zone_dir = "/usr/share/zoneinfo" init_cmd = ["service"] # systemctl, service etc - renderer_configs = {} # type: Mapping[str, Mapping[str, Any]] + renderer_configs: Mapping[str, Mapping[str, Any]] = {} _preferred_ntp_clients = None - networking_cls = LinuxNetworking + networking_cls: Type[Networking] = LinuxNetworking # This is used by self.shutdown_command(), and can be overridden in # subclasses shutdown_options_map = {"halt": "-H", "poweroff": "-P", "reboot": "-r"} @@ -91,7 +91,7 @@ class Distro(persistence.CloudInitPickleMixin, metaclass=abc.ABCMeta): self._paths = paths self._cfg = cfg self.name = name - self.networking = self.networking_cls() + self.networking: Networking = self.networking_cls() def _unpickle(self, ci_pkl_version: int) -> None: """Perform deserialization fixes for Distro.""" diff --git a/cloudinit/distros/networking.py b/cloudinit/distros/networking.py index e18a48ca..b24b6233 100644 --- a/cloudinit/distros/networking.py +++ b/cloudinit/distros/networking.py @@ -1,6 +1,7 @@ import abc import logging import os +from typing import List, Optional from cloudinit import net, subp, util @@ -22,7 +23,7 @@ class Networking(metaclass=abc.ABCMeta): """ def __init__(self): - self.blacklist_drivers = None + self.blacklist_drivers: Optional[List[str]] = None def _get_current_rename_info(self) -> dict: return net._get_current_rename_info() diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index f4be4cda..2566bd8e 100755 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -13,7 +13,7 @@ import re import xml.etree.ElementTree as ET from enum import Enum from time import sleep, time -from typing import List, Optional +from typing import Any, Dict, List, Optional from xml.dom import minidom import requests @@ -83,7 +83,7 @@ class PPSType(Enum): UNKNOWN = "Unknown" -PLATFORM_ENTROPY_SOURCE = "/sys/firmware/acpi/tables/OEM0" +PLATFORM_ENTROPY_SOURCE: Optional[str] = "/sys/firmware/acpi/tables/OEM0" # List of static scripts and network config artifacts created by # stock ubuntu suported images. @@ -155,7 +155,7 @@ def find_busdev_from_disk(camcontrol_out, disk_drv): return None -def find_dev_from_busdev(camcontrol_out, busdev): +def find_dev_from_busdev(camcontrol_out: str, busdev: str) -> Optional[str]: # find the daX from 'camcontrol devlist' output # if busdev matches the specified value, i.e. 'scbus2' """ @@ -172,9 +172,9 @@ def find_dev_from_busdev(camcontrol_out, busdev): return None -def execute_or_debug(cmd, fail_ret=None): +def execute_or_debug(cmd, fail_ret=None) -> str: try: - return subp.subp(cmd)[0] + return subp.subp(cmd)[0] # type: ignore except subp.ProcessExecutionError: LOG.debug("Failed to execute: %s", " ".join(cmd)) return fail_ret @@ -192,7 +192,7 @@ def get_camcontrol_dev(): return execute_or_debug(["camcontrol", "devlist"]) -def get_resource_disk_on_freebsd(port_id): +def get_resource_disk_on_freebsd(port_id) -> Optional[str]: g0 = "00000000" if port_id > 1: g0 = "00000001" @@ -316,7 +316,9 @@ class DataSourceAzure(sources.DataSource): def _get_subplatform(self): """Return the subplatform metadata source details.""" - if self.seed.startswith("/dev"): + if self.seed is None: + subplatform_type = "unknown" + elif self.seed.startswith("/dev"): subplatform_type = "config-disk" elif self.seed.lower() == "imds": subplatform_type = "imds" @@ -541,9 +543,9 @@ class DataSourceAzure(sources.DataSource): if metadata_source == "IMDS" and not crawled_data["files"]: try: contents = build_minimal_ovf( - username=imds_username, - hostname=imds_hostname, - disableSshPwd=imds_disable_password, + username=imds_username, # type: ignore + hostname=imds_hostname, # type: ignore + disableSshPwd=imds_disable_password, # type: ignore ) crawled_data["files"] = {"ovf-env.xml": contents} except Exception as e: @@ -803,7 +805,11 @@ class DataSourceAzure(sources.DataSource): # We don't want Azure to react to an UPPER/lower difference as a new # instance id as it rewrites SSH host keys. # LP: #1835584 - iid = dmi.read_dmi_data("system-uuid").lower() + system_uuid = dmi.read_dmi_data("system-uuid") + if system_uuid is None: + raise RuntimeError("failed to read system-uuid") + + iid = system_uuid.lower() if os.path.exists(prev_iid_path): previous = util.load_file(prev_iid_path).strip() if previous.lower() == iid: @@ -866,7 +872,7 @@ class DataSourceAzure(sources.DataSource): path, "{pid}: {time}\n".format(pid=os.getpid(), time=time()) ) except AssertionError as error: - report_diagnostic_event(error, logger_func=LOG.error) + report_diagnostic_event(str(error), logger_func=LOG.error) raise @azure_ds_telemetry_reporter @@ -888,10 +894,10 @@ class DataSourceAzure(sources.DataSource): attempts = 0 LOG.info("Unbinding and binding the interface %s", ifname) while True: - - devicename = net.read_sys_net(ifname, "device/device_id").strip( - "{}" - ) + device_id = net.read_sys_net(ifname, "device/device_id") + if device_id is False or not isinstance(device_id, str): + raise RuntimeError("Unable to read device ID: %s" % device_id) + devicename = device_id.strip("{}") util.write_file( "/sys/bus/vmbus/drivers/hv_netvsc/unbind", devicename ) @@ -1118,7 +1124,7 @@ class DataSourceAzure(sources.DataSource): break except AssertionError as error: - report_diagnostic_event(error, logger_func=LOG.error) + report_diagnostic_event(str(error), logger_func=LOG.error) @azure_ds_telemetry_reporter def _wait_for_all_nics_ready(self): @@ -1168,7 +1174,7 @@ class DataSourceAzure(sources.DataSource): logger_func=LOG.info, ) except netlink.NetlinkCreateSocketError as e: - report_diagnostic_event(e, logger_func=LOG.warning) + report_diagnostic_event(str(e), logger_func=LOG.warning) raise finally: if nl_sock: @@ -1234,6 +1240,11 @@ class DataSourceAzure(sources.DataSource): self._setup_ephemeral_networking(timeout_minutes=20) try: + if ( + self._ephemeral_dhcp_ctx is None + or self._ephemeral_dhcp_ctx.iface is None + ): + raise RuntimeError("Missing ephemeral context") iface = self._ephemeral_dhcp_ctx.iface nl_sock = netlink.create_bound_netlink_socket() @@ -1873,7 +1884,7 @@ def read_azure_ovf(contents): raise BrokenAzureDataSource("no child nodes of configuration set") md_props = "seedfrom" - md = {"azure_data": {}} + md: Dict[str, Any] = {"azure_data": {}} cfg = {} ud = "" password = None @@ -2084,9 +2095,7 @@ def _get_random_seed(source=PLATFORM_ENTROPY_SOURCE): # string. Same number of bits of entropy, just with 25% more zeroes. # There's no need to undo this base64-encoding when the random seed is # actually used in cc_seed_random.py. - seed = base64.b64encode(seed).decode() - - return seed + return base64.b64encode(seed).decode() # type: ignore @azure_ds_telemetry_reporter @@ -2151,7 +2160,7 @@ def _generate_network_config_from_imds_metadata(imds_metadata) -> dict: @param: imds_metadata: Dict of content read from IMDS network service. @return: Dictionary containing network version 2 standard configuration. """ - netconfig = {"version": 2, "ethernets": {}} + netconfig: Dict[str, Any] = {"version": 2, "ethernets": {}} network_metadata = imds_metadata["network"] for idx, intf in enumerate(network_metadata["interface"]): has_ip_address = False @@ -2160,7 +2169,7 @@ def _generate_network_config_from_imds_metadata(imds_metadata) -> dict: # addresses. nicname = "eth{idx}".format(idx=idx) dhcp_override = {"route-metric": (idx + 1) * 100} - dev_config = { + dev_config: Dict[str, Any] = { "dhcp4": True, "dhcp4-overrides": dhcp_override, "dhcp6": False, @@ -2214,9 +2223,12 @@ def _generate_network_config_from_fallback_config() -> dict: @return: Dictionary containing network version 2 standard configuration. """ - return net.generate_fallback_config( + cfg = net.generate_fallback_config( blacklist_drivers=BLACKLIST_DRIVERS, config_driver=True ) + if cfg is None: + return {} + return cfg @azure_ds_telemetry_reporter diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index ec6ab80c..d07dc3c0 100755 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -334,12 +334,10 @@ def _get_dhcp_endpoint_option_name(): @azure_ds_telemetry_reporter -def http_with_retries(url, **kwargs) -> str: +def http_with_retries(url, **kwargs) -> url_helper.UrlResponse: """Wrapper around url_helper.readurl() with custom telemetry logging that url_helper.readurl() does not provide. """ - exc = None - max_readurl_attempts = 240 default_readurl_timeout = 5 sleep_duration_between_retries = 5 @@ -374,16 +372,18 @@ def http_with_retries(url, **kwargs) -> str: return ret except Exception as e: - exc = e if attempt % periodic_logging_attempts == 0: report_diagnostic_event( "Failed HTTP request with Azure endpoint %s during " "attempt %d with exception: %s" % (url, attempt, e), logger_func=LOG.debug, ) - time.sleep(sleep_duration_between_retries) + if attempt == max_readurl_attempts: + raise + + time.sleep(sleep_duration_between_retries) - raise exc + raise RuntimeError("Failed to return in http_with_retries") def build_minimal_ovf( @@ -433,14 +433,16 @@ class AzureEndpointHttpClient: "x-ms-guest-agent-public-x509-cert": certificate, } - def get(self, url, secure=False): + def get(self, url, secure=False) -> url_helper.UrlResponse: headers = self.headers if secure: headers = self.headers.copy() headers.update(self.extra_secure_headers) return http_with_retries(url, headers=headers) - def post(self, url, data=None, extra_headers=None): + def post( + self, url, data=None, extra_headers=None + ) -> url_helper.UrlResponse: headers = self.headers if extra_headers is not None: headers = self.headers.copy() diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 790e2fbf..5b2f2ef9 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -188,7 +188,7 @@ def readurl( infinite=False, log_req_resp=True, request_method=None, -): +) -> UrlResponse: """Wrapper around requests.Session to read the url and retry if necessary :param url: Mandatory url to request. @@ -339,9 +339,8 @@ def readurl( sec_between, ) time.sleep(sec_between) - if excps: - raise excps[-1] - return None # Should throw before this... + + raise excps[-1] def wait_for_url( diff --git a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test_azure.py index a47ed611..bd525b13 100644 --- a/tests/unittests/sources/test_azure.py +++ b/tests/unittests/sources/test_azure.py @@ -2993,7 +2993,7 @@ class TestPreprovisioningHotAttachNics(CiTestCase): @mock.patch(MOCKPATH + "net.is_up", autospec=True) @mock.patch(MOCKPATH + "util.write_file") - @mock.patch("cloudinit.net.read_sys_net") + @mock.patch("cloudinit.net.read_sys_net", return_value="device-id") @mock.patch("cloudinit.distros.networking.LinuxNetworking.try_set_link_up") def test_wait_for_link_up_checks_link_after_sleep( self, m_try_set_link_up, m_read_sys_net, m_writefile, m_is_up @@ -3023,7 +3023,7 @@ class TestPreprovisioningHotAttachNics(CiTestCase): self.assertEqual(2, m_is_up.call_count) @mock.patch(MOCKPATH + "util.write_file") - @mock.patch("cloudinit.net.read_sys_net") + @mock.patch("cloudinit.net.read_sys_net", return_value="device-id") @mock.patch("cloudinit.distros.networking.LinuxNetworking.try_set_link_up") def test_wait_for_link_up_writes_to_device_file( self, m_is_link_up, m_read_sys_net, m_writefile -- cgit v1.2.3