mypy: introduce type checking (#1254)

All currently failing modules are excluded from reporting
errors using follow-imports=silent and an exclusion list.

Future work can whittle down this failing list. This change will
start enforcing new modules and those currently passing.

Includes some minor alphabetical reordering in tox.ini.

Signed-off-by: Chris Patterson <cpatterson@microsoft.com>

3 files changed, 149 insertions, 6 deletions

diff --git a/.travis.yml b/.travis.yml
index 2e79f2b7..f655fa50 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -133,6 +133,8 @@ matrix:
         - python: 3.6
           env: TOXENV=flake8
         - python: 3.6
+          env: TOXENV=mypy
+        - python: 3.6
           env: TOXENV=pylint
         - python: 3.6
           env: TOXENV=black
diff --git a/pyproject.toml b/pyproject.toml
index 26734e62..a131037f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -6,3 +6,118 @@ profile = "black"
 line_length = 79
 # We patch logging in main.py before certain imports
 skip = ["cloudinit/cmd/main.py", ".tox", "packages", "tools"]
+
+[tool.mypy]
+follow_imports = "silent"
+exclude=[
+  '^cloudinit/apport\.py$',
+  '^cloudinit/cmd/query\.py$',
+  '^cloudinit/config/cc_apk_configure\.py$',
+  '^cloudinit/config/cc_apt_configure\.py$',
+  '^cloudinit/config/cc_apt_pipelining\.py$',
+  '^cloudinit/config/cc_bootcmd\.py$',
+  '^cloudinit/config/cc_byobu\.py$',
+  '^cloudinit/config/cc_ca_certs\.py$',
+  '^cloudinit/config/cc_chef\.py$',
+  '^cloudinit/config/cc_debug\.py$',
+  '^cloudinit/config/cc_disable_ec2_metadata\.py$',
+  '^cloudinit/config/cc_disk_setup\.py$',
+  '^cloudinit/config/cc_install_hotplug\.py$',
+  '^cloudinit/config/cc_keyboard\.py$',
+  '^cloudinit/config/cc_landscape\.py$',
+  '^cloudinit/config/cc_locale\.py$',
+  '^cloudinit/config/cc_mcollective\.py$',
+  '^cloudinit/config/cc_ntp\.py$',
+  '^cloudinit/config/cc_resizefs\.py$',
+  '^cloudinit/config/cc_rsyslog\.py$',
+  '^cloudinit/config/cc_runcmd\.py$',
+  '^cloudinit/config/cc_snap\.py$',
+  '^cloudinit/config/cc_ubuntu_advantage\.py$',
+  '^cloudinit/config/cc_ubuntu_drivers\.py$',
+  '^cloudinit/config/cc_write_files\.py$',
+  '^cloudinit/config/cc_write_files_deferred\.py$',
+  '^cloudinit/config/cc_zypper_add_repo\.py$',
+  '^cloudinit/config/schema\.py$',
+  '^cloudinit/distros/bsd\.py$',
+  '^cloudinit/distros/freebsd\.py$',
+  '^cloudinit/distros/parsers/networkmanager_conf\.py$',
+  '^cloudinit/distros/parsers/resolv_conf\.py$',
+  '^cloudinit/distros/parsers/sys_conf\.py$',
+  '^cloudinit/dmi\.py$',
+  '^cloudinit/features\.py$',
+  '^cloudinit/handlers/cloud_config\.py$',
+  '^cloudinit/handlers/jinja_template\.py$',
+  '^cloudinit/importer\.py$',
+  '^cloudinit/net/__init__\.py$',
+  '^cloudinit/net/dhcp\.py$',
+  '^cloudinit/net/netplan\.py$',
+  '^cloudinit/net/sysconfig\.py$',
+  '^cloudinit/serial\.py$',
+  '^cloudinit/sources/DataSourceAzure\.py$',
+  '^cloudinit/sources/DataSourceAliYun\.py$',
+  '^cloudinit/sources/DataSourceLXD\.py$',
+  '^cloudinit/sources/DataSourceOracle\.py$',
+  '^cloudinit/sources/DataSourceScaleway\.py$',
+  '^cloudinit/sources/DataSourceSmartOS\.py$',
+  '^cloudinit/sources/DataSourceVMware\.py$',
+  '^cloudinit/sources/__init__\.py$',
+  '^cloudinit/sources/helpers/azure\.py$',
+  '^cloudinit/sources/helpers/vmware/imc/config_file\.py$',
+  '^cloudinit/stages\.py$',
+  '^cloudinit/templater\.py$',
+  '^cloudinit/url_helper\.py$',
+  '^conftest\.py$',
+  '^doc/rtd/conf\.py$',
+  '^setup\.py$',
+  '^tests/integration_tests/clouds\.py$',
+  '^tests/integration_tests/conftest\.py$',
+  '^tests/integration_tests/instances\.py$',
+  '^tests/integration_tests/integration_settings\.py$',
+  '^tests/integration_tests/modules/test_disk_setup\.py$',
+  '^tests/integration_tests/modules/test_growpart\.py$',
+  '^tests/integration_tests/modules/test_ssh_keysfile\.py$',
+  '^tests/unittests/__init__\.py$',
+  '^tests/unittests/cmd/devel/test_render\.py$',
+  '^tests/unittests/cmd/test_clean\.py$',
+  '^tests/unittests/cmd/test_cloud_id\.py$',
+  '^tests/unittests/cmd/test_main\.py$',
+  '^tests/unittests/cmd/test_query\.py$',
+  '^tests/unittests/cmd/test_status\.py$',
+  '^tests/unittests/config/test_cc_chef\.py$',
+  '^tests/unittests/config/test_cc_landscape\.py$',
+  '^tests/unittests/config/test_cc_locale\.py$',
+  '^tests/unittests/config/test_cc_mcollective\.py$',
+  '^tests/unittests/config/test_cc_rh_subscription\.py$',
+  '^tests/unittests/config/test_cc_set_hostname\.py$',
+  '^tests/unittests/config/test_cc_snap\.py$',
+  '^tests/unittests/config/test_cc_timezone\.py$',
+  '^tests/unittests/config/test_cc_ubuntu_advantage\.py$',
+  '^tests/unittests/config/test_cc_ubuntu_drivers\.py$',
+  '^tests/unittests/config/test_schema\.py$',
+  '^tests/unittests/helpers\.py$',
+  '^tests/unittests/net/test_dhcp\.py$',
+  '^tests/unittests/net/test_init\.py$',
+  '^tests/unittests/sources/test_aliyun\.py$',
+  '^tests/unittests/sources/test_azure\.py$',
+  '^tests/unittests/sources/test_ec2\.py$',
+  '^tests/unittests/sources/test_exoscale\.py$',
+  '^tests/unittests/sources/test_gce\.py$',
+  '^tests/unittests/sources/test_lxd\.py$',
+  '^tests/unittests/sources/test_opennebula\.py$',
+  '^tests/unittests/sources/test_openstack\.py$',
+  '^tests/unittests/sources/test_rbx\.py$',
+  '^tests/unittests/sources/test_scaleway\.py$',
+  '^tests/unittests/sources/test_smartos\.py$',
+  '^tests/unittests/test_data\.py$',
+  '^tests/unittests/test_ds_identify\.py$',
+  '^tests/unittests/test_ec2_util\.py$',
+  '^tests/unittests/test_net\.py$',
+  '^tests/unittests/test_net_activators\.py$',
+  '^tests/unittests/test_persistence\.py$',
+  '^tests/unittests/test_sshutil\.py$',
+  '^tests/unittests/test_subp\.py$',
+  '^tests/unittests/test_templating\.py$',
+  '^tests/unittests/test_url_helper\.py$',
+  '^tests/unittests/test_util\.py$',
+  '^tools/mock-meta\.py$',
+]
diff --git a/tox.ini b/tox.ini
index aab6d4d4..c494cb94 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = py3, lowest-supported-dev, flake8, pylint, black, isort
+envlist = py3, lowest-supported-dev, black, flake8, isort, mypy, pylint
 recreate = True
 
 [testenv]
@@ -10,10 +10,15 @@ passenv=
     PYTEST_ADDOPTS
 
 [format_deps]
-flake8==3.9.2
-pylint==2.11.1
 black==21.12b0
+flake8==3.9.2
 isort==5.10.1
+mypy==0.931
+pylint==2.11.1
+pytest==7.0.0
+types-PyYAML==6.0.4
+types-requests==2.27.8
+types-setuptools==57.4.9
 
 [testenv:flake8]
 deps =
@@ -37,18 +42,30 @@ deps =
     isort=={[format_deps]isort}
 commands = {envpython} -m isort . --check-only
 
+[testenv:mypy]
+deps =
+    mypy=={[format_deps]mypy}
+    types-pyyaml=={[format_deps]types-PyYAML}
+    types-requests=={[format_deps]types-requests}
+    types-setuptools=={[format_deps]types-setuptools}
+    pytest=={[format_deps]pytest}
+commands = {envpython} -m mypy .
+
 [testenv:check_format]
 deps =
-    flake8=={[format_deps]flake8}
-    pylint=={[format_deps]pylint}
     black=={[format_deps]black}
+    flake8=={[format_deps]flake8}
     isort=={[format_deps]isort}
+    mypy=={[format_deps]mypy}
+    pylint=={[format_deps]pylint}
+    pytest=={[format_deps]pytest}
     -r{toxinidir}/test-requirements.txt
     -r{toxinidir}/integration-requirements.txt
 commands =
     {[testenv:black]commands}
-    {[testenv:isort]commands}
     {[testenv:flake8]commands}
+    {[testenv:isort]commands}
+    {[testenv:mypy]commands}
     {[testenv:pylint]commands}
 
 [testenv:do_format]
@@ -122,6 +139,15 @@ commands =
 commands = {envpython} -m flake8 {posargs:cloudinit/ tests/ tools/ setup.py}
 deps = flake8
 
+[testenv:tip-mypy]
+commands = {envpython} -m mypy --install-types --non-interactive .
+deps =
+    mypy
+    pytest
+    types-PyYAML
+    types-requests
+    types-setuptools
+
 [testenv:tip-pylint]
 commands = {envpython} -m pylint {posargs:cloudinit tests tools}
 deps =