Merge pull request #18 from zdc/T2117-equuleus-20.3

T2117: Cloud-init updated to 20.3

18 files changed, 167 insertions, 121 deletions

diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers
new file mode 100644
index 00000000..c67db436
--- /dev/null
+++ b/tools/.github-cla-signers
@@ -0,0 +1,22 @@
+AlexBaranowski
+beezly
+bipinbachhao
+BirknerAlex
+candlerb
+dermotbradley
+dhensby
+eandersson
+izzyleung
+johnsonshi
+landon912
+lucasmoura
+marlluslustosa
+matthewruffell
+nishigori
+omBratteng
+onitake
+smoser
+sshedi
+TheRealFalcon
+tomponline
+tsanghan
diff --git a/tools/.lp-to-git-user b/tools/.lp-to-git-user
index 6b20d360..89422dbb 100644
--- a/tools/.lp-to-git-user
+++ b/tools/.lp-to-git-user
@@ -6,6 +6,7 @@
  "askon": "ask0n",
  "bitfehler": "bitfehler",
  "chad.smith": "blackboxsw",
+ "chcheng": "chengcheng-chcheng",
  "d-info-e": "do3meli",
  "daniel-thewatkins": "OddBloke",
  "eric-lafontaine1": "elafontaine",
@@ -13,10 +14,13 @@
  "goneri": "goneri",
  "harald-jensas": "hjensas",
  "i.galic": "igalic",
+ "kgarloff": "garloff",
+ "killermoehre": "killermoehre",
  "larsks": "larsks",
  "legovini": "paride",
  "louis": "karibou",
  "madhuri-rai07": "madhuri-rai07",
+ "momousta": "Moustafa-Moustafa",
  "otubo": "otubo",
  "pengpengs": "PengpengSun",
  "powersj": "powersj",
@@ -24,5 +28,6 @@
  "rjschwei": "rjschwei",
  "tribaal": "chrisglass",
  "trstringer": "trstringer",
+ "vtqanh": "anhvoms",
  "xiaofengw": "xiaofengw-vmware"
 }
\ No newline at end of file
diff --git a/tools/build-on-freebsd b/tools/build-on-freebsd
index 876368a9..3211c355 100755
--- a/tools/build-on-freebsd
+++ b/tools/build-on-freebsd
@@ -28,8 +28,7 @@ pkgs="
     $py_prefix-jsonschema
     $py_prefix-oauthlib
     $py_prefix-requests
-    $py_prefix-serial
-    $py_prefix-six
+    $py_prefix-pyserial
     $py_prefix-yaml
     sudo
 "
diff --git a/tools/build-on-netbsd b/tools/build-on-netbsd
new file mode 100755
index 00000000..d2a7067d
--- /dev/null
+++ b/tools/build-on-netbsd
@@ -0,0 +1,36 @@
+#!/bin/sh
+
+fail() { echo "FAILED:" "$@" 1>&2; exit 1; }
+
+# Check dependencies:
+depschecked=/tmp/c-i.dependencieschecked
+pkgs="
+   bash
+   dmidecode
+   py37-configobj
+   py37-jinja2
+   py37-oauthlib
+   py37-requests
+   py37-setuptools
+   py37-yaml
+   sudo
+"
+[ -f "$depschecked" ] || pkg_add ${pkgs} || fail "install packages"
+
+touch $depschecked
+
+# Build the code and install in /usr/pkg/:
+python3.7 setup.py build
+python3.7 setup.py install -O1 --distro netbsd --skip-build --init-system sysvinit_netbsd
+mv -v /usr/local/etc/rc.d/cloud* /etc/rc.d
+
+# Enable cloud-init in /etc/rc.conf:
+sed -i.bak -e "/^cloud.*=.*/d" /etc/rc.conf
+echo '
+# You can safely remove the following lines starting with "cloud"
+cloudinitlocal="YES"
+cloudinit="YES"
+cloudconfig="YES"
+cloudfinal="YES"' >> /etc/rc.conf
+
+echo "Installation completed."
diff --git a/tools/build-on-openbsd b/tools/build-on-openbsd
new file mode 100755
index 00000000..ca028606
--- /dev/null
+++ b/tools/build-on-openbsd
@@ -0,0 +1,27 @@
+#!/bin/sh
+
+fail() { echo "FAILED:" "$@" 1>&2; exit 1; }
+
+# Check dependencies:
+depschecked=/tmp/c-i.dependencieschecked
+pkgs="
+   bash
+   dmidecode
+   py3-configobj
+   py3-jinja2
+   py3-jsonschema
+   py3-oauthlib
+   py3-requests
+   py3-setuptools
+   py3-six
+   py3-yaml
+   sudo--
+"
+[ -f "$depschecked" ] || pkg_add ${pkgs} || fail "install packages"
+
+touch $depschecked
+
+python3 setup.py build
+python3 setup.py install -O1 --distro openbsd --skip-build
+
+echo "Installation completed."
diff --git a/tools/ccfg-merge-debug b/tools/ccfg-merge-debug
index 1f08e0cb..59c573af 100755
--- a/tools/ccfg-merge-debug
+++ b/tools/ccfg-merge-debug
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 
 from cloudinit import handlers
 from cloudinit.handlers import cloud_config as cc_part
diff --git a/tools/ds-identify b/tools/ds-identify
index c93d4a77..4e5700fc 100755
--- a/tools/ds-identify
+++ b/tools/ds-identify
@@ -266,8 +266,9 @@ read_fs_info() {
                     isodevs="${isodevs},${dev}=$label"
                 ftype=""; dev=""; label="";
                 dev=${line#DEVNAME=};;
-            LABEL=*) label="${line#LABEL=}";
-                     labels="${labels}${line#LABEL=}${delim}";;
+            LABEL=*|LABEL_FATBOOT=*)
+                label="${line#*=}";
+                labels="${labels}${label}${delim}";;
             TYPE=*) ftype=${line#TYPE=};;
             UUID=*) uuids="${uuids}${line#UUID=}$delim";;
         esac
@@ -1062,6 +1063,10 @@ dscheck_OpenStack() {
         return ${DS_FOUND}
     fi
 
+    if dmi_chassis_asset_tag_matches "SAP CCloud VM"; then
+        return ${DS_FOUND}
+    fi
+
     # LP: #1669875 : allow identification of OpenStack by asset tag
     if dmi_chassis_asset_tag_matches "$nova"; then
         return ${DS_FOUND}
diff --git a/tools/make-mime.py b/tools/make-mime.py
deleted file mode 100755
index d321479b..00000000
--- a/tools/make-mime.py
+++ /dev/null
@@ -1,62 +0,0 @@
-#!/usr/bin/python
-
-import argparse
-import sys
-
-from email.mime.multipart import MIMEMultipart
-from email.mime.text import MIMEText
-
-KNOWN_CONTENT_TYPES = [
-    'text/x-include-once-url',
-    'text/x-include-url',
-    'text/cloud-config-archive',
-    'text/upstart-job',
-    'text/cloud-config',
-    'text/part-handler',
-    'text/x-shellscript',
-    'text/cloud-boothook',
-]
-
-
-def file_content_type(text):
-    try:
-        filename, content_type = text.split(":", 1)
-        return (open(filename, 'r'), filename, content_type.strip())
-    except ValueError:
-        raise argparse.ArgumentError(text, "Invalid value for %r" % (text))
-
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("-a", "--attach",
-                        dest="files",
-                        type=file_content_type,
-                        action='append',
-                        default=[],
-                        required=True,
-                        metavar="<file>:<content-type>",
-                        help="attach the given file in the specified "
-                             "content type")
-    args = parser.parse_args()
-    sub_messages = []
-    for i, (fh, filename, format_type) in enumerate(args.files):
-        contents = fh.read()
-        sub_message = MIMEText(contents, format_type, sys.getdefaultencoding())
-        sub_message.add_header('Content-Disposition',
-                               'attachment; filename="%s"' % (filename))
-        content_type = sub_message.get_content_type().lower()
-        if content_type not in KNOWN_CONTENT_TYPES:
-            sys.stderr.write(("WARNING: content type %r for attachment %s "
-                             "may be incorrect!\n") % (content_type, i + 1))
-        sub_messages.append(sub_message)
-    combined_message = MIMEMultipart()
-    for msg in sub_messages:
-        combined_message.attach(msg)
-    print(combined_message)
-    return 0
-
-
-if __name__ == '__main__':
-    sys.exit(main())
-
-# vi: ts=4 expandtab
diff --git a/tools/mock-meta.py b/tools/mock-meta.py
index 724f7fc4..9dd067b9 100755
--- a/tools/mock-meta.py
+++ b/tools/mock-meta.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 
 # Provides a somewhat random, somewhat compat, somewhat useful mock version of
 # http://docs.amazonwebservices.com
@@ -258,12 +258,14 @@ class MetaDataHandler(object):
                 try:
                     key_id = int(mybe_key)
                     key_name = key_ids[key_id]
-                except ValueError:
-                    raise WebException(hclient.BAD_REQUEST,
-                                       "%s: not an integer" % mybe_key)
-                except IndexError:
-                    raise WebException(hclient.NOT_FOUND,
-                                       "Unknown key id %r" % mybe_key)
+                except ValueError as e:
+                    raise WebException(
+                        hclient.BAD_REQUEST, "%s: not an integer" % mybe_key
+                    ) from e
+                except IndexError as e:
+                    raise WebException(
+                        hclient.NOT_FOUND, "Unknown key id %r" % mybe_key
+                    ) from e
                 # Extract the possible sub-params
                 result = traverse(nparams[1:], {
                     "openssh-key": "\n".join(avail_keys[key_name]),
diff --git a/tools/pipremove b/tools/pipremove
index f8f4ff11..e1213edd 100755
--- a/tools/pipremove
+++ b/tools/pipremove
@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 import subprocess
 import sys
 
diff --git a/tools/read-dependencies b/tools/read-dependencies
index b4656e69..6ad5f701 100755
--- a/tools/read-dependencies
+++ b/tools/read-dependencies
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 """List pip dependencies or system package dependencies for cloud-init."""
 
 # You might be tempted to rewrite this as a shell script, but you
@@ -9,7 +9,7 @@ try:
     from argparse import ArgumentParser
 except ImportError:
     raise RuntimeError(
-        'Could not import python-argparse. Please install python-argparse '
+        'Could not import argparse. Please install python3-argparse '
         'package to continue')
 
 import json
@@ -34,6 +34,13 @@ MAYBE_RELIABLE_YUM_INSTALL = [
     'sh', '-c',
     """
     error() { echo "$@" 1>&2; }
+    configure_repos_for_proxy_use() {
+        grep -q "^proxy=" /etc/yum.conf || return 0
+        error ":: http proxy in use => forcing the use of fixed URLs in /etc/yum.repos.d/*.repo"
+        sed -i --regexp-extended '/^#baseurl=/s/#// ; /^(mirrorlist|metalink)=/s/^/#/' /etc/yum.repos.d/*.repo
+        sed -i 's/download\.fedoraproject\.org/dl.fedoraproject.org/g' /etc/yum.repos.d/*.repo
+    }
+    configure_repos_for_proxy_use
     n=0; max=10;
     bcmd="yum install --downloadonly --assumeyes --setopt=keepcache=1"
     while n=$(($n+1)); do
@@ -48,6 +55,7 @@ MAYBE_RELIABLE_YUM_INSTALL = [
     done
     error ":: running yum install --cacheonly --assumeyes $*"
     yum install --cacheonly --assumeyes "$@"
+    configure_repos_for_proxy_use
     """,
     'reliable-yum-install']
 
@@ -73,8 +81,8 @@ DISTRO_INSTALL_PKG_CMD = {
 # List of base system packages required to enable ci automation
 CI_SYSTEM_BASE_PKGS = {
     'common': ['make', 'sudo', 'tar'],
-    'redhat': ['python-tox'],
-    'centos': ['python-tox'],
+    'redhat': ['python3-tox'],
+    'centos': ['python3-tox'],
     'ubuntu': ['devscripts', 'python3-dev', 'libssl-dev', 'tox', 'sbuild'],
     'debian': ['devscripts', 'python3-dev', 'libssl-dev', 'tox', 'sbuild']}
 
@@ -93,13 +101,21 @@ def get_parser():
     parser.add_argument(
         '-d', '--distro', type=str, choices=DISTRO_PKG_TYPE_MAP.keys(),
         help='The name of the distro to generate package deps for.')
+    deptype = parser.add_mutually_exclusive_group()
+    deptype.add_argument(
+        '-R', '--runtime-requires', action='store_true', default=False,
+        dest='runtime_requires',
+        help='Print only runtime required packages')
+    deptype.add_argument(
+        '-b', '--build-requires', action='store_true', default=False,
+        dest='build_requires', help='Print only buildtime required packages')
     parser.add_argument(
         '--dry-run', action='store_true', default=False, dest='dry_run',
         help='Dry run the install, making no package changes.')
     parser.add_argument(
         '-s', '--system-pkg-names', action='store_true', default=False,
         dest='system_pkg_names',
-        help='The name of the distro to generate package deps for.')
+        help='Generate distribution package names (python3-pkgname).')
     parser.add_argument(
         '-i', '--install', action='store_true', default=False,
         dest='install',
@@ -109,12 +125,6 @@ def get_parser():
         dest='test_distro',
         help='Additionally install continuous integration system packages '
              'required for build and test automation.')
-    parser.add_argument(
-        '-v', '--python-version', type=str, dest='python_version',
-        default=None, choices=["2", "3"],
-        help='Override the version of python we want to generate system '
-             'package dependencies for. Defaults to the version of python '
-             'this script is called with')
     return parser
 
 
@@ -132,6 +142,9 @@ def get_package_deps_from_json(topdir, distro):
         deps = json.loads(stream.read())
     if distro is None:
         return {}
+    if deps.get(distro):  # If we have a specific distro defined, use it.
+        return deps[distro]
+    # Use generic distro dependency map via DISTRO_PKG_TYPE_MAP
     return deps[DISTRO_PKG_TYPE_MAP[distro]]
 
 
@@ -155,27 +168,20 @@ def parse_pip_requirements(requirements_path):
     return dep_names
 
 
-def translate_pip_to_system_pkg(pip_requires, renames, python_ver):
+def translate_pip_to_system_pkg(pip_requires, renames):
     """Translate pip package names to distro-specific package names.
 
     @param pip_requires: List of versionless pip package names to translate.
     @param renames: Dict containg special case renames from pip name to system
         package name for the distro.
-    @param python_ver: Optional python version string "2" or "3". When None,
-     use the python version that is calling this script via sys.version_info.
     """
-    if python_ver is None:
-        python_ver = str(sys.version_info[0])
-    if python_ver == "2":
-        prefix = "python-"
-    else:
-        prefix = "python3-"
+    prefix = "python3-"
     standard_pkg_name = "{0}{1}"
     translated_names = []
     for pip_name in pip_requires:
         pip_name = pip_name.lower()
         # Find a rename if present for the distro package and python version
-        rename = renames.get(pip_name, {}).get(python_ver, None)
+        rename = renames.get(pip_name, "")
         if rename:
             translated_names.append(rename)
         else:
@@ -222,17 +228,26 @@ def main(distro):
     deps_from_json = get_package_deps_from_json(topd, args.distro)
     renames = deps_from_json.get('renames', {})
     translated_pip_names = translate_pip_to_system_pkg(
-        pip_pkg_names, renames, args.python_version)
+        pip_pkg_names, renames)
     all_deps = []
+    select_requires = [args.build_requires, args.runtime_requires]
     if args.distro:
-        all_deps.extend(
-            translated_pip_names + deps_from_json['requires'] +
-            deps_from_json['build-requires'])
+        if not any(select_requires):
+            all_deps.extend(
+                translated_pip_names + deps_from_json['requires'] +
+                deps_from_json['build-requires'])
+        else:
+            if args.build_requires:
+                all_deps.extend(deps_from_json['build-requires'])
+            else:
+                all_deps.extend(
+                    translated_pip_names + deps_from_json['requires'])
     else:
         if args.system_pkg_names:
             all_deps = translated_pip_names
         else:
             all_deps = pip_pkg_names
+    all_deps = sorted(all_deps)
     if args.install:
         pkg_install(all_deps, args.distro, args.test_distro, args.dry_run)
     else:
diff --git a/tools/read-version b/tools/read-version
index 92e9fc96..02c90643 100755
--- a/tools/read-version
+++ b/tools/read-version
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import os
 import json
diff --git a/tools/render-cloudcfg b/tools/render-cloudcfg
index 3d5fa725..ed454840 100755
--- a/tools/render-cloudcfg
+++ b/tools/render-cloudcfg
@@ -4,8 +4,10 @@ import argparse
 import os
 import sys
 
-VARIANTS = ["amazon", "arch", "centos", "debian", "fedora", "freebsd", "rhel",
-            "suse", "ubuntu", "unknown"]
+VARIANTS = ["alpine", "amazon", "arch", "centos", "debian", "fedora",
+            "freebsd", "netbsd", "openbsd", "rhel", "suse", "ubuntu",
+            "unknown"]
+
 
 if "avoid-pep8-E402-import-not-top-of-file":
     _tdir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
diff --git a/tools/run-container b/tools/run-container
index 23243474..15948e77 100755
--- a/tools/run-container
+++ b/tools/run-container
@@ -287,8 +287,8 @@ prep() {
     install_packages "$@"
 }
 
-nose() {
-    python3 -m nose "$@"
+pytest() {
+    python3 -m pytest "$@"
 }
 
 is_done_cloudinit() {
@@ -351,9 +351,8 @@ wait_for_boot() {
         if [ "$OS_NAME" = "centos" ]; then
             debug 1 "configuring proxy ${http_proxy}"
             inside "$name" sh -c "echo proxy=$http_proxy >> /etc/yum.conf"
-            inside "$name" sed -i s/enabled=1/enabled=0/ \
-                /etc/yum/pluginconf.d/fastestmirror.conf
-            inside "$name" sh -c "sed -i '/^#baseurl=/s/#// ; s/^mirrorlist/#mirrorlist/' /etc/yum.repos.d/*.repo"
+            inside "$name" sh -c "sed -i --regexp-extended '/^#baseurl=/s/#// ; /^(mirrorlist|metalink)=/s/^/#/' /etc/yum.repos.d/*.repo"
+            inside "$name" sh -c "sed -i 's/download\.fedoraproject\.org/dl.fedoraproject.org/g' /etc/yum.repos.d/*.repo"
         else
             debug 1 "do not know how to configure proxy on $OS_NAME"
         fi
@@ -463,9 +462,8 @@ main() {
         return
     }
 
-    inside_as_cd "$name" root "$cdir" \
-        python3 ./tools/read-dependencies "--distro=${OS_NAME}" \
-            --test-distro || {
+    local rdcmd=(python3 tools/read-dependencies "--distro=${OS_NAME}" --install --test-distro)
+    inside_as_cd "$name" root "$cdir" "${rdcmd[@]}" || {
         errorrc "FAIL: failed to install dependencies with read-dependencies"
         return
     }
@@ -478,10 +476,10 @@ main() {
 
     if [ -n "$unittest" ]; then
         debug 1 "running unit tests."
-        run_self_inside_as_cd "$name" "$user" "$cdir" nose \
+        run_self_inside_as_cd "$name" "$user" "$cdir" pytest \
             tests/unittests cloudinit/ || {
-                errorrc "nosetests failed.";
-                errors[${#errors[@]}]="nosetests"
+                errorrc "pytest failed.";
+                errors[${#errors[@]}]="pytest"
             }
     fi
 
@@ -557,7 +555,7 @@ main() {
 }
 
 case "${1:-}" in
-    prep|os_info|wait_inside|nose) _n=$1; shift; "$_n" "$@";;
+    prep|os_info|wait_inside|pytest) _n=$1; shift; "$_n" "$@";;
     *) main "$@";;
 esac
 
diff --git a/tools/run-pyflakes b/tools/run-pyflakes
index b3759a94..179afebe 100755
--- a/tools/run-pyflakes
+++ b/tools/run-pyflakes
@@ -1,6 +1,5 @@
 #!/bin/bash
 
-PYTHON_VERSION=${PYTHON_VERSION:-2}
 CR="
 "
 pycheck_dirs=( "cloudinit/" "tests/" "tools/" )
@@ -12,7 +11,7 @@ else
    files=( "$@" )
 fi
 
-cmd=( "python${PYTHON_VERSION}" -m "pyflakes" "${files[@]}" )
+cmd=( "python3" -m "pyflakes" "${files[@]}" )
 
 echo "Running: " "${cmd[@]}" 1>&2
 exec "${cmd[@]}"
diff --git a/tools/run-pyflakes3 b/tools/run-pyflakes3
deleted file mode 100755
index e9f0863d..00000000
--- a/tools/run-pyflakes3
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-PYTHON_VERSION=3 exec "${0%/*}/run-pyflakes" "$@"
diff --git a/tools/tox-venv b/tools/tox-venv
index a5d21625..9dd02460 100755
--- a/tools/tox-venv
+++ b/tools/tox-venv
@@ -116,7 +116,7 @@ Usage: ${0##*/} [--no-create] tox-environment [command [args]]
    be read from tox.ini.  This allows you to do:
       tox-venv py27 - tests/some/sub/dir
    and have the 'command' read correctly and have that execute:
-      python -m nose tests/some/sub/dir
+      python -m pytest tests/some/sub/dir
 EOF
 
     if [ -f "$tox_ini" ]; then
diff --git a/tools/validate-yaml.py b/tools/validate-yaml.py
index a57ea847..d8bbcfcb 100755
--- a/tools/validate-yaml.py
+++ b/tools/validate-yaml.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 """Try to read a YAML file and report any errors.
 """