merge from master at 18.4

6 files changed, 204 insertions, 120 deletions

diff --git a/tools/Z99-cloud-locale-test.sh b/tools/Z99-cloud-locale-test.sh
index 4978d87e..9ee44bd2 100644
--- a/tools/Z99-cloud-locale-test.sh
+++ b/tools/Z99-cloud-locale-test.sh
@@ -11,8 +11,11 @@
 #  of how to fix them.
 
 locale_warn() {
-    local bad_names="" bad_lcs="" key="" val="" var="" vars="" bad_kv=""
-    local w1 w2 w3 w4 remain
+    command -v local >/dev/null && local _local="local" ||
+        typeset _local="typeset"
+
+    $_local bad_names="" bad_lcs="" key="" val="" var="" vars="" bad_kv=""
+    $_local w1 w2 w3 w4 remain
 
     # if shell is zsh, act like sh only for this function (-L).
     # The behavior change will not permenently affect user's shell.
@@ -53,8 +56,8 @@ locale_warn() {
     printf " This can affect your user experience significantly, including the\n"
     printf " ability to manage packages. You may install the locales by running:\n\n"
 
-    local bad invalid="" to_gen="" sfile="/usr/share/i18n/SUPPORTED"
-    local pkgs=""
+    $_local bad invalid="" to_gen="" sfile="/usr/share/i18n/SUPPORTED"
+    $_local local pkgs=""
     if [ -e "$sfile" ]; then
         for bad in ${bad_lcs}; do
             grep -q -i "${bad}" "$sfile" &&
@@ -67,7 +70,7 @@ locale_warn() {
     fi
     to_gen=${to_gen# }
 
-    local pkgs=""
+    $_local pkgs=""
     for bad in ${to_gen}; do
         pkgs="${pkgs} language-pack-${bad%%_*}"
     done
diff --git a/tools/Z99-cloudinit-warnings.sh b/tools/Z99-cloudinit-warnings.sh
index 1d413374..cb8b4638 100644
--- a/tools/Z99-cloudinit-warnings.sh
+++ b/tools/Z99-cloudinit-warnings.sh
@@ -4,9 +4,11 @@
 # Purpose: show user warnings on login.
 
 cloud_init_warnings() {
-    local warning="" idir="/var/lib/cloud/instance" n=0
-    local warndir="$idir/warnings"
-    local ufile="$HOME/.cloud-warnings.skip" sfile="$warndir/.skip"
+    command -v local >/dev/null && local _local="local" ||
+        typeset _local="typeset"
+    $_local warning="" idir="/var/lib/cloud/instance" n=0
+    $_local warndir="$idir/warnings"
+    $_local ufile="$HOME/.cloud-warnings.skip" sfile="$warndir/.skip"
     [ -d "$warndir" ] || return 0
     [ ! -f "$ufile" ] || return 0
     [ ! -f "$sfile" ] || return 0
diff --git a/tools/ds-identify b/tools/ds-identify
index ce0477a5..5afe5aa1 100755
--- a/tools/ds-identify
+++ b/tools/ds-identify
@@ -2,16 +2,24 @@
 # shellcheck disable=2015,2039,2162,2166
 #
 # ds-identify is configured via /etc/cloud/ds-identify.cfg
-# or on the kernel command line. It takes primarily 2 inputs:
+# or on the kernel command line. It takes the following inputs:
+#
 # datasource: can specify the datasource that should be used.
-#     kernel command line option: ci.datasource=<dsname>
+#   kernel command line option: ci.datasource=<dsname> or ci.ds=<dsname>
+#   example line in /etc/cloud/ds-identify.cfg:
+#      datasource: Ec2
 #
 # policy: a string that indicates how ds-identify should operate.
-#     kernel command line option: ci.di.policy=<policy>
+#
 #   The format is:
 #        <mode>,found=value,maybe=value,notfound=value
 #   default setting is:
-#     search,found=all,maybe=all,notfound=disable
+#     search,found=all,maybe=all,notfound=disabled
+#
+#   kernel command line option: ci.di.policy=<policy>
+#   example line in /etc/cloud/ds-identify.cfg:
+#      policy: search,found=all,maybe=none,notfound=disabled
+#
 #
 #   Mode:
 #     disabled: disable cloud-init
@@ -116,7 +124,7 @@ DI_DSNAME=""
 # be searched if there is no setting found in config.
 DI_DSLIST_DEFAULT="MAAS ConfigDrive NoCloud AltCloud Azure Bigstep \
 CloudSigma CloudStack DigitalOcean AliYun Ec2 GCE OpenNebula OpenStack \
-OVF SmartOS Scaleway Hetzner IBMCloud"
+OVF SmartOS Scaleway Hetzner IBMCloud Oracle"
 DI_DSLIST=""
 DI_MODE=""
 DI_ON_FOUND=""
@@ -1036,6 +1044,12 @@ dscheck_Hetzner() {
     return ${DS_NOT_FOUND}
 }
 
+dscheck_Oracle() {
+    local asset_tag="OracleCloud.com"
+    dmi_chassis_asset_tag_matches "${asset_tag}" && return ${DS_FOUND}
+    return ${DS_NOT_FOUND}
+}
+
 is_ibm_provisioning() {
     local pcfg="${PATH_ROOT}/root/provisioningConfiguration.cfg"
     local logf="${PATH_ROOT}/root/swinstall.log"
diff --git a/tools/net-convert.py b/tools/net-convert.py
deleted file mode 100755
index 68559cbf..00000000
--- a/tools/net-convert.py
+++ /dev/null
@@ -1,84 +0,0 @@
-#!/usr/bin/python3
-# This file is part of cloud-init. See LICENSE file for license information.
-
-import argparse
-import json
-import os
-import yaml
-
-from cloudinit.sources.helpers import openstack
-
-from cloudinit.net import eni
-from cloudinit.net import netplan
-from cloudinit.net import network_state
-from cloudinit.net import sysconfig
-
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--network-data", "-p", type=open,
-                        metavar="PATH", required=True)
-    parser.add_argument("--kind", "-k",
-                        choices=['eni', 'network_data.json', 'yaml'],
-                        required=True)
-    parser.add_argument("-d", "--directory",
-                        metavar="PATH",
-                        help="directory to place output in",
-                        required=True)
-    parser.add_argument("-m", "--mac",
-                        metavar="name,mac",
-                        action='append',
-                        help="interface name to mac mapping")
-    parser.add_argument("--output-kind", "-ok",
-                        choices=['eni', 'netplan', 'sysconfig'],
-                        required=True)
-    args = parser.parse_args()
-
-    if not os.path.isdir(args.directory):
-        os.makedirs(args.directory)
-
-    if args.mac:
-        known_macs = {}
-        for item in args.mac:
-            iface_name, iface_mac = item.split(",", 1)
-            known_macs[iface_mac] = iface_name
-    else:
-        known_macs = None
-
-    net_data = args.network_data.read()
-    if args.kind == "eni":
-        pre_ns = eni.convert_eni_data(net_data)
-        ns = network_state.parse_net_config_data(pre_ns)
-    elif args.kind == "yaml":
-        pre_ns = yaml.load(net_data)
-        if 'network' in pre_ns:
-            pre_ns = pre_ns.get('network')
-        print("Input YAML")
-        print(yaml.dump(pre_ns, default_flow_style=False, indent=4))
-        ns = network_state.parse_net_config_data(pre_ns)
-    else:
-        pre_ns = openstack.convert_net_json(
-            json.loads(net_data), known_macs=known_macs)
-        ns = network_state.parse_net_config_data(pre_ns)
-
-    if not ns:
-        raise RuntimeError("No valid network_state object created from"
-                           "input data")
-
-    print("\nInternal State")
-    print(yaml.dump(ns, default_flow_style=False, indent=4))
-    if args.output_kind == "eni":
-        r_cls = eni.Renderer
-    elif args.output_kind == "netplan":
-        r_cls = netplan.Renderer
-    else:
-        r_cls = sysconfig.Renderer
-
-    r = r_cls()
-    r.render_network_state(network_state=ns, target=args.directory)
-
-
-if __name__ == '__main__':
-    main()
-
-# vi: ts=4 expandtab
diff --git a/tools/read-version b/tools/read-version
index 3ea9e66e..e69c2ce0 100755
--- a/tools/read-version
+++ b/tools/read-version
@@ -76,6 +76,12 @@ if is_gitdir(_tdir) and which("git"):
     if not version.startswith(src_version):
         sys.stderr.write("git describe version (%s) differs from "
                          "cloudinit.version (%s)\n" % (version, src_version))
+        sys.stderr.write(
+            "Please get the latest upstream tags.\n"
+            "As an example, this can be done with the following:\n"
+            "$ git remote add upstream https://git.launchpad.net/cloud-init\n"
+            "$ git fetch upstream --tags\n"
+        )
         sys.exit(1)
 
     version_long = tiny_p(cmd + ["--long"]).strip()
diff --git a/tools/tox-venv b/tools/tox-venv
index 76ed5076..a5d21625 100755
--- a/tools/tox-venv
+++ b/tools/tox-venv
@@ -1,42 +1,185 @@
 #!/bin/sh
+# https://gist.github.com/smoser/2d4100a6a5d230ca937f
 
+CR='
+'
 error() { echo "$@" 1>&2; }
 fail() { [ $# -eq 0 ] || error "$@"; exit 1; }
+get_env_dirs() {
+    # read 'tox --showconfig'. return list of
+    #  envname:dir
+    local key="" equal="" val="" curenv="" out=""
+    while read key equal val; do
+        case "$key" in
+           "[testenv:"*)
+                curenv=${key#*:};
+                curenv=${curenv%%"]"*};
+                continue;;
+        esac
+        if [ "${key#*=}" != "$key" ]; then
+            # older tox shows key=value or key=   value
+            # newer tox shows: key =    value
+            key=${key%%=*}
+            val=${equal}
+        fi
+        [ "$key" = "envdir" ] || continue
+        out="${out:+${out}${CR}}${curenv}:$val"
+    done
+    echo "$out"
+}
+
+load_config() {
+    local tox_ini="$1" out="" envs=""
+    if [ "$tox_ini" = "${CACHED_ENVS_INI}" ]; then
+        _RET="$CACHED_ENVS"
+        return
+    fi
+    out=$(tox -c "$tox_ini" --showconfig) || return 1
+    envs=$(echo "$out" | get_env_dirs) || return 1
+    CACHED_ENVS="$envs"
+    CACHED_ENVS_INI="$tox_ini"
+    _RET="$envs"
+}
+
+list_environments() {
+    local tox_ini="$1" prefix="  " out="" envs="" oifs="$IFS"
+    load_config "$tox_ini" || return 1
+    envs="${_RET}"
+    IFS="$CR"
+    for d in ${envs}; do
+        env=${d%%:*}
+        dir=${d#*:}
+        [ -f "$dir/bin/activate" ] && s="*" || s=""
+        echo "${prefix}$env$s";
+    done
+    IFS="$oifs"
+}
+
+get_command() {
+    local tox_ini="$1" env="$2" out=""
+    shift 2
+    out=$(
+        sed -e ':x; /\\$/ { N; s/\\\n[ ]*//; tx };' "${tox_ini}" |
+        gawk '
+        $1 ~ /^\[testenv.*\]/ {
+            name=$1;
+            sub("\\[", "", name); sub(".*:", "", name);
+            sub("].*", "", name);
+            curenv=name; };
+        $1 == "basepython" && (name == "testenv" || name == n) { python=$3 }
+        $1 == "commands" && (name == "testenv" || name == n) {
+            sub("commands = ", ""); cmd = $0; };
+        END {
+            sub("{envpython}", python, cmd);
+            sub("{toxinidir}", toxinidir, cmd);
+            if (inargs == "") replacement = "\\1"
+            else replacement = inargs
+            cmd = gensub(/{posargs:?([^}]*)}/, replacement, "global", cmd)
+            print(cmd);
+            }' n="$env" toxinidir="$(dirname $tox_ini)" inargs="$*")
+    if [ -z "$out" ]; then
+        error "Failed to find command for $env in $tox_ini"
+        return 1
+    fi
+    echo "$out"
+}
+
+get_env_dir() {
+    local tox_ini="$1" env="$2" oifs="$IFS" t="" d="" envs=""
+    if [ "${TOX_VENV_SHORTCUT:-1}" != "0" ]; then
+        local stox_d="${tox_ini%/*}/.tox/${env}"
+        if [ -e "${stox_d}/bin/activate" ]; then
+            _RET="${stox_d}"
+            return
+        fi
+    fi
+    load_config "$tox_ini" && envs="$_RET" || return 1
+    IFS="$CR"
+    for t in $envs; do
+        [ "$env" = "${t%%:*}" ] && d="${t#*:}" && break
+    done
+    IFS=${oifs}
+    [ -n "$d" ] || return 1
+    _RET="$d"
+}
+
 Usage() {
-   cat <<EOF
-Usage: ${0##*/} tox-environment [command [args]]
+    local tox_ini="$1"
+    cat <<EOF
+Usage: ${0##*/} [--no-create] tox-environment [command [args]]
    run command with provided arguments in the provided tox environment
-   command defaults to \${SHELL:-/bin/sh}.
+   command defaults to 'cmd' (see below).
+
+   run with '--list' to show available environments
 
-   invoke with '--list' to show available environments
+   if 'command' above is literal 'cmd' or '-', then the 'command' will
+   be read from tox.ini.  This allows you to do:
+      tox-venv py27 - tests/some/sub/dir
+   and have the 'command' read correctly and have that execute:
+      python -m nose tests/some/sub/dir
 EOF
-}
-list_toxes() {
-   local td="$1" pre="$2" d=""
-   ( cd "$tox_d" &&
-     for d in *; do [ -f "$d/bin/activate" ] && echo "${pre}$d"; done)
+
+    if [ -f "$tox_ini" ]; then
+        local oini=${tox_ini}
+        [ "${tox_ini}" -ef "$PWD/tox.ini" ] && oini="./tox.ini"
+        echo
+        echo "environments in $oini"
+        list_environments "$tox_ini"
+    fi
 }
 
-[ $# -eq 0 ] && { Usage 1>&2; exit 1; }
-[ "$1" = "-h" -o "$1" = "--help" ] && { Usage; exit 0; }
+if [ -f tox.ini ]; then
+    tox_ini="$PWD/tox.ini"
+else
+    tox_ini="${0%/*}/../tox.ini"
+fi
 
-env="$1"
-shift
-tox_d="${0%/*}/../.tox"
-activate="$tox_d/$env/bin/activate"
+[ $# -eq 0 ] && { Usage "$tox_ini" 1>&2; exit 1; }
+[ "$1" = "-h" -o "$1" = "--help" ] && { Usage "$tox_ini"; exit 0; }
 
+[ -f "$tox_ini" ] || fail "$tox_ini: did not find tox.ini"
 
-[ -d "$tox_d" ] || fail "$tox_d: not a dir. maybe run 'tox'?"
+if [ "$1" = "-l" -o "$1" = "--list" ]; then
+    list_environments "$tox_ini"
+    exit
+fi
 
-[ "$env" = "-l" -o "$env" = "--list" ] && { list_toxes ; exit ; }
+nocreate="false"
+if [ "$1" = "--no-create" ]; then
+    nocreate="true"
+    shift
+fi
 
-if [ ! -f "$activate" ]; then
-   error "$env: not a valid tox environment?"
-   error "try one of:"
-   list_toxes "$tox_d" "  "
-   fail
+env="$1"
+shift
+[ "$1" = "--" ] && shift
+get_env_dir "$tox_ini" "$env" && activate="$_RET/bin/activate" || activate=""
+
+if [ -z "$activate" -o ! -f "$activate" ]; then
+    if $nocreate; then
+        fail "tox env '$env' did not exist, and no-create specified"
+    elif [ -n "$activate" ]; then
+        error "attempting to create $env:"
+        error "    tox -c $tox_ini --recreate --notest -e $env"
+        tox -c "$tox_ini" --recreate --notest -e "$env" ||
+            fail "failed creation of env $env"
+    else
+        error "$env: not a valid tox environment?"
+        error "found tox_ini=$tox_ini"
+        error "try one of:"
+        list_environments "$tox_ini" 1>&2
+        fail
+    fi
 fi
 . "$activate"
 
-[ "$#" -gt 0 ] || set -- ${SHELL:-/bin/bash}
+[ $# -eq 0 ] && set -- cmd
+if [ "$1" = "cmd" -o "$1" = "-" ]; then
+   shift
+   out=$(get_command "$tox_ini" "$env" "$@") || exit
+   eval set -- "$out"
+fi
+echo "inside tox:$env running: $*" 1>&2
 debian_chroot="tox:$env" exec "$@"
+
+# vi: ts=4 expandtab