From f895cb12141281702b34da18f2384deb64c881e7 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Wed, 21 Jan 2015 17:56:53 -0500 Subject: Largely merge lp:~harlowja/cloud-init/py2-3 albeit manually because it seemed to be behind trunk. `tox -e py27` passes full test suite. Now to work on replacing mocker. --- cloudinit/config/cc_apt_configure.py | 2 +- cloudinit/config/cc_debug.py | 7 ++++--- cloudinit/config/cc_landscape.py | 2 +- cloudinit/config/cc_mcollective.py | 15 ++++++++------- cloudinit/config/cc_phone_home.py | 4 ++-- cloudinit/config/cc_puppet.py | 8 ++++---- cloudinit/config/cc_resolv_conf.py | 4 ++-- cloudinit/config/cc_seed_random.py | 3 ++- cloudinit/config/cc_ssh.py | 16 ++++++++-------- cloudinit/config/cc_yum_add_repo.py | 7 ++++--- 10 files changed, 36 insertions(+), 32 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index f10b76a3..de72903f 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -126,7 +126,7 @@ def mirror2lists_fileprefix(mirror): def rename_apt_lists(old_mirrors, new_mirrors, lists_d="/var/lib/apt/lists"): - for (name, omirror) in old_mirrors.iteritems(): + for (name, omirror) in old_mirrors.items(): nmirror = new_mirrors.get(name) if not nmirror: continue diff --git a/cloudinit/config/cc_debug.py b/cloudinit/config/cc_debug.py index 8c489426..bdc32fe6 100644 --- a/cloudinit/config/cc_debug.py +++ b/cloudinit/config/cc_debug.py @@ -34,7 +34,8 @@ It can be configured with the following option structure:: """ import copy -from StringIO import StringIO + +from six import StringIO from cloudinit import type_utils from cloudinit import util @@ -77,7 +78,7 @@ def handle(name, cfg, cloud, log, args): dump_cfg = copy.deepcopy(cfg) for k in SKIP_KEYS: dump_cfg.pop(k, None) - all_keys = list(dump_cfg.keys()) + all_keys = list(dump_cfg) for k in all_keys: if k.startswith("_"): dump_cfg.pop(k, None) @@ -103,6 +104,6 @@ def handle(name, cfg, cloud, log, args): line = "ci-info: %s\n" % (line) content_to_file.append(line) if out_file: - util.write_file(out_file, "".join(content_to_file), 0644, "w") + util.write_file(out_file, "".join(content_to_file), 0o644, "w") else: util.multi_log("".join(content_to_file), console=True, stderr=False) diff --git a/cloudinit/config/cc_landscape.py b/cloudinit/config/cc_landscape.py index 8a709677..0b9d846e 100644 --- a/cloudinit/config/cc_landscape.py +++ b/cloudinit/config/cc_landscape.py @@ -20,7 +20,7 @@ import os -from StringIO import StringIO +from six import StringIO from configobj import ConfigObj diff --git a/cloudinit/config/cc_mcollective.py b/cloudinit/config/cc_mcollective.py index b670390d..425420ae 100644 --- a/cloudinit/config/cc_mcollective.py +++ b/cloudinit/config/cc_mcollective.py @@ -19,7 +19,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from StringIO import StringIO +import six +from six import StringIO # Used since this can maintain comments # and doesn't need a top level section @@ -51,17 +52,17 @@ def handle(name, cfg, cloud, log, _args): # original file in order to be able to mix the rest up mcollective_config = ConfigObj(SERVER_CFG) # See: http://tiny.cc/jh9agw - for (cfg_name, cfg) in mcollective_cfg['conf'].iteritems(): + for (cfg_name, cfg) in mcollective_cfg['conf'].items(): if cfg_name == 'public-cert': - util.write_file(PUBCERT_FILE, cfg, mode=0644) + util.write_file(PUBCERT_FILE, cfg, mode=0o644) mcollective_config['plugin.ssl_server_public'] = PUBCERT_FILE mcollective_config['securityprovider'] = 'ssl' elif cfg_name == 'private-cert': - util.write_file(PRICERT_FILE, cfg, mode=0600) + util.write_file(PRICERT_FILE, cfg, mode=0o600) mcollective_config['plugin.ssl_server_private'] = PRICERT_FILE mcollective_config['securityprovider'] = 'ssl' else: - if isinstance(cfg, (basestring, str)): + if isinstance(cfg, six.string_types): # Just set it in the 'main' section mcollective_config[cfg_name] = cfg elif isinstance(cfg, (dict)): @@ -69,7 +70,7 @@ def handle(name, cfg, cloud, log, _args): # if it is needed and then add/or create items as needed if cfg_name not in mcollective_config.sections: mcollective_config[cfg_name] = {} - for (o, v) in cfg.iteritems(): + for (o, v) in cfg.items(): mcollective_config[cfg_name][o] = v else: # Otherwise just try to convert it to a string @@ -81,7 +82,7 @@ def handle(name, cfg, cloud, log, _args): contents = StringIO() mcollective_config.write(contents) contents = contents.getvalue() - util.write_file(SERVER_CFG, contents, mode=0644) + util.write_file(SERVER_CFG, contents, mode=0o644) # Start mcollective util.subp(['service', 'mcollective', 'start'], capture=False) diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py index 5bc68b83..18a7ddad 100644 --- a/cloudinit/config/cc_phone_home.py +++ b/cloudinit/config/cc_phone_home.py @@ -81,7 +81,7 @@ def handle(name, cfg, cloud, log, args): 'pub_key_ecdsa': '/etc/ssh/ssh_host_ecdsa_key.pub', } - for (n, path) in pubkeys.iteritems(): + for (n, path) in pubkeys.items(): try: all_keys[n] = util.load_file(path) except: @@ -99,7 +99,7 @@ def handle(name, cfg, cloud, log, args): # Get them read to be posted real_submit_keys = {} - for (k, v) in submit_keys.iteritems(): + for (k, v) in submit_keys.items(): if v is None: real_submit_keys[k] = 'N/A' else: diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py index 471a1a8a..6f1b3c57 100644 --- a/cloudinit/config/cc_puppet.py +++ b/cloudinit/config/cc_puppet.py @@ -18,7 +18,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from StringIO import StringIO +from six import StringIO import os import socket @@ -81,13 +81,13 @@ def handle(name, cfg, cloud, log, _args): cleaned_contents = '\n'.join(cleaned_lines) puppet_config.readfp(StringIO(cleaned_contents), filename=PUPPET_CONF_PATH) - for (cfg_name, cfg) in puppet_cfg['conf'].iteritems(): + for (cfg_name, cfg) in puppet_cfg['conf'].items(): # Cert configuration is a special case # Dump the puppet master ca certificate in the correct place if cfg_name == 'ca_cert': # Puppet ssl sub-directory isn't created yet # Create it with the proper permissions and ownership - util.ensure_dir(PUPPET_SSL_DIR, 0771) + util.ensure_dir(PUPPET_SSL_DIR, 0o771) util.chownbyname(PUPPET_SSL_DIR, 'puppet', 'root') util.ensure_dir(PUPPET_SSL_CERT_DIR) util.chownbyname(PUPPET_SSL_CERT_DIR, 'puppet', 'root') @@ -96,7 +96,7 @@ def handle(name, cfg, cloud, log, _args): else: # Iterate throug the config items, we'll use ConfigParser.set # to overwrite or create new items as needed - for (o, v) in cfg.iteritems(): + for (o, v) in cfg.items(): if o == 'certname': # Expand %f as the fqdn # TODO(harlowja) should this use the cloud fqdn?? diff --git a/cloudinit/config/cc_resolv_conf.py b/cloudinit/config/cc_resolv_conf.py index bbaa6c63..71d9e3a7 100644 --- a/cloudinit/config/cc_resolv_conf.py +++ b/cloudinit/config/cc_resolv_conf.py @@ -66,8 +66,8 @@ def generate_resolv_conf(template_fn, params, target_fname="/etc/resolv.conf"): false_flags = [] if 'options' in params: - for key, val in params['options'].iteritems(): - if type(val) == bool: + for key, val in params['options'].items(): + if isinstance(val, bool): if val: flags.append(key) else: diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py index 49a6b3e8..3b7235bf 100644 --- a/cloudinit/config/cc_seed_random.py +++ b/cloudinit/config/cc_seed_random.py @@ -21,7 +21,8 @@ import base64 import os -from StringIO import StringIO + +from six import StringIO from cloudinit.settings import PER_INSTANCE from cloudinit import log as logging diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py index 4c76581c..ab6940fa 100644 --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -34,12 +34,12 @@ DISABLE_ROOT_OPTS = ("no-port-forwarding,no-agent-forwarding," "rather than the user \\\"root\\\".\';echo;sleep 10\"") KEY_2_FILE = { - "rsa_private": ("/etc/ssh/ssh_host_rsa_key", 0600), - "rsa_public": ("/etc/ssh/ssh_host_rsa_key.pub", 0644), - "dsa_private": ("/etc/ssh/ssh_host_dsa_key", 0600), - "dsa_public": ("/etc/ssh/ssh_host_dsa_key.pub", 0644), - "ecdsa_private": ("/etc/ssh/ssh_host_ecdsa_key", 0600), - "ecdsa_public": ("/etc/ssh/ssh_host_ecdsa_key.pub", 0644), + "rsa_private": ("/etc/ssh/ssh_host_rsa_key", 0o600), + "rsa_public": ("/etc/ssh/ssh_host_rsa_key.pub", 0o644), + "dsa_private": ("/etc/ssh/ssh_host_dsa_key", 0o600), + "dsa_public": ("/etc/ssh/ssh_host_dsa_key.pub", 0o644), + "ecdsa_private": ("/etc/ssh/ssh_host_ecdsa_key", 0o600), + "ecdsa_public": ("/etc/ssh/ssh_host_ecdsa_key.pub", 0o644), } PRIV_2_PUB = { @@ -68,13 +68,13 @@ def handle(_name, cfg, cloud, log, _args): if "ssh_keys" in cfg: # if there are keys in cloud-config, use them - for (key, val) in cfg["ssh_keys"].iteritems(): + for (key, val) in cfg["ssh_keys"].items(): if key in KEY_2_FILE: tgt_fn = KEY_2_FILE[key][0] tgt_perms = KEY_2_FILE[key][1] util.write_file(tgt_fn, val, tgt_perms) - for (priv, pub) in PRIV_2_PUB.iteritems(): + for (priv, pub) in PRIV_2_PUB.items(): if pub in cfg['ssh_keys'] or priv not in cfg['ssh_keys']: continue pair = (KEY_2_FILE[priv][0], KEY_2_FILE[pub][0]) diff --git a/cloudinit/config/cc_yum_add_repo.py b/cloudinit/config/cc_yum_add_repo.py index 0d836f28..3b821af9 100644 --- a/cloudinit/config/cc_yum_add_repo.py +++ b/cloudinit/config/cc_yum_add_repo.py @@ -18,9 +18,10 @@ import os -from cloudinit import util - import configobj +import six + +from cloudinit import util def _canonicalize_id(repo_id): @@ -37,7 +38,7 @@ def _format_repo_value(val): # Can handle 'lists' in certain cases # See: http://bit.ly/Qqrf1t return "\n ".join([_format_repo_value(v) for v in val]) - if not isinstance(val, (basestring, str)): + if not isinstance(val, six.string_types): return str(val) return val -- cgit v1.2.3 From 3b798b5d5c3caa5d0e8e534855e29010ca932aaa Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Thu, 22 Jan 2015 21:21:04 -0500 Subject: Low hanging Python 3 fruit. --- cloudinit/config/cc_ca_certs.py | 4 ++-- cloudinit/config/cc_chef.py | 6 ++++-- cloudinit/distros/__init__.py | 12 ++++++++++-- cloudinit/distros/debian.py | 2 +- cloudinit/distros/rhel_util.py | 4 ++-- cloudinit/distros/sles.py | 2 +- cloudinit/sources/DataSourceAltCloud.py | 12 ++++++------ cloudinit/sources/DataSourceAzure.py | 4 ++-- cloudinit/sources/DataSourceMAAS.py | 10 ++++++---- cloudinit/sources/DataSourceOpenNebula.py | 2 +- cloudinit/templater.py | 2 +- cloudinit/util.py | 7 +++++-- templates/resolv.conf.tmpl | 2 +- tests/unittests/helpers.py | 4 ++-- tests/unittests/test_datasource/test_configdrive.py | 2 +- tests/unittests/test_datasource/test_digitalocean.py | 7 +++---- tests/unittests/test_datasource/test_gce.py | 2 +- tests/unittests/test_datasource/test_opennebula.py | 2 +- tests/unittests/test_datasource/test_smartos.py | 4 +++- .../unittests/test_handler/test_handler_apt_configure.py | 2 +- tests/unittests/test_merging.py | 16 +++++++++------- tools/ccfg-merge-debug | 4 ++-- 22 files changed, 65 insertions(+), 47 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_ca_certs.py b/cloudinit/config/cc_ca_certs.py index 4f2a46a1..8248b020 100644 --- a/cloudinit/config/cc_ca_certs.py +++ b/cloudinit/config/cc_ca_certs.py @@ -44,7 +44,7 @@ def add_ca_certs(certs): if certs: # First ensure they are strings... cert_file_contents = "\n".join([str(c) for c in certs]) - util.write_file(CA_CERT_FULL_PATH, cert_file_contents, mode=0644) + util.write_file(CA_CERT_FULL_PATH, cert_file_contents, mode=0o644) # Append cert filename to CA_CERT_CONFIG file. # We have to strip the content because blank lines in the file @@ -63,7 +63,7 @@ def remove_default_ca_certs(): """ util.delete_dir_contents(CA_CERT_PATH) util.delete_dir_contents(CA_CERT_SYSTEM_PATH) - util.write_file(CA_CERT_CONFIG, "", mode=0644) + util.write_file(CA_CERT_CONFIG, "", mode=0o644) debconf_sel = "ca-certificates ca-certificates/trust_new_crts select no" util.subp(('debconf-set-selections', '-'), debconf_sel) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index fc837363..584199e5 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -76,6 +76,8 @@ from cloudinit import templater from cloudinit import url_helper from cloudinit import util +import six + RUBY_VERSION_DEFAULT = "1.8" CHEF_DIRS = tuple([ @@ -261,7 +263,7 @@ def run_chef(chef_cfg, log): cmd_args = chef_cfg['exec_arguments'] if isinstance(cmd_args, (list, tuple)): cmd.extend(cmd_args) - elif isinstance(cmd_args, (str, basestring)): + elif isinstance(cmd_args, six.string_types): cmd.append(cmd_args) else: log.warn("Unknown type %s provided for chef" @@ -300,7 +302,7 @@ def install_chef(cloud, chef_cfg, log): with util.tempdir() as tmpd: # Use tmpdir over tmpfile to avoid 'text file busy' on execute tmpf = "%s/chef-omnibus-install" % tmpd - util.write_file(tmpf, str(content), mode=0700) + util.write_file(tmpf, str(content), mode=0o700) util.subp([tmpf], capture=False) else: log.warn("Unknown chef install type '%s'", install_type) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 4ebccdda..6b96d58c 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -25,7 +25,6 @@ import six from six import StringIO import abc -import itertools import os import re @@ -37,6 +36,15 @@ from cloudinit import util from cloudinit.distros.parsers import hosts +try: + # Python 3 + from six import filter +except ImportError: + # Python 2 + from itertools import ifilter as filter + + + OSFAMILIES = { 'debian': ['debian', 'ubuntu'], 'redhat': ['fedora', 'rhel'], @@ -853,7 +861,7 @@ def extract_default(users, default_name=None, default_config=None): return config['default'] tmp_users = users.items() - tmp_users = dict(itertools.ifilter(safe_find, tmp_users)) + tmp_users = dict(filter(safe_find, tmp_users)) if not tmp_users: return (default_name, default_config) else: diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index b09eb094..6d3a82bf 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -97,7 +97,7 @@ class Distro(distros.Distro): if not conf: conf = HostnameConf('') conf.set_hostname(your_hostname) - util.write_file(out_fn, str(conf), 0644) + util.write_file(out_fn, str(conf), 0o644) def _read_system_hostname(self): sys_hostname = self._read_hostname(self.hostname_conf_fn) diff --git a/cloudinit/distros/rhel_util.py b/cloudinit/distros/rhel_util.py index 063d536e..903d7793 100644 --- a/cloudinit/distros/rhel_util.py +++ b/cloudinit/distros/rhel_util.py @@ -50,7 +50,7 @@ def update_sysconfig_file(fn, adjustments, allow_empty=False): ] if not exists: lines.insert(0, util.make_header()) - util.write_file(fn, "\n".join(lines) + "\n", 0644) + util.write_file(fn, "\n".join(lines) + "\n", 0o644) # Helper function to read a RHEL/SUSE /etc/sysconfig/* file @@ -86,4 +86,4 @@ def update_resolve_conf_file(fn, dns_servers, search_servers): r_conf.add_search_domain(s) except ValueError: util.logexc(LOG, "Failed at adding search domain %s", s) - util.write_file(fn, str(r_conf), 0644) + util.write_file(fn, str(r_conf), 0o644) diff --git a/cloudinit/distros/sles.py b/cloudinit/distros/sles.py index 0c6d1203..620c974c 100644 --- a/cloudinit/distros/sles.py +++ b/cloudinit/distros/sles.py @@ -113,7 +113,7 @@ class Distro(distros.Distro): if not conf: conf = HostnameConf('') conf.set_hostname(hostname) - util.write_file(out_fn, str(conf), 0644) + util.write_file(out_fn, str(conf), 0o644) def _read_system_hostname(self): host_fn = self.hostname_conf_fn diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py index 1e913a6e..69053d0b 100644 --- a/cloudinit/sources/DataSourceAltCloud.py +++ b/cloudinit/sources/DataSourceAltCloud.py @@ -124,11 +124,11 @@ class DataSourceAltCloud(sources.DataSource): cmd = CMD_DMI_SYSTEM try: (cmd_out, _err) = util.subp(cmd) - except ProcessExecutionError, _err: + except ProcessExecutionError as _err: LOG.debug(('Failed command: %s\n%s') % \ (' '.join(cmd), _err.message)) return 'UNKNOWN' - except OSError, _err: + except OSError as _err: LOG.debug(('Failed command: %s\n%s') % \ (' '.join(cmd), _err.message)) return 'UNKNOWN' @@ -211,11 +211,11 @@ class DataSourceAltCloud(sources.DataSource): cmd = CMD_PROBE_FLOPPY (cmd_out, _err) = util.subp(cmd) LOG.debug(('Command: %s\nOutput%s') % (' '.join(cmd), cmd_out)) - except ProcessExecutionError, _err: + except ProcessExecutionError as _err: util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err.message) return False - except OSError, _err: + except OSError as _err: util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err.message) return False @@ -228,11 +228,11 @@ class DataSourceAltCloud(sources.DataSource): cmd.append('--exit-if-exists=' + floppy_dev) (cmd_out, _err) = util.subp(cmd) LOG.debug(('Command: %s\nOutput%s') % (' '.join(cmd), cmd_out)) - except ProcessExecutionError, _err: + except ProcessExecutionError as _err: util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err.message) return False - except OSError, _err: + except OSError as _err: util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err.message) return False diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 09bc196d..29ae2c22 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -151,7 +151,7 @@ class DataSourceAzureNet(sources.DataSource): # walinux agent writes files world readable, but expects # the directory to be protected. - write_files(ddir, files, dirmode=0700) + write_files(ddir, files, dirmode=0o700) # handle the hostname 'publishing' try: @@ -390,7 +390,7 @@ def write_files(datadir, files, dirmode=None): util.ensure_dir(datadir, dirmode) for (name, content) in files.items(): util.write_file(filename=os.path.join(datadir, name), - content=content, mode=0600) + content=content, mode=0o600) def invoke_agent(cmd): diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index 9a3e30c5..8f9c81de 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -18,6 +18,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from __future__ import print_function + from email.utils import parsedate import errno import oauth.oauth as oauth @@ -361,7 +363,7 @@ if __name__ == "__main__": return (urllib2.urlopen(req).read()) def printurl(url, headers_cb): - print "== %s ==\n%s\n" % (url, geturl(url, headers_cb)) + print("== %s ==\n%s\n" % (url, geturl(url, headers_cb))) def crawl(url, headers_cb=None): if url.endswith("/"): @@ -386,9 +388,9 @@ if __name__ == "__main__": version=args.apiver) else: (userdata, metadata) = read_maas_seed_url(args.url) - print "=== userdata ===" - print userdata - print "=== metadata ===" + print("=== userdata ===") + print(userdata) + print("=== metadata ===") pprint.pprint(metadata) elif args.subcmd == "get": diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index e2469f6e..f9dac29e 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -280,7 +280,7 @@ def parse_shell_config(content, keylist=None, bash=None, asuser=None, # allvars expands to all existing variables by using '${!x*}' notation # where x is lower or upper case letters or '_' - allvars = ["${!%s*}" % x for x in string.letters + "_"] + allvars = ["${!%s*}" % x for x in string.ascii_letters + "_"] keylist_in = keylist if keylist is None: diff --git a/cloudinit/templater.py b/cloudinit/templater.py index 4cd3f13d..a9231482 100644 --- a/cloudinit/templater.py +++ b/cloudinit/templater.py @@ -137,7 +137,7 @@ def render_from_file(fn, params): return renderer(content, params) -def render_to_file(fn, outfn, params, mode=0644): +def render_to_file(fn, outfn, params, mode=0o644): contents = render_from_file(fn, params) util.write_file(outfn, contents, mode=mode) diff --git a/cloudinit/util.py b/cloudinit/util.py index 434ba7fb..94fd5c70 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -142,6 +142,9 @@ class ProcessExecutionError(IOError): 'reason': self.reason, } IOError.__init__(self, message) + # For backward compatibility with Python 2. + if not hasattr(self, 'message'): + self.message = message class SeLinuxGuard(object): @@ -260,7 +263,7 @@ def translate_bool(val, addons=None): def rand_str(strlen=32, select_from=None): if not select_from: - select_from = string.letters + string.digits + select_from = string.ascii_letters + string.digits return "".join([random.choice(select_from) for _x in range(0, strlen)]) @@ -1127,7 +1130,7 @@ def pipe_in_out(in_fh, out_fh, chunk_size=1024, chunk_cb=None): bytes_piped = 0 while True: data = in_fh.read(chunk_size) - if data == '': + if len(data) == 0: break else: out_fh.write(data) diff --git a/templates/resolv.conf.tmpl b/templates/resolv.conf.tmpl index 1300156c..bfae80db 100644 --- a/templates/resolv.conf.tmpl +++ b/templates/resolv.conf.tmpl @@ -24,7 +24,7 @@ sortlist {% for sort in sortlist %}{{sort}} {% endfor %} {% if options or flags %} options {% for flag in flags %}{{flag}} {% endfor %} -{% for key, value in options.iteritems() -%} +{% for key, value in options.items() -%} {{key}}:{{value}} {% endfor %} {% endif %} diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 38a2176d..70b8116f 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -65,7 +65,7 @@ if PY26: def assertDictContainsSubset(self, expected, actual, msg=None): missing = [] mismatched = [] - for k, v in expected.iteritems(): + for k, v in expected.items(): if k not in actual: missing.append(k) elif actual[k] != v: @@ -243,7 +243,7 @@ class HttprettyTestCase(TestCase): def populate_dir(path, files): if not os.path.exists(path): os.makedirs(path) - for (name, content) in files.iteritems(): + for (name, content) in files.items(): with open(os.path.join(path, name), "w") as fp: fp.write(content) fp.close() diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index 800c5fd8..258c68e2 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -338,7 +338,7 @@ def populate_ds_from_read_config(cfg_ds, source, results): def populate_dir(seed_dir, files): - for (name, content) in files.iteritems(): + for (name, content) in files.items(): path = os.path.join(seed_dir, name) dirname = os.path.dirname(path) if not os.path.isdir(dirname): diff --git a/tests/unittests/test_datasource/test_digitalocean.py b/tests/unittests/test_datasource/test_digitalocean.py index d1270fc2..98f9cfac 100644 --- a/tests/unittests/test_datasource/test_digitalocean.py +++ b/tests/unittests/test_datasource/test_digitalocean.py @@ -18,8 +18,7 @@ import httpretty import re -from types import ListType -from urlparse import urlparse +from six.moves.urllib_parse import urlparse from cloudinit import settings from cloudinit import helpers @@ -110,7 +109,7 @@ class TestDataSourceDigitalOcean(test_helpers.HttprettyTestCase): self.assertEqual([DO_META.get('public-keys')], self.ds.get_public_ssh_keys()) - self.assertIs(type(self.ds.get_public_ssh_keys()), ListType) + self.assertIsInstance(self.ds.get_public_ssh_keys(), list) @httpretty.activate def test_multiple_ssh_keys(self): @@ -124,4 +123,4 @@ class TestDataSourceDigitalOcean(test_helpers.HttprettyTestCase): self.assertEqual(DO_META.get('public-keys').splitlines(), self.ds.get_public_ssh_keys()) - self.assertIs(type(self.ds.get_public_ssh_keys()), ListType) + self.assertIsInstance(self.ds.get_public_ssh_keys(), list) diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py index 06050bb1..aa60eb33 100644 --- a/tests/unittests/test_datasource/test_gce.py +++ b/tests/unittests/test_datasource/test_gce.py @@ -19,7 +19,7 @@ import httpretty import re from base64 import b64encode, b64decode -from urlparse import urlparse +from six.moves.urllib_parse import urlparse from cloudinit import settings from cloudinit import helpers diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index ddf77265..b79237f0 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -294,7 +294,7 @@ class TestParseShellConfig(unittest.TestCase): def populate_context_dir(path, variables): data = "# Context variables generated by OpenNebula\n" - for (k, v) in variables.iteritems(): + for (k, v) in variables.items(): data += ("%s='%s'\n" % (k.upper(), v.replace(r"'", r"'\''"))) populate_dir(path, {'context.sh': data}) diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 35d7ef5e..01b9b73e 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -22,6 +22,8 @@ # return responses. # +from __future__ import print_function + import base64 from cloudinit import helpers as c_helpers from cloudinit.sources import DataSourceSmartOS @@ -369,7 +371,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): permissions = oct(os.stat(name_f)[stat.ST_MODE])[-3:] if re.match(r'.*\/mdata-user-data$', name_f): found_new = True - print name_f + print(name_f) self.assertEquals(permissions, '400') self.assertFalse(found_new) diff --git a/tests/unittests/test_handler/test_handler_apt_configure.py b/tests/unittests/test_handler/test_handler_apt_configure.py index 2c3dad72..d72fa8c7 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure.py +++ b/tests/unittests/test_handler/test_handler_apt_configure.py @@ -62,7 +62,7 @@ class TestAptProxyConfig(unittest.TestCase): contents = str(util.read_file_or_url(self.pfile)) - for ptype, pval in values.iteritems(): + for ptype, pval in values.items(): self.assertTrue(self._search_apt_config(contents, ptype, pval)) def test_proxy_deleted(self): diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py index 07b610f7..976d8283 100644 --- a/tests/unittests/test_merging.py +++ b/tests/unittests/test_merging.py @@ -11,11 +11,13 @@ import glob import os import random import re +import six import string SOURCE_PAT = "source*.*yaml" EXPECTED_PAT = "expected%s.yaml" -TYPES = [long, int, dict, str, list, tuple, None] +TYPES = [dict, str, list, tuple, None] +TYPES.extend(six.integer_types) def _old_mergedict(src, cand): @@ -25,7 +27,7 @@ def _old_mergedict(src, cand): Nested dictionaries are merged recursively. """ if isinstance(src, dict) and isinstance(cand, dict): - for (k, v) in cand.iteritems(): + for (k, v) in cand.items(): if k not in src: src[k] = v else: @@ -42,8 +44,8 @@ def _old_mergemanydict(*args): def _random_str(rand): base = '' - for _i in xrange(rand.randint(1, 2 ** 8)): - base += rand.choice(string.letters + string.digits) + for _i in range(rand.randint(1, 2 ** 8)): + base += rand.choice(string.ascii_letters + string.digits) return base @@ -64,7 +66,7 @@ def _make_dict(current_depth, max_depth, rand): if t in [dict, list, tuple]: if t in [dict]: amount = rand.randint(0, 5) - keys = [_random_str(rand) for _i in xrange(0, amount)] + keys = [_random_str(rand) for _i in range(0, amount)] base = {} for k in keys: try: @@ -74,14 +76,14 @@ def _make_dict(current_depth, max_depth, rand): elif t in [list, tuple]: base = [] amount = rand.randint(0, 5) - for _i in xrange(0, amount): + for _i in range(0, amount): try: base.append(_make_dict(current_depth + 1, max_depth, rand)) except _NoMoreException: pass if t in [tuple]: base = tuple(base) - elif t in [long, int]: + elif t in six.integer_types: base = rand.randint(0, 2 ** 8) elif t in [str]: base = _random_str(rand) diff --git a/tools/ccfg-merge-debug b/tools/ccfg-merge-debug index 85227da7..1f08e0cb 100755 --- a/tools/ccfg-merge-debug +++ b/tools/ccfg-merge-debug @@ -51,7 +51,7 @@ def main(): c_handlers.register(ccph) called = [] - for (_ctype, mod) in c_handlers.iteritems(): + for (_ctype, mod) in c_handlers.items(): if mod in called: continue handlers.call_begin(mod, data, frequency) @@ -76,7 +76,7 @@ def main(): # Give callbacks opportunity to finalize called = [] - for (_ctype, mod) in c_handlers.iteritems(): + for (_ctype, mod) in c_handlers.items(): if mod in called: continue handlers.call_end(mod, data, frequency) -- cgit v1.2.3 From 926a3df79a10ede61967c60f48ff0670a36e689a Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Mon, 26 Jan 2015 12:41:04 -0500 Subject: More Python 3 test fixes. --- cloudinit/config/cc_write_files.py | 5 +++-- tests/unittests/test_datasource/test_opennebula.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_write_files.py b/cloudinit/config/cc_write_files.py index a73d6f4e..4b03ea91 100644 --- a/cloudinit/config/cc_write_files.py +++ b/cloudinit/config/cc_write_files.py @@ -18,6 +18,7 @@ import base64 import os +import six from cloudinit.settings import PER_INSTANCE from cloudinit import util @@ -25,7 +26,7 @@ from cloudinit import util frequency = PER_INSTANCE DEFAULT_OWNER = "root:root" -DEFAULT_PERMS = 0644 +DEFAULT_PERMS = 0o644 UNKNOWN_ENC = 'text/plain' @@ -79,7 +80,7 @@ def write_files(name, files, log): def decode_perms(perm, default, log): try: - if isinstance(perm, (int, long, float)): + if isinstance(perm, six.integer_types + (float,)): # Just 'downcast' it (if a float) return int(perm) else: diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index 1a8d2122..31c6232f 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -33,7 +33,7 @@ TEST_VARS = { } INVALID_CONTEXT = ';' -USER_DATA = '#cloud-config\napt_upgrade: true' +USER_DATA = b'#cloud-config\napt_upgrade: true' SSH_KEY = 'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460-%i' HOSTNAME = 'foo.example.com' PUBLIC_IP = '10.0.0.3' -- cgit v1.2.3 From 18b35de06432869a9d859e2978e7e9567eba66a2 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Mon, 26 Jan 2015 14:48:23 -0500 Subject: Another handling of b64decode. Also, restore Python 2 compatibility. --- cloudinit/config/cc_seed_random.py | 8 +++++++- cloudinit/sources/DataSourceOpenNebula.py | 11 ++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py index 3b7235bf..981e1b08 100644 --- a/cloudinit/config/cc_seed_random.py +++ b/cloudinit/config/cc_seed_random.py @@ -38,7 +38,13 @@ def _decode(data, encoding=None): if not encoding or encoding.lower() in ['raw']: return data elif encoding.lower() in ['base64', 'b64']: - return base64.b64decode(data) + # Try to give us a native string in both Python 2 and 3, and remember + # that b64decode() returns bytes in Python 3. + decoded = base64.b64decode(data) + try: + return decoded.decode('utf-8') + except UnicodeDecodeError: + return decoded elif encoding.lower() in ['gzip', 'gz']: return util.decomp_gzip(data, quiet=False) else: diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 691b39f8..6da569ec 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -25,6 +25,7 @@ # along with this program. If not, see . import base64 +import codecs import os import pwd import re @@ -34,6 +35,8 @@ from cloudinit import log as logging from cloudinit import sources from cloudinit import util +import six + LOG = logging.getLogger(__name__) DEFAULT_IID = "iid-dsopennebula" @@ -43,6 +46,12 @@ CONTEXT_DISK_FILES = ["context.sh"] VALID_DSMODES = ("local", "net", "disabled") +def utf8_open(path): + if six.PY3: + return open(path, 'r', encoding='utf-8') + return codecs.open(path, 'r', encoding='utf-8') + + class DataSourceOpenNebula(sources.DataSource): def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) @@ -380,7 +389,7 @@ def read_context_disk_dir(source_dir, asuser=None): "does not exist", asuser) try: path = os.path.join(source_dir, 'context.sh') - with open(path, 'r', encoding='utf-8') as f: + with utf8_open(path) as f: content = f.read().strip() context = parse_shell_config(content, asuser=asuser) -- cgit v1.2.3 From 1e76dad45e3bce4dac5a638dda970fc02a044dbb Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Tue, 27 Jan 2015 14:24:22 -0500 Subject: Respond to review: - Remove str() wrappers to second argument to write_files() where it is no longer necessary. Also: Fixed a couple of other octal literals which clearly weren't being tested. --- cloudinit/config/cc_chef.py | 2 +- cloudinit/config/cc_puppet.py | 2 +- cloudinit/config/cc_rightscale_userdata.py | 2 +- cloudinit/config/cc_runcmd.py | 2 +- cloudinit/config/cc_salt_minion.py | 2 +- cloudinit/distros/arch.py | 2 +- cloudinit/distros/gentoo.py | 2 +- cloudinit/distros/rhel_util.py | 2 +- cloudinit/user_data.py | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 584199e5..e18c5405 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -302,7 +302,7 @@ def install_chef(cloud, chef_cfg, log): with util.tempdir() as tmpd: # Use tmpdir over tmpfile to avoid 'text file busy' on execute tmpf = "%s/chef-omnibus-install" % tmpd - util.write_file(tmpf, str(content), mode=0o700) + util.write_file(tmpf, content, mode=0o700) util.subp([tmpf], capture=False) else: log.warn("Unknown chef install type '%s'", install_type) diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py index 6f1b3c57..4501598e 100644 --- a/cloudinit/config/cc_puppet.py +++ b/cloudinit/config/cc_puppet.py @@ -91,7 +91,7 @@ def handle(name, cfg, cloud, log, _args): util.chownbyname(PUPPET_SSL_DIR, 'puppet', 'root') util.ensure_dir(PUPPET_SSL_CERT_DIR) util.chownbyname(PUPPET_SSL_CERT_DIR, 'puppet', 'root') - util.write_file(PUPPET_SSL_CERT_PATH, str(cfg)) + util.write_file(PUPPET_SSL_CERT_PATH, cfg) util.chownbyname(PUPPET_SSL_CERT_PATH, 'puppet', 'root') else: # Iterate throug the config items, we'll use ConfigParser.set diff --git a/cloudinit/config/cc_rightscale_userdata.py b/cloudinit/config/cc_rightscale_userdata.py index 7d2ec10a..1f769c0a 100644 --- a/cloudinit/config/cc_rightscale_userdata.py +++ b/cloudinit/config/cc_rightscale_userdata.py @@ -82,7 +82,7 @@ def handle(name, _cfg, cloud, log, _args): resp = uhelp.readurl(url) # Ensure its a valid http response (and something gotten) if resp.ok() and resp.contents: - util.write_file(fname, str(resp), mode=0700) + util.write_file(fname, resp, mode=0o700) wrote_fns.append(fname) except Exception as e: captured_excps.append(e) diff --git a/cloudinit/config/cc_runcmd.py b/cloudinit/config/cc_runcmd.py index 598c3a3e..66dc3363 100644 --- a/cloudinit/config/cc_runcmd.py +++ b/cloudinit/config/cc_runcmd.py @@ -33,6 +33,6 @@ def handle(name, cfg, cloud, log, _args): cmd = cfg["runcmd"] try: content = util.shellify(cmd) - util.write_file(out_fn, content, 0700) + util.write_file(out_fn, content, 0o700) except: util.logexc(log, "Failed to shellify %s into file %s", cmd, out_fn) diff --git a/cloudinit/config/cc_salt_minion.py b/cloudinit/config/cc_salt_minion.py index 53013dcb..f5786a31 100644 --- a/cloudinit/config/cc_salt_minion.py +++ b/cloudinit/config/cc_salt_minion.py @@ -47,7 +47,7 @@ def handle(name, cfg, cloud, log, _args): # ... copy the key pair if specified if 'public_key' in salt_cfg and 'private_key' in salt_cfg: pki_dir = salt_cfg.get('pki_dir', '/etc/salt/pki') - with util.umask(077): + with util.umask(0o77): util.ensure_dir(pki_dir) pub_name = os.path.join(pki_dir, 'minion.pub') pem_name = os.path.join(pki_dir, 'minion.pem') diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py index e540e0bc..45fcf26f 100644 --- a/cloudinit/distros/arch.py +++ b/cloudinit/distros/arch.py @@ -129,7 +129,7 @@ class Distro(distros.Distro): if not conf: conf = HostnameConf('') conf.set_hostname(your_hostname) - util.write_file(out_fn, str(conf), 0644) + util.write_file(out_fn, conf, 0o644) def _read_system_hostname(self): sys_hostname = self._read_hostname(self.hostname_conf_fn) diff --git a/cloudinit/distros/gentoo.py b/cloudinit/distros/gentoo.py index 09dd0d73..9e80583c 100644 --- a/cloudinit/distros/gentoo.py +++ b/cloudinit/distros/gentoo.py @@ -108,7 +108,7 @@ class Distro(distros.Distro): if not conf: conf = HostnameConf('') conf.set_hostname(your_hostname) - util.write_file(out_fn, str(conf), 0644) + util.write_file(out_fn, conf, 0o644) def _read_system_hostname(self): sys_hostname = self._read_hostname(self.hostname_conf_fn) diff --git a/cloudinit/distros/rhel_util.py b/cloudinit/distros/rhel_util.py index 903d7793..84aad623 100644 --- a/cloudinit/distros/rhel_util.py +++ b/cloudinit/distros/rhel_util.py @@ -86,4 +86,4 @@ def update_resolve_conf_file(fn, dns_servers, search_servers): r_conf.add_search_domain(s) except ValueError: util.logexc(LOG, "Failed at adding search domain %s", s) - util.write_file(fn, str(r_conf), 0o644) + util.write_file(fn, r_conf, 0o644) diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index 3f860f3b..bf5642a5 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -248,7 +248,7 @@ class UserDataProcessor(object): resp = util.read_file_or_url(include_url, ssl_details=self.ssl_details) if include_once_on and resp.ok(): - util.write_file(include_once_fn, str(resp), mode=0o600) + util.write_file(include_once_fn, resp, mode=0o600) if resp.ok(): content = str(resp) else: -- cgit v1.2.3 From 6e742d20e9ed56498925c7c850cd5da65d063b4b Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Tue, 27 Jan 2015 15:03:52 -0500 Subject: Respond to review: - Refactor both the base64 encoding and decoding into utility functions. Also: - Mechanically fix some other broken untested code. --- cloudinit/config/cc_seed_random.py | 8 +------ cloudinit/config/cc_ssh_authkey_fingerprints.py | 2 +- cloudinit/sources/DataSourceOpenNebula.py | 7 +----- cloudinit/sources/DataSourceSmartOS.py | 11 +-------- cloudinit/util.py | 20 ++++++++++++++++ tests/unittests/test_datasource/test_azure.py | 28 ++++++++-------------- tests/unittests/test_datasource/test_opennebula.py | 11 ++------- tests/unittests/test_datasource/test_smartos.py | 14 ++++------- .../test_handler/test_handler_seed_random.py | 12 ++-------- 9 files changed, 42 insertions(+), 71 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py index 981e1b08..bb64b0f5 100644 --- a/cloudinit/config/cc_seed_random.py +++ b/cloudinit/config/cc_seed_random.py @@ -38,13 +38,7 @@ def _decode(data, encoding=None): if not encoding or encoding.lower() in ['raw']: return data elif encoding.lower() in ['base64', 'b64']: - # Try to give us a native string in both Python 2 and 3, and remember - # that b64decode() returns bytes in Python 3. - decoded = base64.b64decode(data) - try: - return decoded.decode('utf-8') - except UnicodeDecodeError: - return decoded + return util.b64d(data) elif encoding.lower() in ['gzip', 'gz']: return util.decomp_gzip(data, quiet=False) else: diff --git a/cloudinit/config/cc_ssh_authkey_fingerprints.py b/cloudinit/config/cc_ssh_authkey_fingerprints.py index 51580633..6ce831bc 100644 --- a/cloudinit/config/cc_ssh_authkey_fingerprints.py +++ b/cloudinit/config/cc_ssh_authkey_fingerprints.py @@ -32,7 +32,7 @@ from cloudinit import util def _split_hash(bin_hash): split_up = [] - for i in xrange(0, len(bin_hash), 2): + for i in range(0, len(bin_hash), 2): split_up.append(bin_hash[i:i + 2]) return split_up diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index a0275cda..61709c1b 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -426,12 +426,7 @@ def read_context_disk_dir(source_dir, asuser=None): context.get('USER_DATA_ENCODING')) if encoding == "base64": try: - userdata = base64.b64decode(results['userdata']) - # In Python 3 we still expect a str, but b64decode will return - # bytes. Convert to str. - if isinstance(userdata, bytes): - userdata = userdata.decode('utf-8') - results['userdata'] = userdata + results['userdata'] = util.b64d(results['userdata']) except TypeError: LOG.warn("Failed base64 decoding of userdata") diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index f59ad3d6..9d48beab 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -351,16 +351,7 @@ def query_data(noun, seed_device, seed_timeout, strip=False, default=None, if b64: try: - # Generally, we want native strings in the values. Python 3's - # b64decode will return bytes though, so decode them to utf-8 if - # possible. If that fails, return the bytes. - decoded = base64.b64decode(resp) - try: - if isinstance(decoded, bytes): - return decoded.decode('utf-8') - except UnicodeDecodeError: - pass - return decoded + return util.b64d(resp) # Bogus input produces different errors in Python 2 and 3; catch both. except (TypeError, binascii.Error): LOG.warn("Failed base64 decoding key '%s'", noun) diff --git a/cloudinit/util.py b/cloudinit/util.py index 766f8e32..8916cc11 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -44,6 +44,7 @@ import sys import tempfile import time +from base64 import b64decode, b64encode from six.moves.urllib import parse as urlparse import six @@ -90,6 +91,25 @@ def encode_text(text, encoding='utf-8'): return text return text.encode(encoding) + +def b64d(source): + # Base64 decode some data, accepting bytes or unicode/str, and returning + # str/unicode if the result is utf-8 compatible, otherwise returning bytes. + decoded = b64decode(source) + if isinstance(decoded, bytes): + try: + return decoded.decode('utf-8') + except UnicodeDecodeError: + return decoded + +def b64e(source): + # Base64 encode some data, accepting bytes or unicode/str, and returning + # str/unicode if the result is utf-8 compatible, otherwise returning bytes. + if not isinstance(source, bytes): + source = source.encode('utf-8') + return b64encode(source).decode('utf-8') + + # Path for DMI Data DMI_SYS_PATH = "/sys/class/dmi/id" diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 97a53bee..965bce4b 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -1,5 +1,5 @@ from cloudinit import helpers -from cloudinit.util import load_file +from cloudinit.util import b64e, load_file from cloudinit.sources import DataSourceAzure from ..helpers import TestCase, populate_dir @@ -12,7 +12,6 @@ try: except ImportError: from contextlib2 import ExitStack -import base64 import crypt import os import stat @@ -22,13 +21,6 @@ import tempfile import unittest -def b64(source): - # In Python 3, b64encode only accepts bytes and returns bytes. - if not isinstance(source, bytes): - source = source.encode('utf-8') - return base64.b64encode(source).decode('us-ascii') - - def construct_valid_ovf_env(data=None, pubkeys=None, userdata=None): if data is None: data = {'HostName': 'FOOHOST'} @@ -58,7 +50,7 @@ def construct_valid_ovf_env(data=None, pubkeys=None, userdata=None): content += "<%s%s>%s\n" % (key, attrs, val, key) if userdata: - content += "%s\n" % (b64(userdata)) + content += "%s\n" % (b64e(userdata)) if pubkeys: content += "\n" @@ -189,7 +181,7 @@ class TestAzureDataSource(TestCase): # set dscfg in via base64 encoded yaml cfg = {'agent_command': "my_command"} odata = {'HostName': "myhost", 'UserName': "myuser", - 'dscfg': {'text': b64(yaml.dump(cfg)), + 'dscfg': {'text': b64e(yaml.dump(cfg)), 'encoding': 'base64'}} data = {'ovfcontent': construct_valid_ovf_env(data=odata)} @@ -241,7 +233,7 @@ class TestAzureDataSource(TestCase): def test_userdata_found(self): mydata = "FOOBAR" - odata = {'UserData': b64(mydata)} + odata = {'UserData': b64e(mydata)} data = {'ovfcontent': construct_valid_ovf_env(data=odata)} dsrc = self._get_ds(data) @@ -289,7 +281,7 @@ class TestAzureDataSource(TestCase): 'command': 'my-bounce-command', 'hostname_command': 'my-hostname-command'}} odata = {'HostName': "xhost", - 'dscfg': {'text': b64(yaml.dump(cfg)), + 'dscfg': {'text': b64e(yaml.dump(cfg)), 'encoding': 'base64'}} data = {'ovfcontent': construct_valid_ovf_env(data=odata)} self._get_ds(data).get_data() @@ -304,7 +296,7 @@ class TestAzureDataSource(TestCase): # config specifying set_hostname off should not bounce cfg = {'set_hostname': False} odata = {'HostName': "xhost", - 'dscfg': {'text': b64(yaml.dump(cfg)), + 'dscfg': {'text': b64e(yaml.dump(cfg)), 'encoding': 'base64'}} data = {'ovfcontent': construct_valid_ovf_env(data=odata)} self._get_ds(data).get_data() @@ -333,7 +325,7 @@ class TestAzureDataSource(TestCase): # Make sure that user can affect disk aliases dscfg = {'disk_aliases': {'ephemeral0': '/dev/sdc'}} odata = {'HostName': "myhost", 'UserName': "myuser", - 'dscfg': {'text': b64(yaml.dump(dscfg)), + 'dscfg': {'text': b64e(yaml.dump(dscfg)), 'encoding': 'base64'}} usercfg = {'disk_setup': {'/dev/sdc': {'something': '...'}, 'ephemeral0': False}} @@ -370,7 +362,7 @@ class TestAzureDataSource(TestCase): def test_existing_ovf_same(self): # waagent/SharedConfig left alone if found ovf-env.xml same as cached - odata = {'UserData': b64("SOMEUSERDATA")} + odata = {'UserData': b64e("SOMEUSERDATA")} data = {'ovfcontent': construct_valid_ovf_env(data=odata)} populate_dir(self.waagent_d, @@ -394,9 +386,9 @@ class TestAzureDataSource(TestCase): # 'get_data' should remove SharedConfig.xml in /var/lib/waagent # if ovf-env.xml differs. cached_ovfenv = construct_valid_ovf_env( - {'userdata': b64("FOO_USERDATA")}) + {'userdata': b64e("FOO_USERDATA")}) new_ovfenv = construct_valid_ovf_env( - {'userdata': b64("NEW_USERDATA")}) + {'userdata': b64e("NEW_USERDATA")}) populate_dir(self.waagent_d, {'ovf-env.xml': cached_ovfenv, diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index e5a4bd18..27adf21b 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -3,19 +3,12 @@ from cloudinit.sources import DataSourceOpenNebula as ds from cloudinit import util from ..helpers import TestCase, populate_dir -from base64 import b64encode import os import pwd import shutil import tempfile import unittest -def b64(source): - # In Python 3, b64encode only accepts bytes and returns bytes. - if not isinstance(source, bytes): - source = source.encode('utf-8') - return b64encode(source).decode('us-ascii') - TEST_VARS = { 'VAR1': 'single', @@ -186,7 +179,7 @@ class TestOpenNebulaDataSource(TestCase): self.assertEqual(USER_DATA, results['userdata']) def test_user_data_encoding_required_for_decode(self): - b64userdata = b64(USER_DATA) + b64userdata = util.b64e(USER_DATA) for k in ('USER_DATA', 'USERDATA'): my_d = os.path.join(self.tmp, k) populate_context_dir(my_d, {k: b64userdata}) @@ -198,7 +191,7 @@ class TestOpenNebulaDataSource(TestCase): def test_user_data_base64_encoding(self): for k in ('USER_DATA', 'USERDATA'): my_d = os.path.join(self.tmp, k) - populate_context_dir(my_d, {k: b64(USER_DATA), + populate_context_dir(my_d, {k: util.b64e(USER_DATA), 'USERDATA_ENCODING': 'base64'}) results = ds.read_context_disk_dir(my_d) diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index b5ebf94d..8b62b1b1 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -24,9 +24,9 @@ from __future__ import print_function -import base64 from cloudinit import helpers as c_helpers from cloudinit.sources import DataSourceSmartOS +from cloudinit.util import b64e from .. import helpers import os import os.path @@ -36,12 +36,6 @@ import tempfile import stat import uuid -def b64(source): - # In Python 3, b64encode only accepts bytes and returns bytes. - if not isinstance(source, bytes): - source = source.encode('utf-8') - return base64.b64encode(source).decode('us-ascii') - MOCK_RETURNS = { 'hostname': 'test-host', @@ -239,7 +233,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): my_returns = MOCK_RETURNS.copy() my_returns['base64_all'] = "true" for k in ('hostname', 'cloud-init:user-data'): - my_returns[k] = b64(my_returns[k]) + my_returns[k] = b64e(my_returns[k]) dsrc = self._get_ds(mockdata=my_returns) ret = dsrc.get_data() @@ -260,7 +254,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): my_returns['b64-cloud-init:user-data'] = "true" my_returns['b64-hostname'] = "true" for k in ('hostname', 'cloud-init:user-data'): - my_returns[k] = b64(my_returns[k]) + my_returns[k] = b64e(my_returns[k]) dsrc = self._get_ds(mockdata=my_returns) ret = dsrc.get_data() @@ -276,7 +270,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): my_returns = MOCK_RETURNS.copy() my_returns['base64_keys'] = 'hostname,ignored' for k in ('hostname',): - my_returns[k] = b64(my_returns[k]) + my_returns[k] = b64e(my_returns[k]) dsrc = self._get_ds(mockdata=my_returns) ret = dsrc.get_data() diff --git a/tests/unittests/test_handler/test_handler_seed_random.py b/tests/unittests/test_handler/test_handler_seed_random.py index d3f18fa0..0bcdcb31 100644 --- a/tests/unittests/test_handler/test_handler_seed_random.py +++ b/tests/unittests/test_handler/test_handler_seed_random.py @@ -18,7 +18,6 @@ from cloudinit.config import cc_seed_random -import base64 import gzip import tempfile @@ -38,13 +37,6 @@ import logging LOG = logging.getLogger(__name__) -def b64(source): - # In Python 3, b64encode only accepts bytes and returns bytes. - if not isinstance(source, bytes): - source = source.encode('utf-8') - return base64.b64encode(source).decode('us-ascii') - - class TestRandomSeed(t_help.TestCase): def setUp(self): super(TestRandomSeed, self).setUp() @@ -141,7 +133,7 @@ class TestRandomSeed(t_help.TestCase): self.assertEquals("big-toe", contents) def test_append_random_base64(self): - data = b64('bubbles') + data = util.b64e('bubbles') cfg = { 'random_seed': { 'file': self._seed_file, @@ -154,7 +146,7 @@ class TestRandomSeed(t_help.TestCase): self.assertEquals("bubbles", contents) def test_append_random_b64(self): - data = b64('kit-kat') + data = util.b64e('kit-kat') cfg = { 'random_seed': { 'file': self._seed_file, -- cgit v1.2.3 From c8e75ddf808f4ac017f609bafc101648b7568935 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 10 Feb 2015 17:17:48 +0000 Subject: fix use of 'letters' and translate --- cloudinit/config/cc_set_passwords.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index 4ca85e21..8b705d90 100644 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -28,11 +28,11 @@ from cloudinit import distros as ds from cloudinit import ssh_util from cloudinit import util -from string import letters, digits +from string import ascii_letters, digits # We are removing certain 'painful' letters/numbers -PW_SET = (letters.translate(None, 'loLOI') + - digits.translate(None, '01')) +PW_SET = (''.join([x for x in ascii_letters + digits + if x not in 'loLOI01'])) def handle(_name, cfg, cloud, log, args): -- cgit v1.2.3 From 888db3e6bb9076973d2f6a73e0c4f691caa89603 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 10 Feb 2015 17:25:01 +0000 Subject: fix parse_qs usage --- cloudinit/config/cc_rightscale_userdata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_rightscale_userdata.py b/cloudinit/config/cc_rightscale_userdata.py index 1f769c0a..24880d13 100644 --- a/cloudinit/config/cc_rightscale_userdata.py +++ b/cloudinit/config/cc_rightscale_userdata.py @@ -41,7 +41,7 @@ from cloudinit.settings import PER_INSTANCE from cloudinit import url_helper as uhelp from cloudinit import util -from urlparse import parse_qs +from six.moves.urllib_parse import parse_qs frequency = PER_INSTANCE -- cgit v1.2.3 From b8eb55f9acdf92a58d3c72b0c5e5437c4f0272c1 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 10 Feb 2015 21:33:11 +0000 Subject: use encode_text --- bin/cloud-init | 2 +- cloudinit/config/cc_bootcmd.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'cloudinit/config') diff --git a/bin/cloud-init b/bin/cloud-init index d67b2b6d..6c83c2e7 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -428,7 +428,7 @@ def atomic_write_json(path, data): try: tf = tempfile.NamedTemporaryFile(dir=os.path.dirname(path), delete=False) - tf.write((json.dumps(data, indent=1) + "\n").encode()) + tf.write(util.encode_text(json.dumps(data, indent=1) + "\n")) tf.close() os.rename(tf.name, path) except Exception as e: diff --git a/cloudinit/config/cc_bootcmd.py b/cloudinit/config/cc_bootcmd.py index 3ac22967..a295cc4e 100644 --- a/cloudinit/config/cc_bootcmd.py +++ b/cloudinit/config/cc_bootcmd.py @@ -36,7 +36,7 @@ def handle(name, cfg, cloud, log, _args): with util.ExtendedTemporaryFile(suffix=".sh") as tmpf: try: content = util.shellify(cfg["bootcmd"]) - tmpf.write(content) + tmpf.write(util.encode_text(content)) tmpf.flush() except: util.logexc(log, "Failed to shellify bootcmd") -- cgit v1.2.3 From f62b86bd45c8df78ada32ab4040a639c9d096202 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 11 Feb 2015 01:09:34 +0000 Subject: fix random_seed module --- cloudinit/config/cc_seed_random.py | 16 ++++++++-------- cloudinit/sources/DataSourceAzure.py | 3 ++- cloudinit/util.py | 16 +++++++++------- 3 files changed, 19 insertions(+), 16 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py index bb64b0f5..3288a853 100644 --- a/cloudinit/config/cc_seed_random.py +++ b/cloudinit/config/cc_seed_random.py @@ -22,7 +22,7 @@ import base64 import os -from six import StringIO +from six import BytesIO from cloudinit.settings import PER_INSTANCE from cloudinit import log as logging @@ -34,13 +34,13 @@ LOG = logging.getLogger(__name__) def _decode(data, encoding=None): if not data: - return '' + return b'' if not encoding or encoding.lower() in ['raw']: - return data + return util.encode_text(data) elif encoding.lower() in ['base64', 'b64']: - return util.b64d(data) + return base64.b64decode(data) elif encoding.lower() in ['gzip', 'gz']: - return util.decomp_gzip(data, quiet=False) + return util.decomp_gzip(data, quiet=False, decode=None) else: raise IOError("Unknown random_seed encoding: %s" % (encoding)) @@ -65,9 +65,9 @@ def handle_random_seed_command(command, required, env=None): def handle(name, cfg, cloud, log, _args): mycfg = cfg.get('random_seed', {}) seed_path = mycfg.get('file', '/dev/urandom') - seed_data = mycfg.get('data', '') + seed_data = mycfg.get('data', b'') - seed_buf = StringIO() + seed_buf = BytesIO() if seed_data: seed_buf.write(_decode(seed_data, encoding=mycfg.get('encoding'))) @@ -75,7 +75,7 @@ def handle(name, cfg, cloud, log, _args): # openstack meta_data.json metadata = cloud.datasource.metadata if metadata and 'random_seed' in metadata: - seed_buf.write(metadata['random_seed']) + seed_buf.write(util.encode_text(metadata['random_seed'])) seed_data = seed_buf.getvalue() if len(seed_data): diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 29ae2c22..c599d50f 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -124,7 +124,8 @@ class DataSourceAzureNet(sources.DataSource): LOG.debug("using files cached in %s", ddir) # azure / hyper-v provides random data here - seed = util.load_file("/sys/firmware/acpi/tables/OEM0", quiet=True) + seed = util.load_file("/sys/firmware/acpi/tables/OEM0", + quiet=True, decode=False) if seed: self.metadata['random_seed'] = seed diff --git a/cloudinit/util.py b/cloudinit/util.py index 3a921afe..c998154a 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -96,11 +96,10 @@ def b64d(source): # Base64 decode some data, accepting bytes or unicode/str, and returning # str/unicode if the result is utf-8 compatible, otherwise returning bytes. decoded = b64decode(source) - if isinstance(decoded, bytes): - try: - return decoded.decode('utf-8') - except UnicodeDecodeError: - return decoded + try: + return decoded.decode('utf-8') + except UnicodeDecodeError: + return decoded def b64e(source): # Base64 encode some data, accepting bytes or unicode/str, and returning @@ -354,11 +353,14 @@ def clean_filename(fn): return fn -def decomp_gzip(data, quiet=True): +def decomp_gzip(data, quiet=True, decode=True): try: buf = six.BytesIO(encode_text(data)) with contextlib.closing(gzip.GzipFile(None, "rb", 1, buf)) as gh: - return decode_binary(gh.read()) + if decode: + return decode_binary(gh.read()) + else: + return gh.read() except Exception as e: if quiet: return data -- cgit v1.2.3 From f67d459da3d81f3b4c4c4171eaf5940dbc73ea25 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 10 Feb 2015 20:50:45 -0500 Subject: pep8 --- cloudinit/config/cc_set_passwords.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index 8b705d90..0c315361 100644 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -31,7 +31,7 @@ from cloudinit import util from string import ascii_letters, digits # We are removing certain 'painful' letters/numbers -PW_SET = (''.join([x for x in ascii_letters + digits +PW_SET = (''.join([x for x in ascii_letters + digits if x not in 'loLOI01'])) -- cgit v1.2.3