diff options
| -rw-r--r-- | cloudinit/config/cc_growpart.py | 3 | ||||
| -rw-r--r-- | cloudinit/distros/freebsd.py | 47 | ||||
| -rw-r--r-- | cloudinit/ec2_utils.py | 27 | ||||
| -rw-r--r-- | cloudinit/netinfo.py | 30 | ||||
| -rw-r--r-- | cloudinit/url_helper.py | 17 | ||||
| -rw-r--r-- | cloudinit/user_data.py | 3 | ||||
| -rw-r--r-- | cloudinit/util.py | 17 | ||||
| -rw-r--r-- | doc/rtd/conf.py | 3 | ||||
| -rwxr-xr-x | setup.py | 6 | ||||
| -rw-r--r-- | tests/unittests/test_datasource/test_configdrive.py | 2 | ||||
| -rw-r--r-- | tests/unittests/test_datasource/test_maas.py | 3 | ||||
| -rw-r--r-- | tests/unittests/test_ec2_util.py | 8 | ||||
| -rwxr-xr-x | tools/run-pep8 | 11 | ||||
| -rwxr-xr-x | tools/run-pylint | 3 |
14 files changed, 103 insertions, 77 deletions
diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index b81951ad..f52c41f0 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -223,7 +223,8 @@ def resize_devices(resizer, devices): "stat of '%s' failed: %s" % (blockdev, e),)) continue - if not stat.S_ISBLK(statret.st_mode) and not stat.S_ISCHR(statret.st_mode): + if (not stat.S_ISBLK(statret.st_mode) and + not stat.S_ISCHR(statret.st_mode)): info.append((devent, RESIZE.SKIPPED, "device '%s' not a block device" % blockdev,)) continue diff --git a/cloudinit/distros/freebsd.py b/cloudinit/distros/freebsd.py index d28860eb..4c0c6d29 100644 --- a/cloudinit/distros/freebsd.py +++ b/cloudinit/distros/freebsd.py @@ -16,15 +16,14 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +import re + from cloudinit import distros from cloudinit import helpers from cloudinit import log as logging -from cloudinit import netinfo from cloudinit import ssh_util from cloudinit import util -from cloudinit.settings import PER_INSTANCE - LOG = logging.getLogger(__name__) @@ -39,26 +38,27 @@ class Distro(distros.Distro): # Updates a key in /etc/rc.conf. def updatercconf(self, key, value): - LOG.debug("updatercconf: %s => %s" % (key, value)) + LOG.debug("updatercconf: %s => %s", key, value) conf = self.loadrcconf() configchanged = False for item in conf: if item == key and conf[item] != value: conf[item] = value - LOG.debug("[rc.conf]: Value %s for key %s needs to be changed" % (value, key)) + LOG.debug("[rc.conf]: Value %s for key %s needs to be changed", + value, key) configchanged = True if configchanged: LOG.debug("Writing new /etc/rc.conf file") - with open('/etc/rc.conf', 'w') as file: + with open('/etc/rc.conf', 'w') as fp: for keyval in conf.items(): - file.write("%s=%s\n" % keyval) + fp.write("%s=%s\n" % keyval) # Load the contents of /etc/rc.conf and store all keys in a dict. def loadrcconf(self): conf = {} - with open("/etc/rc.conf") as file: - for line in file: + with open("/etc/rc.conf") as fp: + for line in fp: tok = line.split('=') conf[tok[0]] = tok[1].rstrip() return conf @@ -75,7 +75,7 @@ class Distro(distros.Distro): sys_hostname = self._read_hostname() return ('rc.conf', sys_hostname) - def _read_hostname(self, default=None): + def _read_hostname(self, filename, default=None): hostname = None try: hostname = self.readrcconf('hostname') @@ -90,17 +90,17 @@ class Distro(distros.Distro): return fqdn return hostname - def _write_hostname(self, your_hostname, out_fn): - self.updatercconf('hostname', your_hostname) + def _write_hostname(self, hostname, filename): + self.updatercconf('hostname', hostname) def create_group(self, name, members): group_add_cmd = ['pw', '-n', name] if util.is_group(name): - LOG.warn("Skipping creation of existing group '%s'" % name) + LOG.warn("Skipping creation of existing group '%s'", name) else: try: util.subp(group_add_cmd) - LOG.info("Created new group %s" % name) + LOG.info("Created new group %s", name) except Exception: util.logexc("Failed to create group %s", name) @@ -111,11 +111,11 @@ class Distro(distros.Distro): "; user does not exist.", member, name) continue util.subp(['pw', 'usermod', '-n', name, '-G', member]) - LOG.info("Added user '%s' to group '%s'" % (member, name)) + LOG.info("Added user '%s' to group '%s'", member, name) def add_user(self, name, **kwargs): if util.is_user(name): - LOG.info("User %s already exists, skipping." % name) + LOG.info("User %s already exists, skipping.", name) return False adduser_cmd = ['pw', 'useradd', '-n', name] @@ -170,7 +170,7 @@ class Distro(distros.Distro): raise e # TODO: - def set_passwd(self, name, **kwargs): + def set_passwd(self, user, passwd, hashed=False): return False def lock_passwd(self, name): @@ -182,7 +182,7 @@ class Distro(distros.Distro): # TODO: def write_sudo_rules(self, name, rules, sudo_file=None): - LOG.debug("[write_sudo_rules] Name: %s" % name) + LOG.debug("[write_sudo_rules] Name: %s", name) def create_user(self, name, **kwargs): self.add_user(name, **kwargs) @@ -217,7 +217,8 @@ class Distro(distros.Distro): origconf = open(loginconf, 'r') for line in origconf: - newconf.write(re.sub('^default:', r'default:lang=%s:' % locale, line)) + newconf.write(re.sub(r'^default:', + r'default:lang=%s:' % locale, line)) newconf.close() origconf.close() # Make a backup of login.conf. @@ -233,14 +234,14 @@ class Distro(distros.Distro): util.logexc("Failed to apply locale %s", locale) copyfile(backupconf, loginconf) - def install_packages(): + def install_packages(self, pkglist): return - def package_command(): + def package_command(self, cmd, args=None, pkgs=None): return - def set_timezone(): + def set_timezone(self, tz): return - def update_package_sources(): + def update_package_sources(self): return diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index 92a22747..7f4c0443 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -16,6 +16,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +import httplib from urlparse import (urlparse, urlunparse) import functools @@ -23,9 +24,11 @@ import json import urllib from cloudinit import log as logging +from cloudinit import url_helper from cloudinit import util LOG = logging.getLogger(__name__) +SKIP_USERDATA_CODES = frozenset([httplib.NOT_FOUND]) def maybe_json_object(text): @@ -138,20 +141,38 @@ class MetadataMaterializer(object): return joined +def _skip_retry_on_codes(status_codes, _request_args, cause): + """Returns if a request should retry based on a given set of codes that + case retrying to be stopped/skipped. + """ + if cause.code in status_codes: + return False + return True + + def get_instance_userdata(api_version='latest', metadata_address='http://169.254.169.254', ssl_details=None, timeout=5, retries=5): ud_url = combine_url(metadata_address, api_version) ud_url = combine_url(ud_url, 'user-data') + user_data = '' try: + # It is ok for userdata to not exist (thats why we are stopping if + # NOT_FOUND occurs) and just in that case returning an empty string. + exception_cb = functools.partial(_skip_retry_on_codes, + SKIP_USERDATA_CODES) response = util.read_file_or_url(ud_url, ssl_details=ssl_details, timeout=timeout, - retries=retries) - return str(response) + retries=retries, + exception_cb=exception_cb) + user_data = str(response) + except url_helper.UrlError as e: + if e.code not in SKIP_USERDATA_CODES: + util.logexc(LOG, "Failed fetching userdata from url %s", ud_url) except Exception: util.logexc(LOG, "Failed fetching userdata from url %s", ud_url) - return '' + return user_data def get_instance_metadata(api_version='latest', diff --git a/cloudinit/netinfo.py b/cloudinit/netinfo.py index 63f720e4..ac3c011f 100644 --- a/cloudinit/netinfo.py +++ b/cloudinit/netinfo.py @@ -44,7 +44,7 @@ def netdev_info(empty=""): # If the output of ifconfig doesn't contain the required info in the # obvious place, use a regex filter to be sure. elif len(toks) > 1: - if re.search("flags=\d+<up,", toks[1]): + if re.search(r"flags=\d+<up,", toks[1]): devs[curdev]['up'] = True fieldpost = "" @@ -58,11 +58,8 @@ def netdev_info(empty=""): except IndexError: pass - """ - Couple the different items we're interested in with the correct field - since FreeBSD/CentOS/Fedora differ in the output. - """ - + # Couple the different items we're interested in with the correct + # field since FreeBSD/CentOS/Fedora differ in the output. ifconfigfields = { "addr:": "addr", "inet": "addr", "bcast:": "bcast", "broadcast": "bcast", @@ -98,17 +95,16 @@ def route_info(): continue toks = line.split() - """ - FreeBSD shows 6 items in the routing table: - Destination Gateway Flags Refs Use Netif Expire - default 10.65.0.1 UGS 0 34920 vtnet0 - - Linux netstat shows 2 more: - Destination Gateway Genmask Flags MSS Window irtt Iface - 0.0.0.0 10.65.0.1 0.0.0.0 UG 0 0 0 eth0 - """ - - if len(toks) < 6 or toks[0] == "Kernel" or toks[0] == "Destination" or toks[0] == "Internet" or toks[0] == "Internet6" or toks[0] == "Routing": + # FreeBSD shows 6 items in the routing table: + # Destination Gateway Flags Refs Use Netif Expire + # default 10.65.0.1 UGS 0 34920 vtnet0 + # + # Linux netstat shows 2 more: + # Destination Gateway Genmask Flags MSS Window irtt Iface + # 0.0.0.0 10.65.0.1 0.0.0.0 UG 0 0 0 eth0 + if (len(toks) < 6 or toks[0] == "Kernel" or + toks[0] == "Destination" or toks[0] == "Internet" or + toks[0] == "Internet6" or toks[0] == "Routing"): continue if len(toks) < 8: diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 19a30409..227983f3 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -103,7 +103,7 @@ class UrlError(IOError): def readurl(url, data=None, timeout=None, retries=0, sec_between=1, headers=None, headers_cb=None, ssl_details=None, - check_status=True, allow_redirects=True): + check_status=True, allow_redirects=True, exception_cb=None): url = _cleanurl(url) req_args = { 'url': url, @@ -163,14 +163,13 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, # Handle retrying ourselves since the built-in support # doesn't handle sleeping between tries... for i in range(0, manual_tries): + req_args['headers'] = headers_cb(url) + filtered_req_args = {} + for (k, v) in req_args.items(): + if k == 'data': + continue + filtered_req_args[k] = v try: - req_args['headers'] = headers_cb(url) - filtered_req_args = {} - for (k, v) in req_args.items(): - if k == 'data': - continue - filtered_req_args[k] = v - LOG.debug("[%s/%s] open '%s' with %s configuration", i, manual_tries, url, filtered_req_args) @@ -196,6 +195,8 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, # ssl exceptions are not going to get fixed by waiting a # few seconds break + if exception_cb and not exception_cb(filtered_req_args, excps[-1]): + break if i + 1 < manual_tries and sec_between > 0: LOG.debug("Please wait %s seconds while we wait to try again", sec_between) diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index 3032ef70..de6487d8 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -311,7 +311,8 @@ class UserDataProcessor(object): def _attach_part(self, outer_msg, part): """ Attach a message to an outer message. outermsg must be a MIMEMultipart. - Modifies a header in the outer message to keep track of number of attachments. + Modifies a header in the outer message to keep track of number of + attachments. """ part_count = self._multi_part_count(outer_msg) self._process_before_attach(part, part_count + 1) diff --git a/cloudinit/util.py b/cloudinit/util.py index 5f64cb69..bf4006cb 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -691,7 +691,7 @@ def fetch_ssl_details(paths=None): def read_file_or_url(url, timeout=5, retries=10, headers=None, data=None, sec_between=1, ssl_details=None, - headers_cb=None): + headers_cb=None, exception_cb=None): url = url.lstrip() if url.startswith("/"): url = "file://%s" % url @@ -708,7 +708,8 @@ def read_file_or_url(url, timeout=5, retries=10, headers_cb=headers_cb, data=data, sec_between=sec_between, - ssl_details=ssl_details) + ssl_details=ssl_details, + exception_cb=exception_cb) def load_yaml(blob, default=None, allowed=(dict,)): @@ -962,7 +963,7 @@ def is_resolvable(name): pass _DNS_REDIRECT_IP = badips if badresults: - LOG.debug("detected dns redirection: %s" % badresults) + LOG.debug("detected dns redirection: %s", badresults) try: result = socket.getaddrinfo(name, None) @@ -1321,6 +1322,7 @@ def mounts(): (mountoutput, _err) = subp("mount") mount_locs = mountoutput.splitlines() method = 'mount' + mountre = r'^(/dev/[\S]+) on (/.*) \((.+), .+, (.+)\)$' for mpline in mount_locs: # Linux: /dev/sda1 on /boot type ext4 (rw,relatime,data=ordered) # FreeBSD: /dev/vtbd0p2 on / (ufs, local, journaled soft-updates) @@ -1328,7 +1330,7 @@ def mounts(): if method == 'proc': (dev, mp, fstype, opts, _freq, _passno) = mpline.split() else: - m = re.search('^(/dev/[\S]+) on (/.*) \((.+), .+, (.+)\)$', mpline) + m = re.search(mountre, mpline) dev = m.group(1) mp = m.group(2) fstype = m.group(3) @@ -1402,7 +1404,7 @@ def get_builtin_cfg(): def sym_link(source, link): - LOG.debug("Creating symbolic link from %r => %r" % (link, source)) + LOG.debug("Creating symbolic link from %r => %r", link, source) os.symlink(source, link) @@ -1443,7 +1445,8 @@ def uptime(): size = ctypes.c_size_t() buf = ctypes.c_int() size.value = ctypes.sizeof(buf) - libc.sysctlbyname("kern.boottime", ctypes.byref(buf), ctypes.byref(size), None, 0) + libc.sysctlbyname("kern.boottime", ctypes.byref(buf), + ctypes.byref(size), None, 0) now = time.time() bootup = buf.value uptime_str = now - bootup @@ -1792,7 +1795,7 @@ def parse_mount(path): (mountoutput, _err) = subp("mount") mount_locs = mountoutput.splitlines() for line in mount_locs: - m = re.search('^(/dev/[\S]+) on (/.*) \((.+), .+, (.+)\)$', line) + m = re.search(r'^(/dev/[\S]+) on (/.*) \((.+), .+, (.+)\)$', line) devpth = m.group(1) mount_point = m.group(2) fs_type = m.group(3) diff --git a/doc/rtd/conf.py b/doc/rtd/conf.py index c9ae79f4..52a8f92b 100644 --- a/doc/rtd/conf.py +++ b/doc/rtd/conf.py @@ -1,4 +1,5 @@ -import sys, os +import os +import sys # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -63,7 +63,7 @@ def tiny_p(cmd, capture=True): (out, err) = sp.communicate() ret = sp.returncode # pylint: disable=E1101 if ret not in [0]: - raise RuntimeError("Failed running %s [rc=%s] (%s, %s)" + raise RuntimeError("Failed running %s [rc=%s] (%s, %s)" % (cmd, ret, out, err)) return (out, err) @@ -102,7 +102,7 @@ class InitsysInstallData(install): " specifying a init system!") % (", ".join(INITSYS_TYPES))) elif self.init_system: self.distribution.data_files.append( - (INITSYS_ROOTS[self.init_system], + (INITSYS_ROOTS[self.init_system], INITSYS_FILES[self.init_system])) # Force that command to reinitalize (with new file list) self.distribution.reinitialize_command('install_data', True) @@ -134,7 +134,7 @@ setuptools.setup(name='cloud-init', [f for f in glob('doc/examples/seed/*') if is_f(f)]), ], install_requires=read_requires(), - cmdclass = { + cmdclass={ # Use a subclass for install that handles # adding on the right init system configuration files 'install': InitsysInstallData, diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index 3c1e8add..1f4a0a0b 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -285,7 +285,7 @@ class TestConfigDriveDataSource(MockerTestCase): self.assertEqual(["/dev/vdb", "/dev/zdd"], ds.find_candidate_devs()) - # verify that partitions are considered, but only if they have a label. + # verify that partitions are considered, that have correct label. devs_with_answers = {"TYPE=vfat": ["/dev/sda1"], "TYPE=iso9660": [], "LABEL=config-2": ["/dev/vdb3"]} self.assertEqual(["/dev/vdb3"], diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index 2007a6df..ebfb24da 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -119,7 +119,8 @@ class TestMAASDataSource(mocker.MockerTestCase): mock_request(url, headers=None, timeout=mocker.ANY, data=mocker.ANY, sec_between=mocker.ANY, ssl_details=mocker.ANY, retries=mocker.ANY, - headers_cb=my_headers_cb) + headers_cb=my_headers_cb, + exception_cb=mocker.ANY) resp = valid.get(key) self.mocker.result(util.StringResponse(resp)) self.mocker.replay() diff --git a/tests/unittests/test_ec2_util.py b/tests/unittests/test_ec2_util.py index dd588aca..957dc3f2 100644 --- a/tests/unittests/test_ec2_util.py +++ b/tests/unittests/test_ec2_util.py @@ -34,6 +34,14 @@ class TestEc2Util(helpers.TestCase): self.assertEquals('', userdata) @hp.activate + def test_userdata_fetch_fail_server_not_found(self): + hp.register_uri(hp.GET, + 'http://169.254.169.254/%s/user-data' % (self.VERSION), + status=404) + userdata = eu.get_instance_userdata(self.VERSION) + self.assertEquals('', userdata) + + @hp.activate def test_metadata_fetch_no_keys(self): base_url = 'http://169.254.169.254/%s/meta-data' % (self.VERSION) hp.register_uri(hp.GET, base_url, status=200, diff --git a/tools/run-pep8 b/tools/run-pep8 index 20e594bc..cfce5edd 100755 --- a/tools/run-pep8 +++ b/tools/run-pep8 @@ -1,15 +1,7 @@ #!/bin/bash -ci_files='cloudinit/*.py cloudinit/config/*.py' -test_files=$(find tests -name "*.py") -def_files="$ci_files $test_files" - if [ $# -eq 0 ]; then - files=( ) - for f in $def_files; do - [ -f "$f" ] || { echo "failed, $f not a file" 1>&2; exit 1; } - files[${#files[@]}]=${f} - done + files=( bin/cloud-init $(find * -name "*.py" -type f) ) else files=( "$@" ); fi @@ -44,4 +36,3 @@ cmd=( echo -e "\nRunning 'cloudinit' pep8:" echo "${cmd[@]}" "${cmd[@]}" - diff --git a/tools/run-pylint b/tools/run-pylint index b74efda9..0fe0c64a 100755 --- a/tools/run-pylint +++ b/tools/run-pylint @@ -1,7 +1,7 @@ #!/bin/bash if [ $# -eq 0 ]; then - files=( $(find * -name "*.py" -type f) ) + files=( bin/cloud-init $(find * -name "*.py" -type f) ) else files=( "$@" ); fi @@ -16,6 +16,7 @@ cmd=( --rcfile=$RC_FILE --disable=R --disable=I + --dummy-variables-rgx="_" "${files[@]}" ) |