From 66490ebb92af59d148f79aae42a2eddc1ecedb7e Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 30 Jul 2013 14:23:10 -0400 Subject: add util.log_time helper The reason for this is that more and more things I was wanting to be able to see how long they took. This puts that time logic into a single place. It also supports (by default) reading from /proc/uptime as the timing mechanism. While that is almost certainly slower than time.time(), it does give millisecond granularity and is not affected by 'ntpdate' having run in between the two events. --- cloudinit/config/cc_growpart.py | 3 ++- cloudinit/config/cc_resizefs.py | 11 +++++------ 2 files changed, 7 insertions(+), 7 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index 4f8c8f80..ba6c58af 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -264,7 +264,8 @@ def handle(_name, cfg, _cloud, log, _args): raise e return - resized = resize_devices(resizer, devices) + resized = util.log_time(logfunc=log.debug, msg="resize_devices", + func=resize_devices, args=(resizer, devices)) for (entry, action, msg) in resized: if action == RESIZE.CHANGED: log.info("'%s' resized: %s" % (entry, msg)) diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index b4ee16b2..56040fdd 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -21,7 +21,6 @@ import errno import os import stat -import time from cloudinit.settings import PER_ALWAYS from cloudinit import util @@ -120,9 +119,12 @@ def handle(name, cfg, _cloud, log, args): if resize_root == NOBLOCK: # Fork to a child that will run # the resize command - util.fork_cb(do_resize, resize_cmd, log) + util.fork_cb( + util.log_time(logfunc=log.debug, msg="backgrounded Resizing", + func=do_resize, args=(resize_cmd, log))) else: - do_resize(resize_cmd, log) + util.log_time(logfunc=log.debug, msg="Resizing", + func=do_resize, args=(resize_cmd, log)) action = 'Resized' if resize_root == NOBLOCK: @@ -132,13 +134,10 @@ def handle(name, cfg, _cloud, log, args): def do_resize(resize_cmd, log): - start = time.time() try: util.subp(resize_cmd) except util.ProcessExecutionError: util.logexc(log, "Failed to resize filesystem (cmd=%s)", resize_cmd) raise - tot_time = time.time() - start - log.debug("Resizing took %.3f seconds", tot_time) # TODO(harlowja): Should we add a fsck check after this to make # sure we didn't corrupt anything? -- cgit v1.2.3 From 1e84d5dad7dcd21a521d0977963450424ec35c9c Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 6 Aug 2013 07:15:36 +0100 Subject: initially add support for apt_ftp_proxy and apt_https_proxy. tests to come. LP: #1057195 --- cloudinit/config/cc_apt_configure.py | 41 ++++++++++++++++++++++++------------ doc/examples/cloud-config-TODO.txt | 20 ------------------ doc/examples/cloud-config.txt | 3 +++ 3 files changed, 31 insertions(+), 33 deletions(-) delete mode 100644 doc/examples/cloud-config-TODO.txt (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 3ce3b351..0c26050a 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -21,13 +21,17 @@ import glob import os +from cloudinit import log as logging from cloudinit import templater from cloudinit import util +LOG = logging.getLogger(__name__) + distros = ['ubuntu', 'debian'] PROXY_TPL = "Acquire::HTTP::Proxy \"%s\";\n" -PROXY_FN = "/etc/apt/apt.conf.d/95cloud-init-proxy" +APT_CONFIG_FN = "/etc/apt/apt.conf.d/94cloud-init-config" +APT_PROXY_FN = "/etc/apt/apt.conf.d/95cloud-init-proxy" # A temporary shell program to get a given gpg key # from a given keyserver @@ -67,18 +71,10 @@ def handle(name, cfg, cloud, log, _args): "security": "security.ubuntu.com/ubuntu"}) rename_apt_lists(old_mirrors, mirrors) - # Set up any apt proxy - proxy = cfg.get("apt_proxy", None) - proxy_filename = PROXY_FN - if proxy: - try: - # See man 'apt.conf' - contents = PROXY_TPL % (proxy) - util.write_file(proxy_filename, contents) - except Exception as e: - util.logexc(log, "Failed to write proxy to %s", proxy_filename) - elif os.path.isfile(proxy_filename): - util.del_file(proxy_filename) + try: + apply_apt_config(cfg, APT_PROXY_FN, APT_CONFIG_FN) + except Exception as e: + log.warn("failed to proxy or apt config info: %s", e) # Process 'apt_sources' if 'apt_sources' in cfg: @@ -256,3 +252,22 @@ def find_apt_mirror_info(cloud, cfg): mirror_info.update({'primary': mirror}) return mirror_info + + +def apply_apt_config(cfg, proxy_fname, config_fname): + # Set up any apt proxy + cfgs = (('apt_proxy', 'Acquire::HTTP::Proxy "%s";'), + ('apt_http_proxy', 'Acquire::HTTP::Proxy "%s";'), + ('apt_ftp_proxy', 'Acquire::FTP::Proxy "%s";'), + ('apt_https_proxy', 'Acquire::HTTPS::Proxy "%s";')) + + proxies = [fmt % cfg.get(name) for (name, fmt) in cfgs if cfg.get(name)] + if len(proxies): + util.write_file(proxy_fname, '\n'.join(proxies) + '\n') + elif os.path.isfile(proxy_fname): + util.del_file(proxy_fname) + + if cfg.get('apt_config', None): + util.write_file(config_fname, cfg.get('apt_config')) + elif os.path.isfile(config_fname): + util.del_file(config_fname) diff --git a/doc/examples/cloud-config-TODO.txt b/doc/examples/cloud-config-TODO.txt deleted file mode 100644 index c7ed54ab..00000000 --- a/doc/examples/cloud-config-TODO.txt +++ /dev/null @@ -1,20 +0,0 @@ -# Add apt configuration files -# Add an apt.conf.d/ file with the relevant content -# -# See apt.conf man page for more information. -# -# Defaults: -# + filename: 00-boot-conf -# -apt_conf: - - # Creates an apt proxy configuration in /etc/apt/apt.conf.d/01-proxy - - filename: "01-proxy" - content: | - Acquire::http::Proxy "http://proxy.example.org:3142/ubuntu"; - - # Add the following line to /etc/apt/apt.conf.d/00-boot-conf - # (run debconf at a critical priority) - - content: | - DPkg::Pre-Install-Pkgs:: "/usr/sbin/dpkg-preconfigure --apt -p critical|| true"; - diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index 24b4b36c..bcfd7917 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -53,6 +53,9 @@ apt_mirror_search: apt_mirror_search_dns: False # apt_proxy (configure Acquire::HTTP::Proxy) +# 'apt_http_proxy' is an alias for 'apt_proxy'. +# Also, available are 'apt_ftp_proxy' and 'apt_https_proxy'. +# These affect Acquire::FTP::Proxy and Acquire::HTTPS::Proxy respectively apt_proxy: http://my.apt.proxy:3128 # apt_pipelining (configure Acquire::http::Pipeline-Depth) -- cgit v1.2.3 From b4c0bde830fbe56a0d59a68f4ed97a2b6cbdece3 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 6 Aug 2013 07:18:45 +0100 Subject: remove unused LOG --- cloudinit/config/cc_apt_configure.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 0c26050a..73733f3d 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -25,8 +25,6 @@ from cloudinit import log as logging from cloudinit import templater from cloudinit import util -LOG = logging.getLogger(__name__) - distros = ['ubuntu', 'debian'] PROXY_TPL = "Acquire::HTTP::Proxy \"%s\";\n" -- cgit v1.2.3 From 58cfc129165d10dc00c7dc2bdd1422a43f270443 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 6 Aug 2013 11:31:49 +0100 Subject: remove unused import --- cloudinit/config/cc_apt_configure.py | 1 - 1 file changed, 1 deletion(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 73733f3d..5a407016 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -21,7 +21,6 @@ import glob import os -from cloudinit import log as logging from cloudinit import templater from cloudinit import util -- cgit v1.2.3 From cc0b78533dd1f863901119c420f5779a66396d27 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 14 Aug 2013 21:31:54 -0400 Subject: cc_growpart: prefer growpart over parted 'resizepart' the resizepart code was not functional. We will re-favor it later under bug 1212492. For now, we'll just favor the 'growpart' resizer. Both will be found in Ubuntu cloud images. LP: #1212444 --- cloudinit/config/cc_growpart.py | 6 ++++-- tests/unittests/test_handler/test_handler_growpart.py | 3 +++ 2 files changed, 7 insertions(+), 2 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index ba6c58af..2d54aabf 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -96,7 +96,7 @@ class ResizeParted(object): def resize(self, diskdev, partnum, partdev): before = get_size(partdev) try: - util.subp(["parted", "resizepart", diskdev, partnum]) + util.subp(["parted", diskdev, "resizepart", partnum]) except util.ProcessExecutionError as e: raise ResizeFailedException(e) @@ -272,4 +272,6 @@ def handle(_name, cfg, _cloud, log, _args): else: log.debug("'%s' %s: %s" % (entry, action, msg)) -RESIZERS = (('parted', ResizeParted), ('growpart', ResizeGrowPart)) +# LP: 1212444 FIXME re-order and favor ResizeParted +#RESIZERS = (('growpart', ResizeGrowPart),) +RESIZERS = (('growpart', ResizeGrowPart), ('parted', ResizeParted)) diff --git a/tests/unittests/test_handler/test_handler_growpart.py b/tests/unittests/test_handler/test_handler_growpart.py index b1b872b0..c0497e08 100644 --- a/tests/unittests/test_handler/test_handler_growpart.py +++ b/tests/unittests/test_handler/test_handler_growpart.py @@ -9,6 +9,7 @@ import errno import logging import os import re +import unittest # growpart: # mode: auto # off, on, auto, 'growpart', 'parted' @@ -121,6 +122,7 @@ class TestConfig(MockerTestCase): # Order must be correct self.mocker.order() + @unittest.skip("until LP: #1212444 fixed") def test_no_resizers_auto_is_fine(self): subp = self.mocker.replace(util.subp, passthrough=False) subp(['parted', '--help'], env={'LANG': 'C'}) @@ -142,6 +144,7 @@ class TestConfig(MockerTestCase): self.assertRaises(ValueError, self.handle, self.name, config, self.cloud_init, self.log, self.args) + @unittest.skip("until LP: #1212444 fixed") def test_mode_auto_prefers_parted(self): subp = self.mocker.replace(util.subp, passthrough=False) subp(['parted', '--help'], env={'LANG': 'C'}) -- cgit v1.2.3