From acf41ea8717646dedc9ddebed85d360996edfddc Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Wed, 7 Mar 2012 16:05:17 -0700 Subject: Added ability of cloud-init to manage apt http pipelining - cloud-config option of "apt-pipelining" - Address LP: 948461 --- cloudinit/CloudConfig/cc_apt_pipelining.py | 112 +++++++++++++++++++++++++++++ config/cloud.cfg | 1 + doc/examples/cloud-config.txt | 9 +++ 3 files changed, 122 insertions(+) create mode 100644 cloudinit/CloudConfig/cc_apt_pipelining.py diff --git a/cloudinit/CloudConfig/cc_apt_pipelining.py b/cloudinit/CloudConfig/cc_apt_pipelining.py new file mode 100644 index 00000000..8282bb0a --- /dev/null +++ b/cloudinit/CloudConfig/cc_apt_pipelining.py @@ -0,0 +1,112 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# +# Author: Ben Howard +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import cloudinit.SshUtil as sshutil +import re +import os +from cloudinit.CloudConfig import per_always + +frequency = per_always +default_file = "/etc/apt/apt.conf.d/90cloud-init-tweaks" + +def handle(_name, cfg, cloud, log, _args): + + apt_pipelining_enabled = util.get_cfg_option_str(cfg, "apt-pipelining", False) + + if apt_pipelining_enabled in ("False", "false", False): + write_apt_snippet(0, log) + + elif apt_pipelining_enabled in ("Default", "default", "True", "true", True): + revert_os_default(log) + + else: + write_apt_snippet(apt_pipelining_enabled, log) + +def revert_os_default(log, f_name=default_file): + try: + + if os.path.exists(f_name): + os.unlink(f_name) + + except OSError: + log.debug("Unable to remove %s" % f_name) + + +def write_apt_snippet(setting, log, f_name=default_file): + """ + Reads f_name and determines if the setting matches or not. Sets to + desired value + """ + + acquire_pipeline_depth = 'Acquire::http::Pipeline-Depth "%s";\n' + try: + if os.path.exists(f_name): + update_file = False + skip_re = re.compile('^//CLOUD-INIT-IGNORE.*') + enabled_re = re.compile('Acquire::http::Pipeline-Depth.*') + + local_override = False + tweak = open(f_name, 'r') + new_file = [] + + for line in tweak.readlines(): + if skip_re.match(line): + local_override = True + continue + + if enabled_re.match(line): + + try: + value = line.replace('"','') + value = value.replace(';','') + enabled = value.split()[1] + + if enabled != setting: + update_file = True + line = acquire_pipeline_depth % setting + + except IndexError: + log.debug("Unable to determine current setting of 'Acquire::http::Pipeline-Depth'\n%s" % e) + return + + new_file.append(line) + + tweak.close() + + if local_override: + log.debug("Not updating apt pipelining settings due to local override in %s" % f_name) + return + + if update_file: + tweak = open(f_name, 'w') + for line in new_file: + tweak.write(line) + tweak.close() + + return + + tweak = open(f_name, 'w') + tweak.write("""//Cloud-init Tweaks\n//Disables APT HTTP pipelining\n""") + tweak.write(acquire_pipeline_depth % setting) + tweak.close() + + log.debug("Wrote %s with APT pipeline setting" % f_name ) + + except IOError as e: + log.debug("Unable to update pipeline settings in %s\n%s" % (f_name, e)) diff --git a/config/cloud.cfg b/config/cloud.cfg index fe197ca8..72a2d8e4 100644 --- a/config/cloud.cfg +++ b/config/cloud.cfg @@ -19,6 +19,7 @@ cloud_config_modules: - locale - set-passwords - grub-dpkg + - apt-pipelining - apt-update-upgrade - landscape - timezone diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index 4f621274..ce188756 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -45,6 +45,15 @@ apt_mirror_search: # apt_proxy (configure Acquire::HTTP::Proxy) apt_proxy: http://my.apt.proxy:3128 +# apt_pipelining (confiugure Acquire::http::Pipeline-Depth) +# Default: disables HTTP pipelining. Certain web servers, such +# as S3 do not pipeline properly. +# Valid options: +# True/Default: Enables OS default +# False: Disables pipelining all-together +# Number: Set pipelining to some number (not recommended) +apt_pipelining: False + # Preserve existing /etc/apt/sources.list # Default: overwrite sources_list with mirror. If this is true # then apt_mirror above will have no effect -- cgit v1.2.3 From 162762d916c18ad83a6775cc1bd9a86cb78a5dd7 Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Thu, 8 Mar 2012 16:22:16 -0700 Subject: Simplified proposed patch - Changed values to be more simplistic and intuitive - Only allow pipelining values up to 5 - Changed to per_instance over per_always to remove need for tracking the values - Fixed Python style --- cloudinit/CloudConfig/cc_apt_pipelining.py | 76 +++++++----------------------- doc/examples/cloud-config.txt | 6 +-- 2 files changed, 20 insertions(+), 62 deletions(-) diff --git a/cloudinit/CloudConfig/cc_apt_pipelining.py b/cloudinit/CloudConfig/cc_apt_pipelining.py index 8282bb0a..c1e65847 100644 --- a/cloudinit/CloudConfig/cc_apt_pipelining.py +++ b/cloudinit/CloudConfig/cc_apt_pipelining.py @@ -17,36 +17,29 @@ # along with this program. If not, see . import cloudinit.util as util -import cloudinit.SshUtil as sshutil import re import os -from cloudinit.CloudConfig import per_always +from cloudinit.CloudConfig import per_instance -frequency = per_always -default_file = "/etc/apt/apt.conf.d/90cloud-init-tweaks" +frequency = per_instance +default_file = "/etc/apt/apt.conf.d/90cloud-init-pipeling" def handle(_name, cfg, cloud, log, _args): - apt_pipelining_enabled = util.get_cfg_option_str(cfg, "apt-pipelining", False) + apt_pipe_value = util.get_cfg_option_str(cfg, "apt_pipelining", False) + apt_pipe_value = str(apt_pipe_value).lower() - if apt_pipelining_enabled in ("False", "false", False): + if apt_pipe_value in ("false", "default", False): write_apt_snippet(0, log) - elif apt_pipelining_enabled in ("Default", "default", "True", "true", True): - revert_os_default(log) + elif apt_pipe_value in ("none", "unchanged", "os"): + return - else: - write_apt_snippet(apt_pipelining_enabled, log) - -def revert_os_default(log, f_name=default_file): - try: - - if os.path.exists(f_name): - os.unlink(f_name) - - except OSError: - log.debug("Unable to remove %s" % f_name) + elif apt_pipe_value in str(range(1, 5)): + write_apt_snippet(apt_pipe_value, log) + else: + log.warn("Invalid option for apt_pipeling") def write_apt_snippet(setting, log, f_name=default_file): """ @@ -57,54 +50,19 @@ def write_apt_snippet(setting, log, f_name=default_file): acquire_pipeline_depth = 'Acquire::http::Pipeline-Depth "%s";\n' try: if os.path.exists(f_name): - update_file = False skip_re = re.compile('^//CLOUD-INIT-IGNORE.*') - enabled_re = re.compile('Acquire::http::Pipeline-Depth.*') - - local_override = False - tweak = open(f_name, 'r') - new_file = [] for line in tweak.readlines(): if skip_re.match(line): - local_override = True - continue - - if enabled_re.match(line): - - try: - value = line.replace('"','') - value = value.replace(';','') - enabled = value.split()[1] - - if enabled != setting: - update_file = True - line = acquire_pipeline_depth % setting - - except IndexError: - log.debug("Unable to determine current setting of 'Acquire::http::Pipeline-Depth'\n%s" % e) - return - - new_file.append(line) + tweak.close() + return tweak.close() - if local_override: - log.debug("Not updating apt pipelining settings due to local override in %s" % f_name) - return - - if update_file: - tweak = open(f_name, 'w') - for line in new_file: - tweak.write(line) - tweak.close() - - return + file_contents = ("//Cloud-init Tweaks\n//Disables APT HTTP pipelining"\ + "\n" + (acquire_pipeline_depth % setting)) - tweak = open(f_name, 'w') - tweak.write("""//Cloud-init Tweaks\n//Disables APT HTTP pipelining\n""") - tweak.write(acquire_pipeline_depth % setting) - tweak.close() + util.write_file(f_name, file_contents) log.debug("Wrote %s with APT pipeline setting" % f_name ) diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index ce188756..542a49c7 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -45,12 +45,12 @@ apt_mirror_search: # apt_proxy (configure Acquire::HTTP::Proxy) apt_proxy: http://my.apt.proxy:3128 -# apt_pipelining (confiugure Acquire::http::Pipeline-Depth) +# apt_pipelining (configure Acquire::http::Pipeline-Depth) # Default: disables HTTP pipelining. Certain web servers, such # as S3 do not pipeline properly. # Valid options: -# True/Default: Enables OS default -# False: Disables pipelining all-together +# False/default: Disables pipelining for APT +# None/Unchanged: Use OS default # Number: Set pipelining to some number (not recommended) apt_pipelining: False -- cgit v1.2.3 From 210dfb8840baded8a2282e8b0b49c3a034222bd8 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 9 Mar 2012 09:50:04 -0500 Subject: Some cleanups before merge. * removed the 'CLOUD-INIT-IGNORE' section, as we're just blindly writing the file now. removed the now-unnecessary import of 're' and 'os' * removed try/except block around write_apt_snippet. This will bubble up and cloud-init will let it through even to the console. Catching it and turning it into a debug would just hide it. * removed 'default' as a synonym for 'whatever cloud-init thinks is best' If people are going to change this, I'd rather they be specific. * supported value of "0" * fixed some complaints from ./tools/run-pylint cloudinit/CloudConfig/cc_apt_pipelining.py --- cloudinit/CloudConfig/cc_apt_pipelining.py | 41 +++++++++--------------------- 1 file changed, 12 insertions(+), 29 deletions(-) diff --git a/cloudinit/CloudConfig/cc_apt_pipelining.py b/cloudinit/CloudConfig/cc_apt_pipelining.py index c1e65847..7ca93e66 100644 --- a/cloudinit/CloudConfig/cc_apt_pipelining.py +++ b/cloudinit/CloudConfig/cc_apt_pipelining.py @@ -17,54 +17,37 @@ # along with this program. If not, see . import cloudinit.util as util -import re -import os from cloudinit.CloudConfig import per_instance frequency = per_instance default_file = "/etc/apt/apt.conf.d/90cloud-init-pipeling" -def handle(_name, cfg, cloud, log, _args): + +def handle(_name, cfg, _cloud, log, _args): apt_pipe_value = util.get_cfg_option_str(cfg, "apt_pipelining", False) apt_pipe_value = str(apt_pipe_value).lower() - if apt_pipe_value in ("false", "default", False): - write_apt_snippet(0, log) + if apt_pipe_value == "false": + write_apt_snippet("0", log) elif apt_pipe_value in ("none", "unchanged", "os"): return - elif apt_pipe_value in str(range(1, 5)): + elif apt_pipe_value in str(range(0, 6)): write_apt_snippet(apt_pipe_value, log) else: - log.warn("Invalid option for apt_pipeling") + log.warn("Invalid option for apt_pipeling: %s" % apt_pipe_value) + def write_apt_snippet(setting, log, f_name=default_file): - """ - Reads f_name and determines if the setting matches or not. Sets to - desired value - """ + """ Writes f_name with apt pipeline depth 'setting' """ acquire_pipeline_depth = 'Acquire::http::Pipeline-Depth "%s";\n' - try: - if os.path.exists(f_name): - skip_re = re.compile('^//CLOUD-INIT-IGNORE.*') - - for line in tweak.readlines(): - if skip_re.match(line): - tweak.close() - return - - tweak.close() - - file_contents = ("//Cloud-init Tweaks\n//Disables APT HTTP pipelining"\ - "\n" + (acquire_pipeline_depth % setting)) - - util.write_file(f_name, file_contents) + file_contents = ("//Written by cloud-init per 'apt_pipelining'\n" + + (acquire_pipeline_depth % setting)) - log.debug("Wrote %s with APT pipeline setting" % f_name ) + util.write_file(f_name, file_contents) - except IOError as e: - log.debug("Unable to update pipeline settings in %s\n%s" % (f_name, e)) + log.debug("Wrote %s with APT pipeline setting" % f_name) -- cgit v1.2.3 From 91c1ef651077dbdbe7e8a85cb7b48926fbeb0aab Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 9 Mar 2012 09:51:26 -0500 Subject: mention bug number in cloud-config.txt --- doc/examples/cloud-config.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index 542a49c7..171802cc 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -47,7 +47,7 @@ apt_proxy: http://my.apt.proxy:3128 # apt_pipelining (configure Acquire::http::Pipeline-Depth) # Default: disables HTTP pipelining. Certain web servers, such -# as S3 do not pipeline properly. +# as S3 do not pipeline properly (LP: #948461). # Valid options: # False/default: Disables pipelining for APT # None/Unchanged: Use OS default -- cgit v1.2.3