From 5e17bf1066b53359681f5164c662f779b268561b Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 5 Mar 2012 16:08:15 -0500 Subject: commit initial config information for maas datasource --- doc/examples/cloud-config-datasources.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'doc/examples') diff --git a/doc/examples/cloud-config-datasources.txt b/doc/examples/cloud-config-datasources.txt index b86c5ba6..f147aaf6 100644 --- a/doc/examples/cloud-config-datasources.txt +++ b/doc/examples/cloud-config-datasources.txt @@ -13,3 +13,18 @@ datasource: metadata_urls: - http://169.254.169.254:80 - http://instance-data:8773 + + MaaS: + timeout : 50 + max_wait : 120 + + # there are no default values for metadata_url or credentials + # credentials are for oath. + + # If no credentials are present, + # then non-authed attempts will be made. + # will be made + metadata_url: http://mass-host.localdomain/source + consumer_key: Xh234sdkljf + token_key: kjfhgb3n + token_secret: 24uysdfx1w4 -- cgit v1.2.3 From acf41ea8717646dedc9ddebed85d360996edfddc Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Wed, 7 Mar 2012 16:05:17 -0700 Subject: Added ability of cloud-init to manage apt http pipelining - cloud-config option of "apt-pipelining" - Address LP: 948461 --- cloudinit/CloudConfig/cc_apt_pipelining.py | 112 +++++++++++++++++++++++++++++ config/cloud.cfg | 1 + doc/examples/cloud-config.txt | 9 +++ 3 files changed, 122 insertions(+) create mode 100644 cloudinit/CloudConfig/cc_apt_pipelining.py (limited to 'doc/examples') diff --git a/cloudinit/CloudConfig/cc_apt_pipelining.py b/cloudinit/CloudConfig/cc_apt_pipelining.py new file mode 100644 index 00000000..8282bb0a --- /dev/null +++ b/cloudinit/CloudConfig/cc_apt_pipelining.py @@ -0,0 +1,112 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# +# Author: Ben Howard +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import cloudinit.SshUtil as sshutil +import re +import os +from cloudinit.CloudConfig import per_always + +frequency = per_always +default_file = "/etc/apt/apt.conf.d/90cloud-init-tweaks" + +def handle(_name, cfg, cloud, log, _args): + + apt_pipelining_enabled = util.get_cfg_option_str(cfg, "apt-pipelining", False) + + if apt_pipelining_enabled in ("False", "false", False): + write_apt_snippet(0, log) + + elif apt_pipelining_enabled in ("Default", "default", "True", "true", True): + revert_os_default(log) + + else: + write_apt_snippet(apt_pipelining_enabled, log) + +def revert_os_default(log, f_name=default_file): + try: + + if os.path.exists(f_name): + os.unlink(f_name) + + except OSError: + log.debug("Unable to remove %s" % f_name) + + +def write_apt_snippet(setting, log, f_name=default_file): + """ + Reads f_name and determines if the setting matches or not. Sets to + desired value + """ + + acquire_pipeline_depth = 'Acquire::http::Pipeline-Depth "%s";\n' + try: + if os.path.exists(f_name): + update_file = False + skip_re = re.compile('^//CLOUD-INIT-IGNORE.*') + enabled_re = re.compile('Acquire::http::Pipeline-Depth.*') + + local_override = False + tweak = open(f_name, 'r') + new_file = [] + + for line in tweak.readlines(): + if skip_re.match(line): + local_override = True + continue + + if enabled_re.match(line): + + try: + value = line.replace('"','') + value = value.replace(';','') + enabled = value.split()[1] + + if enabled != setting: + update_file = True + line = acquire_pipeline_depth % setting + + except IndexError: + log.debug("Unable to determine current setting of 'Acquire::http::Pipeline-Depth'\n%s" % e) + return + + new_file.append(line) + + tweak.close() + + if local_override: + log.debug("Not updating apt pipelining settings due to local override in %s" % f_name) + return + + if update_file: + tweak = open(f_name, 'w') + for line in new_file: + tweak.write(line) + tweak.close() + + return + + tweak = open(f_name, 'w') + tweak.write("""//Cloud-init Tweaks\n//Disables APT HTTP pipelining\n""") + tweak.write(acquire_pipeline_depth % setting) + tweak.close() + + log.debug("Wrote %s with APT pipeline setting" % f_name ) + + except IOError as e: + log.debug("Unable to update pipeline settings in %s\n%s" % (f_name, e)) diff --git a/config/cloud.cfg b/config/cloud.cfg index fe197ca8..72a2d8e4 100644 --- a/config/cloud.cfg +++ b/config/cloud.cfg @@ -19,6 +19,7 @@ cloud_config_modules: - locale - set-passwords - grub-dpkg + - apt-pipelining - apt-update-upgrade - landscape - timezone diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index 4f621274..ce188756 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -45,6 +45,15 @@ apt_mirror_search: # apt_proxy (configure Acquire::HTTP::Proxy) apt_proxy: http://my.apt.proxy:3128 +# apt_pipelining (confiugure Acquire::http::Pipeline-Depth) +# Default: disables HTTP pipelining. Certain web servers, such +# as S3 do not pipeline properly. +# Valid options: +# True/Default: Enables OS default +# False: Disables pipelining all-together +# Number: Set pipelining to some number (not recommended) +apt_pipelining: False + # Preserve existing /etc/apt/sources.list # Default: overwrite sources_list with mirror. If this is true # then apt_mirror above will have no effect -- cgit v1.2.3 From 62623b8c5ac10b2bff9f6205f696b4277bce9451 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 8 Mar 2012 14:59:17 -0500 Subject: doc fixes --- doc/examples/cloud-config-datasources.txt | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'doc/examples') diff --git a/doc/examples/cloud-config-datasources.txt b/doc/examples/cloud-config-datasources.txt index f147aaf6..b3a26114 100644 --- a/doc/examples/cloud-config-datasources.txt +++ b/doc/examples/cloud-config-datasources.txt @@ -18,12 +18,8 @@ datasource: timeout : 50 max_wait : 120 - # there are no default values for metadata_url or credentials - # credentials are for oath. - - # If no credentials are present, - # then non-authed attempts will be made. - # will be made + # there are no default values for metadata_url or oauth credentials + # If no credentials are present, non-authed attempts will be made. metadata_url: http://mass-host.localdomain/source consumer_key: Xh234sdkljf token_key: kjfhgb3n -- cgit v1.2.3 From 162762d916c18ad83a6775cc1bd9a86cb78a5dd7 Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Thu, 8 Mar 2012 16:22:16 -0700 Subject: Simplified proposed patch - Changed values to be more simplistic and intuitive - Only allow pipelining values up to 5 - Changed to per_instance over per_always to remove need for tracking the values - Fixed Python style --- cloudinit/CloudConfig/cc_apt_pipelining.py | 76 +++++++----------------------- doc/examples/cloud-config.txt | 6 +-- 2 files changed, 20 insertions(+), 62 deletions(-) (limited to 'doc/examples') diff --git a/cloudinit/CloudConfig/cc_apt_pipelining.py b/cloudinit/CloudConfig/cc_apt_pipelining.py index 8282bb0a..c1e65847 100644 --- a/cloudinit/CloudConfig/cc_apt_pipelining.py +++ b/cloudinit/CloudConfig/cc_apt_pipelining.py @@ -17,36 +17,29 @@ # along with this program. If not, see . import cloudinit.util as util -import cloudinit.SshUtil as sshutil import re import os -from cloudinit.CloudConfig import per_always +from cloudinit.CloudConfig import per_instance -frequency = per_always -default_file = "/etc/apt/apt.conf.d/90cloud-init-tweaks" +frequency = per_instance +default_file = "/etc/apt/apt.conf.d/90cloud-init-pipeling" def handle(_name, cfg, cloud, log, _args): - apt_pipelining_enabled = util.get_cfg_option_str(cfg, "apt-pipelining", False) + apt_pipe_value = util.get_cfg_option_str(cfg, "apt_pipelining", False) + apt_pipe_value = str(apt_pipe_value).lower() - if apt_pipelining_enabled in ("False", "false", False): + if apt_pipe_value in ("false", "default", False): write_apt_snippet(0, log) - elif apt_pipelining_enabled in ("Default", "default", "True", "true", True): - revert_os_default(log) + elif apt_pipe_value in ("none", "unchanged", "os"): + return - else: - write_apt_snippet(apt_pipelining_enabled, log) - -def revert_os_default(log, f_name=default_file): - try: - - if os.path.exists(f_name): - os.unlink(f_name) - - except OSError: - log.debug("Unable to remove %s" % f_name) + elif apt_pipe_value in str(range(1, 5)): + write_apt_snippet(apt_pipe_value, log) + else: + log.warn("Invalid option for apt_pipeling") def write_apt_snippet(setting, log, f_name=default_file): """ @@ -57,54 +50,19 @@ def write_apt_snippet(setting, log, f_name=default_file): acquire_pipeline_depth = 'Acquire::http::Pipeline-Depth "%s";\n' try: if os.path.exists(f_name): - update_file = False skip_re = re.compile('^//CLOUD-INIT-IGNORE.*') - enabled_re = re.compile('Acquire::http::Pipeline-Depth.*') - - local_override = False - tweak = open(f_name, 'r') - new_file = [] for line in tweak.readlines(): if skip_re.match(line): - local_override = True - continue - - if enabled_re.match(line): - - try: - value = line.replace('"','') - value = value.replace(';','') - enabled = value.split()[1] - - if enabled != setting: - update_file = True - line = acquire_pipeline_depth % setting - - except IndexError: - log.debug("Unable to determine current setting of 'Acquire::http::Pipeline-Depth'\n%s" % e) - return - - new_file.append(line) + tweak.close() + return tweak.close() - if local_override: - log.debug("Not updating apt pipelining settings due to local override in %s" % f_name) - return - - if update_file: - tweak = open(f_name, 'w') - for line in new_file: - tweak.write(line) - tweak.close() - - return + file_contents = ("//Cloud-init Tweaks\n//Disables APT HTTP pipelining"\ + "\n" + (acquire_pipeline_depth % setting)) - tweak = open(f_name, 'w') - tweak.write("""//Cloud-init Tweaks\n//Disables APT HTTP pipelining\n""") - tweak.write(acquire_pipeline_depth % setting) - tweak.close() + util.write_file(f_name, file_contents) log.debug("Wrote %s with APT pipeline setting" % f_name ) diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index ce188756..542a49c7 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -45,12 +45,12 @@ apt_mirror_search: # apt_proxy (configure Acquire::HTTP::Proxy) apt_proxy: http://my.apt.proxy:3128 -# apt_pipelining (confiugure Acquire::http::Pipeline-Depth) +# apt_pipelining (configure Acquire::http::Pipeline-Depth) # Default: disables HTTP pipelining. Certain web servers, such # as S3 do not pipeline properly. # Valid options: -# True/Default: Enables OS default -# False: Disables pipelining all-together +# False/default: Disables pipelining for APT +# None/Unchanged: Use OS default # Number: Set pipelining to some number (not recommended) apt_pipelining: False -- cgit v1.2.3 From 91c1ef651077dbdbe7e8a85cb7b48926fbeb0aab Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 9 Mar 2012 09:51:26 -0500 Subject: mention bug number in cloud-config.txt --- doc/examples/cloud-config.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc/examples') diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index 542a49c7..171802cc 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -47,7 +47,7 @@ apt_proxy: http://my.apt.proxy:3128 # apt_pipelining (configure Acquire::http::Pipeline-Depth) # Default: disables HTTP pipelining. Certain web servers, such -# as S3 do not pipeline properly. +# as S3 do not pipeline properly (LP: #948461). # Valid options: # False/default: Disables pipelining for APT # None/Unchanged: Use OS default -- cgit v1.2.3