From 47d15924ea38f249a2c54d33dd24c3c284c4eb72 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 9 Oct 2013 12:22:06 -0700 Subject: Log message around import failure In certain cases import failure is expected and in certain cases it is not expected, in either case it is useful to at least log the failure. --- cloudinit/importer.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cloudinit/importer.py b/cloudinit/importer.py index 71cf2726..a094141a 100644 --- a/cloudinit/importer.py +++ b/cloudinit/importer.py @@ -36,6 +36,7 @@ def find_module(base_name, search_paths, required_attrs=None): found_places = [] if not required_attrs: required_attrs = [] + # NOTE(harlowja): translate the search paths to include the base name. real_paths = [] for path in search_paths: real_path = [] @@ -50,8 +51,9 @@ def find_module(base_name, search_paths, required_attrs=None): mod = None try: mod = import_module(full_path) - except ImportError: - pass + except ImportError as e: + LOG.debug("Failed at attempted import of '%s' due to: %s", + full_path, e) if not mod: continue found_attrs = 0 -- cgit v1.2.3 From 9c762cc3fa13782397a15bd3c68e9c62a3cba689 Mon Sep 17 00:00:00 2001 From: Shraddha Pandhe Date: Fri, 6 Dec 2013 11:16:17 -0800 Subject: This is a new debug module thats supposed to print out some information about the instance being created. The module can be included at any stage of the process - init/config/final LP: #1258619 --- cloudinit/config/cc_debug.py | 82 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 cloudinit/config/cc_debug.py diff --git a/cloudinit/config/cc_debug.py b/cloudinit/config/cc_debug.py new file mode 100644 index 00000000..86c61d68 --- /dev/null +++ b/cloudinit/config/cc_debug.py @@ -0,0 +1,82 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2013 Yahoo! Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from StringIO import StringIO + +from cloudinit import util + +import copy + +import yaml + + +def _format_yaml(obj): + try: + formatted = yaml.safe_dump(obj, + line_break="\n", + indent=4, + explicit_start=True, + explicit_end=True, + default_flow_style=False) + return formatted.strip() + except: + return "???" + + +def _make_header(text): + header = StringIO() + header.write("-" * 80) + header.write("\n") + header.write(text.center(80, ' ')) + header.write("\n") + header.write("-" * 80) + header.write("\n") + return header.getvalue() + + +def handle(name, cfg, cloud, log, _args): + verbose = util.get_cfg_option_bool(cfg, 'verbose', default=True) + if not verbose: + log.debug(("Skipping module named %s," + " verbose printing disabled"), name) + return + # Clean out some keys that we just don't care about showing... + dump_cfg = copy.deepcopy(cfg) + for k in ['log_cfgs']: + dump_cfg.pop(k, None) + all_keys = list(dump_cfg.keys()) + for k in all_keys: + if k.startswith("_"): + dump_cfg.pop(k, None) + # Now dump it... + to_print = StringIO() + to_print.write(_make_header("Config")) + to_print.write(_format_yaml(dump_cfg)) + to_print.write("\n") + to_print.write(_make_header("MetaData")) + to_print.write(_format_yaml(cloud.datasource.metadata)) + to_print.write("\n") + to_print.write(_make_header("Misc")) + to_print.write("Datasource: %s\n" % (util.obj_name(cloud.datasource))) + to_print.write("Distro: %s\n" % (util.obj_name(cloud.distro))) + to_print.write("Hostname: %s\n" % (cloud.get_hostname(True))) + to_print.write("Instance ID: %s\n" % (cloud.get_instance_id())) + to_print.write("Locale: %s\n" % (cloud.get_locale())) + to_print.write("Launch IDX: %s\n" % (cloud.launch_index)) + contents = to_print.getvalue() + for line in contents.splitlines(): + line = "ci-info: %s\n" % (line) + util.multi_log(line, console=True, stderr=False) -- cgit v1.2.3 From c0d263968d93e44fbaaa98d284690ac93a6ce024 Mon Sep 17 00:00:00 2001 From: Shraddha Pandhe Date: Wed, 11 Dec 2013 14:21:18 -0800 Subject: bug: 1258619 added namespace for config options, output file, commandline argument for output file --- cloudinit/config/cc_debug.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/cloudinit/config/cc_debug.py b/cloudinit/config/cc_debug.py index 86c61d68..85dc5c58 100644 --- a/cloudinit/config/cc_debug.py +++ b/cloudinit/config/cc_debug.py @@ -15,11 +15,9 @@ # along with this program. If not, see . from StringIO import StringIO - from cloudinit import util - +from cloudinit import type_utils import copy - import yaml @@ -47,12 +45,17 @@ def _make_header(text): return header.getvalue() -def handle(name, cfg, cloud, log, _args): - verbose = util.get_cfg_option_bool(cfg, 'verbose', default=True) +def handle(name, cfg, cloud, log, args): + verbose = util.get_cfg_by_path(cfg, ('debug','verbose'), default=True) if not verbose: log.debug(("Skipping module named %s," " verbose printing disabled"), name) return + out_file = None + if args: + out_file = args[0] + else: + out_file = util.get_cfg_by_path(cfg, ('debug','output')) # Clean out some keys that we just don't care about showing... dump_cfg = copy.deepcopy(cfg) for k in ['log_cfgs']: @@ -70,8 +73,8 @@ def handle(name, cfg, cloud, log, _args): to_print.write(_format_yaml(cloud.datasource.metadata)) to_print.write("\n") to_print.write(_make_header("Misc")) - to_print.write("Datasource: %s\n" % (util.obj_name(cloud.datasource))) - to_print.write("Distro: %s\n" % (util.obj_name(cloud.distro))) + to_print.write("Datasource: %s\n" % (type_utils.obj_name(cloud.datasource))) + to_print.write("Distro: %s\n" % (type_utils.obj_name(cloud.distro))) to_print.write("Hostname: %s\n" % (cloud.get_hostname(True))) to_print.write("Instance ID: %s\n" % (cloud.get_instance_id())) to_print.write("Locale: %s\n" % (cloud.get_locale())) @@ -79,4 +82,7 @@ def handle(name, cfg, cloud, log, _args): contents = to_print.getvalue() for line in contents.splitlines(): line = "ci-info: %s\n" % (line) - util.multi_log(line, console=True, stderr=False) + if out_file: + util.write_file(out_file, line, 0644, "a") + else: + util.multi_log(line, console=True, stderr=False) -- cgit v1.2.3 From 7c6f2de4f7f66874d6c9131c04cb84637955e5ce Mon Sep 17 00:00:00 2001 From: Shraddha Pandhe Date: Wed, 11 Dec 2013 15:41:14 -0800 Subject: essage --- cloudinit/config/cc_debug.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/cloudinit/config/cc_debug.py b/cloudinit/config/cc_debug.py index 85dc5c58..a3d99da8 100644 --- a/cloudinit/config/cc_debug.py +++ b/cloudinit/config/cc_debug.py @@ -67,10 +67,10 @@ def handle(name, cfg, cloud, log, args): # Now dump it... to_print = StringIO() to_print.write(_make_header("Config")) - to_print.write(_format_yaml(dump_cfg)) + to_print.write(util.yaml_dumps(dump_cfg)) to_print.write("\n") to_print.write(_make_header("MetaData")) - to_print.write(_format_yaml(cloud.datasource.metadata)) + to_print.write(util.yaml_dumps(cloud.datasource.metadata)) to_print.write("\n") to_print.write(_make_header("Misc")) to_print.write("Datasource: %s\n" % (type_utils.obj_name(cloud.datasource))) @@ -80,9 +80,11 @@ def handle(name, cfg, cloud, log, args): to_print.write("Locale: %s\n" % (cloud.get_locale())) to_print.write("Launch IDX: %s\n" % (cloud.launch_index)) contents = to_print.getvalue() + content_to_file = [] for line in contents.splitlines(): line = "ci-info: %s\n" % (line) - if out_file: - util.write_file(out_file, line, 0644, "a") - else: - util.multi_log(line, console=True, stderr=False) + content_to_file.append(line) + if out_file: + util.write_file(out_file, "".join(content_to_file), 0644, "w") + else: + util.multi_log("".join(content_to_file), console=True, stderr=False) -- cgit v1.2.3 From bce8220f1688af1e155661bfd57e73fe23c42522 Mon Sep 17 00:00:00 2001 From: Shraddha Pandhe Date: Wed, 11 Dec 2013 15:51:40 -0800 Subject: Removed method _format_yaml --- cloudinit/config/cc_debug.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/cloudinit/config/cc_debug.py b/cloudinit/config/cc_debug.py index a3d99da8..971af71b 100644 --- a/cloudinit/config/cc_debug.py +++ b/cloudinit/config/cc_debug.py @@ -21,19 +21,6 @@ import copy import yaml -def _format_yaml(obj): - try: - formatted = yaml.safe_dump(obj, - line_break="\n", - indent=4, - explicit_start=True, - explicit_end=True, - default_flow_style=False) - return formatted.strip() - except: - return "???" - - def _make_header(text): header = StringIO() header.write("-" * 80) -- cgit v1.2.3 From 67ede943d3a02878061be2314300644c636b14f8 Mon Sep 17 00:00:00 2001 From: Shraddha Pandhe Date: Wed, 11 Dec 2013 15:56:08 -0800 Subject: Removed yaml import --- cloudinit/config/cc_debug.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cloudinit/config/cc_debug.py b/cloudinit/config/cc_debug.py index 971af71b..c0a447cb 100644 --- a/cloudinit/config/cc_debug.py +++ b/cloudinit/config/cc_debug.py @@ -18,7 +18,6 @@ from StringIO import StringIO from cloudinit import util from cloudinit import type_utils import copy -import yaml def _make_header(text): -- cgit v1.2.3 From 5414797f1b9286ddbe82c8637dfff74cf352b967 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 12 Dec 2013 11:47:53 -0500 Subject: fix pep8 and pylint warnings This fixes warnings raised by: ./tools/run-pep8 cloudinit/config/cc_debug.py ./tools/run-pylint cloudinit/config/cc_debug.py --- cloudinit/config/cc_debug.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cloudinit/config/cc_debug.py b/cloudinit/config/cc_debug.py index c0a447cb..55fdf2dd 100644 --- a/cloudinit/config/cc_debug.py +++ b/cloudinit/config/cc_debug.py @@ -32,7 +32,7 @@ def _make_header(text): def handle(name, cfg, cloud, log, args): - verbose = util.get_cfg_by_path(cfg, ('debug','verbose'), default=True) + verbose = util.get_cfg_by_path(cfg, ('debug', 'verbose'), default=True) if not verbose: log.debug(("Skipping module named %s," " verbose printing disabled"), name) @@ -41,7 +41,7 @@ def handle(name, cfg, cloud, log, args): if args: out_file = args[0] else: - out_file = util.get_cfg_by_path(cfg, ('debug','output')) + out_file = util.get_cfg_by_path(cfg, ('debug', 'output')) # Clean out some keys that we just don't care about showing... dump_cfg = copy.deepcopy(cfg) for k in ['log_cfgs']: @@ -59,7 +59,8 @@ def handle(name, cfg, cloud, log, args): to_print.write(util.yaml_dumps(cloud.datasource.metadata)) to_print.write("\n") to_print.write(_make_header("Misc")) - to_print.write("Datasource: %s\n" % (type_utils.obj_name(cloud.datasource))) + to_print.write("Datasource: %s\n" % + (type_utils.obj_name(cloud.datasource))) to_print.write("Distro: %s\n" % (type_utils.obj_name(cloud.distro))) to_print.write("Hostname: %s\n" % (cloud.get_hostname(True))) to_print.write("Instance ID: %s\n" % (cloud.get_instance_id())) -- cgit v1.2.3 From fd5231ae771cd3b87c26ac2b0839fb672bf0acee Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 12 Dec 2013 11:50:55 -0500 Subject: be verbose explicitly if run from cmdline. Let the command line (or module args) that set outfile explicitly override a config'd value of 'verbose'. Ie, if /etc/cloud/cloud.cfg.d/my.cfg had: debug: verbose: False but the user ran: cloud-init single --frequency=always --name=debug output.txt Then they probably wanted to have the debug in output.txt even though verbose was configured to False. --- cloudinit/config/cc_debug.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/cloudinit/config/cc_debug.py b/cloudinit/config/cc_debug.py index 55fdf2dd..cfd31fa1 100644 --- a/cloudinit/config/cc_debug.py +++ b/cloudinit/config/cc_debug.py @@ -33,15 +33,17 @@ def _make_header(text): def handle(name, cfg, cloud, log, args): verbose = util.get_cfg_by_path(cfg, ('debug', 'verbose'), default=True) - if not verbose: - log.debug(("Skipping module named %s," - " verbose printing disabled"), name) - return - out_file = None if args: + # if args are provided (from cmdline) then explicitly set verbose out_file = args[0] + verbose = True else: out_file = util.get_cfg_by_path(cfg, ('debug', 'output')) + + if not verbose: + log.debug(("Skipping module named %s," + " verbose printing disabled"), name) + return # Clean out some keys that we just don't care about showing... dump_cfg = copy.deepcopy(cfg) for k in ['log_cfgs']: -- cgit v1.2.3 From 9a922bfa70306d89535574c6580296c9b02ec43e Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 12 Dec 2013 14:22:14 -0500 Subject: fix 'make yaml'. --- doc/examples/cloud-config-datasources.txt | 1 - doc/examples/cloud-config-disk-setup.txt | 426 +++++++++++++++--------------- doc/examples/cloud-config.txt | 2 +- 3 files changed, 214 insertions(+), 215 deletions(-) diff --git a/doc/examples/cloud-config-datasources.txt b/doc/examples/cloud-config-datasources.txt index 65a3cdf5..3bde4aac 100644 --- a/doc/examples/cloud-config-datasources.txt +++ b/doc/examples/cloud-config-datasources.txt @@ -49,7 +49,6 @@ datasource: hostname_bounce: interface: eth0 policy: on # [can be 'on', 'off' or 'force'] - } SmartOS: # Smart OS datasource works over a serial console interacting with diff --git a/doc/examples/cloud-config-disk-setup.txt b/doc/examples/cloud-config-disk-setup.txt index 6ad61c33..0dfef8e8 100644 --- a/doc/examples/cloud-config-disk-setup.txt +++ b/doc/examples/cloud-config-disk-setup.txt @@ -1,24 +1,24 @@ -Cloud-init supports the creation of simple partition tables and file systems -on devices. +# Cloud-init supports the creation of simple partition tables and file systems +# on devices. -Default disk definitions for AWS --------------------------------- -(Not implemented yet, but provided for future documentation) +# Default disk definitions for AWS +# -------------------------------- +# (Not implemented yet, but provided for future documentation) - disk_setup: - ephmeral0: - type: 'mbr' - layout: True - overwrite: False +disk_setup: + ephmeral0: + type: 'mbr' + layout: True + overwrite: False - fs_setup: - - label: None, - filesystem: ext3 - device: ephemeral0 - partition: auto +fs_setup: + - label: None, + filesystem: ext3 + device: ephemeral0 + partition: auto -Default disk definitions for Windows Azure ------------------------------------------- +# Default disk definitions for Windows Azure +# ------------------------------------------ device_aliases: {'ephemeral0': '/dev/sdb'} disk_setup: @@ -34,8 +34,8 @@ fs_setup: replace_fs: ntfs -Default disk definitions for SmartOS ------------------------------------- +# Default disk definitions for SmartOS +# ------------------------------------ device_aliases: {'ephemeral0': '/dev/sdb'} disk_setup: @@ -49,203 +49,203 @@ fs_setup: filesystem: ext3 device: ephemeral0.0 -Cavaut for SmartOS: if ephemeral disk is not defined, then the disk will - not be automatically added to the mounts. - - -The default definition is used to make sure that the ephemeral storage is -setup properly. - -"disk_setup": disk partitioning --------------------------------- - -The disk_setup directive instructs Cloud-init to partition a disk. The format is: - - disk_setup: - ephmeral0: - type: 'mbr' - layout: 'auto' - /dev/xvdh: - type: 'mbr' - layout: - - 33 - - [33, 82] - - 33 - overwrite: True - -The format is a list of dicts of dicts. The first value is the name of the -device and the subsequent values define how to create and layout the partition. - -The general format is: - disk_setup: - : - type: 'mbr' - layout: - overwrite: - -Where: - : The name of the device. 'ephemeralX' and 'swap' are special - values which are specific to the cloud. For these devices - Cloud-init will look up what the real devices is and then - use it. - - For other devices, the kernel device name is used. At this - time only simply kernel devices are supported, meaning - that device mapper and other targets may not work. - - Note: At this time, there is no handling or setup of - device mapper targets. +# Cavaut for SmartOS: if ephemeral disk is not defined, then the disk will +# not be automatically added to the mounts. - type=: Currently the following are supported: - 'mbr': default and setups a MS-DOS partition table - Note: At this time only 'mbr' partition tables are allowed. - It is anticipated in the future that we'll have GPT as - option in the future, or even "RAID" to create a mdadm - RAID. +# The default definition is used to make sure that the ephemeral storage is +# setup properly. - layout={...}: The device layout. This is a list of values, with the - percentage of disk that partition will take. - Valid options are: - [, [, is the _percentage_ of the disk to use, while - is the numerical value of the partition type. +# The disk_setup directive instructs Cloud-init to partition a disk. The format is: - The following setups two partitions, with the first - partition having a swap label, taking 1/3 of the disk space - and the remainder being used as the second partition. - /dev/xvdh': - type: 'mbr' - layout: - - [33,82] - - 66 - overwrite: True - - When layout is "true" it means single partition the entire - device. - - When layout is "false" it means don't partition or ignore - existing partitioning. - - If layout is set to "true" and overwrite is set to "false", - it will skip partitioning the device without a failure. - - overwrite=: This describes whether to ride with saftey's on and - everything holstered. - - 'false' is the default, which means that: - 1. The device will be checked for a partition table - 2. The device will be checked for a file system - 3. If either a partition of file system is found, then - the operation will be _skipped_. - - 'true' is cowboy mode. There are no checks and things are - done blindly. USE with caution, you can do things you - really, really don't want to do. - - -fs_setup: Setup the file system -------------------------------- - -fs_setup describes the how the file systems are supposed to look. +disk_setup: + ephmeral0: + type: 'mbr' + layout: 'auto' + /dev/xvdh: + type: 'mbr' + layout: + - 33 + - [33, 82] + - 33 + overwrite: True + +# The format is a list of dicts of dicts. The first value is the name of the +# device and the subsequent values define how to create and layout the +# partition. +# The general format is: +# disk_setup: +# : +# type: 'mbr' +# layout: +# overwrite: +# +# Where: +# : The name of the device. 'ephemeralX' and 'swap' are special +# values which are specific to the cloud. For these devices +# Cloud-init will look up what the real devices is and then +# use it. +# +# For other devices, the kernel device name is used. At this +# time only simply kernel devices are supported, meaning +# that device mapper and other targets may not work. +# +# Note: At this time, there is no handling or setup of +# device mapper targets. +# +# type=: Currently the following are supported: +# 'mbr': default and setups a MS-DOS partition table +# +# Note: At this time only 'mbr' partition tables are allowed. +# It is anticipated in the future that we'll have GPT as +# option in the future, or even "RAID" to create a mdadm +# RAID. +# +# layout={...}: The device layout. This is a list of values, with the +# percentage of disk that partition will take. +# Valid options are: +# [, [, is the _percentage_ of the disk to use, while +# is the numerical value of the partition type. +# +# The following setups two partitions, with the first +# partition having a swap label, taking 1/3 of the disk space +# and the remainder being used as the second partition. +# /dev/xvdh': +# type: 'mbr' +# layout: +# - [33,82] +# - 66 +# overwrite: True +# +# When layout is "true" it means single partition the entire +# device. +# +# When layout is "false" it means don't partition or ignore +# existing partitioning. +# +# If layout is set to "true" and overwrite is set to "false", +# it will skip partitioning the device without a failure. +# +# overwrite=: This describes whether to ride with saftey's on and +# everything holstered. +# +# 'false' is the default, which means that: +# 1. The device will be checked for a partition table +# 2. The device will be checked for a file system +# 3. If either a partition of file system is found, then +# the operation will be _skipped_. +# +# 'true' is cowboy mode. There are no checks and things are +# done blindly. USE with caution, you can do things you +# really, really don't want to do. +# +# +# fs_setup: Setup the file system +# ------------------------------- +# +# fs_setup describes the how the file systems are supposed to look. - fs_setup: - - label: ephemeral0 - filesystem: 'ext3' - device: 'ephemeral0' - partition: 'auto' - - label: mylabl2 - filesystem: 'ext4' - device: '/dev/xvda1' - - special: - cmd: mkfs -t %(FILESYSTEM)s -L %(LABEL)s %(DEVICE)s - filesystem: 'btrfs' - device: '/dev/xvdh' - -The general format is: - fs_setup: - - label: