From 7b24230437a27779bafa0828e73ab595c5aeb202 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 10 Jul 2012 17:06:45 -0700 Subject: Fixes 1012854 by implementing file writing, adjusts other code to have user/group parsing in util instead of in stages.py, renames decomp_str to decomp_gzip since it is more meaningful when named that (as thats all it can decompress). --- cloudinit/config/cc_write_files.py | 69 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 cloudinit/config/cc_write_files.py (limited to 'cloudinit/config/cc_write_files.py') diff --git a/cloudinit/config/cc_write_files.py b/cloudinit/config/cc_write_files.py new file mode 100644 index 00000000..683eac27 --- /dev/null +++ b/cloudinit/config/cc_write_files.py @@ -0,0 +1,69 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import base64 +import os + +from cloudinit import util +from cloudinit.settings import PER_INSTANCE + +frequency = PER_INSTANCE + +DEFAULT_PERMS = 0644 + + +def handle(name, cfg, _cloud, log, _args): + files = cfg.get('files') + if not files: + log.debug(("Skipping module named %s," + " no/empty 'files' key in configuration"), name) + return + write_files(name, files, log) + + +def write_files(name, files, log): + if not files: + return + + for (i, f_info) in enumerate(files): + path = f_info.get('path') + if not path: + log.warn("No path provided to write for entry %s in module %s", + i + 1, name) + continue + path = os.path.abspath(path) + contents = decode_string(f_info.get('content', ''), + f_info.get('compression')) + (u, g) = util.extract_usergroup(f_info.get('owner')) + perms = safe_int(f_info.get('permissions'), DEFAULT_PERMS) + util.write_file(path, contents, mode=perms) + util.chownbyname(path, u, g) + + +def safe_int(text, default): + try: + return int(text) + except (TypeError, ValueError): + return default + + +def decode_string(contents, content_type): + if util.is_true(content_type, addons=['gzip', 'gz']): + contents_dec = base64.b64decode(contents) + contents = util.decomp_gzip(contents_dec, quiet=False) + return contents -- cgit v1.2.3 From eb939e7765f490529aade46e38ee0f85d8a911dd Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 11 Jul 2012 11:20:24 -0700 Subject: Adjust the decoding of the files that are given so that a canonicalize happens first, which will examine the incoming encoding, and decide the neccasary decoding types needed to get the final resultant string and then use these normalized decoding types to actually do the final decode. Also change the name of the config key that is looked up to 'write_files' since 'files' is pretty generic and could have clashes with other modules. Add an example that shows how to use this in the different encoding formats that are supported. --- cloudinit/config/cc_write_files.py | 49 ++++++++++++++++++++++++++----- doc/examples/cloud-config-write-files.txt | 32 ++++++++++++++++++++ 2 files changed, 73 insertions(+), 8 deletions(-) create mode 100644 doc/examples/cloud-config-write-files.txt (limited to 'cloudinit/config/cc_write_files.py') diff --git a/cloudinit/config/cc_write_files.py b/cloudinit/config/cc_write_files.py index 683eac27..9e8f63a5 100644 --- a/cloudinit/config/cc_write_files.py +++ b/cloudinit/config/cc_write_files.py @@ -25,10 +25,11 @@ from cloudinit.settings import PER_INSTANCE frequency = PER_INSTANCE DEFAULT_PERMS = 0644 +UNKNOWN_ENC = 'text/plain' def handle(name, cfg, _cloud, log, _args): - files = cfg.get('files') + files = cfg.get('write_files') if not files: log.debug(("Skipping module named %s," " no/empty 'files' key in configuration"), name) @@ -36,6 +37,29 @@ def handle(name, cfg, _cloud, log, _args): write_files(name, files, log) +def canonicalize_decoding(enc): + if not enc: + enc = '' + enc = enc.lower().strip() + # Translate to a mime-type (or set of) that will be understood + # when decoding (for now we only support a limited set of known mime-types) + # See: http://tiny.cc/m4kahw + # See: http://www.iana.org/assignments/media-types/index.html + if enc in ['gz', 'gzip']: + # Should we assume that this is 'always' base64? + # Someone might of got lucky and not had to encode it? + return ['application/x-gzip'] + if enc in ['gz+base64', 'gzip+base64', 'gz+b64', 'gzip+b64']: + return ['application/base64', 'application/x-gzip'] + if enc in ['base64', 'b64']: + return ['application/base64'] + if enc in ['base32', 'b32']: + return ['application/base32'] + if enc in ['base16', 'b16']: + return ['application/base16'] + return [UNKNOWN_ENC] + + def write_files(name, files, log): if not files: return @@ -47,8 +71,8 @@ def write_files(name, files, log): i + 1, name) continue path = os.path.abspath(path) - contents = decode_string(f_info.get('content', ''), - f_info.get('compression')) + decodings = canonicalize_decoding(f_info.get('encoding')) + contents = decode_contents(f_info.get('content', ''), decodings) (u, g) = util.extract_usergroup(f_info.get('owner')) perms = safe_int(f_info.get('permissions'), DEFAULT_PERMS) util.write_file(path, contents, mode=perms) @@ -62,8 +86,17 @@ def safe_int(text, default): return default -def decode_string(contents, content_type): - if util.is_true(content_type, addons=['gzip', 'gz']): - contents_dec = base64.b64decode(contents) - contents = util.decomp_gzip(contents_dec, quiet=False) - return contents +def decode_contents(contents, decodings): + result = str(contents) + for enc in decodings: + if enc == 'application/x-gzip': + result = util.decomp_gzip(result, quiet=False) + elif enc == 'application/base64': + result = base64.b64decode(result) + elif enc == 'application/base32': + result = base64.b32decode(result) + elif enc == 'application/base16': + result = base64.b16decode(result) + elif enc == UNKNOWN_ENC: + pass + return result diff --git a/doc/examples/cloud-config-write-files.txt b/doc/examples/cloud-config-write-files.txt new file mode 100644 index 00000000..fb0c1541 --- /dev/null +++ b/doc/examples/cloud-config-write-files.txt @@ -0,0 +1,32 @@ +#cloud-config +# vim: syntax=yaml +# +# This is the configuration syntax that the write_files module +# will know how to understand, it can be given b64, b32, b16, or +# gz (or gz+b64) encoded strings which will be decoded accordingly +# and then written to the path that is provided. +# +# Note: Content strings here are truncated for example purposes. +# +write_files: +- content: CiMgVGhpcyBmaWxlIGNvbnRyb2xzIHRoZSBzdGF0ZSBvZiBTRUxpbnV4IG9u..... + encoding: b64 + path: /etc/sysconfig/selinux +- content: ' + + # My new /etc/sysconfig/samba file + + SMBDOPTIONS="-D" + + ' + path: /etc/sysconfig/samba +- content: H4sIADXC/U8C/+1Yf2wbVx1/d7YT122TQEMpadS6kgOZqL30J+..... + encoding: gz+b64 + path: /usr/bin/uptime +- content: BIRSAU3FOR2GS3THOMQGM33SEB2GQZJAINJE6TRAMRQWK3LPNYXAU=== + encoding: b32 + path: /etc/sysconfig/crond +- content: 0A232053657474696E677320666F7220746865204E4653206461656D6F6E2E0A + encoding: b16 + path: /etc/sysconfig/nfs + -- cgit v1.2.3 From f79570474118e8e26fa7e6db324434f2980eae8f Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 11 Jul 2012 11:25:01 -0700 Subject: Fix log message after 'write_files' key change. --- cloudinit/config/cc_write_files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/config/cc_write_files.py') diff --git a/cloudinit/config/cc_write_files.py b/cloudinit/config/cc_write_files.py index 9e8f63a5..ed2abe9e 100644 --- a/cloudinit/config/cc_write_files.py +++ b/cloudinit/config/cc_write_files.py @@ -32,7 +32,7 @@ def handle(name, cfg, _cloud, log, _args): files = cfg.get('write_files') if not files: log.debug(("Skipping module named %s," - " no/empty 'files' key in configuration"), name) + " no/empty 'write_files' key in configuration"), name) return write_files(name, files, log) -- cgit v1.2.3 From 50a5728db977ec11f3448c473a396995ea29319e Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 11 Jul 2012 13:02:24 -0700 Subject: Update the write files with the new content/compression handling. Adjust the examples file to reflect this. --- cloudinit/config/cc_write_files.py | 64 +++++++++++++++---------------- doc/examples/cloud-config-write-files.txt | 27 +++++++------ 2 files changed, 48 insertions(+), 43 deletions(-) (limited to 'cloudinit/config/cc_write_files.py') diff --git a/cloudinit/config/cc_write_files.py b/cloudinit/config/cc_write_files.py index ed2abe9e..061b9810 100644 --- a/cloudinit/config/cc_write_files.py +++ b/cloudinit/config/cc_write_files.py @@ -24,6 +24,7 @@ from cloudinit.settings import PER_INSTANCE frequency = PER_INSTANCE +DEFAULT_OWNER = "root:root" DEFAULT_PERMS = 0644 UNKNOWN_ENC = 'text/plain' @@ -37,26 +38,23 @@ def handle(name, cfg, _cloud, log, _args): write_files(name, files, log) -def canonicalize_decoding(enc): - if not enc: - enc = '' - enc = enc.lower().strip() - # Translate to a mime-type (or set of) that will be understood - # when decoding (for now we only support a limited set of known mime-types) - # See: http://tiny.cc/m4kahw - # See: http://www.iana.org/assignments/media-types/index.html - if enc in ['gz', 'gzip']: - # Should we assume that this is 'always' base64? - # Someone might of got lucky and not had to encode it? +def canonicalize_extraction(compression_type, log): + if not compression_type: + compression_type = '' + compression_type = compression_type.lower().strip() + if compression_type in ['gz', 'gzip']: return ['application/x-gzip'] - if enc in ['gz+base64', 'gzip+base64', 'gz+b64', 'gzip+b64']: + if compression_type in ['gz+base64', 'gzip+base64', 'gz+b64', 'gzip+b64']: return ['application/base64', 'application/x-gzip'] - if enc in ['base64', 'b64']: + # Yaml already encodes binary data as base64 if it is given to the + # yaml file as binary, so those will be automatically decoded for you. + # But the above b64 is just for people that are more 'comfortable' + # specifing it manually (which might be a possiblity) + if compression_type in ['b64', 'base64']: return ['application/base64'] - if enc in ['base32', 'b32']: - return ['application/base32'] - if enc in ['base16', 'b16']: - return ['application/base16'] + if compression_type: + log.warn("Unknown compression type %s, assuming %s", + compression_type, UNKNOWN_ENC) return [UNKNOWN_ENC] @@ -71,32 +69,34 @@ def write_files(name, files, log): i + 1, name) continue path = os.path.abspath(path) - decodings = canonicalize_decoding(f_info.get('encoding')) - contents = decode_contents(f_info.get('content', ''), decodings) - (u, g) = util.extract_usergroup(f_info.get('owner')) - perms = safe_int(f_info.get('permissions'), DEFAULT_PERMS) + extractions = canonicalize_extraction(f_info.get('compression'), log) + contents = extract_contents(f_info.get('content', ''), extractions) + (u, g) = util.extract_usergroup(f_info.get('owner', DEFAULT_OWNER)) + perms = decode_perms(f_info.get('permissions'), DEFAULT_PERMS, log) util.write_file(path, contents, mode=perms) util.chownbyname(path, u, g) -def safe_int(text, default): +def decode_perms(perm, default, log): try: - return int(text) + if isinstance(perm, (int, long, float)): + # Just 'downcast' it (if a float) + return int(perm) + else: + # Force to string and try octal conversion + return int(str(perm), 8) except (TypeError, ValueError): + log.warn("Undecodable permissions %s, assuming %s", perm, default) return default -def decode_contents(contents, decodings): +def extract_contents(contents, extraction_types): result = str(contents) - for enc in decodings: - if enc == 'application/x-gzip': + for t in extraction_types: + if t == 'application/x-gzip': result = util.decomp_gzip(result, quiet=False) - elif enc == 'application/base64': + elif t == 'application/base64': result = base64.b64decode(result) - elif enc == 'application/base32': - result = base64.b32decode(result) - elif enc == 'application/base16': - result = base64.b16decode(result) - elif enc == UNKNOWN_ENC: + elif t == UNKNOWN_ENC: pass return result diff --git a/doc/examples/cloud-config-write-files.txt b/doc/examples/cloud-config-write-files.txt index fb0c1541..09ec12c2 100644 --- a/doc/examples/cloud-config-write-files.txt +++ b/doc/examples/cloud-config-write-files.txt @@ -9,9 +9,11 @@ # Note: Content strings here are truncated for example purposes. # write_files: -- content: CiMgVGhpcyBmaWxlIGNvbnRyb2xzIHRoZSBzdGF0ZSBvZiBTRUxpbnV4IG9u..... - encoding: b64 +- compression: b64 + content: CiMgVGhpcyBmaWxlIGNvbnRyb2xzIHRoZSBzdGF0ZSBvZiBTRUxpbnV4... + owner: root:root path: /etc/sysconfig/selinux + perms: '0644' - content: ' # My new /etc/sysconfig/samba file @@ -20,13 +22,16 @@ write_files: ' path: /etc/sysconfig/samba -- content: H4sIADXC/U8C/+1Yf2wbVx1/d7YT122TQEMpadS6kgOZqL30J+..... - encoding: gz+b64 - path: /usr/bin/uptime -- content: BIRSAU3FOR2GS3THOMQGM33SEB2GQZJAINJE6TRAMRQWK3LPNYXAU=== - encoding: b32 - path: /etc/sysconfig/crond -- content: 0A232053657474696E677320666F7220746865204E4653206461656D6F6E2E0A - encoding: b16 - path: /etc/sysconfig/nfs +- content: !!binary | + f0VMRgIBAQAAAAAAAAAAAAIAPgABAAAAwARAAAAAAABAAAAAAAAAAJAVAAAAAAAAAAAAAEAAOAAI + AEAAHgAdAAYAAAAFAAAAQAAAAAAAAABAAEAAAAAAAEAAQAAAAAAAwAEAAAAAAADAAQAAAAAAAAgA + AAAAAAAAAwAAAAQAAAAAAgAAAAAAAAACQAAAAAAAAAJAAAAAAAAcAAAAAAAAABwAAAAAAAAAAQAA + .... + path: /bin/arch + perms: '0555' +- compression: gzip + content: !!binary | + H4sIAIDb/U8C/1NW1E/KzNMvzuBKTc7IV8hIzcnJVyjPL8pJ4QIA6N+MVxsAAAA= + path: /usr/bin/hello + perms: '0755' -- cgit v1.2.3