From 3b3b91960225d6525b8db9cc2789da0a62be2e14 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 22 Dec 2011 03:02:08 -0500 Subject: fix issue with part-handlers and base64 encoding (LP: #874342) Previously, * if content came into cloud-init for processing came in via a multipart mime file, and was already base64 encoded, it would get base64 encoded again before being handed to a part-handler. * if it came in via a '#include'd file then it would not be encoded at all. This drops the internal 'parts' array, that was just converted to and then from. Instead, we keep MIME format throughout and keep headers along the way. That means that a message that comes in with 'Content-Transfer-Encoding' set to 'base64' will be decoded before being handed to a part-handler. It also reduces the chance of failure due to content appearing to be an actual email. Previously if content contained colon separated fields, it might be read as headers (email.message_from_string(open("/etc/passwd","r")) would come back as all headers, no payload) The weak point right now is that '#include'd data cannot have mime types associated with it (unless it is a mime formatted content). I had hoped to read user headers and possibly set 'Content-Type' from that. --- cloudinit/UserDataHandler.py | 168 ++++++++++++++++++++++++------------------- 1 file changed, 94 insertions(+), 74 deletions(-) diff --git a/cloudinit/UserDataHandler.py b/cloudinit/UserDataHandler.py index 982e57f0..84ad9c9b 100644 --- a/cloudinit/UserDataHandler.py +++ b/cloudinit/UserDataHandler.py @@ -25,6 +25,8 @@ import yaml import cloudinit import cloudinit.util as util import hashlib +import os +import urllib starts_with_mappings={ '#include' : 'text/x-include-url', @@ -47,13 +49,12 @@ def decomp_str(str): except: return(str) -def do_include(str,parts): - import urllib +def do_include(content, appendmsg): import os # is just a list of urls, one per line # also support '#include ' includeonce = False - for line in str.splitlines(): + for line in content.splitlines(): if line == "#include": continue if line == "#include-once": includeonce = True @@ -81,31 +82,70 @@ def do_include(str,parts): except Exception as e: raise - process_includes(email.message_from_string(decomp_str(content)),parts) + process_includes(message_from_string(decomp_str(content)), appendmsg) -def explode_cc_archive(archive,parts): +def explode_cc_archive(archive, appendmsg): for ent in yaml.load(archive): # ent can be one of: # dict { 'filename' : 'value' , 'content' : 'value', 'type' : 'value' } # filename and type not be present # or # scalar(payload) - filename = 'part-%03d' % len(parts['content']) + def_type = "text/cloud-config" if isinstance(ent,str): - content = ent - mtype = type_from_startswith(content,def_type) + ent = { 'content': ent } + + content = ent.get('content', '') + mtype = ent.get('type', None) + if mtype == None: + mtype = type_from_startswith(content, def_type) + + maintype, subtype = mtype.split('/', 1) + if maintype == "text": + msg = MIMEText(content, _subtype=subtype) else: - content = ent.get('content', '') - filename = ent.get('filename', filename) - mtype = ent.get('type', None) - if mtype == None: - mtype = type_from_startswith(payload,def_type) - - parts['content'].append(content) - parts['names'].append(filename) - parts['types'].append(mtype) + msg = MIMEBase(maintype, subtype) + msg.set_payload(content) + + if 'filename' in ent: + msg.add_header('Content-Disposition', 'attachment', + filename=ent['filename']) + + for header in ent.keys(): + if header in ('content', 'filename', 'type'): + continue + msg.add_header(header, ent['header']) + + _attach_part(appendmsg,msg) + + +def multi_part_count(outermsg, newcount=None): + """ + Return the number of attachments to this MIMEMultipart by looking + at its 'Number-Attachments' header. + """ + nfield = 'Number-Attachments' + if nfield not in outermsg: + outermsg[nfield] = "0" + + if newcount != None: + outermsg.replace_header(nfield, str(newcount)) + + return(int(outermsg.get('Number-Attachments', 0))) + +def _attach_part(outermsg, part): + """ + Attach an part to an outer message. outermsg must be a MIMEMultipart. + Modifies a header in outermsg to keep track of number of attachments. + """ + cur = multi_part_count(outermsg) + if not part.get_filename(None): + part.add_header('Content-Disposition', 'attachment', + filename = 'part-%03d' % (cur+1)) + outermsg.attach(part) + multi_part_count(outermsg, cur+1) def type_from_startswith(payload, default=None): # slist is sorted longest first @@ -115,85 +155,65 @@ def type_from_startswith(payload, default=None): return(starts_with_mappings[sstr]) return default -def process_includes(msg,parts): - # parts is a dictionary of arrays - # parts['content'] - # parts['names'] - # parts['types'] - for t in ( 'content', 'names', 'types' ): - if not parts.has_key(t): - parts[t]=[ ] +def process_includes(msg, appendmsg=None): + if appendmsg == None: + appendmsg = MIMEMultipart() + for part in msg.walk(): # multipart/* are just containers if part.get_content_maintype() == 'multipart': continue - payload = part.get_payload() - ctype = None ctype_orig = part.get_content_type() + + payload = part.get_payload(decode=True) + if ctype_orig == "text/plain": ctype = type_from_startswith(payload) if ctype is None: ctype = ctype_orig - if ctype == 'text/x-include-url': - do_include(payload,parts) - continue - - if ctype == 'text/x-include-once-url': - do_include(payload,parts) + if ctype in ('text/x-include-url', 'text/x-include-once-url'): + do_include(payload, appendmsg) continue if ctype == "text/cloud-config-archive": - explode_cc_archive(payload,parts) + explode_cc_archive(payload, appendmsg) continue - filename = part.get_filename() - if not filename: - filename = 'part-%03d' % len(parts['content']) - - parts['content'].append(payload) - parts['types'].append(ctype) - parts['names'].append(filename) - -def parts2mime(parts): - outer = MIMEMultipart() + part['Content-Type'] = ctype + _attach_part(appendmsg, part) - i = 0 - while i < len(parts['content']): - if parts['types'][i] is None: - # No guess could be made, or the file is encoded (compressed), so - # use a generic bag-of-bits type. - ctype = 'application/octet-stream' - else: ctype = parts['types'][i] - maintype, subtype = ctype.split('/', 1) - if maintype == 'text': - msg = MIMEText(parts['content'][i], _subtype=subtype) - else: - msg = MIMEBase(maintype, subtype) - msg.set_payload(parts['content'][i]) - # Encode the payload using Base64 - encoders.encode_base64(msg) - # Set the filename parameter - msg.add_header('Content-Disposition', 'attachment', - filename=parts['names'][i]) - outer.attach(msg) +def message_from_string(data, headers={}): + if "mime-version:" in data[0:4096].lower(): + was_mime = True + msg = email.message_from_string(data) + for (key,val) in headers.items(): + if key in msg: + msg.replace_header(key,val) + else: + msg[key] = val + else: + was_mime = False + mtype = headers.get("Content-Type","text/plain") + maintype, subtype = mtype.split("/", 1) + msg = MIMEBase(maintype, subtype, *headers) + msg.set_payload(data) - i=i+1 - return(outer.as_string()) + return(msg) # this is heavily wasteful, reads through userdata string input def preprocess_userdata(data): - parts = { } - process_includes(email.message_from_string(decomp_str(data)),parts) - return(parts2mime(parts)) + newmsg = MIMEMultipart() + process_includes(message_from_string(decomp_str(data)), newmsg) + return(newmsg.as_string()) # callback is a function that will be called with (data, content_type, filename, payload) def walk_userdata(istr, callback, data = None): partnum = 0 - for part in email.message_from_string(istr).walk(): + for part in message_from_string(istr).walk(): # multipart/* are just containers if part.get_content_maintype() == 'multipart': continue @@ -206,14 +226,14 @@ def walk_userdata(istr, callback, data = None): if not filename: filename = 'part-%03d' % partnum - callback(data, ctype, filename, part.get_payload()) + callback(data, ctype, filename, part.get_payload(decode=True)) partnum = partnum+1 if __name__ == "__main__": import sys data = decomp_str(file(sys.argv[1]).read()) - parts = { } - process_includes(email.message_from_string(data),parts) - print "#found %s parts" % len(parts['content']) - print parts2mime(parts) + newmsg = MIMEMultipart() + process_includes(message_from_string(data), newmsg) + print newmsg + print "#found %s parts" % multi_part_count(newmsg) -- cgit v1.2.3 From 585284eca9e9219c6517eebc02d45edc800cf72d Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 22 Dec 2011 03:40:52 -0500 Subject: fix issue with multiple content-types --- cloudinit/UserDataHandler.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cloudinit/UserDataHandler.py b/cloudinit/UserDataHandler.py index 84ad9c9b..541ee87a 100644 --- a/cloudinit/UserDataHandler.py +++ b/cloudinit/UserDataHandler.py @@ -183,7 +183,11 @@ def process_includes(msg, appendmsg=None): explode_cc_archive(payload, appendmsg) continue - part['Content-Type'] = ctype + if 'Content-Type' in msg: + msg.replace_header('Content-Type', ctype) + else: + msg['Content-Type'] = ctype + _attach_part(appendmsg, part) def message_from_string(data, headers={}): -- cgit v1.2.3