diff options
| -rw-r--r-- | cloudinit/templater.py | 113 | ||||
| -rw-r--r-- | requirements.txt | 1 | ||||
| -rw-r--r-- | templates/chef_client.rb.tmpl | 30 | ||||
| -rw-r--r-- | templates/hosts.debian.tmpl | 21 | ||||
| -rw-r--r-- | templates/hosts.redhat.tmpl | 17 | ||||
| -rw-r--r-- | templates/hosts.suse.tmpl | 14 | ||||
| -rw-r--r-- | templates/resolv.conf.tmpl | 51 | ||||
| -rw-r--r-- | templates/sources.list.debian.tmpl | 50 | ||||
| -rw-r--r-- | templates/sources.list.ubuntu.tmpl | 102 | ||||
| -rw-r--r-- | tests/unittests/test_templating.py | 106 |
10 files changed, 359 insertions, 146 deletions
diff --git a/cloudinit/templater.py b/cloudinit/templater.py index 77af1270..02f6261d 100644 --- a/cloudinit/templater.py +++ b/cloudinit/templater.py @@ -20,13 +20,119 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -from Cheetah.Template import Template +import collections +import re +try: + from Cheetah.Template import Template as CTemplate + CHEETAH_AVAILABLE = True +except (ImportError, AttributeError): + CHEETAH_AVAILABLE = False + +try: + import jinja2 + from jinja2 import Template as JTemplate + JINJA_AVAILABLE = True +except (ImportError, AttributeError): + JINJA_AVAILABLE = False + +from cloudinit import log as logging +from cloudinit import type_utils as tu from cloudinit import util +LOG = logging.getLogger(__name__) +TYPE_MATCHER = re.compile(r"##\s*template:(.*)", re.I) +BASIC_MATCHER = re.compile(r'\$\{([A-Za-z0-9_.]+)\}|\$([A-Za-z0-9_.]+)') + + +def basic_render(content, params): + """This does simple replacement of bash variable like templates. + + It identifies patterns like ${a} or $a and can also identify patterns like + ${a.b} or $a.b which will look for a key 'b' in the dictionary rooted + by key 'a'. + """ + + def replacer(match): + # Only 1 of the 2 groups will actually have a valid entry. + name = match.group(1) + if name is None: + name = match.group(2) + if name is None: + raise RuntimeError("Match encountered but no valid group present") + path = collections.deque(name.split(".")) + selected_params = params + while len(path) > 1: + key = path.popleft() + if not isinstance(selected_params, dict): + raise TypeError("Can not traverse into" + " non-dictionary '%s' of type %s while" + " looking for subkey '%s'" + % (selected_params, + tu.obj_name(selected_params), + key)) + selected_params = selected_params[key] + key = path.popleft() + if not isinstance(selected_params, dict): + raise TypeError("Can not extract key '%s' from non-dictionary" + " '%s' of type %s" + % (key, selected_params, + tu.obj_name(selected_params))) + return str(selected_params[key]) + + return BASIC_MATCHER.sub(replacer, content) + + +def detect_template(text): + + def cheetah_render(content, params): + return CTemplate(content, searchList=[params]).respond() + + def jinja_render(content, params): + return JTemplate(content, + undefined=jinja2.StrictUndefined, + trim_blocks=True).render(**params) + + if text.find("\n") != -1: + ident, rest = text.split("\n", 1) + else: + ident = text + rest = '' + type_match = TYPE_MATCHER.match(ident) + if not type_match: + if not CHEETAH_AVAILABLE: + LOG.warn("Cheetah not available as the default renderer for" + " unknown template, reverting to the basic renderer.") + return ('basic', basic_render, text) + else: + return ('cheetah', cheetah_render, text) + else: + template_type = type_match.group(1).lower().strip() + if template_type not in ('jinja', 'cheetah', 'basic'): + raise ValueError("Unknown template rendering type '%s' requested" + % template_type) + if template_type == 'jinja' and not JINJA_AVAILABLE: + LOG.warn("Jinja not available as the selected renderer for" + " desired template, reverting to the basic renderer.") + return ('basic', basic_render, rest) + elif template_type == 'jinja' and JINJA_AVAILABLE: + return ('jinja', jinja_render, rest) + if template_type == 'cheetah' and not CHEETAH_AVAILABLE: + LOG.warn("Cheetah not available as the selected renderer for" + " desired template, reverting to the basic renderer.") + return ('basic', basic_render, rest) + elif template_type == 'cheetah' and CHEETAH_AVAILABLE: + return ('cheetah', cheetah_render, rest) + # Only thing left over is the basic renderer (it is always available). + return ('basic', basic_render, rest) + def render_from_file(fn, params): - return render_string(util.load_file(fn), params) + if not params: + params = {} + template_type, renderer, content = detect_template(util.load_file(fn)) + LOG.debug("Rendering content of '%s' using renderer %s", fn, template_type) + return renderer(content, params) def render_to_file(fn, outfn, params, mode=0644): @@ -37,4 +143,5 @@ def render_to_file(fn, outfn, params, mode=0644): def render_string(content, params): if not params: params = {} - return Template(content, searchList=[params]).respond() + template_type, renderer, content = detect_template(content) + return renderer(content, params) diff --git a/requirements.txt b/requirements.txt index fdcbd143..943dbef7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ # Used for untemplating any files or strings with parameters. cheetah +jinja2 # This is used for any pretty printing of tabular data. PrettyTable diff --git a/templates/chef_client.rb.tmpl b/templates/chef_client.rb.tmpl index 7981cba7..538850ca 100644 --- a/templates/chef_client.rb.tmpl +++ b/templates/chef_client.rb.tmpl @@ -1,25 +1,25 @@ -#* - This file is only utilized if the module 'cc_chef' is enabled in - cloud-config. Specifically, in order to enable it - you need to add the following to config: - chef: - validation_key: XYZ - validation_cert: XYZ - validation_name: XYZ - server_url: XYZ -*# +## template:jinja +{# +This file is only utilized if the module 'cc_chef' is enabled in +cloud-config. Specifically, in order to enable it +you need to add the following to config: + chef: + validation_key: XYZ + validation_cert: XYZ + validation_name: XYZ + server_url: XYZ +-#} log_level :info log_location "/var/log/chef/client.log" ssl_verify_mode :verify_none -validation_client_name "$validation_name" +validation_client_name "{{validation_name}}" validation_key "/etc/chef/validation.pem" client_key "/etc/chef/client.pem" -chef_server_url "$server_url" -environment "$environment" -node_name "$node_name" +chef_server_url "{{server_url}}" +environment "{{environment}}" +node_name "{{node_name}}" json_attribs "/etc/chef/firstboot.json" file_cache_path "/var/cache/chef" file_backup_path "/var/backups/chef" pid_file "/var/run/chef/client.pid" Chef::Log::Formatter.show_time = true - diff --git a/templates/hosts.debian.tmpl b/templates/hosts.debian.tmpl index ae120b02..a1d97212 100644 --- a/templates/hosts.debian.tmpl +++ b/templates/hosts.debian.tmpl @@ -1,19 +1,19 @@ -## This file (/etc/cloud/templates/hosts.tmpl) is only utilized -## if enabled in cloud-config. Specifically, in order to enable it -## you need to add the following to config: -## manage_etc_hosts: True -## -## Note, double-hash commented lines will not appear in /etc/hosts -# +## template:jinja +{# +This file (/etc/cloud/templates/hosts.tmpl) is only utilized +if enabled in cloud-config. Specifically, in order to enable it +you need to add the following to config: + manage_etc_hosts: True +-#} # Your system has configured 'manage_etc_hosts' as True. # As a result, if you wish for changes to this file to persist # then you will need to either # a.) make changes to the master file in /etc/cloud/templates/hosts.tmpl # b.) change or remove the value of 'manage_etc_hosts' in # /etc/cloud/cloud.cfg or cloud-config from user-data -# -## The value '$hostname' will be replaced with the local-hostname -127.0.1.1 $fqdn $hostname +# +{# The value '{{hostname}}' will be replaced with the local-hostname -#} +127.0.1.1 {{fqdn}} {{hostname}} 127.0.0.1 localhost # The following lines are desirable for IPv6 capable hosts @@ -23,3 +23,4 @@ ff00::0 ip6-mcastprefix ff02::1 ip6-allnodes ff02::2 ip6-allrouters ff02::3 ip6-allhosts + diff --git a/templates/hosts.redhat.tmpl b/templates/hosts.redhat.tmpl index 80459d95..bc5da32c 100644 --- a/templates/hosts.redhat.tmpl +++ b/templates/hosts.redhat.tmpl @@ -1,9 +1,10 @@ -#* - This file /etc/cloud/templates/hosts.redhat.tmpl is only utilized - if enabled in cloud-config. Specifically, in order to enable it - you need to add the following to config: - manage_etc_hosts: True -*# +## template:jinja +{# +This file /etc/cloud/templates/hosts.redhat.tmpl is only utilized +if enabled in cloud-config. Specifically, in order to enable it +you need to add the following to config: + manage_etc_hosts: True +-#} # Your system has configured 'manage_etc_hosts' as True. # As a result, if you wish for changes to this file to persist # then you will need to either @@ -12,12 +13,12 @@ # /etc/cloud/cloud.cfg or cloud-config from user-data # # The following lines are desirable for IPv4 capable hosts -127.0.0.1 ${fqdn} ${hostname} +127.0.0.1 {{fqdn}} {{hostname}} 127.0.0.1 localhost.localdomain localhost 127.0.0.1 localhost4.localdomain4 localhost4 # The following lines are desirable for IPv6 capable hosts -::1 ${fqdn} ${hostname} +::1 {{fqdn}} {{hostname}} ::1 localhost.localdomain localhost ::1 localhost6.localdomain6 localhost6 diff --git a/templates/hosts.suse.tmpl b/templates/hosts.suse.tmpl index 5d3d57e4..b6082692 100644 --- a/templates/hosts.suse.tmpl +++ b/templates/hosts.suse.tmpl @@ -1,9 +1,10 @@ -#* - This file /etc/cloud/templates/hosts.suse.tmpl is only utilized - if enabled in cloud-config. Specifically, in order to enable it - you need to add the following to config: - manage_etc_hosts: True -*# +## template:jinja +{# +This file /etc/cloud/templates/hosts.suse.tmpl is only utilized +if enabled in cloud-config. Specifically, in order to enable it +you need to add the following to config: + manage_etc_hosts: True +-#} # Your system has configured 'manage_etc_hosts' as True. # As a result, if you wish for changes to this file to persist # then you will need to either @@ -22,3 +23,4 @@ ff00::0 ipv6-mcastprefix ff02::1 ipv6-allnodes ff02::2 ipv6-allrouters ff02::3 ipv6-allhosts + diff --git a/templates/resolv.conf.tmpl b/templates/resolv.conf.tmpl index b7e97b13..6f908f30 100644 --- a/templates/resolv.conf.tmpl +++ b/templates/resolv.conf.tmpl @@ -1,39 +1,30 @@ -# +## template:jinja # Your system has been configured with 'manage-resolv-conf' set to true. # As a result, cloud-init has written this file with configuration data # that it has been provided. Cloud-init, by default, will write this file # a single time (PER_ONCE). # +{% if nameservers is defined %} +{% for server in nameservers %} +nameserver {{server}} +{% endfor %} -#if $varExists('nameservers') -#for $server in $nameservers -nameserver $server -#end for -#end if -#if $varExists('searchdomains') -search #slurp -#for $search in $searchdomains -$search #slurp -#end for +{% endif -%} +{% if searchdomains is defined %} +search {% for search in searchdomains %}{{search}} {% endfor %} -#end if -#if $varExists('domain') -domain $domain -#end if -#if $varExists('sortlist') -sortlist #slurp -#for $sort in $sortlist -$sort #slurp -#end for +{% endif %} +{% if domain is defined %} +domain {{domain}} +{% endif %} +{% if sortlist is defined %} -#end if -#if $varExists('options') or $varExists('flags') -options #slurp -#for $flag in $flags -$flag #slurp -#end for -#for $key, $value in $options.items() -$key:$value #slurp -#end for +sortlist {% for sort in sortlist %}{{sort}} {% endfor %} +{% endif %} +{% if options is defined or flags is defined %} -#end if +options {% for flag in flags %}{{flag}} {% endfor %} +{% for key, value in options.iteritems() -%} + {{key}}:{{value}} +{% endfor %} +{% endif %} diff --git a/templates/sources.list.debian.tmpl b/templates/sources.list.debian.tmpl index 609bc6bd..c8043f76 100644 --- a/templates/sources.list.debian.tmpl +++ b/templates/sources.list.debian.tmpl @@ -1,28 +1,32 @@ -\## Note, this file is written by cloud-init on first boot of an instance -\## modifications made here will not survive a re-bundle. -\## if you wish to make changes you can: -\## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg -\## or do the same in user-data -\## b.) add sources in /etc/apt/sources.list.d -\## c.) make changes to template file /etc/cloud/templates/sources.list.debian.tmpl -\### +## template:jinja +## Note, this file is written by cloud-init on first boot of an instance +## modifications made here will not survive a re-bundle. +## if you wish to make changes you can: +## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg +## or do the same in user-data +## b.) add sources in /etc/apt/sources.list.d +## c.) make changes to template file /etc/cloud/templates/sources.list.debian.tmpl +### # See http://www.debian.org/releases/stable/i386/release-notes/ch-upgrading.html # for how to upgrade to newer versions of the distribution. -deb $mirror $codename main contrib non-free -deb-src $mirror $codename main contrib non-free +deb {{mirror}} {{codename}} main contrib non-free +deb-src {{mirror}} {{codename}} main contrib non-free -\## Major bug fix updates produced after the final release of the -\## distribution. -deb $security $codename/updates main contrib non-free -deb-src $security $codename/updates main contrib non-free -deb $mirror $codename-updates main contrib non-free -deb-src $mirror $codename-updates main contrib non-free +## Major bug fix updates produced after the final release of the +## distribution. +deb {{security}} {{codename}}/updates main contrib non-free +deb-src {{security}} {{codename}}/updates main contrib non-free +deb {{mirror}} {{codename}}-updates main contrib non-free +deb-src {{mirror}} {{codename}}-updates main contrib non-free -\## Uncomment the following two lines to add software from the 'backports' -\## repository. -\## N.B. software from this repository may not have been tested as -\## extensively as that contained in the main release, although it includes -\## newer versions of some applications which may provide useful features. -# deb http://backports.debian.org/debian-backports $codename-backports main contrib non-free -# deb-src http://backports.debian.org/debian-backports $codename-backports main contrib non-free +## Uncomment the following two lines to add software from the 'backports' +## repository. +## +## N.B. software from this repository may not have been tested as +## extensively as that contained in the main release, although it includes +## newer versions of some applications which may provide useful features. +{# +deb http://backports.debian.org/debian-backports {{codename}}-backports main contrib non-free +deb-src http://backports.debian.org/debian-backports {{codename}}-backports main contrib non-free +-#} diff --git a/templates/sources.list.ubuntu.tmpl b/templates/sources.list.ubuntu.tmpl index ce395b3d..4b1b019a 100644 --- a/templates/sources.list.ubuntu.tmpl +++ b/templates/sources.list.ubuntu.tmpl @@ -1,60 +1,60 @@ -\## Note, this file is written by cloud-init on first boot of an instance -\## modifications made here will not survive a re-bundle. -\## if you wish to make changes you can: -\## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg -\## or do the same in user-data -\## b.) add sources in /etc/apt/sources.list.d -\## c.) make changes to template file /etc/cloud/templates/sources.list.tmpl -\### +## template:jinja +## Note, this file is written by cloud-init on first boot of an instance +## modifications made here will not survive a re-bundle. +## if you wish to make changes you can: +## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg +## or do the same in user-data +## b.) add sources in /etc/apt/sources.list.d +## c.) make changes to template file /etc/cloud/templates/sources.list.tmpl # See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to # newer versions of the distribution. -deb $mirror $codename main -deb-src $mirror $codename main +deb {{mirror}} {{codename}} main +deb-src {{mirror}} {{codename}} main -\## Major bug fix updates produced after the final release of the -\## distribution. -deb $mirror $codename-updates main -deb-src $mirror $codename-updates main +## Major bug fix updates produced after the final release of the +## distribution. +deb {{mirror}} {{codename}}-updates main +deb-src {{mirror}} {{codename}}-updates main -\## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu -\## team. Also, please note that software in universe WILL NOT receive any -\## review or updates from the Ubuntu security team. -deb $mirror $codename universe -deb-src $mirror $codename universe -deb $mirror $codename-updates universe -deb-src $mirror $codename-updates universe +## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu +## team. Also, please note that software in universe WILL NOT receive any +## review or updates from the Ubuntu security team. +deb {{mirror}} {{codename}} universe +deb-src {{mirror}} {{codename}} universe +deb {{mirror}} {{codename}}-updates universe +deb-src {{mirror}} {{codename}}-updates universe -\## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu -\## team, and may not be under a free licence. Please satisfy yourself as to -\## your rights to use the software. Also, please note that software in -\## multiverse WILL NOT receive any review or updates from the Ubuntu -\## security team. -# deb $mirror $codename multiverse -# deb-src $mirror $codename multiverse -# deb $mirror $codename-updates multiverse -# deb-src $mirror $codename-updates multiverse +## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu +## team, and may not be under a free licence. Please satisfy yourself as to +## your rights to use the software. Also, please note that software in +## multiverse WILL NOT receive any review or updates from the Ubuntu +## security team. +# deb {{mirror}} {{codename}} multiverse +# deb-src {{mirror}} {{codename}} multiverse +# deb {{mirror}} {{codename}}-updates multiverse +# deb-src {{mirror}} {{codename}}-updates multiverse -\## Uncomment the following two lines to add software from the 'backports' -\## repository. -\## N.B. software from this repository may not have been tested as -\## extensively as that contained in the main release, although it includes -\## newer versions of some applications which may provide useful features. -\## Also, please note that software in backports WILL NOT receive any review -\## or updates from the Ubuntu security team. -# deb $mirror $codename-backports main restricted universe multiverse -# deb-src $mirror $codename-backports main restricted universe multiverse +## Uncomment the following two lines to add software from the 'backports' +## repository. +## N.B. software from this repository may not have been tested as +## extensively as that contained in the main release, although it includes +## newer versions of some applications which may provide useful features. +## Also, please note that software in backports WILL NOT receive any review +## or updates from the Ubuntu security team. +# deb {{mirror}} {{codename}}-backports main restricted universe multiverse +# deb-src {{mirror}} {{codename}}-backports main restricted universe multiverse -\## Uncomment the following two lines to add software from Canonical's -\## 'partner' repository. -\## This software is not part of Ubuntu, but is offered by Canonical and the -\## respective vendors as a service to Ubuntu users. -# deb http://archive.canonical.com/ubuntu $codename partner -# deb-src http://archive.canonical.com/ubuntu $codename partner +## Uncomment the following two lines to add software from Canonical's +## 'partner' repository. +## This software is not part of Ubuntu, but is offered by Canonical and the +## respective vendors as a service to Ubuntu users. +# deb http://archive.canonical.com/ubuntu {{codename}} partner +# deb-src http://archive.canonical.com/ubuntu {{codename}} partner -deb $security $codename-security main -deb-src $security $codename-security main -deb $security $codename-security universe -deb-src $security $codename-security universe -# deb $security $codename-security multiverse -# deb-src $security $codename-security multiverse +deb {{security}} {{codename}}-security main +deb-src {{security}} {{codename}}-security main +deb {{security}} {{codename}}-security universe +deb-src {{security}} {{codename}}-security universe +# deb {{security}} {{codename}}-security multiverse +# deb-src {{security}} {{codename}}-security multiverse diff --git a/tests/unittests/test_templating.py b/tests/unittests/test_templating.py new file mode 100644 index 00000000..5601a2e1 --- /dev/null +++ b/tests/unittests/test_templating.py @@ -0,0 +1,106 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2014 Yahoo! Inc. +# +# Author: Joshua Harlow <harlowja@yahoo-inc.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +from tests.unittests import helpers as test_helpers + +from cloudinit import templater + + +class TestTemplates(test_helpers.TestCase): + def test_render_basic(self): + in_data = """ +${b} + +c = d +""" + in_data = in_data.strip() + expected_data = """ +2 + +c = d +""" + out_data = templater.basic_render(in_data, {'b': 2}) + self.assertEqual(expected_data.strip(), out_data) + + def test_detection(self): + blob = "## template:cheetah" + + (template_type, renderer, contents) = templater.detect_template(blob) + self.assertIn("cheetah", template_type) + self.assertEqual("", contents.strip()) + + blob = "blahblah $blah" + (template_type, renderer, contents) = templater.detect_template(blob) + self.assertIn("cheetah", template_type) + self.assertEquals(blob, contents) + + blob = '##template:something-new' + self.assertRaises(ValueError, templater.detect_template, blob) + + def test_render_cheetah(self): + blob = '''## template:cheetah +$a,$b''' + c = templater.render_string(blob, {"a": 1, "b": 2}) + self.assertEquals("1,2", c) + + def test_render_jinja(self): + blob = '''## template:jinja +{{a}},{{b}}''' + c = templater.render_string(blob, {"a": 1, "b": 2}) + self.assertEquals("1,2", c) + + def test_render_default(self): + blob = '''$a,$b''' + c = templater.render_string(blob, {"a": 1, "b": 2}) + self.assertEquals("1,2", c) + + def test_render_basic_deeper(self): + hn = 'myfoohost.yahoo.com' + expected_data = "h=%s\nc=d\n" % hn + in_data = "h=$hostname.canonical_name\nc=d\n" + params = { + "hostname": { + "canonical_name": hn, + }, + } + out_data = templater.render_string(in_data, params) + self.assertEqual(expected_data, out_data) + + def test_render_basic_no_parens(self): + hn = "myfoohost" + in_data = "h=$hostname\nc=d\n" + expected_data = "h=%s\nc=d\n" % hn + out_data = templater.basic_render(in_data, {'hostname': hn}) + self.assertEqual(expected_data, out_data) + + def test_render_basic_parens(self): + hn = "myfoohost" + in_data = "h = ${hostname}\nc=d\n" + expected_data = "h = %s\nc=d\n" % hn + out_data = templater.basic_render(in_data, {'hostname': hn}) + self.assertEqual(expected_data, out_data) + + def test_render_basic2(self): + mirror = "mymirror" + codename = "zany" + in_data = "deb $mirror $codename-updates main contrib non-free" + ex_data = "deb %s %s-updates main contrib non-free" % (mirror, codename) + + out_data = templater.basic_render(in_data, + {'mirror': mirror, 'codename': codename}) + self.assertEqual(ex_data, out_data) |