From b740568ddfe6194af7c2c5d7123b3cc2a07fa7f7 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 7 Mar 2014 19:33:41 -0800 Subject: Add some basic template tests --- tests/unittests/test_templating.py | 56 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 tests/unittests/test_templating.py (limited to 'tests') diff --git a/tests/unittests/test_templating.py b/tests/unittests/test_templating.py new file mode 100644 index 00000000..b4f425a8 --- /dev/null +++ b/tests/unittests/test_templating.py @@ -0,0 +1,56 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2014 Yahoo! Inc. +# +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from tests.unittests import helpers as test_helpers + + +from cloudinit import templater + + +class TestTemplates(test_helpers.TestCase): + def test_detection(self): + blob = "## template:cheetah" + + (template_type, contents) = templater.detect_template(blob) + self.assertEqual("cheetah", template_type) + self.assertEqual("", contents.strip()) + + blob = "blahblah $blah" + (template_type, contents) = templater.detect_template(blob) + self.assertEqual("cheetah", template_type) + self.assertEquals(blob, contents) + + blob = '##template:something-new' + self.assertRaises(ValueError, templater.detect_template, blob) + + def test_render_cheetah(self): + blob = '''## template:cheetah +$a,$b''' + c = templater.render_string(blob, {"a": 1, "b": 2}) + self.assertEquals("1,2", c) + + def test_render_jinja(self): + blob = '''## template:jinja +{{a}},{{b}}''' + c = templater.render_string(blob, {"a": 1, "b": 2}) + self.assertEquals("1,2", c) + + def test_render_default(self): + blob = '''$a,$b''' + c = templater.render_string(blob, {"a": 1, "b": 2}) + self.assertEquals("1,2", c) -- cgit v1.2.3 From 7dfabbefbadf626503bed8fe4c6e79243bde73ce Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 16 Jul 2014 11:31:31 -0700 Subject: Add basic renderer support and more robust import handling --- cloudinit/templater.py | 107 ++++++++++++++++++++++++++++--------- tests/unittests/test_templating.py | 24 +++++++-- 2 files changed, 100 insertions(+), 31 deletions(-) (limited to 'tests') diff --git a/cloudinit/templater.py b/cloudinit/templater.py index 9922c633..459f241a 100644 --- a/cloudinit/templater.py +++ b/cloudinit/templater.py @@ -20,30 +20,72 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import collections import re -from Cheetah.Template import Template as CTemplate +try: + from Cheetah.Template import Template as CTemplate + CHEETAH_AVAILABLE = True +except (ImportError, AttributeError): + CHEETAH_AVAILABLE = False -import jinja2 -from jinja2 import Template as JTemplate +try: + import jinja2 + from jinja2 import Template as JTemplate + JINJA_AVAILABLE = True +except (ImportError, AttributeError): + JINJA_AVAILABLE = False from cloudinit import log as logging +from cloudinit import type_utils as tu from cloudinit import util LOG = logging.getLogger(__name__) -DEF_RENDERER = 'cheetah' -RENDERERS = { - 'jinja': (lambda content, params: - JTemplate(content, - undefined=jinja2.StrictUndefined, - trim_blocks=True).render(**params)), - 'cheetah': (lambda content, params: - CTemplate(content, searchList=[params]).respond()), -} TYPE_MATCHER = re.compile(r"##\s*template:(.*)", re.I) +BASIC_MATCHER = re.compile(r'\$\{([A-Za-z0-9_.]+)\}') + + +def basic_render(content, params): + """This does simple replacement of bash variable like templates. + + It identifies patterns like ${a} and can also identify patterns like + ${a.b} which will look for a key 'b' in the dictionary rooted by key 'a'. + """ + + def replacer(match): + path = collections.deque(match.group(1).split(".")) + selected_params = params + while len(path) > 1: + key = path.popleft() + if not isinstance(selected_params, dict): + raise TypeError("Can not traverse into" + " non-dictionary '%s' of type %s while" + " looking for subkey '%s'" + % (selected_params, + tu.obj_name(selected_params), + key)) + selected_params = selected_params[key] + key = path.popleft() + if not isinstance(selected_params, dict): + raise TypeError("Can not extract key '%s' from non-dictionary" + " '%s' of type %s" + % (key, selected_params, + tu.obj_name(selected_params))) + return str(selected_params[key]) + + return BASIC_MATCHER.sub(replacer, content) def detect_template(text): + + def cheetah_render(content, params): + return CTemplate(content, searchList=[params]).respond() + + def jinja_render(content, params): + return JTemplate(content, + undefined=jinja2.StrictUndefined, + trim_blocks=True).render(**params) + if text.find("\n") != -1: ident, rest = text.split("\n", 1) else: @@ -51,14 +93,32 @@ def detect_template(text): rest = '' type_match = TYPE_MATCHER.match(ident) if not type_match: - return (DEF_RENDERER, text) - template_type = type_match.group(1).lower().strip() - if template_type not in RENDERERS: - raise ValueError("Unknown template type '%s' requested" - % template_type) + if not CHEETAH_AVAILABLE: + LOG.warn("Cheetah not available as the default renderer for" + " unknown template, reverting to the basic renderer.") + return ('basic', basic_render, text) + else: + return ('cheetah', cheetah_render, text) else: - return (template_type, rest) - + template_type = type_match.group(1).lower().strip() + if template_type not in ('jinja', 'cheetah', 'basic'): + raise ValueError("Unknown template rendering type '%s' requested" + % template_type) + if template_type == 'jinja' and not JINJA_AVAILABLE: + LOG.warn("Jinja not available as the selected renderer for" + " desired template, reverting to the basic renderer.") + return ('basic', basic_render, rest) + elif template_type == 'jinja' and JINJA_AVAILABLE: + return ('jinja', jinja_render, rest) + if template_type == 'cheetah' and not CHEETAH_AVAILABLE: + LOG.warn("Cheetah not available as the selected renderer for" + " desired template, reverting to the basic renderer.") + return ('basic', basic_render, rest) + elif template_type == 'cheetah' and CHEETAH_AVAILABLE: + return ('cheetah', cheetah_render, rest) + # Only thing left over is the basic renderer (it is always available). + return ('basic', basic_render, rest) + def render_from_file(fn, params): return render_string(util.load_file(fn), params) @@ -72,10 +132,5 @@ def render_to_file(fn, outfn, params, mode=0644): def render_string(content, params): if not params: params = {} - try: - renderer, content = detect_template(content) - except ValueError as e: - renderer = DEF_RENDERER - LOG.warn("%s, using renderer %s", e, renderer) - LOG.debug("Rendering %s using renderer '%s'", content, renderer) - return RENDERERS[renderer](content, params) + template_type, renderer, content = detect_template(content) + return renderer(content, params) diff --git a/tests/unittests/test_templating.py b/tests/unittests/test_templating.py index b4f425a8..c3faac3d 100644 --- a/tests/unittests/test_templating.py +++ b/tests/unittests/test_templating.py @@ -18,21 +18,35 @@ from tests.unittests import helpers as test_helpers - from cloudinit import templater class TestTemplates(test_helpers.TestCase): + def test_render_basic(self): + in_data = """ +${b} + +c = d +""" + in_data = in_data.strip() + expected_data = """ +2 + +c = d +""" + out_data = templater.basic_render(in_data, {'b': 2}) + self.assertEqual(expected_data.strip(), out_data) + def test_detection(self): blob = "## template:cheetah" - (template_type, contents) = templater.detect_template(blob) - self.assertEqual("cheetah", template_type) + (template_type, renderer, contents) = templater.detect_template(blob) + self.assertIn("cheetah", template_type) self.assertEqual("", contents.strip()) blob = "blahblah $blah" - (template_type, contents) = templater.detect_template(blob) - self.assertEqual("cheetah", template_type) + (template_type, renderer, contents) = templater.detect_template(blob) + self.assertIn("cheetah", template_type) self.assertEquals(blob, contents) blob = '##template:something-new' -- cgit v1.2.3 From f9d18e2ed747a6d44e60547fbcc0bbff780f351f Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 18 Jul 2014 10:52:11 -0700 Subject: Add non braces matching and a few more tests --- cloudinit/templater.py | 15 +++++++++++---- tests/unittests/test_templating.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 4 deletions(-) (limited to 'tests') diff --git a/cloudinit/templater.py b/cloudinit/templater.py index 95916dff..02f6261d 100644 --- a/cloudinit/templater.py +++ b/cloudinit/templater.py @@ -42,18 +42,25 @@ from cloudinit import util LOG = logging.getLogger(__name__) TYPE_MATCHER = re.compile(r"##\s*template:(.*)", re.I) -BASIC_MATCHER = re.compile(r'\$\{([A-Za-z0-9_.]+)\}') +BASIC_MATCHER = re.compile(r'\$\{([A-Za-z0-9_.]+)\}|\$([A-Za-z0-9_.]+)') def basic_render(content, params): """This does simple replacement of bash variable like templates. - It identifies patterns like ${a} and can also identify patterns like - ${a.b} which will look for a key 'b' in the dictionary rooted by key 'a'. + It identifies patterns like ${a} or $a and can also identify patterns like + ${a.b} or $a.b which will look for a key 'b' in the dictionary rooted + by key 'a'. """ def replacer(match): - path = collections.deque(match.group(1).split(".")) + # Only 1 of the 2 groups will actually have a valid entry. + name = match.group(1) + if name is None: + name = match.group(2) + if name is None: + raise RuntimeError("Match encountered but no valid group present") + path = collections.deque(name.split(".")) selected_params = params while len(path) > 1: key = path.popleft() diff --git a/tests/unittests/test_templating.py b/tests/unittests/test_templating.py index c3faac3d..5601a2e1 100644 --- a/tests/unittests/test_templating.py +++ b/tests/unittests/test_templating.py @@ -68,3 +68,39 @@ $a,$b''' blob = '''$a,$b''' c = templater.render_string(blob, {"a": 1, "b": 2}) self.assertEquals("1,2", c) + + def test_render_basic_deeper(self): + hn = 'myfoohost.yahoo.com' + expected_data = "h=%s\nc=d\n" % hn + in_data = "h=$hostname.canonical_name\nc=d\n" + params = { + "hostname": { + "canonical_name": hn, + }, + } + out_data = templater.render_string(in_data, params) + self.assertEqual(expected_data, out_data) + + def test_render_basic_no_parens(self): + hn = "myfoohost" + in_data = "h=$hostname\nc=d\n" + expected_data = "h=%s\nc=d\n" % hn + out_data = templater.basic_render(in_data, {'hostname': hn}) + self.assertEqual(expected_data, out_data) + + def test_render_basic_parens(self): + hn = "myfoohost" + in_data = "h = ${hostname}\nc=d\n" + expected_data = "h = %s\nc=d\n" % hn + out_data = templater.basic_render(in_data, {'hostname': hn}) + self.assertEqual(expected_data, out_data) + + def test_render_basic2(self): + mirror = "mymirror" + codename = "zany" + in_data = "deb $mirror $codename-updates main contrib non-free" + ex_data = "deb %s %s-updates main contrib non-free" % (mirror, codename) + + out_data = templater.basic_render(in_data, + {'mirror': mirror, 'codename': codename}) + self.assertEqual(ex_data, out_data) -- cgit v1.2.3 From 96e9e1b0509a9a000303ad72b2a332dc8821191e Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 21 Jul 2014 14:41:24 -0400 Subject: use textwrap, simple formatting improvement. --- tests/unittests/test_templating.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) (limited to 'tests') diff --git a/tests/unittests/test_templating.py b/tests/unittests/test_templating.py index 5601a2e1..1ec3004b 100644 --- a/tests/unittests/test_templating.py +++ b/tests/unittests/test_templating.py @@ -17,23 +17,24 @@ # along with this program. If not, see . from tests.unittests import helpers as test_helpers +import textwrap from cloudinit import templater class TestTemplates(test_helpers.TestCase): def test_render_basic(self): - in_data = """ -${b} + in_data = textwrap.dedent(""" + ${b} -c = d -""" + c = d + """) in_data = in_data.strip() - expected_data = """ -2 + expected_data = textwrap.dedent(""" + 2 -c = d -""" + c = d + """) out_data = templater.basic_render(in_data, {'b': 2}) self.assertEqual(expected_data.strip(), out_data) @@ -102,5 +103,5 @@ $a,$b''' ex_data = "deb %s %s-updates main contrib non-free" % (mirror, codename) out_data = templater.basic_render(in_data, - {'mirror': mirror, 'codename': codename}) + {'mirror': mirror, 'codename': codename}) self.assertEqual(ex_data, out_data) -- cgit v1.2.3