From c6dff581a9c253170d5e3f12fb83d16a8dec8257 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Thu, 12 Apr 2018 15:32:25 -0400 Subject: Implement ntp client spec with auto support for distro selection Add a base NTP client configuration dictionary and allow Distro specific changes to be merged. Add a select client function which implements logic to preferr installed clients over clients which need to be installed. Also allow distributions to override the cloud-init defaults. LP: #1749722 --- templates/chrony.conf.debian.tmpl | 39 ++++++++++++++++++++++++++++++ templates/chrony.conf.fedora.tmpl | 48 +++++++++++++++++++++++++++++++++++++ templates/chrony.conf.opensuse.tmpl | 38 +++++++++++++++++++++++++++++ templates/chrony.conf.rhel.tmpl | 45 ++++++++++++++++++++++++++++++++++ templates/chrony.conf.sles.tmpl | 38 +++++++++++++++++++++++++++++ templates/chrony.conf.ubuntu.tmpl | 42 ++++++++++++++++++++++++++++++++ 6 files changed, 250 insertions(+) create mode 100644 templates/chrony.conf.debian.tmpl create mode 100644 templates/chrony.conf.fedora.tmpl create mode 100644 templates/chrony.conf.opensuse.tmpl create mode 100644 templates/chrony.conf.rhel.tmpl create mode 100644 templates/chrony.conf.sles.tmpl create mode 100644 templates/chrony.conf.ubuntu.tmpl (limited to 'templates') diff --git a/templates/chrony.conf.debian.tmpl b/templates/chrony.conf.debian.tmpl new file mode 100644 index 00000000..e72bfee1 --- /dev/null +++ b/templates/chrony.conf.debian.tmpl @@ -0,0 +1,39 @@ +## template:jinja +# Welcome to the chrony configuration file. See chrony.conf(5) for more +# information about usuable directives. +{% if pools %}# pools +{% endif %} +{% for pool in pools -%} +pool {{pool}} iburst +{% endfor %} +{%- if servers %}# servers +{% endif %} +{% for server in servers -%} +server {{server}} iburst +{% endfor %} + +# This directive specify the location of the file containing ID/key pairs for +# NTP authentication. +keyfile /etc/chrony/chrony.keys + +# This directive specify the file into which chronyd will store the rate +# information. +driftfile /var/lib/chrony/chrony.drift + +# Uncomment the following line to turn logging on. +#log tracking measurements statistics + +# Log files location. +logdir /var/log/chrony + +# Stop bad estimates upsetting machine clock. +maxupdateskew 100.0 + +# This directive enables kernel synchronisation (every 11 minutes) of the +# real-time clock. Note that it can't be used along with the 'rtcfile' directive. +rtcsync + +# Step the system clock instead of slewing it if the adjustment is larger than +# one second, but only in the first three clock updates. +makestep 1 3 + diff --git a/templates/chrony.conf.fedora.tmpl b/templates/chrony.conf.fedora.tmpl new file mode 100644 index 00000000..8551f793 --- /dev/null +++ b/templates/chrony.conf.fedora.tmpl @@ -0,0 +1,48 @@ +## template:jinja +# Use public servers from the pool.ntp.org project. +# Please consider joining the pool (http://www.pool.ntp.org/join.html). +{% if pools %}# pools +{% endif %} +{% for pool in pools -%} +pool {{pool}} iburst +{% endfor %} +{%- if servers %}# servers +{% endif %} +{% for server in servers -%} +server {{server}} iburst +{% endfor %} + +# Record the rate at which the system clock gains/losses time. +driftfile /var/lib/chrony/drift + +# Allow the system clock to be stepped in the first three updates +# if its offset is larger than 1 second. +makestep 1.0 3 + +# Enable kernel synchronization of the real-time clock (RTC). +rtcsync + +# Enable hardware timestamping on all interfaces that support it. +#hwtimestamp * + +# Increase the minimum number of selectable sources required to adjust +# the system clock. +#minsources 2 + +# Allow NTP client access from local network. +#allow 192.168.0.0/16 + +# Serve time even if not synchronized to a time source. +#local stratum 10 + +# Specify file containing keys for NTP authentication. +#keyfile /etc/chrony.keys + +# Get TAI-UTC offset and leap seconds from the system tz database. +leapsectz right/UTC + +# Specify directory for log files. +logdir /var/log/chrony + +# Select which information is logged. +#log measurements statistics tracking diff --git a/templates/chrony.conf.opensuse.tmpl b/templates/chrony.conf.opensuse.tmpl new file mode 100644 index 00000000..a3d3e0ec --- /dev/null +++ b/templates/chrony.conf.opensuse.tmpl @@ -0,0 +1,38 @@ +## template:jinja +# Use public servers from the pool.ntp.org project. +# Please consider joining the pool (http://www.pool.ntp.org/join.html). +{% if pools %}# pools +{% endif %} +{% for pool in pools -%} +pool {{pool}} iburst +{% endfor %} +{%- if servers %}# servers +{% endif %} +{% for server in servers -%} +server {{server}} iburst +{% endfor %} + +# Record the rate at which the system clock gains/losses time. +driftfile /var/lib/chrony/drift + +# In first three updates step the system clock instead of slew +# if the adjustment is larger than 1 second. +makestep 1.0 3 + +# Enable kernel synchronization of the real-time clock (RTC). +rtcsync + +# Allow NTP client access from local network. +#allow 192.168/16 + +# Serve time even if not synchronized to any NTP server. +#local stratum 10 + +# Specify file containing keys for NTP authentication. +#keyfile /etc/chrony.keys + +# Specify directory for log files. +logdir /var/log/chrony + +# Select which information is logged. +#log measurements statistics tracking diff --git a/templates/chrony.conf.rhel.tmpl b/templates/chrony.conf.rhel.tmpl new file mode 100644 index 00000000..5b3542ef --- /dev/null +++ b/templates/chrony.conf.rhel.tmpl @@ -0,0 +1,45 @@ +## template:jinja +# Use public servers from the pool.ntp.org project. +# Please consider joining the pool (http://www.pool.ntp.org/join.html). +{% if pools %}# pools +{% endif %} +{% for pool in pools -%} +pool {{pool}} iburst +{% endfor %} +{%- if servers %}# servers +{% endif %} +{% for server in servers -%} +server {{server}} iburst +{% endfor %} + +# Record the rate at which the system clock gains/losses time. +driftfile /var/lib/chrony/drift + +# Allow the system clock to be stepped in the first three updates +# if its offset is larger than 1 second. +makestep 1.0 3 + +# Enable kernel synchronization of the real-time clock (RTC). +rtcsync + +# Enable hardware timestamping on all interfaces that support it. +#hwtimestamp * + +# Increase the minimum number of selectable sources required to adjust +# the system clock. +#minsources 2 + +# Allow NTP client access from local network. +#allow 192.168.0.0/16 + +# Serve time even if not synchronized to a time source. +#local stratum 10 + +# Specify file containing keys for NTP authentication. +#keyfile /etc/chrony.keys + +# Specify directory for log files. +logdir /var/log/chrony + +# Select which information is logged. +#log measurements statistics tracking diff --git a/templates/chrony.conf.sles.tmpl b/templates/chrony.conf.sles.tmpl new file mode 100644 index 00000000..a3d3e0ec --- /dev/null +++ b/templates/chrony.conf.sles.tmpl @@ -0,0 +1,38 @@ +## template:jinja +# Use public servers from the pool.ntp.org project. +# Please consider joining the pool (http://www.pool.ntp.org/join.html). +{% if pools %}# pools +{% endif %} +{% for pool in pools -%} +pool {{pool}} iburst +{% endfor %} +{%- if servers %}# servers +{% endif %} +{% for server in servers -%} +server {{server}} iburst +{% endfor %} + +# Record the rate at which the system clock gains/losses time. +driftfile /var/lib/chrony/drift + +# In first three updates step the system clock instead of slew +# if the adjustment is larger than 1 second. +makestep 1.0 3 + +# Enable kernel synchronization of the real-time clock (RTC). +rtcsync + +# Allow NTP client access from local network. +#allow 192.168/16 + +# Serve time even if not synchronized to any NTP server. +#local stratum 10 + +# Specify file containing keys for NTP authentication. +#keyfile /etc/chrony.keys + +# Specify directory for log files. +logdir /var/log/chrony + +# Select which information is logged. +#log measurements statistics tracking diff --git a/templates/chrony.conf.ubuntu.tmpl b/templates/chrony.conf.ubuntu.tmpl new file mode 100644 index 00000000..da7f16a4 --- /dev/null +++ b/templates/chrony.conf.ubuntu.tmpl @@ -0,0 +1,42 @@ +## template:jinja +# Welcome to the chrony configuration file. See chrony.conf(5) for more +# information about usuable directives. + +# Use servers from the NTP Pool Project. Approved by Ubuntu Technical Board +# on 2011-02-08 (LP: #104525). See http://www.pool.ntp.org/join.html for +# more information. +{% if pools %}# pools +{% endif %} +{% for pool in pools -%} +pool {{pool}} iburst +{% endfor %} +{%- if servers %}# servers +{% endif %} +{% for server in servers -%} +server {{server}} iburst +{% endfor %} + +# This directive specify the location of the file containing ID/key pairs for +# NTP authentication. +keyfile /etc/chrony/chrony.keys + +# This directive specify the file into which chronyd will store the rate +# information. +driftfile /var/lib/chrony/chrony.drift + +# Uncomment the following line to turn logging on. +#log tracking measurements statistics + +# Log files location. +logdir /var/log/chrony + +# Stop bad estimates upsetting machine clock. +maxupdateskew 100.0 + +# This directive enables kernel synchronisation (every 11 minutes) of the +# real-time clock. Note that it can't be used along with the 'rtcfile' directive. +rtcsync + +# Step the system clock instead of slewing it if the adjustment is larger than +# one second, but only in the first three clock updates. +makestep 1 3 -- cgit v1.2.3 From 4b86ab9a25b512420ecfe98953a3f3a6e4b4bba1 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 12 Apr 2018 15:51:07 -0600 Subject: renderer: support unicode in render_from_file. If a file passed to render_from_file had non-ascii text then jinja in python2 would decode as ascii, which would cause UnicodeDecodeError. This issue can be re-created in python2 with just: 'can\xe2\x80\x99t'.decode() The solution here is to explicitly pass in unicode supporting type (py3 str, py2 unicode). Those are six.text_type. Then jinja does not try to decode. The reason we hit this is that load_file calls decode_binary. decode_binary believes it has no work to do if it got a six.string_types. isinstance('can\xe2\x80\x99t', six.string_types) == True So it returns the original string which will blow up for jinja. Our fix here then is to load the file in binary mode and explicitly decode it to utf-8. Then in python2 we'll have a unicode type and in python3 we'll have a string type. --- cloudinit/templater.py | 10 +++++- templates/chrony.conf.debian.tmpl | 2 +- templates/chrony.conf.ubuntu.tmpl | 2 +- tests/unittests/test_handler/test_handler_ntp.py | 4 +-- tests/unittests/test_templating.py | 41 +++++++++++++++++++++++- 5 files changed, 53 insertions(+), 6 deletions(-) (limited to 'templates') diff --git a/cloudinit/templater.py b/cloudinit/templater.py index b3ea64e4..9a087e1c 100644 --- a/cloudinit/templater.py +++ b/cloudinit/templater.py @@ -121,7 +121,11 @@ def detect_template(text): def render_from_file(fn, params): if not params: params = {} - template_type, renderer, content = detect_template(util.load_file(fn)) + # jinja in python2 uses unicode internally. All py2 str will be decoded. + # If it is given a str that has non-ascii then it will raise a + # UnicodeDecodeError. So we explicitly convert to unicode type here. + template_type, renderer, content = detect_template( + util.load_file(fn, decode=False).decode('utf-8')) LOG.debug("Rendering content of '%s' using renderer %s", fn, template_type) return renderer(content, params) @@ -132,11 +136,15 @@ def render_to_file(fn, outfn, params, mode=0o644): def render_string_to_file(content, outfn, params, mode=0o644): + """Render string (or py2 unicode) to file. + Warning: py2 str with non-ascii chars will cause UnicodeDecodeError.""" contents = render_string(content, params) util.write_file(outfn, contents, mode=mode) def render_string(content, params): + """Render string (or py2 unicode). + Warning: py2 str with non-ascii chars will cause UnicodeDecodeError.""" if not params: params = {} template_type, renderer, content = detect_template(content) diff --git a/templates/chrony.conf.debian.tmpl b/templates/chrony.conf.debian.tmpl index e72bfee1..661bf04e 100644 --- a/templates/chrony.conf.debian.tmpl +++ b/templates/chrony.conf.debian.tmpl @@ -30,7 +30,7 @@ logdir /var/log/chrony maxupdateskew 100.0 # This directive enables kernel synchronisation (every 11 minutes) of the -# real-time clock. Note that it can't be used along with the 'rtcfile' directive. +# real-time clock. Note that it can’t be used along with the 'rtcfile' directive. rtcsync # Step the system clock instead of slewing it if the adjustment is larger than diff --git a/templates/chrony.conf.ubuntu.tmpl b/templates/chrony.conf.ubuntu.tmpl index da7f16a4..50a6f518 100644 --- a/templates/chrony.conf.ubuntu.tmpl +++ b/templates/chrony.conf.ubuntu.tmpl @@ -34,7 +34,7 @@ logdir /var/log/chrony maxupdateskew 100.0 # This directive enables kernel synchronisation (every 11 minutes) of the -# real-time clock. Note that it can't be used along with the 'rtcfile' directive. +# real-time clock. Note that it can’t be used along with the 'rtcfile' directive. rtcsync # Step the system clock instead of slewing it if the adjustment is larger than diff --git a/tests/unittests/test_handler/test_handler_ntp.py b/tests/unittests/test_handler/test_handler_ntp.py index 1b3ca570..02676aa6 100644 --- a/tests/unittests/test_handler/test_handler_ntp.py +++ b/tests/unittests/test_handler/test_handler_ntp.py @@ -272,12 +272,12 @@ class TestNtp(FilesystemMockingTestCase): expected_servers = '\n'.join([ 'server {0} iburst'.format(srv) for srv in servers]) print('distro=%s client=%s' % (distro, client)) - self.assertIn(expected_servers, content.decode(), + self.assertIn(expected_servers, content.decode('utf-8'), ('failed to render {0} conf' ' for distro:{1}'.format(client, distro))) expected_pools = '\n'.join([ 'pool {0} iburst'.format(pool) for pool in pools]) - self.assertIn(expected_pools, content.decode(), + self.assertIn(expected_pools, content.decode('utf-8'), ('failed to render {0} conf' ' for distro:{1}'.format(client, distro))) elif client == 'systemd-timesyncd': diff --git a/tests/unittests/test_templating.py b/tests/unittests/test_templating.py index 53154d33..1080e135 100644 --- a/tests/unittests/test_templating.py +++ b/tests/unittests/test_templating.py @@ -10,6 +10,7 @@ from cloudinit.tests import helpers as test_helpers import textwrap from cloudinit import templater +from cloudinit.util import load_file, write_file try: import Cheetah @@ -19,7 +20,17 @@ except ImportError: HAS_CHEETAH = False -class TestTemplates(test_helpers.TestCase): +class TestTemplates(test_helpers.CiTestCase): + jinja_utf8 = b'It\xe2\x80\x99s not ascii, {{name}}\n' + jinja_utf8_rbob = b'It\xe2\x80\x99s not ascii, bob\n'.decode('utf-8') + + @staticmethod + def add_header(renderer, data): + """Return text (py2 unicode/py3 str) with template header.""" + if isinstance(data, bytes): + data = data.decode('utf-8') + return "## template: %s\n" % renderer + data + def test_render_basic(self): in_data = textwrap.dedent(""" ${b} @@ -106,4 +117,32 @@ $a,$b''' 'codename': codename}) self.assertEqual(ex_data, out_data) + def test_jinja_nonascii_render_to_string(self): + """Test jinja render_to_string with non-ascii content.""" + self.assertEqual( + templater.render_string( + self.add_header("jinja", self.jinja_utf8), {"name": "bob"}), + self.jinja_utf8_rbob) + + def test_jinja_nonascii_render_to_file(self): + """Test jinja render_to_file of a filename with non-ascii content.""" + tmpl_fn = self.tmp_path("j-render-to-file.template") + out_fn = self.tmp_path("j-render-to-file.out") + write_file(filename=tmpl_fn, omode="wb", + content=self.add_header( + "jinja", self.jinja_utf8).encode('utf-8')) + templater.render_to_file(tmpl_fn, out_fn, {"name": "bob"}) + result = load_file(out_fn, decode=False).decode('utf-8') + self.assertEqual(result, self.jinja_utf8_rbob) + + def test_jinja_nonascii_render_from_file(self): + """Test jinja render_from_file with non-ascii content.""" + tmpl_fn = self.tmp_path("j-render-from-file.template") + write_file(tmpl_fn, omode="wb", + content=self.add_header( + "jinja", self.jinja_utf8).encode('utf-8')) + result = templater.render_from_file(tmpl_fn, {"name": "bob"}) + self.assertEqual(result, self.jinja_utf8_rbob) + + # vi: ts=4 expandtab -- cgit v1.2.3