rsyslog: add new format of config

new format of config is namespaced under top level 'rsyslog'
rather than having 3 tokens in the top level namespace.

Additionally added here are:
 * tests
 * support for legacy config
 * support for 'remotes' configuration allowing simple config like
   rsyslog: remotes: {'myremote': "192.168.1.1"}

4 files changed, 513 insertions, 39 deletions

diff --git a/ChangeLog b/ChangeLog
index bef5f77d..8a1c77bc 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -57,6 +57,7 @@
    GCE regional mirrors. (LP: #1470890)
  - add udev rules that identify ephemeral device on Azure (LP: #1411582)
  - _read_dmi_syspath: fix bad log message causing unintended exception
+ - rsyslog: add additional configuration mode (LP: #1478103)
 0.7.6:
  - open 0.7.6
  - Enable vendordata on CloudSigma datasource (LP: #1303986)
diff --git a/cloudinit/config/cc_rsyslog.py b/cloudinit/config/cc_rsyslog.py
index 57486edc..5ecf1629 100644
--- a/cloudinit/config/cc_rsyslog.py
+++ b/cloudinit/config/cc_rsyslog.py
@@ -17,37 +17,165 @@
 #
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+rsyslog module allows configuration of syslog logging via rsyslog
+Configuration is done under the cloud-config top level 'rsyslog'.
+
+Under 'rsyslog' you can define:
+  - configs:  [default=[]]
+    this is a list.  entries in it are a string or a dictionary.
+    each entry has 2 parts:
+       * content
+       * filename
+    if the entry is a string, then it is assigned to 'content'.
+    for each entry, content is written to the provided filename.
+    if filename is not provided, its default is read from 'config_filename'
+
+    Content here can be any valid rsyslog configuration.  No format
+    specific format is enforced.
+
+    For simply logging to an existing remote syslog server, via udp:
+      configs: ["*.* @192.168.1.1"]
+
+  - remotes: [default={}]
+    This is a dictionary of name / value pairs.
+    In comparison to 'config's, it is more focused in that it only supports
+    remote syslog configuration.  It is not rsyslog specific, and could
+    convert to other syslog implementations.
+
+    Each entry in remotes is a 'name' and a 'value'.
+     * name: an string identifying the entry.  good practice would indicate
+       using a consistent and identifiable string for the producer.
+       For example, the MAAS service could use 'maas' as the key.
+     * value consists of the following parts:
+       * optional filter for log messages
+         default if not present: *.*
+       * optional leading '@' or '@@' (indicates udp or tcp respectively).
+         default if not present (udp): @
+         This is rsyslog format for that. if not present, is '@'.
+       * ipv4 or ipv6 or hostname
+         ipv6 addresses must be in [::1] format. (@[fd00::1]:514)
+       * optional port
+         port defaults to 514
+
+  - config_filename: [default=20-cloud-config.conf]
+    this is the file name to use if none is provided in a config entry.
+
+  - config_dir: [default=/etc/rsyslog.d]
+    this directory is used for filenames that are not absolute paths.
+
+  - service_reload_command: [default="auto"]
+    this command is executed if files have been written and thus the syslog
+    daemon needs to be told.
+
+Note, since cloud-init 0.5 a legacy version of rsyslog config has been
+present and is still supported. See below for the mappings between old
+value and new value:
+   old value           -> new value
+   'rsyslog'           -> rsyslog/configs
+   'rsyslog_filename'  -> rsyslog/config_filename
+   'rsyslog_dir'       -> rsyslog/config_dir
+
+the legacy config does not support 'service_reload_command'.
+
+Example config:
+  #cloud-config
+  rsyslog:
+    configs:
+      - "*.* @@192.158.1.1"
+      - content: "*.*   @@192.0.2.1:10514"
+        filename: 01-example.conf
+      - content: |
+        *.*   @@syslogd.example.com
+    remotes:
+      maas: "192.168.1.1"
+      juju: "10.0.4.1"
+    config_dir: config_dir
+    config_filename: config_filename
+    service_reload_command: [your, syslog, restart, command]
+
+Example Legacy config:
+  #cloud-config
+  rsyslog:
+    - "*.* @@192.158.1.1"
+  rsyslog_dir: /etc/rsyslog-config.d/
+  rsyslog_filename: 99-local.conf
+"""
 
 import os
+import re
+import six
 
+from cloudinit import log as logging
 from cloudinit import util
 
 DEF_FILENAME = "20-cloud-config.conf"
 DEF_DIR = "/etc/rsyslog.d"
+DEF_RELOAD = "auto"
+DEF_REMOTES = {}
 
+KEYNAME_CONFIGS = 'configs'
+KEYNAME_FILENAME = 'config_filename'
+KEYNAME_DIR = 'config_dir'
+KEYNAME_RELOAD = 'service_reload_command'
+KEYNAME_LEGACY_FILENAME = 'rsyslog_filename'
+KEYNAME_LEGACY_DIR = 'rsyslog_dir'
+KEYNAME_REMOTES = 'remotes'
 
-def handle(name, cfg, cloud, log, _args):
-    # rsyslog:
-    #  - "*.* @@192.158.1.1"
-    #  - content: "*.*   @@192.0.2.1:10514"
-    #  - filename: 01-examplecom.conf
-    #    content: |
-    #      *.*   @@syslogd.example.com
-
-    # process 'rsyslog'
-    if 'rsyslog' not in cfg:
-        log.debug(("Skipping module named %s,"
-                   " no 'rsyslog' key in configuration"), name)
-        return
+LOG = logging.getLogger(__name__)
 
-    def_dir = cfg.get('rsyslog_dir', DEF_DIR)
-    def_fname = cfg.get('rsyslog_filename', DEF_FILENAME)
+COMMENT_RE = re.compile(r'[ ]*[#]+[ ]*')
+HOST_PORT_RE = re.compile(
+    r'^(?P<proto>[@]{0,2})'
+    '(([[](?P<bracket_addr>[^\]]*)[\]])|(?P<addr>[^:]*))'
+    '([:](?P<port>[0-9]+))?$')
 
+def reload_syslog(command=DEF_RELOAD, systemd=False):
+    service = 'rsyslog'
+    if command == DEF_RELOAD:
+        if systemd:
+            cmd = ['systemctl', 'reload-or-try-restart', service]
+        else:
+            cmd = ['service', service, 'restart']
+    else:
+        cmd = command
+    util.subp(cmd, capture=True)
+
+
+def load_config(cfg):
+    # return an updated config with entries of the correct type
+    # support converting the old top level format into new format
+    mycfg = cfg.get('rsyslog', {})
+
+    if isinstance(cfg.get('rsyslog'), list):
+        mycfg = {KEYNAME_CONFIGS: cfg.get('rsyslog')}
+        if KEYNAME_LEGACY_FILENAME in cfg:
+            mycfg[KEYNAME_FILENAME] = cfg[KEYNAME_LEGACY_FILENAME]
+        if KEYNAME_LEGACY_DIR in cfg:
+            mycfg[KEYNAME_DIR] = cfg[KEYNAME_LEGACY_DIR]
+
+    fillup = (
+        (KEYNAME_CONFIGS, [], list),
+        (KEYNAME_DIR, DEF_DIR, six.string_types),
+        (KEYNAME_FILENAME, DEF_FILENAME, six.string_types),
+        (KEYNAME_RELOAD, DEF_RELOAD, six.string_types + (list,)),
+        (KEYNAME_REMOTES, DEF_REMOTES, dict))
+
+    for key, default, vtypes in fillup:
+        if key not in mycfg or not isinstance(mycfg[key], vtypes):
+            mycfg[key] = default
+
+    return mycfg
+
+
+def apply_rsyslog_changes(configs, def_fname, cfg_dir):
+    # apply the changes in 'configs' to the paths in def_fname and cfg_dir
+    # return a list of the files changed
     files = []
-    for i, ent in enumerate(cfg['rsyslog']):
+    for cur_pos, ent in enumerate(configs):
         if isinstance(ent, dict):
             if "content" not in ent:
-                log.warn("No 'content' entry in config entry %s", i + 1)
+                LOG.warn("No 'content' entry in config entry %s", cur_pos + 1)
                 continue
             content = ent['content']
             filename = ent.get("filename", def_fname)
@@ -57,11 +185,10 @@ def handle(name, cfg, cloud, log, _args):
 
         filename = filename.strip()
         if not filename:
-            log.warn("Entry %s has an empty filename", i + 1)
+            LOG.warn("Entry %s has an empty filename", cur_pos + 1)
             continue
 
-        if not filename.startswith("/"):
-            filename = os.path.join(def_dir, filename)
+        filename = os.path.join(cfg_dir, filename)
 
         # Truncate filename first time you see it
         omode = "ab"
@@ -70,27 +197,162 @@ def handle(name, cfg, cloud, log, _args):
             files.append(filename)
 
         try:
-            contents = "%s\n" % (content)
-            util.write_file(filename, contents, omode=omode)
+            endl = ""
+            if not content.endswith("\n"):
+                endl = "\n"
+            util.write_file(filename, content + endl, omode=omode)
         except Exception:
-            util.logexc(log, "Failed to write to %s", filename)
+            util.logexc(LOG, "Failed to write to %s", filename)
+
+    return files
+
+
+def parse_remotes_line(line, name=None):
+    try:
+        data, comment = COMMENT_RE.split(line)
+        comment = comment.strip()
+    except ValueError:
+        data, comment = (line, None)
+
+    toks = data.strip().split()
+    match = None
+    if len(toks) == 1:
+        host_port = data
+    elif len(toks) == 2:
+        match, host_port = toks
+    else:
+        raise ValueError("line had multiple spaces: %s" % data)
+
+    toks = HOST_PORT_RE.match(host_port)
+
+    if not toks:
+        raise ValueError("Invalid host specification '%s'" % host_port)
+
+    proto = toks.group('proto')
+    addr = toks.group('addr') or toks.group('bracket_addr')
+    port = toks.group('port')
+
+    if addr.startswith("[") and not addr.endswith("]"):
+        raise ValueError("host spec had invalid brackets: %s" % addr)
+
+    if comment and not name:
+        name = comment
+
+    t = SyslogRemotesLine(name=name, match=match, proto=proto,
+                          addr=addr, port=port)
+    t.validate()
+    return t
+
+
+class SyslogRemotesLine(object):
+    def __init__(self, name=None, match=None, proto=None, addr=None,
+                 port=None):
+        if not match:
+            match = "*.*"
+        self.name = name
+        self.match = match
+        if not proto:
+            proto = "udp"
+        if proto == "@":
+            proto = "udp"
+        elif proto == "@@":
+            proto = "tcp"
+        self.proto = proto
+
+        self.addr = addr
+        if port:
+            self.port = int(port)
+        else:
+            self.port = None
+
+    def validate(self):
+        if self.port:
+            try:
+                int(self.port)
+            except ValueError:
+                raise ValueError("port '%s' is not an integer" % self.port)
+
+        if not self.addr:
+            raise ValueError("address is required")
+
+    def __repr__(self):
+        return "[name=%s match=%s proto=%s address=%s port=%s]" % (
+            self.name, self.match, self.proto, self.addr, self.port
+        )
+
+    def __str__(self):
+        buf = self.match + " "
+        if self.proto == "udp":
+            buf += "@"
+        elif self.proto == "tcp":
+            buf += "@@"
+
+        if ":" in self.addr:
+            buf += "[" + self.addr + "]"
+        else:
+            buf += self.addr
+
+        if self.port:
+            buf += ":%s" % self.port
+
+        if self.name:
+            buf += " # %s" % self.name
+        return buf
+
+
+def remotes_to_rsyslog_cfg(remotes, header=None, footer=None):
+    if not remotes:
+        return None
+    lines = []
+    if header is not None:
+        lines.append(header)
+    for name, line in remotes.items():
+        try:
+            lines.append(str(parse_remotes_line(line, name=name)))
+        except ValueError as e:
+            LOG.warn("failed loading remote %s: %s [%s]", name, line, e)
+    if footer is not None:
+        lines.append(footer)
+    return '\n'.join(lines) + "\n"
+
+
+def handle(name, cfg, cloud, log, _args):
+    if 'rsyslog' not in cfg:
+        log.debug(("Skipping module named %s,"
+                   " no 'rsyslog' key in configuration"), name)
+        return
+
+    mycfg = load_config(cfg)
+    configs = mycfg[KEYNAME_CONFIGS]
+
+    if mycfg[KEYNAME_REMOTES]:
+        configs.append(
+            remotes_to_rsyslog_cfg(
+                mycfg[KEYNAME_REMOTES],
+                header="# begin remotes",
+                footer="# end remotes",
+            ))
+
+    if not mycfg['configs']:
+        log.debug("Empty config rsyslog['configs'], nothing to do")
+        return
+
+    changes = apply_rsyslog_changes(
+        configs=mycfg[KEYNAME_CONFIGS],
+        def_fname=mycfg[KEYNAME_FILENAME],
+        cfg_dir=mycfg[KEYNAME_DIR])
+
+    if not changes:
+        log.debug("restart of syslog not necessary, no changes made")
+        return
 
-    # Attempt to restart syslogd
-    restarted = False
     try:
-        # If this config module is running at cloud-init time
-        # (before rsyslog is running) we don't actually have to
-        # restart syslog.
-        #
-        # Upstart actually does what we want here, in that it doesn't
-        # start a service that wasn't running already on 'restart'
-        # it will also return failure on the attempt, so 'restarted'
-        # won't get set.
-        log.debug("Restarting rsyslog")
-        util.subp(['service', 'rsyslog', 'restart'])
-        restarted = True
-    except Exception:
-        util.logexc(log, "Failed restarting rsyslog")
+        restarted = reload_syslog(
+            command=mycfg[KEYNAME_RELOAD],
+            systemd=cloud.distro.uses_systemd()),
+    except util.ProcessExecutionError as e:
+        restarted = False
+        log.warn("Failed to reload syslog", e)
 
     if restarted:
         # This only needs to run if we *actually* restarted
@@ -98,4 +360,4 @@ def handle(name, cfg, cloud, log, _args):
         cloud.cycle_logging()
         # This should now use rsyslog if
         # the logging was setup to use it...
-        log.debug("%s configured %s files", name, files)
+        log.debug("%s configured %s files", name, changes)
diff --git a/doc/examples/cloud-config-rsyslog.txt b/doc/examples/cloud-config-rsyslog.txt
new file mode 100644
index 00000000..d54960e8
--- /dev/null
+++ b/doc/examples/cloud-config-rsyslog.txt
@@ -0,0 +1,46 @@
+## the rsyslog module allows you to configure the systems syslog.
+## configuration of syslog is under the top level cloud-config
+## entry 'rsyslog'.
+##
+## Example:
+#cloud-config
+rsyslog:
+ remotes:
+  # udp to host 'maas.mydomain' port 514
+  maashost: maas.mydomain
+  # udp to ipv4 host on port 514
+  maas: "@[10.5.1.56]:514"
+  # tcp to host ipv6 host on port 555
+  maasipv6: "*.* @@[FE80::0202:B3FF:FE1E:8329]:555"
+ configs:
+   - "*.* @@192.158.1.1"
+   - content: "*.*   @@192.0.2.1:10514"
+     filename: 01-example.conf
+   - content: |
+     *.*   @@syslogd.example.com
+ config_dir: /etc/rsyslog.d
+ config_filename: 20-cloud-config.conf
+ service_reload_command: [your, syslog, reload, command]
+
+## Additionally the following legacy format is supported
+## it is converted into the format above before use.
+##  rsyslog_filename -> rsyslog/config_filename
+##  rsyslog_dir -> rsyslog/config_dir
+##  rsyslog -> rsyslog/configs
+# rsyslog:
+#  - "*.* @@192.158.1.1"
+#  - content: "*.*   @@192.0.2.1:10514"
+#    filename: 01-example.conf
+#  - content: |
+#    *.*   @@syslogd.example.com
+# rsyslog_filename: 20-cloud-config.conf
+# rsyslog_dir: /etc/rsyslog.d
+
+## to configure rsyslog to accept remote logging on Ubuntu
+## write the following into /etc/rsyslog.d/20-remote-udp.conf
+##   $ModLoad imudp
+##   $UDPServerRun 514
+##   $template LogRemote,"/var/log/maas/rsyslog/%HOSTNAME%/messages"
+##   :fromhost-ip, !isequal, "127.0.0.1" ?LogRemote
+## then:
+## sudo service rsyslog restart
diff --git a/tests/unittests/test_handler/test_handler_rsyslog.py b/tests/unittests/test_handler/test_handler_rsyslog.py
new file mode 100644
index 00000000..7bfa65a9
--- /dev/null
+++ b/tests/unittests/test_handler/test_handler_rsyslog.py
@@ -0,0 +1,165 @@
+import os
+import shutil
+import tempfile
+
+from cloudinit.config.cc_rsyslog import (
+    apply_rsyslog_changes, DEF_DIR, DEF_FILENAME, DEF_RELOAD, load_config,
+    parse_remotes_line, remotes_to_rsyslog_cfg)
+from cloudinit import util
+
+from .. import helpers as t_help
+
+
+class TestLoadConfig(t_help.TestCase):
+    def setUp(self):
+        super(TestLoadConfig, self).setUp()
+        self.basecfg = {
+            'config_filename': DEF_FILENAME,
+            'config_dir': DEF_DIR,
+            'service_reload_command': DEF_RELOAD,
+            'configs': [],
+            'remotes': {},
+        }
+
+    def test_legacy_full(self):
+        found = load_config({
+            'rsyslog': ['*.* @192.168.1.1'],
+            'rsyslog_dir': "mydir",
+            'rsyslog_filename': "myfilename"})
+        self.basecfg.update({
+            'configs': ['*.* @192.168.1.1'],
+            'config_dir': "mydir",
+            'config_filename': 'myfilename',
+            'service_reload_command': 'auto'}
+            )
+
+        self.assertEqual(found, self.basecfg)
+
+    def test_legacy_defaults(self):
+        found = load_config({
+            'rsyslog': ['*.* @192.168.1.1']})
+        self.basecfg.update({
+            'configs': ['*.* @192.168.1.1']})
+        self.assertEqual(found, self.basecfg)
+
+    def test_new_defaults(self):
+        self.assertEqual(load_config({}), self.basecfg)
+
+    def test_new_configs(self):
+        cfgs = ['*.* myhost', '*.* my2host']
+        self.basecfg.update({'configs': cfgs})
+        self.assertEqual(
+            load_config({'rsyslog': {'configs': cfgs}}),
+            self.basecfg)
+
+
+class TestApplyChanges(t_help.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, self.tmp)
+
+    def test_simple(self):
+        cfgline = "*.* foohost"
+        changed = apply_rsyslog_changes(
+            configs=[cfgline], def_fname="foo.cfg", cfg_dir=self.tmp)
+
+        fname = os.path.join(self.tmp, "foo.cfg")
+        self.assertEqual([fname], changed)
+        self.assertEqual(
+            util.load_file(fname), cfgline + "\n")
+
+    def test_multiple_files(self):
+        configs = [
+            '*.* foohost',
+            {'content': 'abc', 'filename': 'my.cfg'},
+            {'content': 'filefoo-content',
+             'filename': os.path.join(self.tmp, 'mydir/mycfg')},
+        ]
+
+        changed = apply_rsyslog_changes(
+            configs=configs, def_fname="default.cfg", cfg_dir=self.tmp)
+
+        expected = [
+            (os.path.join(self.tmp, "default.cfg"),
+             "*.* foohost\n"),
+            (os.path.join(self.tmp, "my.cfg"), "abc\n"),
+            (os.path.join(self.tmp, "mydir/mycfg"), "filefoo-content\n"),
+        ]
+        self.assertEqual([f[0] for f in expected], changed)
+        actual = []
+        for fname, _content in expected:
+            util.load_file(fname)
+            actual.append((fname, util.load_file(fname),))
+        self.assertEqual(expected, actual)
+
+    def test_repeat_def(self):
+        configs = ['*.* foohost', "*.warn otherhost"]
+
+        changed = apply_rsyslog_changes(
+            configs=configs, def_fname="default.cfg", cfg_dir=self.tmp)
+
+        fname = os.path.join(self.tmp, "default.cfg")
+        self.assertEqual([fname], changed)
+
+        expected_content = '\n'.join([c for c in configs]) + '\n'
+        found_content = util.load_file(fname)
+        self.assertEqual(expected_content, found_content)
+
+    def test_multiline_content(self):
+        configs = ['line1', 'line2\nline3\n']
+
+        apply_rsyslog_changes(
+            configs=configs, def_fname="default.cfg", cfg_dir=self.tmp)
+
+        fname = os.path.join(self.tmp, "default.cfg")
+        expected_content = '\n'.join([c for c in configs])
+        found_content = util.load_file(fname)
+        self.assertEqual(expected_content, found_content)
+
+
+class TestParseRemotesLine(t_help.TestCase):
+    def test_valid_port(self):
+        r = parse_remotes_line("foo:9")
+        self.assertEqual(9, r.port)
+
+    def test_invalid_port(self):
+        with self.assertRaises(ValueError):
+            parse_remotes_line("*.* foo:abc")
+
+    def test_valid_ipv6(self):
+        r = parse_remotes_line("*.* [::1]")
+        self.assertEqual("*.* @[::1]", str(r))
+
+    def test_valid_ipv6_with_port(self):
+        r = parse_remotes_line("*.* [::1]:100")
+        self.assertEqual(r.port, 100)
+        self.assertEqual(r.addr, "::1")
+        self.assertEqual("*.* @[::1]:100", str(r))
+
+    def test_invalid_multiple_colon(self):
+        with self.assertRaises(ValueError):
+            parse_remotes_line("*.* ::1:100")
+
+    def test_name_in_string(self):
+        r = parse_remotes_line("syslog.host", name="foobar")
+        self.assertEqual("*.* @syslog.host # foobar", str(r))
+
+
+class TestRemotesToSyslog(t_help.TestCase):
+    def test_simple(self):
+        # str rendered line must appear in remotes_to_ryslog_cfg return
+        mycfg = "*.* myhost"
+        myline = str(parse_remotes_line(mycfg, name="myname"))
+        r = remotes_to_rsyslog_cfg({'myname': mycfg})
+        lines = r.splitlines()
+        self.assertEqual(1, len(lines))
+        self.assertTrue(myline in r.splitlines())
+
+    def test_header_footer(self):
+        header = "#foo head"
+        footer = "#foo foot"
+        r = remotes_to_rsyslog_cfg(
+            {'myname': "*.* myhost"}, header=header, footer=footer)
+        lines = r.splitlines()
+        self.assertTrue(header, lines[0])
+        self.assertTrue(footer, lines[-1])