diff options
| -rw-r--r-- | ChangeLog | 1 | ||||
| -rw-r--r-- | cloudinit/config/cc_rsyslog.py | 340 | ||||
| -rw-r--r-- | doc/examples/cloud-config-rsyslog.txt | 46 | ||||
| -rw-r--r-- | tests/unittests/test_handler/test_handler_rsyslog.py | 165 |
4 files changed, 513 insertions, 39 deletions
@@ -57,6 +57,7 @@ GCE regional mirrors. (LP: #1470890) - add udev rules that identify ephemeral device on Azure (LP: #1411582) - _read_dmi_syspath: fix bad log message causing unintended exception + - rsyslog: add additional configuration mode (LP: #1478103) 0.7.6: - open 0.7.6 - Enable vendordata on CloudSigma datasource (LP: #1303986) diff --git a/cloudinit/config/cc_rsyslog.py b/cloudinit/config/cc_rsyslog.py index 57486edc..5ecf1629 100644 --- a/cloudinit/config/cc_rsyslog.py +++ b/cloudinit/config/cc_rsyslog.py @@ -17,37 +17,165 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +rsyslog module allows configuration of syslog logging via rsyslog +Configuration is done under the cloud-config top level 'rsyslog'. + +Under 'rsyslog' you can define: + - configs: [default=[]] + this is a list. entries in it are a string or a dictionary. + each entry has 2 parts: + * content + * filename + if the entry is a string, then it is assigned to 'content'. + for each entry, content is written to the provided filename. + if filename is not provided, its default is read from 'config_filename' + + Content here can be any valid rsyslog configuration. No format + specific format is enforced. + + For simply logging to an existing remote syslog server, via udp: + configs: ["*.* @192.168.1.1"] + + - remotes: [default={}] + This is a dictionary of name / value pairs. + In comparison to 'config's, it is more focused in that it only supports + remote syslog configuration. It is not rsyslog specific, and could + convert to other syslog implementations. + + Each entry in remotes is a 'name' and a 'value'. + * name: an string identifying the entry. good practice would indicate + using a consistent and identifiable string for the producer. + For example, the MAAS service could use 'maas' as the key. + * value consists of the following parts: + * optional filter for log messages + default if not present: *.* + * optional leading '@' or '@@' (indicates udp or tcp respectively). + default if not present (udp): @ + This is rsyslog format for that. if not present, is '@'. + * ipv4 or ipv6 or hostname + ipv6 addresses must be in [::1] format. (@[fd00::1]:514) + * optional port + port defaults to 514 + + - config_filename: [default=20-cloud-config.conf] + this is the file name to use if none is provided in a config entry. + + - config_dir: [default=/etc/rsyslog.d] + this directory is used for filenames that are not absolute paths. + + - service_reload_command: [default="auto"] + this command is executed if files have been written and thus the syslog + daemon needs to be told. + +Note, since cloud-init 0.5 a legacy version of rsyslog config has been +present and is still supported. See below for the mappings between old +value and new value: + old value -> new value + 'rsyslog' -> rsyslog/configs + 'rsyslog_filename' -> rsyslog/config_filename + 'rsyslog_dir' -> rsyslog/config_dir + +the legacy config does not support 'service_reload_command'. + +Example config: + #cloud-config + rsyslog: + configs: + - "*.* @@192.158.1.1" + - content: "*.* @@192.0.2.1:10514" + filename: 01-example.conf + - content: | + *.* @@syslogd.example.com + remotes: + maas: "192.168.1.1" + juju: "10.0.4.1" + config_dir: config_dir + config_filename: config_filename + service_reload_command: [your, syslog, restart, command] + +Example Legacy config: + #cloud-config + rsyslog: + - "*.* @@192.158.1.1" + rsyslog_dir: /etc/rsyslog-config.d/ + rsyslog_filename: 99-local.conf +""" import os +import re +import six +from cloudinit import log as logging from cloudinit import util DEF_FILENAME = "20-cloud-config.conf" DEF_DIR = "/etc/rsyslog.d" +DEF_RELOAD = "auto" +DEF_REMOTES = {} +KEYNAME_CONFIGS = 'configs' +KEYNAME_FILENAME = 'config_filename' +KEYNAME_DIR = 'config_dir' +KEYNAME_RELOAD = 'service_reload_command' +KEYNAME_LEGACY_FILENAME = 'rsyslog_filename' +KEYNAME_LEGACY_DIR = 'rsyslog_dir' +KEYNAME_REMOTES = 'remotes' -def handle(name, cfg, cloud, log, _args): - # rsyslog: - # - "*.* @@192.158.1.1" - # - content: "*.* @@192.0.2.1:10514" - # - filename: 01-examplecom.conf - # content: | - # *.* @@syslogd.example.com - - # process 'rsyslog' - if 'rsyslog' not in cfg: - log.debug(("Skipping module named %s," - " no 'rsyslog' key in configuration"), name) - return +LOG = logging.getLogger(__name__) - def_dir = cfg.get('rsyslog_dir', DEF_DIR) - def_fname = cfg.get('rsyslog_filename', DEF_FILENAME) +COMMENT_RE = re.compile(r'[ ]*[#]+[ ]*') +HOST_PORT_RE = re.compile( + r'^(?P<proto>[@]{0,2})' + '(([[](?P<bracket_addr>[^\]]*)[\]])|(?P<addr>[^:]*))' + '([:](?P<port>[0-9]+))?$') +def reload_syslog(command=DEF_RELOAD, systemd=False): + service = 'rsyslog' + if command == DEF_RELOAD: + if systemd: + cmd = ['systemctl', 'reload-or-try-restart', service] + else: + cmd = ['service', service, 'restart'] + else: + cmd = command + util.subp(cmd, capture=True) + + +def load_config(cfg): + # return an updated config with entries of the correct type + # support converting the old top level format into new format + mycfg = cfg.get('rsyslog', {}) + + if isinstance(cfg.get('rsyslog'), list): + mycfg = {KEYNAME_CONFIGS: cfg.get('rsyslog')} + if KEYNAME_LEGACY_FILENAME in cfg: + mycfg[KEYNAME_FILENAME] = cfg[KEYNAME_LEGACY_FILENAME] + if KEYNAME_LEGACY_DIR in cfg: + mycfg[KEYNAME_DIR] = cfg[KEYNAME_LEGACY_DIR] + + fillup = ( + (KEYNAME_CONFIGS, [], list), + (KEYNAME_DIR, DEF_DIR, six.string_types), + (KEYNAME_FILENAME, DEF_FILENAME, six.string_types), + (KEYNAME_RELOAD, DEF_RELOAD, six.string_types + (list,)), + (KEYNAME_REMOTES, DEF_REMOTES, dict)) + + for key, default, vtypes in fillup: + if key not in mycfg or not isinstance(mycfg[key], vtypes): + mycfg[key] = default + + return mycfg + + +def apply_rsyslog_changes(configs, def_fname, cfg_dir): + # apply the changes in 'configs' to the paths in def_fname and cfg_dir + # return a list of the files changed files = [] - for i, ent in enumerate(cfg['rsyslog']): + for cur_pos, ent in enumerate(configs): if isinstance(ent, dict): if "content" not in ent: - log.warn("No 'content' entry in config entry %s", i + 1) + LOG.warn("No 'content' entry in config entry %s", cur_pos + 1) continue content = ent['content'] filename = ent.get("filename", def_fname) @@ -57,11 +185,10 @@ def handle(name, cfg, cloud, log, _args): filename = filename.strip() if not filename: - log.warn("Entry %s has an empty filename", i + 1) + LOG.warn("Entry %s has an empty filename", cur_pos + 1) continue - if not filename.startswith("/"): - filename = os.path.join(def_dir, filename) + filename = os.path.join(cfg_dir, filename) # Truncate filename first time you see it omode = "ab" @@ -70,27 +197,162 @@ def handle(name, cfg, cloud, log, _args): files.append(filename) try: - contents = "%s\n" % (content) - util.write_file(filename, contents, omode=omode) + endl = "" + if not content.endswith("\n"): + endl = "\n" + util.write_file(filename, content + endl, omode=omode) except Exception: - util.logexc(log, "Failed to write to %s", filename) + util.logexc(LOG, "Failed to write to %s", filename) + + return files + + +def parse_remotes_line(line, name=None): + try: + data, comment = COMMENT_RE.split(line) + comment = comment.strip() + except ValueError: + data, comment = (line, None) + + toks = data.strip().split() + match = None + if len(toks) == 1: + host_port = data + elif len(toks) == 2: + match, host_port = toks + else: + raise ValueError("line had multiple spaces: %s" % data) + + toks = HOST_PORT_RE.match(host_port) + + if not toks: + raise ValueError("Invalid host specification '%s'" % host_port) + + proto = toks.group('proto') + addr = toks.group('addr') or toks.group('bracket_addr') + port = toks.group('port') + + if addr.startswith("[") and not addr.endswith("]"): + raise ValueError("host spec had invalid brackets: %s" % addr) + + if comment and not name: + name = comment + + t = SyslogRemotesLine(name=name, match=match, proto=proto, + addr=addr, port=port) + t.validate() + return t + + +class SyslogRemotesLine(object): + def __init__(self, name=None, match=None, proto=None, addr=None, + port=None): + if not match: + match = "*.*" + self.name = name + self.match = match + if not proto: + proto = "udp" + if proto == "@": + proto = "udp" + elif proto == "@@": + proto = "tcp" + self.proto = proto + + self.addr = addr + if port: + self.port = int(port) + else: + self.port = None + + def validate(self): + if self.port: + try: + int(self.port) + except ValueError: + raise ValueError("port '%s' is not an integer" % self.port) + + if not self.addr: + raise ValueError("address is required") + + def __repr__(self): + return "[name=%s match=%s proto=%s address=%s port=%s]" % ( + self.name, self.match, self.proto, self.addr, self.port + ) + + def __str__(self): + buf = self.match + " " + if self.proto == "udp": + buf += "@" + elif self.proto == "tcp": + buf += "@@" + + if ":" in self.addr: + buf += "[" + self.addr + "]" + else: + buf += self.addr + + if self.port: + buf += ":%s" % self.port + + if self.name: + buf += " # %s" % self.name + return buf + + +def remotes_to_rsyslog_cfg(remotes, header=None, footer=None): + if not remotes: + return None + lines = [] + if header is not None: + lines.append(header) + for name, line in remotes.items(): + try: + lines.append(str(parse_remotes_line(line, name=name))) + except ValueError as e: + LOG.warn("failed loading remote %s: %s [%s]", name, line, e) + if footer is not None: + lines.append(footer) + return '\n'.join(lines) + "\n" + + +def handle(name, cfg, cloud, log, _args): + if 'rsyslog' not in cfg: + log.debug(("Skipping module named %s," + " no 'rsyslog' key in configuration"), name) + return + + mycfg = load_config(cfg) + configs = mycfg[KEYNAME_CONFIGS] + + if mycfg[KEYNAME_REMOTES]: + configs.append( + remotes_to_rsyslog_cfg( + mycfg[KEYNAME_REMOTES], + header="# begin remotes", + footer="# end remotes", + )) + + if not mycfg['configs']: + log.debug("Empty config rsyslog['configs'], nothing to do") + return + + changes = apply_rsyslog_changes( + configs=mycfg[KEYNAME_CONFIGS], + def_fname=mycfg[KEYNAME_FILENAME], + cfg_dir=mycfg[KEYNAME_DIR]) + + if not changes: + log.debug("restart of syslog not necessary, no changes made") + return - # Attempt to restart syslogd - restarted = False try: - # If this config module is running at cloud-init time - # (before rsyslog is running) we don't actually have to - # restart syslog. - # - # Upstart actually does what we want here, in that it doesn't - # start a service that wasn't running already on 'restart' - # it will also return failure on the attempt, so 'restarted' - # won't get set. - log.debug("Restarting rsyslog") - util.subp(['service', 'rsyslog', 'restart']) - restarted = True - except Exception: - util.logexc(log, "Failed restarting rsyslog") + restarted = reload_syslog( + command=mycfg[KEYNAME_RELOAD], + systemd=cloud.distro.uses_systemd()), + except util.ProcessExecutionError as e: + restarted = False + log.warn("Failed to reload syslog", e) if restarted: # This only needs to run if we *actually* restarted @@ -98,4 +360,4 @@ def handle(name, cfg, cloud, log, _args): cloud.cycle_logging() # This should now use rsyslog if # the logging was setup to use it... - log.debug("%s configured %s files", name, files) + log.debug("%s configured %s files", name, changes) diff --git a/doc/examples/cloud-config-rsyslog.txt b/doc/examples/cloud-config-rsyslog.txt new file mode 100644 index 00000000..d54960e8 --- /dev/null +++ b/doc/examples/cloud-config-rsyslog.txt @@ -0,0 +1,46 @@ +## the rsyslog module allows you to configure the systems syslog. +## configuration of syslog is under the top level cloud-config +## entry 'rsyslog'. +## +## Example: +#cloud-config +rsyslog: + remotes: + # udp to host 'maas.mydomain' port 514 + maashost: maas.mydomain + # udp to ipv4 host on port 514 + maas: "@[10.5.1.56]:514" + # tcp to host ipv6 host on port 555 + maasipv6: "*.* @@[FE80::0202:B3FF:FE1E:8329]:555" + configs: + - "*.* @@192.158.1.1" + - content: "*.* @@192.0.2.1:10514" + filename: 01-example.conf + - content: | + *.* @@syslogd.example.com + config_dir: /etc/rsyslog.d + config_filename: 20-cloud-config.conf + service_reload_command: [your, syslog, reload, command] + +## Additionally the following legacy format is supported +## it is converted into the format above before use. +## rsyslog_filename -> rsyslog/config_filename +## rsyslog_dir -> rsyslog/config_dir +## rsyslog -> rsyslog/configs +# rsyslog: +# - "*.* @@192.158.1.1" +# - content: "*.* @@192.0.2.1:10514" +# filename: 01-example.conf +# - content: | +# *.* @@syslogd.example.com +# rsyslog_filename: 20-cloud-config.conf +# rsyslog_dir: /etc/rsyslog.d + +## to configure rsyslog to accept remote logging on Ubuntu +## write the following into /etc/rsyslog.d/20-remote-udp.conf +## $ModLoad imudp +## $UDPServerRun 514 +## $template LogRemote,"/var/log/maas/rsyslog/%HOSTNAME%/messages" +## :fromhost-ip, !isequal, "127.0.0.1" ?LogRemote +## then: +## sudo service rsyslog restart diff --git a/tests/unittests/test_handler/test_handler_rsyslog.py b/tests/unittests/test_handler/test_handler_rsyslog.py new file mode 100644 index 00000000..7bfa65a9 --- /dev/null +++ b/tests/unittests/test_handler/test_handler_rsyslog.py @@ -0,0 +1,165 @@ +import os +import shutil +import tempfile + +from cloudinit.config.cc_rsyslog import ( + apply_rsyslog_changes, DEF_DIR, DEF_FILENAME, DEF_RELOAD, load_config, + parse_remotes_line, remotes_to_rsyslog_cfg) +from cloudinit import util + +from .. import helpers as t_help + + +class TestLoadConfig(t_help.TestCase): + def setUp(self): + super(TestLoadConfig, self).setUp() + self.basecfg = { + 'config_filename': DEF_FILENAME, + 'config_dir': DEF_DIR, + 'service_reload_command': DEF_RELOAD, + 'configs': [], + 'remotes': {}, + } + + def test_legacy_full(self): + found = load_config({ + 'rsyslog': ['*.* @192.168.1.1'], + 'rsyslog_dir': "mydir", + 'rsyslog_filename': "myfilename"}) + self.basecfg.update({ + 'configs': ['*.* @192.168.1.1'], + 'config_dir': "mydir", + 'config_filename': 'myfilename', + 'service_reload_command': 'auto'} + ) + + self.assertEqual(found, self.basecfg) + + def test_legacy_defaults(self): + found = load_config({ + 'rsyslog': ['*.* @192.168.1.1']}) + self.basecfg.update({ + 'configs': ['*.* @192.168.1.1']}) + self.assertEqual(found, self.basecfg) + + def test_new_defaults(self): + self.assertEqual(load_config({}), self.basecfg) + + def test_new_configs(self): + cfgs = ['*.* myhost', '*.* my2host'] + self.basecfg.update({'configs': cfgs}) + self.assertEqual( + load_config({'rsyslog': {'configs': cfgs}}), + self.basecfg) + + +class TestApplyChanges(t_help.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp) + + def test_simple(self): + cfgline = "*.* foohost" + changed = apply_rsyslog_changes( + configs=[cfgline], def_fname="foo.cfg", cfg_dir=self.tmp) + + fname = os.path.join(self.tmp, "foo.cfg") + self.assertEqual([fname], changed) + self.assertEqual( + util.load_file(fname), cfgline + "\n") + + def test_multiple_files(self): + configs = [ + '*.* foohost', + {'content': 'abc', 'filename': 'my.cfg'}, + {'content': 'filefoo-content', + 'filename': os.path.join(self.tmp, 'mydir/mycfg')}, + ] + + changed = apply_rsyslog_changes( + configs=configs, def_fname="default.cfg", cfg_dir=self.tmp) + + expected = [ + (os.path.join(self.tmp, "default.cfg"), + "*.* foohost\n"), + (os.path.join(self.tmp, "my.cfg"), "abc\n"), + (os.path.join(self.tmp, "mydir/mycfg"), "filefoo-content\n"), + ] + self.assertEqual([f[0] for f in expected], changed) + actual = [] + for fname, _content in expected: + util.load_file(fname) + actual.append((fname, util.load_file(fname),)) + self.assertEqual(expected, actual) + + def test_repeat_def(self): + configs = ['*.* foohost', "*.warn otherhost"] + + changed = apply_rsyslog_changes( + configs=configs, def_fname="default.cfg", cfg_dir=self.tmp) + + fname = os.path.join(self.tmp, "default.cfg") + self.assertEqual([fname], changed) + + expected_content = '\n'.join([c for c in configs]) + '\n' + found_content = util.load_file(fname) + self.assertEqual(expected_content, found_content) + + def test_multiline_content(self): + configs = ['line1', 'line2\nline3\n'] + + apply_rsyslog_changes( + configs=configs, def_fname="default.cfg", cfg_dir=self.tmp) + + fname = os.path.join(self.tmp, "default.cfg") + expected_content = '\n'.join([c for c in configs]) + found_content = util.load_file(fname) + self.assertEqual(expected_content, found_content) + + +class TestParseRemotesLine(t_help.TestCase): + def test_valid_port(self): + r = parse_remotes_line("foo:9") + self.assertEqual(9, r.port) + + def test_invalid_port(self): + with self.assertRaises(ValueError): + parse_remotes_line("*.* foo:abc") + + def test_valid_ipv6(self): + r = parse_remotes_line("*.* [::1]") + self.assertEqual("*.* @[::1]", str(r)) + + def test_valid_ipv6_with_port(self): + r = parse_remotes_line("*.* [::1]:100") + self.assertEqual(r.port, 100) + self.assertEqual(r.addr, "::1") + self.assertEqual("*.* @[::1]:100", str(r)) + + def test_invalid_multiple_colon(self): + with self.assertRaises(ValueError): + parse_remotes_line("*.* ::1:100") + + def test_name_in_string(self): + r = parse_remotes_line("syslog.host", name="foobar") + self.assertEqual("*.* @syslog.host # foobar", str(r)) + + +class TestRemotesToSyslog(t_help.TestCase): + def test_simple(self): + # str rendered line must appear in remotes_to_ryslog_cfg return + mycfg = "*.* myhost" + myline = str(parse_remotes_line(mycfg, name="myname")) + r = remotes_to_rsyslog_cfg({'myname': mycfg}) + lines = r.splitlines() + self.assertEqual(1, len(lines)) + self.assertTrue(myline in r.splitlines()) + + def test_header_footer(self): + header = "#foo head" + footer = "#foo foot" + r = remotes_to_rsyslog_cfg( + {'myname': "*.* myhost"}, header=header, footer=footer) + lines = r.splitlines() + self.assertTrue(header, lines[0]) + self.assertTrue(footer, lines[-1]) |