From ac73200e4f0cf4020dfcd0a11e6004bca479253f Mon Sep 17 00:00:00 2001 From: zsdc Date: Tue, 7 Dec 2021 22:41:27 +0200 Subject: logs: T3774: Added CLI options to control atop logs rotation Added CLI options to generate logrotate configuration file for atop logs --- data/templates/logs/logrotate/vyos-atop.tmpl | 20 ++++++ interface-definitions/system-logs.xml.in | 57 +++++++++++++++++ smoketest/scripts/cli/test_system_logs.py | 91 ++++++++++++++++++++++++++++ src/conf_mode/system-logs.py | 76 +++++++++++++++++++++++ 4 files changed, 244 insertions(+) create mode 100644 data/templates/logs/logrotate/vyos-atop.tmpl create mode 100644 interface-definitions/system-logs.xml.in create mode 100755 smoketest/scripts/cli/test_system_logs.py create mode 100755 src/conf_mode/system-logs.py diff --git a/data/templates/logs/logrotate/vyos-atop.tmpl b/data/templates/logs/logrotate/vyos-atop.tmpl new file mode 100644 index 000000000..797bf237f --- /dev/null +++ b/data/templates/logs/logrotate/vyos-atop.tmpl @@ -0,0 +1,20 @@ +/var/log/atop/atop.log { + daily + dateext + dateformat _%Y-%m-%d_%H-%M-%S + maxsize {{ maxsize|default('10') }}M + missingok + nocompress + nocreate + nomail + rotate {{ rotate|default('10') }} + prerotate + # stop the service + systemctl stop atop.service + endscript + postrotate + # start atop service again + systemctl start atop.service + endscript +} + diff --git a/interface-definitions/system-logs.xml.in b/interface-definitions/system-logs.xml.in new file mode 100644 index 000000000..677f476ff --- /dev/null +++ b/interface-definitions/system-logs.xml.in @@ -0,0 +1,57 @@ + + + + + + + Logging options + 9999 + + + + + Logrotate options + + + + + Atop logs options + + + + + Size of a single log file that triggers rotation + + u32:1-1024 + Size in MB + + + + + The size must be between 1 and 1024 MB + + 10 + + + + Count of rotations before old logs will be deleted + + u32:1-100 + Rotations + + + + + The count must be between 1 and 100 + + 10 + + + + + + + + + + diff --git a/smoketest/scripts/cli/test_system_logs.py b/smoketest/scripts/cli/test_system_logs.py new file mode 100755 index 000000000..bb23dcb1d --- /dev/null +++ b/smoketest/scripts/cli/test_system_logs.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2021 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import re +import unittest +from base_vyostest_shim import VyOSUnitTestSHIM +from vyos.util import read_file + +# path to logrotate config for atop +logrotate_atop_file = '/etc/logrotate.d/vyos-atop' +# default values +default_atop_maxsize = '10M' +default_atop_rotate = '10' + +base_path = ['system', 'logs'] + + +def logrotate_config_parse(file_path): + # read the file + logrotate_config = read_file(file_path) + # create regex for parsing options + regex_options = re.compile( + r'(^\s+(?Ppostrotate|prerotate|firstaction|lastaction|preremove)\n(?P((?!endscript).)*)\n\s+endscript\n)|(^\s+(?P[\S]+)([ \t]+(?P\S+))*$)', + re.M | re.S) + # create empty dict for config + logrotate_config_dict = {} + # fill dictionary with actual config + for option in regex_options.finditer(logrotate_config): + option_name = option.group('option_name') + option_value = option.group('option_value') + option_name_script = option.group('option_name_script') + option_value_script = option.group('option_value_script') + if option_name: + logrotate_config_dict[option_name] = option_value + if option_name_script: + logrotate_config_dict[option_name_script] = option_value_script + + # return config dictionary + return (logrotate_config_dict) + + +class TestSystemLogs(VyOSUnitTestSHIM.TestCase): + + def tearDown(self): + self.cli_delete(base_path) + self.cli_commit() + + def test_logs_defaults(self): + # test with empty section for default values + self.cli_set(base_path) + self.cli_commit() + + # read the config file and check content + logrotate_config = logrotate_config_parse(logrotate_atop_file) + self.assertEqual(logrotate_config['maxsize'], default_atop_maxsize) + self.assertEqual(logrotate_config['rotate'], default_atop_rotate) + + def test_logs_atop_maxsize(self): + # test for maxsize option + self.cli_set(base_path + ['logrotate', 'atop', 'maxsize', '50']) + self.cli_commit() + + # read the config file and check content + logrotate_config = logrotate_config_parse(logrotate_atop_file) + self.assertEqual(logrotate_config['maxsize'], '50M') + + def test_logs_atop_rotate(self): + # test for rotate option + self.cli_set(base_path + ['logrotate', 'atop', 'rotate', '50']) + self.cli_commit() + + # read the config file and check content + logrotate_config = logrotate_config_parse(logrotate_atop_file) + self.assertEqual(logrotate_config['rotate'], '50') + + +if __name__ == '__main__': + unittest.main(verbosity=2, failfast=True) diff --git a/src/conf_mode/system-logs.py b/src/conf_mode/system-logs.py new file mode 100755 index 000000000..8eb95e543 --- /dev/null +++ b/src/conf_mode/system-logs.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2021 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from sys import exit + +from vyos import ConfigError +from vyos import airbag +from vyos.config import Config +from vyos.logger import syslog +from vyos.template import render_to_string +from vyos.util import read_file, write_file +airbag.enable() + +# path to logrotate config for atop +logrotate_atop_file = '/etc/logrotate.d/vyos-atop' + + +def get_config(config=None): + if config: + conf = config + else: + conf = Config() + + base = ['system', 'logs'] + logs_config = conf.get_config_dict(base) + + return logs_config + + +def verify(logs_config): + # Nothing to verify here + pass + + +def generate(logs_config): + # get configuration for logrotate atop + logrotate_atop = logs_config.get('logs', {}).get('logrotate', + {}).get('atop', {}) + # read current config file for atop + logrotate_atop_current = read_file(logrotate_atop_file) + # generate new config file for atop + logrotate_atop_new = render_to_string('logs/logrotate/vyos-atop.tmpl', + logrotate_atop) + # update configuration files if this is necessary + if logrotate_atop_new != logrotate_atop_current: + syslog.debug('Adding logrotate config for atop') + write_file(logrotate_atop_file, logrotate_atop_new) + + +def apply(logs_config): + # No further actions needed + pass + + +if __name__ == '__main__': + try: + c = get_config() + verify(c) + generate(c) + apply(c) + except ConfigError as e: + print(e) + exit(1) -- cgit v1.2.3 From 89fdb4fbfa05e11a7e4ad7f169b5753b4fc60d71 Mon Sep 17 00:00:00 2001 From: zsdc Date: Wed, 8 Dec 2021 12:23:59 +0200 Subject: logs: T3774: Improved logs CLI Added description to the Atop section and information about default values --- interface-definitions/system-logs.xml.in | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/interface-definitions/system-logs.xml.in b/interface-definitions/system-logs.xml.in index 677f476ff..5f81b8f00 100644 --- a/interface-definitions/system-logs.xml.in +++ b/interface-definitions/system-logs.xml.in @@ -15,7 +15,7 @@ - Atop logs options + Atop logs options (system resources usage) @@ -23,7 +23,7 @@ Size of a single log file that triggers rotation u32:1-1024 - Size in MB + Size in MB (default: 10) @@ -37,7 +37,7 @@ Count of rotations before old logs will be deleted u32:1-100 - Rotations + Rotations (default: 10) -- cgit v1.2.3 From a22ba14999a38217155a7a999f61e855d813cc41 Mon Sep 17 00:00:00 2001 From: zsdc Date: Mon, 13 Dec 2021 19:28:27 +0200 Subject: logs: T3774: Improved logs config rendering Switched to `vyos.util.dict_search()` to keep the style common with the rest components. Removed config file comparison - almost the same result may be reached by removing a configuration file with each boot, we already have such a feature in the `vyos-router`. --- src/conf_mode/system-logs.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/conf_mode/system-logs.py b/src/conf_mode/system-logs.py index 8eb95e543..7b5af240f 100755 --- a/src/conf_mode/system-logs.py +++ b/src/conf_mode/system-logs.py @@ -20,8 +20,8 @@ from vyos import ConfigError from vyos import airbag from vyos.config import Config from vyos.logger import syslog -from vyos.template import render_to_string -from vyos.util import read_file, write_file +from vyos.template import render +from vyos.util import dict_search airbag.enable() # path to logrotate config for atop @@ -47,17 +47,13 @@ def verify(logs_config): def generate(logs_config): # get configuration for logrotate atop - logrotate_atop = logs_config.get('logs', {}).get('logrotate', - {}).get('atop', {}) - # read current config file for atop - logrotate_atop_current = read_file(logrotate_atop_file) + logrotate_atop = dict_search('logs.logrotate.atop', logs_config) + # provide an empty dictionary if there is no config + if not logrotate_atop: + logrotate_atop = {} # generate new config file for atop - logrotate_atop_new = render_to_string('logs/logrotate/vyos-atop.tmpl', - logrotate_atop) - # update configuration files if this is necessary - if logrotate_atop_new != logrotate_atop_current: - syslog.debug('Adding logrotate config for atop') - write_file(logrotate_atop_file, logrotate_atop_new) + syslog.debug('Adding logrotate config for atop') + render(logrotate_atop_file, 'logs/logrotate/vyos-atop.tmpl', logrotate_atop) def apply(logs_config): -- cgit v1.2.3 From 945ab070b72ebd9f5ccfe0052ed138a93b83b297 Mon Sep 17 00:00:00 2001 From: zsdc Date: Mon, 13 Dec 2021 20:05:25 +0200 Subject: logs: T3774: Added new CLI item Added the ability to control the `/var/log/messages` rotation. Renamed the option `maxsize` to `max-size`. --- data/templates/logs/logrotate/vyos-atop.tmpl | 2 +- data/templates/logs/logrotate/vyos-rsyslog.tmpl | 13 +++++++++ interface-definitions/system-logs.xml.in | 37 ++++++++++++++++++++++++- smoketest/scripts/cli/test_system_logs.py | 36 ++++++++++++++++++++---- src/conf_mode/system-logs.py | 15 ++++++++-- 5 files changed, 94 insertions(+), 9 deletions(-) create mode 100644 data/templates/logs/logrotate/vyos-rsyslog.tmpl diff --git a/data/templates/logs/logrotate/vyos-atop.tmpl b/data/templates/logs/logrotate/vyos-atop.tmpl index 797bf237f..444d1da36 100644 --- a/data/templates/logs/logrotate/vyos-atop.tmpl +++ b/data/templates/logs/logrotate/vyos-atop.tmpl @@ -2,7 +2,7 @@ daily dateext dateformat _%Y-%m-%d_%H-%M-%S - maxsize {{ maxsize|default('10') }}M + maxsize {{ max_size|default('10') }}M missingok nocompress nocreate diff --git a/data/templates/logs/logrotate/vyos-rsyslog.tmpl b/data/templates/logs/logrotate/vyos-rsyslog.tmpl new file mode 100644 index 000000000..3af1bfd8e --- /dev/null +++ b/data/templates/logs/logrotate/vyos-rsyslog.tmpl @@ -0,0 +1,13 @@ +/var/log/messages { + create + missingok + nomail + notifempty + rotate {{ rotate|default('10') }} + size {{ max_size|default('1') }}M + postrotate + # restart rsyslog service + systemctl restart rsyslog.service + endscript +} + diff --git a/interface-definitions/system-logs.xml.in b/interface-definitions/system-logs.xml.in index 5f81b8f00..dc5751372 100644 --- a/interface-definitions/system-logs.xml.in +++ b/interface-definitions/system-logs.xml.in @@ -18,7 +18,7 @@ Atop logs options (system resources usage) - + Size of a single log file that triggers rotation @@ -48,6 +48,41 @@ + + + The /var/log/messages file rotation + + + + + Size of a single log file that triggers rotation + + u32:1-1024 + Size in MB (default: 1) + + + + + The size must be between 1 and 1024 MB + + 10 + + + + Count of rotations before old logs will be deleted + + u32:1-100 + Rotations (default: 10) + + + + + The count must be between 1 and 100 + + 10 + + + diff --git a/smoketest/scripts/cli/test_system_logs.py b/smoketest/scripts/cli/test_system_logs.py index bb23dcb1d..0c11c4663 100755 --- a/smoketest/scripts/cli/test_system_logs.py +++ b/smoketest/scripts/cli/test_system_logs.py @@ -19,11 +19,14 @@ import unittest from base_vyostest_shim import VyOSUnitTestSHIM from vyos.util import read_file -# path to logrotate config for atop +# path to logrotate configs logrotate_atop_file = '/etc/logrotate.d/vyos-atop' +logrotate_rsyslog_file = '/etc/logrotate.d/vyos-rsyslog' # default values default_atop_maxsize = '10M' default_atop_rotate = '10' +default_rsyslog_size = '1M' +default_rsyslog_rotate = '10' base_path = ['system', 'logs'] @@ -64,13 +67,18 @@ class TestSystemLogs(VyOSUnitTestSHIM.TestCase): self.cli_commit() # read the config file and check content - logrotate_config = logrotate_config_parse(logrotate_atop_file) - self.assertEqual(logrotate_config['maxsize'], default_atop_maxsize) - self.assertEqual(logrotate_config['rotate'], default_atop_rotate) + logrotate_config_atop = logrotate_config_parse(logrotate_atop_file) + logrotate_config_rsyslog = logrotate_config_parse( + logrotate_rsyslog_file) + self.assertEqual(logrotate_config_atop['maxsize'], default_atop_maxsize) + self.assertEqual(logrotate_config_atop['rotate'], default_atop_rotate) + self.assertEqual(logrotate_config_rsyslog['size'], default_rsyslog_size) + self.assertEqual(logrotate_config_rsyslog['rotate'], + default_rsyslog_rotate) def test_logs_atop_maxsize(self): # test for maxsize option - self.cli_set(base_path + ['logrotate', 'atop', 'maxsize', '50']) + self.cli_set(base_path + ['logrotate', 'atop', 'max-size', '50']) self.cli_commit() # read the config file and check content @@ -86,6 +94,24 @@ class TestSystemLogs(VyOSUnitTestSHIM.TestCase): logrotate_config = logrotate_config_parse(logrotate_atop_file) self.assertEqual(logrotate_config['rotate'], '50') + def test_logs_rsyslog_size(self): + # test for size option + self.cli_set(base_path + ['logrotate', 'messages', 'max-size', '50']) + self.cli_commit() + + # read the config file and check content + logrotate_config = logrotate_config_parse(logrotate_rsyslog_file) + self.assertEqual(logrotate_config['size'], '50M') + + def test_logs_rsyslog_rotate(self): + # test for rotate option + self.cli_set(base_path + ['logrotate', 'messages', 'rotate', '50']) + self.cli_commit() + + # read the config file and check content + logrotate_config = logrotate_config_parse(logrotate_rsyslog_file) + self.assertEqual(logrotate_config['rotate'], '50') + if __name__ == '__main__': unittest.main(verbosity=2, failfast=True) diff --git a/src/conf_mode/system-logs.py b/src/conf_mode/system-logs.py index 7b5af240f..a9f2da476 100755 --- a/src/conf_mode/system-logs.py +++ b/src/conf_mode/system-logs.py @@ -24,8 +24,9 @@ from vyos.template import render from vyos.util import dict_search airbag.enable() -# path to logrotate config for atop +# path to logrotate configs logrotate_atop_file = '/etc/logrotate.d/vyos-atop' +logrotate_rsyslog_file = '/etc/logrotate.d/vyos-rsyslog' def get_config(config=None): @@ -35,7 +36,7 @@ def get_config(config=None): conf = Config() base = ['system', 'logs'] - logs_config = conf.get_config_dict(base) + logs_config = conf.get_config_dict(base, key_mangling=('-', '_')) return logs_config @@ -55,6 +56,16 @@ def generate(logs_config): syslog.debug('Adding logrotate config for atop') render(logrotate_atop_file, 'logs/logrotate/vyos-atop.tmpl', logrotate_atop) + # get configuration for logrotate rsyslog + logrotate_rsyslog = dict_search('logs.logrotate.messages', logs_config) + # provide an empty dictionary if there is no config + if not logrotate_rsyslog: + logrotate_rsyslog = {} + # generate new config file for rsyslog + syslog.debug('Adding logrotate config for rsyslog') + render(logrotate_rsyslog_file, 'logs/logrotate/vyos-rsyslog.tmpl', + logrotate_rsyslog) + def apply(logs_config): # No further actions needed -- cgit v1.2.3 From 86bbab75ae414f19305970e2aa8aaeb05e6990ae Mon Sep 17 00:00:00 2001 From: zsdc Date: Fri, 17 Dec 2021 16:41:50 +0200 Subject: logs: T3774: Optimization for logrotate configs * Added proper handling of default values from CLI. * Replaced rsyslog restart postrotate action to native `rsyslog-rotate` script. * Removed unnecessary checks for `None` instead `dict` - with default values the situation becomes impossible. * Fixed default value from 10 to 1 in the rsyslog CLI. --- data/templates/logs/logrotate/vyos-atop.tmpl | 4 ++-- data/templates/logs/logrotate/vyos-rsyslog.tmpl | 8 ++++---- interface-definitions/system-logs.xml.in | 2 +- src/conf_mode/system-logs.py | 18 +++++++++--------- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/data/templates/logs/logrotate/vyos-atop.tmpl b/data/templates/logs/logrotate/vyos-atop.tmpl index 444d1da36..2d078f379 100644 --- a/data/templates/logs/logrotate/vyos-atop.tmpl +++ b/data/templates/logs/logrotate/vyos-atop.tmpl @@ -2,12 +2,12 @@ daily dateext dateformat _%Y-%m-%d_%H-%M-%S - maxsize {{ max_size|default('10') }}M + maxsize {{ max_size }}M missingok nocompress nocreate nomail - rotate {{ rotate|default('10') }} + rotate {{ rotate }} prerotate # stop the service systemctl stop atop.service diff --git a/data/templates/logs/logrotate/vyos-rsyslog.tmpl b/data/templates/logs/logrotate/vyos-rsyslog.tmpl index 3af1bfd8e..f2e4d2ab2 100644 --- a/data/templates/logs/logrotate/vyos-rsyslog.tmpl +++ b/data/templates/logs/logrotate/vyos-rsyslog.tmpl @@ -3,11 +3,11 @@ missingok nomail notifempty - rotate {{ rotate|default('10') }} - size {{ max_size|default('1') }}M + rotate {{ rotate }} + size {{ max_size }}M postrotate - # restart rsyslog service - systemctl restart rsyslog.service + # inform rsyslog service about rotation + /usr/lib/rsyslog/rsyslog-rotate endscript } diff --git a/interface-definitions/system-logs.xml.in b/interface-definitions/system-logs.xml.in index dc5751372..8b6c7c399 100644 --- a/interface-definitions/system-logs.xml.in +++ b/interface-definitions/system-logs.xml.in @@ -65,7 +65,7 @@ The size must be between 1 and 1024 MB - 10 + 1 diff --git a/src/conf_mode/system-logs.py b/src/conf_mode/system-logs.py index a9f2da476..e6296656d 100755 --- a/src/conf_mode/system-logs.py +++ b/src/conf_mode/system-logs.py @@ -19,9 +19,11 @@ from sys import exit from vyos import ConfigError from vyos import airbag from vyos.config import Config +from vyos.configdict import dict_merge from vyos.logger import syslog from vyos.template import render from vyos.util import dict_search +from vyos.xml import defaults airbag.enable() # path to logrotate configs @@ -36,7 +38,11 @@ def get_config(config=None): conf = Config() base = ['system', 'logs'] - logs_config = conf.get_config_dict(base, key_mangling=('-', '_')) + default_values = defaults(base) + logs_config = conf.get_config_dict(base, + key_mangling=('-', '_'), + get_first_key=True) + logs_config = dict_merge(default_values, logs_config) return logs_config @@ -48,19 +54,13 @@ def verify(logs_config): def generate(logs_config): # get configuration for logrotate atop - logrotate_atop = dict_search('logs.logrotate.atop', logs_config) - # provide an empty dictionary if there is no config - if not logrotate_atop: - logrotate_atop = {} + logrotate_atop = dict_search('logrotate.atop', logs_config) # generate new config file for atop syslog.debug('Adding logrotate config for atop') render(logrotate_atop_file, 'logs/logrotate/vyos-atop.tmpl', logrotate_atop) # get configuration for logrotate rsyslog - logrotate_rsyslog = dict_search('logs.logrotate.messages', logs_config) - # provide an empty dictionary if there is no config - if not logrotate_rsyslog: - logrotate_rsyslog = {} + logrotate_rsyslog = dict_search('logrotate.messages', logs_config) # generate new config file for rsyslog syslog.debug('Adding logrotate config for rsyslog') render(logrotate_rsyslog_file, 'logs/logrotate/vyos-rsyslog.tmpl', -- cgit v1.2.3