diff options
| -rw-r--r-- | data/templates/monitoring/syslog_telegraf.tmpl | 15 | ||||
| -rw-r--r-- | data/templates/monitoring/telegraf.tmpl | 11 | ||||
| -rw-r--r-- | debian/control | 1 | ||||
| -rw-r--r-- | debian/vyos-1x.install | 1 | ||||
| -rw-r--r-- | interface-definitions/service_monitoring_telegraf.xml.in | 113 | ||||
| -rwxr-xr-x | smoketest/scripts/cli/test_service_monitoring_telegraf.py | 65 | ||||
| -rwxr-xr-x | src/conf_mode/service_monitoring_telegraf.py (renamed from src/conf_mode/service_monitoring.py) | 53 | ||||
| -rwxr-xr-x | src/etc/telegraf/custom_scripts/show_interfaces_input_filter.py | 47 | ||||
| -rwxr-xr-x | src/etc/telegraf/custom_scripts/vyos_services_input_filter.py | 61 |
9 files changed, 330 insertions, 37 deletions
diff --git a/data/templates/monitoring/syslog_telegraf.tmpl b/data/templates/monitoring/syslog_telegraf.tmpl index 122bdf9cc..cdcbd92a4 100644 --- a/data/templates/monitoring/syslog_telegraf.tmpl +++ b/data/templates/monitoring/syslog_telegraf.tmpl @@ -1,12 +1,5 @@ -# Generated by /usr/libexec/vyos/conf_mode/service_monitoring.py +# Generated by /usr/libexec/vyos/conf_mode/service_monitoring_telegraf.py -$ModLoad imjournal -$ImjournalDefaultSeverity 4 -$WorkDirectory /var/spool/rsyslog - -$ActionQueueType LinkedList # use asynchronous processing -$ActionQueueFileName srvrfwd # set file name, also enables disk mode -$ActionResumeRetryCount -1 # infinite retries on insert failure -$ActionQueueSaveOnShutdown on # save in-memory data if rsyslog shuts down - -*.* @127.0.0.1:6514;RSYSLOG_SyslogProtocol23Format +$ModLoad omuxsock +$OMUxSockSocket /run/telegraf/telegraf_syslog.sock +*.notice :omuxsock: diff --git a/data/templates/monitoring/telegraf.tmpl b/data/templates/monitoring/telegraf.tmpl index 631e59d74..62fa4df7a 100644 --- a/data/templates/monitoring/telegraf.tmpl +++ b/data/templates/monitoring/telegraf.tmpl @@ -1,4 +1,4 @@ -# Generated by /usr/libexec/vyos/conf_mode/service_monitoring.py +# Generated by /usr/libexec/vyos/conf_mode/service_monitoring_telegraf.py [agent] interval = "10s" @@ -15,10 +15,10 @@ hostname = "" omit_hostname = false [[outputs.influxdb_v2]] - urls = ["{{ url }}"] + urls = ["{{ url }}:{{ port }}"] insecure_skip_verify = true token = "{{ authentication.token }}" - organization = "{{ authentication.login }}" + organization = "{{ authentication.organization }}" bucket = "{{ bucket }}" [[inputs.cpu]] percpu = true @@ -51,8 +51,9 @@ [[inputs.internal]] [[inputs.nstat]] [[inputs.syslog]] - server = "udp://localhost:6514" -# server = "unix:///var/run/telegraf-syslog.sock" + server = "unixgram:///run/telegraf/telegraf_syslog.sock" + best_effort = true + syslog_standard = "RFC3164" [[inputs.exec]] commands = [ "{{ custom_scripts_dir }}/show_interfaces_input_filter.py", diff --git a/debian/control b/debian/control index f3a26e73e..e407e05b0 100644 --- a/debian/control +++ b/debian/control @@ -149,6 +149,7 @@ Depends: strongswan-swanctl (>= 5.9), sudo, systemd, + telegraf (>= 1.20), tcpdump, tcptraceroute, telnet, diff --git a/debian/vyos-1x.install b/debian/vyos-1x.install index d332e0d36..20c119e63 100644 --- a/debian/vyos-1x.install +++ b/debian/vyos-1x.install @@ -10,6 +10,7 @@ etc/security etc/sudoers.d etc/systemd etc/sysctl.d +etc/telegraf etc/udev etc/update-motd.d etc/vyos diff --git a/interface-definitions/service_monitoring_telegraf.xml.in b/interface-definitions/service_monitoring_telegraf.xml.in new file mode 100644 index 000000000..0db9052ff --- /dev/null +++ b/interface-definitions/service_monitoring_telegraf.xml.in @@ -0,0 +1,113 @@ +<?xml version="1.0"?> +<interfaceDefinition> + <node name="service"> + <children> + <node name="monitoring"> + <properties> + <help>Monitoring services</help> + <priority>1280</priority> + </properties> + <children> + <node name="telegraf" owner="${vyos_conf_scripts_dir}/service_monitoring_telegraf.py"> + <properties> + <help>Telegraf monitoring</help> + </properties> + <children> + <node name="authentication"> + <properties> + <help>Authentication parameters</help> + </properties> + <children> + <leafNode name="organization"> + <properties> + <help>Authentication organization for InfluxDB v2 [REQUIRED]</help> + <constraint> + <regex>^[a-zA-Z][1-9a-zA-Z@_\-.]{2,50}$</regex> + </constraint> + <constraintErrorMessage>Organization name must be alphanumeric and can contain hyphens, underscores and at symbol.</constraintErrorMessage> + </properties> + </leafNode> + <leafNode name="token"> + <properties> + <help>Authentication token for InfluxDB v2 [REQUIRED]</help> + <valueHelp> + <format>txt</format> + <description>Authentication token</description> + </valueHelp> + <constraint> + <regex>^[a-zA-Z0-9-_]{86}==$</regex> + </constraint> + <constraintErrorMessage>Token must be 88 characters long and must contain only [a-zA-Z0-9-_] and '==' characters.</constraintErrorMessage> + </properties> + </leafNode> + </children> + </node> + <leafNode name="bucket"> + <properties> + <help>Remote bucket, by default (main)</help> + </properties> + <defaultValue>main</defaultValue> + </leafNode> + <leafNode name="source"> + <properties> + <help>Source parameters for monitoring (default: all)</help> + <completionHelp> + <list>all hardware-utilization logs network system telegraf</list> + </completionHelp> + <valueHelp> + <format>all</format> + <description>All parameters (default)</description> + </valueHelp> + <valueHelp> + <format>hardware-utilization</format> + <description>Hardware-utilization parameters (CPU, disk, memory)</description> + </valueHelp> + <valueHelp> + <format>logs</format> + <description>Logs parameters</description> + </valueHelp> + <valueHelp> + <format>network</format> + <description>Network parameters (net, netstat, nftables)</description> + </valueHelp> + <valueHelp> + <format>system</format> + <description>System parameters (system, processes, interrupts)</description> + </valueHelp> + <valueHelp> + <format>telegraf</format> + <description>Telegraf internal statistics</description> + </valueHelp> + <constraint> + <regex>^(all|hardware-utilization|logs|network|system|telegraf)$</regex> + </constraint> + <multi/> + </properties> + <defaultValue>all</defaultValue> + </leafNode> + <leafNode name="url"> + <properties> + <help>Remote URL [REQUIRED]</help> + <valueHelp> + <format>url</format> + <description>Remote URL to InfluxDB v2</description> + </valueHelp> + <constraint> + <regex>^(http:\/\/www\.|https:\/\/www\.|http:\/\/|https:\/\/)?[a-z0-9]+([\-\.]{1}[a-z0-9]+)*\.[a-z]{2,5}?(\/.*)?$</regex> + </constraint> + <constraintErrorMessage>Incorrect URL format.</constraintErrorMessage> + </properties> + </leafNode> + <leafNode name="port"> + <properties> + <help>Remote port (default: 8086)</help> + </properties> + <defaultValue>8086</defaultValue> + </leafNode> + </children> + </node> + </children> + </node> + </children> + </node> +</interfaceDefinition> diff --git a/smoketest/scripts/cli/test_service_monitoring_telegraf.py b/smoketest/scripts/cli/test_service_monitoring_telegraf.py new file mode 100755 index 000000000..b857926e2 --- /dev/null +++ b/smoketest/scripts/cli/test_service_monitoring_telegraf.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2021 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import unittest + +from base_vyostest_shim import VyOSUnitTestSHIM + +from vyos.configsession import ConfigSession +from vyos.configsession import ConfigSessionError +from vyos.util import process_named_running +from vyos.util import read_file + +PROCESS_NAME = 'telegraf' +TELEGRAF_CONF = '/run/telegraf/vyos-telegraf.conf' +base_path = ['service', 'monitoring', 'telegraf'] +org = 'log@in.local' +token = 'GuRJc12tIzfjnYdKRAIYbxdWd2aTpOT9PVYNddzDnFV4HkAcD7u7-kndTFXjGuXzJN6TTxmrvPODB4mnFcseDV==' +port = '8888' +url = 'https://foo.local' +bucket = 'main' +inputs = ['cpu', 'disk', 'mem', 'net', 'system', 'kernel', 'interrupts', 'syslog'] + +class TestMonitoringTelegraf(VyOSUnitTestSHIM.TestCase): + def tearDown(self): + self.cli_delete(base_path) + self.cli_commit() + + def test_01_basic_config(self): + self.cli_set(base_path + ['authentication', 'organization', org]) + self.cli_set(base_path + ['authentication', 'token', token]) + self.cli_set(base_path + ['port', port]) + self.cli_set(base_path + ['url', url]) + + # commit changes + self.cli_commit() + + # Check for running process + self.assertTrue(process_named_running(PROCESS_NAME)) + + config = read_file(TELEGRAF_CONF) + + # Check telegraf config + self.assertIn(f'organization = "{org}"', config) + self.assertIn(token, config) + self.assertIn(f'urls = ["{url}:{port}"]', config) + self.assertIn(f'bucket = "{bucket}"', config) + + for input in inputs: + self.assertIn(input, config) + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/src/conf_mode/service_monitoring.py b/src/conf_mode/service_monitoring_telegraf.py index e14ad78fc..a1e7a7286 100755 --- a/src/conf_mode/service_monitoring.py +++ b/src/conf_mode/service_monitoring_telegraf.py @@ -18,19 +18,23 @@ import os import json from sys import exit +from shutil import rmtree from vyos.config import Config +from vyos.configdict import dict_merge from vyos.template import render from vyos.util import call from vyos.util import chown from vyos.util import cmd +from vyos.xml import defaults from vyos import ConfigError from vyos import airbag airbag.enable() -cache_dir = '/run/telegraf/.cache' -config_telegraf = '/run/telegraf/vyos-telegraf.conf' +base_dir = '/run/telegraf' +cache_dir = f'/etc/telegraf/.cache' +config_telegraf = f'{base_dir}/vyos-telegraf.conf' custom_scripts_dir = '/etc/telegraf/custom_scripts' syslog_telegraf = '/etc/rsyslog.d/50-telegraf.conf' systemd_telegraf_service = '/etc/systemd/system/vyos-telegraf.service' @@ -59,18 +63,20 @@ def get_config(config=None): conf = config else: conf = Config() - base = ['service', 'monitoring'] + base = ['service', 'monitoring', 'telegraf'] + if not conf.exists(base): + return None + monitoring = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True, no_tag_node_value_mangle=True) - if monitoring: - # Set default value for bucket if not configured - if 'bucket' not in monitoring: - monitoring['bucket'] = cmd('cat /etc/hostname') + # We have gathered the dict representation of the CLI, but there are default + # options which we need to update into the dictionary retrived. + default_values = defaults(base) + monitoring = dict_merge(default_values, monitoring) - # Get nftables chains - monitoring['nft_chains'] = get_nft_filter_chains() - monitoring['custom_scripts_dir'] = custom_scripts_dir + monitoring['nft_chains'] = get_nft_filter_chains() + monitoring['custom_scripts_dir'] = custom_scripts_dir return monitoring @@ -79,8 +85,10 @@ def verify(monitoring): if not monitoring: return None - if 'login' not in monitoring['authentication'] or 'token' not in monitoring['authentication']: - raise ConfigError(f'Authentication "login and token" are mandatory!') + if 'authentication' not in monitoring or \ + 'organization' not in monitoring['authentication'] or \ + 'token' not in monitoring['authentication']: + raise ConfigError(f'Authentication "organization and token" are mandatory!') if 'url' not in monitoring: raise ConfigError(f'Monitoring "url" is mandatory!') @@ -89,22 +97,23 @@ def verify(monitoring): def generate(monitoring): if not monitoring: - # Delete config file - if os.path.isfile(config_telegraf): - os.unlink(config_telegraf) - if os.path.isfile(systemd_override): - os.unlink(systemd_override) - if os.path.isfile(syslog_telegraf): - os.unlink(syslog_telegraf) + # Delete config and systemd files + config_files = [config_telegraf, systemd_telegraf_service, systemd_override, syslog_telegraf] + for file in config_files: + if os.path.isfile(file): + os.unlink(file) + # Delete old directories + if os.path.isdir(cache_dir): + rmtree(cache_dir, ignore_errors=True) return None # Create telegraf cache dir if not os.path.exists(cache_dir): - os.mkdir(cache_dir) + os.makedirs(cache_dir) - chown(cache_dir, 'telegraf', 'vyattacfg') + chown(cache_dir, 'telegraf', 'telegraf') # Create systemd override dir if not os.path.exists(systemd_telegraf_override_dir): @@ -120,6 +129,8 @@ def generate(monitoring): render(systemd_override, 'monitoring/override.conf.tmpl', monitoring, permission=0o640) render(syslog_telegraf, 'monitoring/syslog_telegraf.tmpl', monitoring) + chown(base_dir, 'telegraf', 'telegraf') + return None def apply(monitoring): diff --git a/src/etc/telegraf/custom_scripts/show_interfaces_input_filter.py b/src/etc/telegraf/custom_scripts/show_interfaces_input_filter.py new file mode 100755 index 000000000..0f5e366cd --- /dev/null +++ b/src/etc/telegraf/custom_scripts/show_interfaces_input_filter.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 + +import subprocess +import time + +def status_to_int(status): + switcher={ + 'u':'0', + 'D':'1', + 'A':'2' + } + return switcher.get(status,"") + +def description_check(line): + desc=" ".join(line[3:]) + if desc == "": + return "empty" + else: + return desc + +def gen_ip_list(index,interfaces): + line=interfaces[index].split() + ip_list=line[1] + if index < len(interfaces): + index += 1 + while len(interfaces[index].split())==1: + ip = interfaces[index].split() + ip_list = ip_list + " " + ip[0] + index += 1 + if index == len(interfaces): + break + return ip_list + +interfaces = subprocess.check_output("/usr/libexec/vyos/op_mode/show_interfaces.py --action=show-brief", shell=True).decode('utf-8').splitlines() +del interfaces[:3] +lines_count=len(interfaces) +index=0 +while index<lines_count: + line=interfaces[index].split() + if len(line)>1: + print(f'show_interfaces,interface={line[0]} ' + f'ip_addresses="{gen_ip_list(index,interfaces)}",' + f'state={status_to_int(line[2][0])}i,' + f'link={status_to_int(line[2][2])}i,' + f'description="{description_check(line)}" ' + f'{str(int(time.time()))}000000000') + index += 1 diff --git a/src/etc/telegraf/custom_scripts/vyos_services_input_filter.py b/src/etc/telegraf/custom_scripts/vyos_services_input_filter.py new file mode 100755 index 000000000..df4eed131 --- /dev/null +++ b/src/etc/telegraf/custom_scripts/vyos_services_input_filter.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2021 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +import time +from vyos.configquery import ConfigTreeQuery +from vyos.util import is_systemd_service_running, process_named_running + +# Availible services and prouceses +# 1 - service +# 2 - process +services = { + "protocols bgp" : "bgpd", + "protocols ospf" : "ospfd", + "protocols ospfv3" : "ospf6d", + "protocols rip" : "ripd", + "protocols ripng" : "ripngd", + "protocols isis" : "isisd", + "service pppoe" : "accel-ppp@pppoe.service", + "vpn l2tp remote-access" : "accel-ppp@l2tp.service", + "vpn pptp remote-access" : "accel-ppp@pptp.service", + "vpn sstp" : "accel-ppp@sstp.service", + "vpn ipsec" : "charon" +} + +# Configured services +conf_services = { + 'zebra' : 0, + 'staticd' : 0, +} +# Get configured service and create list to check if process running +config = ConfigTreeQuery() +for service in services: + if config.exists(service): + conf_services[services[service]] = 0 + +for conf_service in conf_services: + status = 0 + if ".service" in conf_service: + # Check systemd service + if is_systemd_service_running(conf_service): + status = 1 + else: + # Check process + if process_named_running(conf_service): + status = 1 + print(f'vyos_services,service="{conf_service}" ' + f'status={str(status)}i {str(int(time.time()))}000000000') |