From 015e26acc8ed65b6a7d778107a83ee8604950f90 Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Fri, 24 Dec 2021 22:12:19 +0100 Subject: flow-accounting: T4097: move to get_config_dict() --- data/configd-include.json | 1 + data/templates/netflow/uacctd.conf.tmpl | 111 ++++++++++++++++---------------- 2 files changed, 55 insertions(+), 57 deletions(-) (limited to 'data') diff --git a/data/configd-include.json b/data/configd-include.json index 6893aaa86..ee4cb0d42 100644 --- a/data/configd-include.json +++ b/data/configd-include.json @@ -6,6 +6,7 @@ "dhcpv6_relay.py", "dns_forwarding.py", "dynamic_dns.py", +"flow_accounting_conf.py", "host_name.py", "https.py", "igmp_proxy.py", diff --git a/data/templates/netflow/uacctd.conf.tmpl b/data/templates/netflow/uacctd.conf.tmpl index 11fc76769..2a43b3ce2 100644 --- a/data/templates/netflow/uacctd.conf.tmpl +++ b/data/templates/netflow/uacctd.conf.tmpl @@ -5,71 +5,68 @@ pidfile: /var/run/uacctd.pid uacctd_group: 2 uacctd_nl_size: 2097152 snaplen: {{ snaplen }} -{% if templatecfg['enable-egress'] != none %} -aggregate: in_iface,out_iface,src_mac,dst_mac,vlan,src_host,dst_host,src_port,dst_port,proto,tos,flows -{% else %} -aggregate: in_iface,src_mac,dst_mac,vlan,src_host,dst_host,src_port,dst_port,proto,tos,flows +aggregate: in_iface{{ ',out_iface' if enable_egress is defined }},src_mac,dst_mac,vlan,src_host,dst_host,src_port,dst_port,proto,tos,flows +{% set pipe_size = buffer_size | int *1024 *1024 %} +plugin_pipe_size: {{ pipe_size }} +{# We need an integer division (//) without any remainder or fraction #} +plugin_buffer_size: {{ pipe_size // 1000 }} +{% if syslog_facility is defined and syslog_facility is not none %} +syslog: {{ syslog_facility }} {% endif %} -plugin_pipe_size: {{ templatecfg['plugin_pipe_size'] }} -plugin_buffer_size: {{ templatecfg['plugin_buffer_size'] }} -{% if templatecfg['syslog-facility'] != none %} -syslog: {{ templatecfg['syslog-facility'] }} -{% endif %} -{% if templatecfg['disable-imt'] == none %} +{% if disable_imt is not defined %} imt_path: /tmp/uacctd.pipe imt_mem_pools_number: 169 {% endif %} -plugins: {% if templatecfg['netflow']['servers'] != none %} -{% for server in templatecfg['netflow']['servers'] %} -{% if loop.last %}nfprobe[nf_{{ server['address'] }}]{% else %}nfprobe[nf_{{ server['address'] }}],{% endif %} -{% endfor %} -{% set plugins_presented = true %} -{% endif %} -{% if templatecfg['sflow']['servers'] != none %} -{% if plugins_presented %} -{% for server in templatecfg['sflow']['servers'] %},sfprobe[sf_{{ server['address'] }}]{% endfor %} -{% else %} -{% for server in templatecfg['sflow']['servers'] %} -{% if loop.last %}sfprobe[sf_{{ server['address'] }}]{% else %}sfprobe[sf_{{ server['address'] }}],{% endif %} -{% endfor %} -{% endif %} -{% set plugins_presented = true %} -{% endif %} -{% if templatecfg['disable-imt'] == none %} -{% if plugins_presented %},memory{% else %}memory{% endif %} -{% endif %} -{% if templatecfg['netflow']['servers'] != none %} -{% for server in templatecfg['netflow']['servers'] %} -nfprobe_receiver[nf_{{ server['address'] }}]: {{ server['address'] }}:{{ server['port'] }} -nfprobe_version[nf_{{ server['address'] }}]: {{ templatecfg['netflow']['version'] }} -{% if templatecfg['netflow']['engine-id'] != none %} -nfprobe_engine[nf_{{ server['address'] }}]: {{ templatecfg['netflow']['engine-id'] }} -{% endif %} -{% if templatecfg['netflow']['max-flows'] != none %} -nfprobe_maxflows[nf_{{ server['address'] }}]: {{ templatecfg['netflow']['max-flows'] }} -{% endif %} -{% if templatecfg['netflow']['sampling-rate'] != none %} -sampling_rate[nf_{{ server['address'] }}]: {{ templatecfg['netflow']['sampling-rate'] }} +{% set plugin = [] %} +{% if disable_imt is not defined %} +{% set plugin = ['memory'] %} {% endif %} -{% if templatecfg['netflow']['source-ip'] != none %} -nfprobe_source_ip[nf_{{ server['address'] }}]: {{ templatecfg['netflow']['source-ip'] }} -{% endif %} -{% if templatecfg['netflow']['timeout_string'] != '' %} -nfprobe_timeouts[nf_{{ server['address'] }}]: {{ templatecfg['netflow']['timeout_string'] }} +{% if netflow is defined and netflow.server is defined and netflow.server is not none %} +{% for server in netflow.server %} +{% set plugin = plugin.append('nfprobe[nf_' ~ server ~ ']') %} +{% endfor %} {% endif %} -{% endfor %} +{% if sflow is defined and sflow.server is defined and sflow.server is not none %} +{% for server in sflow.server %} +{% set plugin = plugin.append('sfprobe[sf_' ~ server ~ ']') %} +{% endfor %} {% endif %} +plugins: {{ plugin | join(',') }} -{% if templatecfg['sflow']['servers'] != none %} -{% for server in templatecfg['sflow']['servers'] %} -sfprobe_receiver[sf_{{ server['address'] }}]: {{ server['address'] }}:{{ server['port'] }} -sfprobe_agentip[sf_{{ server['address'] }}]: {{ templatecfg['sflow']['agent-address'] }} -{% if templatecfg['sflow']['sampling-rate'] != none %} -sampling_rate[sf_{{ server['address'] }}]: {{ templatecfg['sflow']['sampling-rate'] }} -{% endif %} -{% if templatecfg['sflow']['source-address'] != none %} -sfprobe_source_ip[sf_{{ server['address'] }}]: {{ templatecfg['sflow']['source-address'] }} +{% if netflow is defined and netflow.server is defined and netflow.server is not none %} +# NetFlow servers +{% for server, server_config in netflow.server.items() %} +nfprobe_receiver[nf_{{ server }}]: {{ server }}:{{ server_config.port }} +nfprobe_version[nf_{{ server }}]: {{ netflow.version }} +{% if netflow.engine_id is defined and netflow.engine_id is not none %} +nfprobe_engine[nf_{{ server }}]: {{ netflow.engine_id }} +{% endif %} +{% if netflow.max_flows is defined and netflow.max_flows is not none %} +nfprobe_maxflows[nf_{{ server }}]: {{ netflow.max_flows }} +{% endif %} +{% if netflow.sampling_rate is defined and netflow.sampling_rate is not none %} +sampling_rate[nf_{{ server }}]: {{ netflow.sampling_rate }} +{% endif %} +{% if netflow.source_ip is defined and netflow.source_ip is not none %} +nfprobe_source_ip[nf_{{ server }}]: {{ netflow.source_ip }} +{% endif %} +{% if netflow.timeout is defined and netflow.timeout is not none %} +nfprobe_timeouts[nf_{{ server }}]: expint={{ netflow.timeout.expiry_interval }}:general={{ netflow.timeout.flow_generic }}:icmp={{ netflow.timeout.icmp }}:maxlife={{ netflow.timeout.max_active_life }}:tcp.fin={{ netflow.timeout.tcp_fin }}:tcp={{ netflow.timeout.tcp_generic }}:tcp.rst={{ netflow.timeout.tcp_rst }}:udp={{ netflow.timeout.udp }} +{% endif %} +{% endfor %} {% endif %} -{% endfor %} + +{% if sflow is defined and sflow.server is defined and sflow.server is not none %} +# sFlow servers +{% for server, server_config in sflow.server.items() %} +sfprobe_receiver[sf_{{ server }}]: {{ server }}:{{ server_config.port }} +sfprobe_agentip[sf_{{ server }}]: {{ sflow.agent_address }} +{% if sflow.sampling_rate is defined and sflow.sampling_rate is not none %} +sampling_rate[sf_{{ server }}]: {{ sflow.sampling_rate }} +{% endif %} +{% if sflow.source_address is defined and sflow.source_address is not none %} +sfprobe_source_ip[sf_{{ server }}]: {{ sflow.source_address }} +{% endif %} +{% endfor %} {% endif %} -- cgit v1.2.3 From 0030f9fc844036a0d1a0381a9096b1b9d368e35f Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Fri, 24 Dec 2021 22:49:36 +0100 Subject: flow-accounting: T4099: rename "netflow source-ip" to source-address sFlow uses the source-address CLI node and netflow uses source-ip this is just confusing and should be synced to the common source-address CLI node. --- data/templates/netflow/uacctd.conf.tmpl | 4 +- interface-definitions/flow-accounting-conf.xml.in | 18 +-------- src/conf_mode/flow_accounting_conf.py | 6 +-- src/migration-scripts/flow-accounting/0-to-1 | 49 +++++++++++++++++++++++ 4 files changed, 55 insertions(+), 22 deletions(-) create mode 100755 src/migration-scripts/flow-accounting/0-to-1 (limited to 'data') diff --git a/data/templates/netflow/uacctd.conf.tmpl b/data/templates/netflow/uacctd.conf.tmpl index 2a43b3ce2..27a157531 100644 --- a/data/templates/netflow/uacctd.conf.tmpl +++ b/data/templates/netflow/uacctd.conf.tmpl @@ -48,8 +48,8 @@ nfprobe_maxflows[nf_{{ server }}]: {{ netflow.max_flows }} {% if netflow.sampling_rate is defined and netflow.sampling_rate is not none %} sampling_rate[nf_{{ server }}]: {{ netflow.sampling_rate }} {% endif %} -{% if netflow.source_ip is defined and netflow.source_ip is not none %} -nfprobe_source_ip[nf_{{ server }}]: {{ netflow.source_ip }} +{% if netflow.source_address is defined and netflow.source_address is not none %} +nfprobe_source_ip[nf_{{ server }}]: {{ netflow.source_address }} {% endif %} {% if netflow.timeout is defined and netflow.timeout is not none %} nfprobe_timeouts[nf_{{ server }}]: expint={{ netflow.timeout.expiry_interval }}:general={{ netflow.timeout.flow_generic }}:icmp={{ netflow.timeout.icmp }}:maxlife={{ netflow.timeout.max_active_life }}:tcp.fin={{ netflow.timeout.tcp_fin }}:tcp={{ netflow.timeout.tcp_generic }}:tcp.rst={{ netflow.timeout.tcp_rst }}:udp={{ netflow.timeout.udp }} diff --git a/interface-definitions/flow-accounting-conf.xml.in b/interface-definitions/flow-accounting-conf.xml.in index 02364425f..7b110e733 100644 --- a/interface-definitions/flow-accounting-conf.xml.in +++ b/interface-definitions/flow-accounting-conf.xml.in @@ -183,23 +183,7 @@ - - - IPv4 or IPv6 source address of NetFlow packets - - ipv4 - IPv4 source address of NetFlow packets - - - ipv6 - IPv6 source address of NetFlow packets - - - - - - - + #include NetFlow version to export diff --git a/src/conf_mode/flow_accounting_conf.py b/src/conf_mode/flow_accounting_conf.py index 5f903bf69..9467e805c 100755 --- a/src/conf_mode/flow_accounting_conf.py +++ b/src/conf_mode/flow_accounting_conf.py @@ -239,9 +239,9 @@ def verify(flow_config): raise ConfigError('You need to configure at least one NetFlow server!') # check if configured netflow source-ip exist in the system - if 'source_ip' in flow_config['netflow']: - if not is_addr_assigned(flow_config['netflow']['source_ip']): - print(f'Warning: your "netflow source-ip" does not exist in the system!') + if 'source_address' in flow_config['netflow']: + if not is_addr_assigned(flow_config['netflow']['source_address']): + print(f'Warning: Configured "netflow source-address" does not exist on the system!') # check if engine-id compatible with selected protocol version if 'engine_id' in flow_config['netflow']: diff --git a/src/migration-scripts/flow-accounting/0-to-1 b/src/migration-scripts/flow-accounting/0-to-1 new file mode 100755 index 000000000..9e9d7132d --- /dev/null +++ b/src/migration-scripts/flow-accounting/0-to-1 @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2021 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# T4099: flow-accounting: sync "source-ip" and "source-address" between netflow +# and sflow ion CLI + +from sys import argv +from vyos.configtree import ConfigTree + +if (len(argv) < 1): + print("Must specify file name!") + exit(1) + +file_name = argv[1] + +with open(file_name, 'r') as f: + config_file = f.read() + +base = ['system', 'flow-accounting'] +config = ConfigTree(config_file) + +if not config.exists(base): + # Nothing to do + exit(0) + +# T4099 +tmp = base + ['netflow', 'source-ip'] +if config.exists(tmp): + config.rename(tmp, 'source-address') + +try: + with open(file_name, 'w') as f: + f.write(config.to_string()) +except OSError as e: + print("Failed to save the modified config: {}".format(e)) + exit(1) -- cgit v1.2.3 From 344c2776bd6e157d33ea81f548d1eacde1d3e644 Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Sat, 25 Dec 2021 23:26:36 +0100 Subject: flow-accounting: T4106: support specification of capture packet length --- data/templates/netflow/uacctd.conf.tmpl | 2 +- interface-definitions/flow-accounting-conf.xml.in | 13 +++++++++++++ src/conf_mode/flow_accounting_conf.py | 19 +++++++------------ 3 files changed, 21 insertions(+), 13 deletions(-) (limited to 'data') diff --git a/data/templates/netflow/uacctd.conf.tmpl b/data/templates/netflow/uacctd.conf.tmpl index 27a157531..b6d31746f 100644 --- a/data/templates/netflow/uacctd.conf.tmpl +++ b/data/templates/netflow/uacctd.conf.tmpl @@ -4,7 +4,7 @@ promisc: false pidfile: /var/run/uacctd.pid uacctd_group: 2 uacctd_nl_size: 2097152 -snaplen: {{ snaplen }} +snaplen: {{ packet_length }} aggregate: in_iface{{ ',out_iface' if enable_egress is defined }},src_mac,dst_mac,vlan,src_host,dst_host,src_port,dst_port,proto,tos,flows {% set pipe_size = buffer_size | int *1024 *1024 %} plugin_pipe_size: {{ pipe_size }} diff --git a/interface-definitions/flow-accounting-conf.xml.in b/interface-definitions/flow-accounting-conf.xml.in index ba5c70979..1b57d706c 100644 --- a/interface-definitions/flow-accounting-conf.xml.in +++ b/interface-definitions/flow-accounting-conf.xml.in @@ -22,6 +22,19 @@ 10 + + + Specifies the maximum number of bytes to capture for each packet + + u32:128-750 + Packet length in bytes (default: 128) + + + + + + 128 + Enable egress flow accounting diff --git a/src/conf_mode/flow_accounting_conf.py b/src/conf_mode/flow_accounting_conf.py index 86fbd96b1..3d3b03e10 100755 --- a/src/conf_mode/flow_accounting_conf.py +++ b/src/conf_mode/flow_accounting_conf.py @@ -34,9 +34,6 @@ from vyos import ConfigError from vyos import airbag airbag.enable() -# default values -default_captured_packet_size = 128 - uacctd_conf_path = '/etc/pmacct/uacctd.conf' iptables_nflog_table = 'raw' iptables_nflog_chain = 'VYATTA_CT_PREROUTING_HOOK' @@ -67,7 +64,7 @@ def _iptables_get_nflog(chain, table): return rules # modify iptables rules -def _iptables_config(configured_ifaces, direction): +def _iptables_config(configured_ifaces, direction, length): # define list of iptables commands to modify settings iptable_commands = [] iptables_chain = iptables_nflog_chain @@ -114,7 +111,7 @@ def _iptables_config(configured_ifaces, direction): if direction == "egress": iptables_op = "-o" - rule_definition = f'{iptables_chain} {iptables_op} {iface} -m comment --comment FLOW_ACCOUNTING_RULE -j NFLOG --nflog-group 2 --nflog-size {default_captured_packet_size} --nflog-threshold 100' + rule_definition = f'{iptables_chain} {iptables_op} {iface} -m comment --comment FLOW_ACCOUNTING_RULE -j NFLOG --nflog-group 2 --nflog-size {length} --nflog-threshold 100' iptable_commands.append(f'{iptables} -t {iptables_table} -I {rule_definition}') # change iptables @@ -158,8 +155,6 @@ def get_config(config=None): flow_accounting[flow_type]['server'][server] = dict_merge( default_values,flow_accounting[flow_type]['server'][server]) - flow_accounting['snaplen'] = default_captured_packet_size - return flow_accounting def verify(flow_config): @@ -253,8 +248,8 @@ def apply(flow_config): action = 'restart' # Check if flow-accounting was removed and define command if not flow_config: - _iptables_config([], 'ingress') - _iptables_config([], 'egress') + _iptables_config([], 'ingress', flow_config['packet_length']) + _iptables_config([], 'egress', flow_config['packet_length']) # Stop flow-accounting daemon cmd('systemctl stop uacctd.service') @@ -265,13 +260,13 @@ def apply(flow_config): # configure iptables rules for defined interfaces if 'interface' in flow_config: - _iptables_config(flow_config['interface'], 'ingress') + _iptables_config(flow_config['interface'], 'ingress', flow_config['packet_length']) # configure egress the same way if configured otherwise remove it if 'enable_egress' in flow_config: - _iptables_config(flow_config['interface'], 'egress') + _iptables_config(flow_config['interface'], 'egress', flow_config['packet_length']) else: - _iptables_config([], 'egress') + _iptables_config([], 'egress', flow_config['packet_length']) if __name__ == '__main__': try: -- cgit v1.2.3