From 015e26acc8ed65b6a7d778107a83ee8604950f90 Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Fri, 24 Dec 2021 22:12:19 +0100 Subject: flow-accounting: T4097: move to get_config_dict() --- interface-definitions/flow-accounting-conf.xml.in | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'interface-definitions') diff --git a/interface-definitions/flow-accounting-conf.xml.in b/interface-definitions/flow-accounting-conf.xml.in index 9f13c295b..02364425f 100644 --- a/interface-definitions/flow-accounting-conf.xml.in +++ b/interface-definitions/flow-accounting-conf.xml.in @@ -14,12 +14,13 @@ Buffer size u32 - Buffer size in MiB + Buffer size in MiB (default: 10) + 10 @@ -218,6 +219,7 @@ Internet Protocol Flow Information Export (IPFIX) + 9 @@ -247,6 +249,7 @@ + 2055 @@ -266,6 +269,7 @@ + 60 @@ -278,6 +282,7 @@ + 3600 @@ -290,6 +295,7 @@ + 300 @@ -302,6 +308,7 @@ + 604800 @@ -314,6 +321,7 @@ + 300 @@ -326,6 +334,7 @@ + 3600 @@ -338,6 +347,7 @@ + 120 @@ -350,6 +360,7 @@ + 300 @@ -380,6 +391,7 @@ ^auto$ + auto @@ -421,6 +433,7 @@ + 6343 -- cgit v1.2.3 From 0030f9fc844036a0d1a0381a9096b1b9d368e35f Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Fri, 24 Dec 2021 22:49:36 +0100 Subject: flow-accounting: T4099: rename "netflow source-ip" to source-address sFlow uses the source-address CLI node and netflow uses source-ip this is just confusing and should be synced to the common source-address CLI node. --- data/templates/netflow/uacctd.conf.tmpl | 4 +- interface-definitions/flow-accounting-conf.xml.in | 18 +-------- src/conf_mode/flow_accounting_conf.py | 6 +-- src/migration-scripts/flow-accounting/0-to-1 | 49 +++++++++++++++++++++++ 4 files changed, 55 insertions(+), 22 deletions(-) create mode 100755 src/migration-scripts/flow-accounting/0-to-1 (limited to 'interface-definitions') diff --git a/data/templates/netflow/uacctd.conf.tmpl b/data/templates/netflow/uacctd.conf.tmpl index 2a43b3ce2..27a157531 100644 --- a/data/templates/netflow/uacctd.conf.tmpl +++ b/data/templates/netflow/uacctd.conf.tmpl @@ -48,8 +48,8 @@ nfprobe_maxflows[nf_{{ server }}]: {{ netflow.max_flows }} {% if netflow.sampling_rate is defined and netflow.sampling_rate is not none %} sampling_rate[nf_{{ server }}]: {{ netflow.sampling_rate }} {% endif %} -{% if netflow.source_ip is defined and netflow.source_ip is not none %} -nfprobe_source_ip[nf_{{ server }}]: {{ netflow.source_ip }} +{% if netflow.source_address is defined and netflow.source_address is not none %} +nfprobe_source_ip[nf_{{ server }}]: {{ netflow.source_address }} {% endif %} {% if netflow.timeout is defined and netflow.timeout is not none %} nfprobe_timeouts[nf_{{ server }}]: expint={{ netflow.timeout.expiry_interval }}:general={{ netflow.timeout.flow_generic }}:icmp={{ netflow.timeout.icmp }}:maxlife={{ netflow.timeout.max_active_life }}:tcp.fin={{ netflow.timeout.tcp_fin }}:tcp={{ netflow.timeout.tcp_generic }}:tcp.rst={{ netflow.timeout.tcp_rst }}:udp={{ netflow.timeout.udp }} diff --git a/interface-definitions/flow-accounting-conf.xml.in b/interface-definitions/flow-accounting-conf.xml.in index 02364425f..7b110e733 100644 --- a/interface-definitions/flow-accounting-conf.xml.in +++ b/interface-definitions/flow-accounting-conf.xml.in @@ -183,23 +183,7 @@ - - - IPv4 or IPv6 source address of NetFlow packets - - ipv4 - IPv4 source address of NetFlow packets - - - ipv6 - IPv6 source address of NetFlow packets - - - - - - - + #include NetFlow version to export diff --git a/src/conf_mode/flow_accounting_conf.py b/src/conf_mode/flow_accounting_conf.py index 5f903bf69..9467e805c 100755 --- a/src/conf_mode/flow_accounting_conf.py +++ b/src/conf_mode/flow_accounting_conf.py @@ -239,9 +239,9 @@ def verify(flow_config): raise ConfigError('You need to configure at least one NetFlow server!') # check if configured netflow source-ip exist in the system - if 'source_ip' in flow_config['netflow']: - if not is_addr_assigned(flow_config['netflow']['source_ip']): - print(f'Warning: your "netflow source-ip" does not exist in the system!') + if 'source_address' in flow_config['netflow']: + if not is_addr_assigned(flow_config['netflow']['source_address']): + print(f'Warning: Configured "netflow source-address" does not exist on the system!') # check if engine-id compatible with selected protocol version if 'engine_id' in flow_config['netflow']: diff --git a/src/migration-scripts/flow-accounting/0-to-1 b/src/migration-scripts/flow-accounting/0-to-1 new file mode 100755 index 000000000..9e9d7132d --- /dev/null +++ b/src/migration-scripts/flow-accounting/0-to-1 @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2021 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# T4099: flow-accounting: sync "source-ip" and "source-address" between netflow +# and sflow ion CLI + +from sys import argv +from vyos.configtree import ConfigTree + +if (len(argv) < 1): + print("Must specify file name!") + exit(1) + +file_name = argv[1] + +with open(file_name, 'r') as f: + config_file = f.read() + +base = ['system', 'flow-accounting'] +config = ConfigTree(config_file) + +if not config.exists(base): + # Nothing to do + exit(0) + +# T4099 +tmp = base + ['netflow', 'source-ip'] +if config.exists(tmp): + config.rename(tmp, 'source-address') + +try: + with open(file_name, 'w') as f: + f.write(config.to_string()) +except OSError as e: + print("Failed to save the modified config: {}".format(e)) + exit(1) -- cgit v1.2.3 From b9b5070203c3c3b31a7b297c5ddba8934b1ca34d Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Sat, 25 Dec 2021 22:57:59 +0100 Subject: flow-accounting: T4105: drop "sflow agent-address auto" The implementation of the "auto" option to specify the sflow/netflow agent-address is very error prone. The current implementation will determine the IP address used for the "auto" value as follow: Get BGP router-id 1) If not found use OSPF router-id 2) If not found use OSPFv3 router-id 3) If not found use "the first IP address found on the system Well, what is the "first IP address found"? Also this changes if DHCP is in use. Also another disadvantage is when the BGP/OSPF/OSPFv3 router-id is changed, the agent-address is not updated upon the next reboot of the system. This task is about removing the "auto" keyword from the CLI at all and make it either entirely configurable by the user and hardcode the value in CLI, or not use this at all. If "auto" is specified we will query the system in the above order and set the proper router-id in the CLI. If none can be found the CLI node is removed. --- interface-definitions/flow-accounting-conf.xml.in | 6 ---- smoketest/configs/bgp-big-as-cloud | 6 ++++ src/conf_mode/flow_accounting_conf.py | 38 ++++++----------------- src/migration-scripts/flow-accounting/0-to-1 | 20 ++++++++++++ 4 files changed, 35 insertions(+), 35 deletions(-) (limited to 'interface-definitions') diff --git a/interface-definitions/flow-accounting-conf.xml.in b/interface-definitions/flow-accounting-conf.xml.in index 7b110e733..ba5c70979 100644 --- a/interface-definitions/flow-accounting-conf.xml.in +++ b/interface-definitions/flow-accounting-conf.xml.in @@ -362,20 +362,14 @@ auto - - auto - auto select sFlow agent-address (default) - ipv4 sFlow IPv4 agent address - ^auto$ - auto diff --git a/smoketest/configs/bgp-big-as-cloud b/smoketest/configs/bgp-big-as-cloud index 694243d1e..10660ec87 100644 --- a/smoketest/configs/bgp-big-as-cloud +++ b/smoketest/configs/bgp-big-as-cloud @@ -1819,6 +1819,12 @@ system { } version 9 } + sflow { + agent-address auto + server 1.2.3.4 { + port 1234 + } + } syslog-facility daemon } host-name vyos diff --git a/src/conf_mode/flow_accounting_conf.py b/src/conf_mode/flow_accounting_conf.py index 9467e805c..86fbd96b1 100755 --- a/src/conf_mode/flow_accounting_conf.py +++ b/src/conf_mode/flow_accounting_conf.py @@ -43,29 +43,6 @@ iptables_nflog_chain = 'VYATTA_CT_PREROUTING_HOOK' egress_iptables_nflog_table = 'mangle' egress_iptables_nflog_chain = 'FORWARD' -# get sFlow agent-ip if agent-address is "auto" (default behaviour) -def _sflow_default_agentip(config): - # check if any of BGP, OSPF, OSPFv3 protocols are configured and use router-id from there - if config.exists('protocols bgp'): - bgp_router_id = config.return_value("protocols bgp {} parameters router-id".format(config.list_nodes('protocols bgp')[0])) - if bgp_router_id: - return bgp_router_id - if config.return_value('protocols ospf parameters router-id'): - return config.return_value('protocols ospf parameters router-id') - if config.return_value('protocols ospfv3 parameters router-id'): - return config.return_value('protocols ospfv3 parameters router-id') - - # if router-id was not found, use first available ip of any interface - for iface in Section.interfaces(): - for address in Interface(iface).get_addr(): - # return an IP, if this is not loopback - regex_filter = re.compile('^(?!(127)|(::1)|(fe80))(?P[a-f\d\.:]+)/\d+$') - if regex_filter.search(address): - return regex_filter.search(address).group('ipaddr') - - # return nothing by default - return None - # get iptables rule dict for chain in table def _iptables_get_nflog(chain, table): # define list with rules @@ -223,14 +200,16 @@ def verify(flow_config): # check agent-id for sFlow: we should avoid mixing IPv4 agent-id with IPv6 collectors and vice-versa for server in flow_config['sflow']['server']: - if flow_config['sflow']['agent_address'] != 'auto': + if 'agent_address' in flow_config['sflow']: if ip_address(server).version != ip_address(flow_config['sflow']['agent_address']).version: - raise ConfigError("Different IP address versions cannot be mixed in \"sflow agent-address\" and \"sflow server\". You need to set manually the same IP version for \"agent-address\" as for all sFlow servers") + raise ConfigError('IPv4 and IPv6 addresses can not be mixed in "sflow agent-address" and "sflow '\ + 'server". You need to set the same IP version for both "agent-address" and '\ + 'all sFlow servers') if 'agent_address' in flow_config['sflow']: - agent_address = flow_config['sflow']['agent_address'] - if agent_address != 'auto' and not is_addr_assigned(agent_address): - print(f'Warning: Configured "sflow agent-address" does not exist in the system!') + if not is_addr_assigned(agent_address): + tmp = flow_config['sflow']['agent_address'] + print(f'Warning: Configured "sflow agent-address {tmp}" does not exist in the system!') # check NetFlow configuration if 'netflow' in flow_config: @@ -241,7 +220,8 @@ def verify(flow_config): # check if configured netflow source-ip exist in the system if 'source_address' in flow_config['netflow']: if not is_addr_assigned(flow_config['netflow']['source_address']): - print(f'Warning: Configured "netflow source-address" does not exist on the system!') + tmp = flow_config['netflow']['source_address'] + print(f'Warning: Configured "netflow source-address {tmp}" does not exist on the system!') # check if engine-id compatible with selected protocol version if 'engine_id' in flow_config['netflow']: diff --git a/src/migration-scripts/flow-accounting/0-to-1 b/src/migration-scripts/flow-accounting/0-to-1 index 9e9d7132d..72cce77b0 100755 --- a/src/migration-scripts/flow-accounting/0-to-1 +++ b/src/migration-scripts/flow-accounting/0-to-1 @@ -16,6 +16,7 @@ # T4099: flow-accounting: sync "source-ip" and "source-address" between netflow # and sflow ion CLI +# T4105: flow-accounting: drop "sflow agent-address auto" from sys import argv from vyos.configtree import ConfigTree @@ -41,6 +42,25 @@ tmp = base + ['netflow', 'source-ip'] if config.exists(tmp): config.rename(tmp, 'source-address') +# T4105 +tmp = base + ['sflow', 'agent-address'] +if config.exists(tmp): + value = config.return_value(tmp) + if value == 'auto': + # delete the "auto" + config.delete(tmp) + + # 1) check if BGP router-id is set + # 2) check if OSPF router-id is set + # 3) check if OSPFv3 router-id is set + router_id = None + for protocol in ['bgp', 'ospf', 'ospfv3']: + if config.exists(['protocols', protocol, 'parameters', 'router-id']): + router_id = config.return_value(['protocols', protocol, 'parameters', 'router-id']) + break + if router_id: + config.set(tmp, value=router_id) + try: with open(file_name, 'w') as f: f.write(config.to_string()) -- cgit v1.2.3 From 344c2776bd6e157d33ea81f548d1eacde1d3e644 Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Sat, 25 Dec 2021 23:26:36 +0100 Subject: flow-accounting: T4106: support specification of capture packet length --- data/templates/netflow/uacctd.conf.tmpl | 2 +- interface-definitions/flow-accounting-conf.xml.in | 13 +++++++++++++ src/conf_mode/flow_accounting_conf.py | 19 +++++++------------ 3 files changed, 21 insertions(+), 13 deletions(-) (limited to 'interface-definitions') diff --git a/data/templates/netflow/uacctd.conf.tmpl b/data/templates/netflow/uacctd.conf.tmpl index 27a157531..b6d31746f 100644 --- a/data/templates/netflow/uacctd.conf.tmpl +++ b/data/templates/netflow/uacctd.conf.tmpl @@ -4,7 +4,7 @@ promisc: false pidfile: /var/run/uacctd.pid uacctd_group: 2 uacctd_nl_size: 2097152 -snaplen: {{ snaplen }} +snaplen: {{ packet_length }} aggregate: in_iface{{ ',out_iface' if enable_egress is defined }},src_mac,dst_mac,vlan,src_host,dst_host,src_port,dst_port,proto,tos,flows {% set pipe_size = buffer_size | int *1024 *1024 %} plugin_pipe_size: {{ pipe_size }} diff --git a/interface-definitions/flow-accounting-conf.xml.in b/interface-definitions/flow-accounting-conf.xml.in index ba5c70979..1b57d706c 100644 --- a/interface-definitions/flow-accounting-conf.xml.in +++ b/interface-definitions/flow-accounting-conf.xml.in @@ -22,6 +22,19 @@ 10 + + + Specifies the maximum number of bytes to capture for each packet + + u32:128-750 + Packet length in bytes (default: 128) + + + + + + 128 + Enable egress flow accounting diff --git a/src/conf_mode/flow_accounting_conf.py b/src/conf_mode/flow_accounting_conf.py index 86fbd96b1..3d3b03e10 100755 --- a/src/conf_mode/flow_accounting_conf.py +++ b/src/conf_mode/flow_accounting_conf.py @@ -34,9 +34,6 @@ from vyos import ConfigError from vyos import airbag airbag.enable() -# default values -default_captured_packet_size = 128 - uacctd_conf_path = '/etc/pmacct/uacctd.conf' iptables_nflog_table = 'raw' iptables_nflog_chain = 'VYATTA_CT_PREROUTING_HOOK' @@ -67,7 +64,7 @@ def _iptables_get_nflog(chain, table): return rules # modify iptables rules -def _iptables_config(configured_ifaces, direction): +def _iptables_config(configured_ifaces, direction, length): # define list of iptables commands to modify settings iptable_commands = [] iptables_chain = iptables_nflog_chain @@ -114,7 +111,7 @@ def _iptables_config(configured_ifaces, direction): if direction == "egress": iptables_op = "-o" - rule_definition = f'{iptables_chain} {iptables_op} {iface} -m comment --comment FLOW_ACCOUNTING_RULE -j NFLOG --nflog-group 2 --nflog-size {default_captured_packet_size} --nflog-threshold 100' + rule_definition = f'{iptables_chain} {iptables_op} {iface} -m comment --comment FLOW_ACCOUNTING_RULE -j NFLOG --nflog-group 2 --nflog-size {length} --nflog-threshold 100' iptable_commands.append(f'{iptables} -t {iptables_table} -I {rule_definition}') # change iptables @@ -158,8 +155,6 @@ def get_config(config=None): flow_accounting[flow_type]['server'][server] = dict_merge( default_values,flow_accounting[flow_type]['server'][server]) - flow_accounting['snaplen'] = default_captured_packet_size - return flow_accounting def verify(flow_config): @@ -253,8 +248,8 @@ def apply(flow_config): action = 'restart' # Check if flow-accounting was removed and define command if not flow_config: - _iptables_config([], 'ingress') - _iptables_config([], 'egress') + _iptables_config([], 'ingress', flow_config['packet_length']) + _iptables_config([], 'egress', flow_config['packet_length']) # Stop flow-accounting daemon cmd('systemctl stop uacctd.service') @@ -265,13 +260,13 @@ def apply(flow_config): # configure iptables rules for defined interfaces if 'interface' in flow_config: - _iptables_config(flow_config['interface'], 'ingress') + _iptables_config(flow_config['interface'], 'ingress', flow_config['packet_length']) # configure egress the same way if configured otherwise remove it if 'enable_egress' in flow_config: - _iptables_config(flow_config['interface'], 'egress') + _iptables_config(flow_config['interface'], 'egress', flow_config['packet_length']) else: - _iptables_config([], 'egress') + _iptables_config([], 'egress', flow_config['packet_length']) if __name__ == '__main__': try: -- cgit v1.2.3