Merge branch 't4097-flow-accounting' into current

* t4097-flow-accounting:
  flow-accounting: T4106: support specification of capture packet length
  flow-accounting: T4105: drop "sflow agent-address auto"
  flow-accounting: T4099: rename "netflow source-ip" to source-address
  flow-accounting: T4097: move to get_config_dict()

7 files changed, 278 insertions, 314 deletions

diff --git a/data/configd-include.json b/data/configd-include.json
index 6893aaa86..ee4cb0d42 100644
--- a/data/configd-include.json
+++ b/data/configd-include.json
@@ -6,6 +6,7 @@
 "dhcpv6_relay.py",
 "dns_forwarding.py",
 "dynamic_dns.py",
+"flow_accounting_conf.py",
 "host_name.py",
 "https.py",
 "igmp_proxy.py",
diff --git a/data/templates/netflow/uacctd.conf.tmpl b/data/templates/netflow/uacctd.conf.tmpl
index 11fc76769..b6d31746f 100644
--- a/data/templates/netflow/uacctd.conf.tmpl
+++ b/data/templates/netflow/uacctd.conf.tmpl
@@ -4,72 +4,69 @@ promisc: false
 pidfile: /var/run/uacctd.pid
 uacctd_group: 2
 uacctd_nl_size: 2097152
-snaplen: {{ snaplen }}
-{% if templatecfg['enable-egress'] != none %}
-aggregate: in_iface,out_iface,src_mac,dst_mac,vlan,src_host,dst_host,src_port,dst_port,proto,tos,flows
-{% else %}
-aggregate: in_iface,src_mac,dst_mac,vlan,src_host,dst_host,src_port,dst_port,proto,tos,flows
+snaplen: {{ packet_length }}
+aggregate: in_iface{{ ',out_iface' if enable_egress is defined }},src_mac,dst_mac,vlan,src_host,dst_host,src_port,dst_port,proto,tos,flows
+{% set pipe_size = buffer_size | int *1024 *1024 %}
+plugin_pipe_size: {{ pipe_size }}
+{# We need an integer division (//) without any remainder or fraction #}
+plugin_buffer_size: {{ pipe_size // 1000 }}
+{% if syslog_facility is defined and syslog_facility is not none %}
+syslog: {{ syslog_facility }}
 {% endif %}
-plugin_pipe_size: {{ templatecfg['plugin_pipe_size'] }}
-plugin_buffer_size: {{ templatecfg['plugin_buffer_size'] }}
-{% if templatecfg['syslog-facility'] != none %}
-syslog: {{ templatecfg['syslog-facility'] }}
-{% endif %}
-{% if templatecfg['disable-imt'] == none %}
+{% if disable_imt is not defined %}
 imt_path: /tmp/uacctd.pipe
 imt_mem_pools_number: 169
 {% endif %}
-plugins: {% if templatecfg['netflow']['servers'] != none %}
-{%   for server in templatecfg['netflow']['servers'] %}
-{%     if loop.last %}nfprobe[nf_{{ server['address'] }}]{% else %}nfprobe[nf_{{ server['address'] }}],{% endif %}
-{%   endfor %}
-{% set plugins_presented = true %}
-{% endif %}
-{% if templatecfg['sflow']['servers'] != none %}
-{%   if plugins_presented %}
-{%     for server in templatecfg['sflow']['servers'] %},sfprobe[sf_{{ server['address'] }}]{% endfor %}
-{%   else %}
-{%     for server in templatecfg['sflow']['servers'] %}
-{%       if loop.last %}sfprobe[sf_{{ server['address'] }}]{% else %}sfprobe[sf_{{ server['address'] }}],{% endif %}
-{%     endfor %}
-{%   endif %}
-{% set plugins_presented = true %}
-{% endif %}
-{% if templatecfg['disable-imt'] == none %}
-{%   if plugins_presented %},memory{% else %}memory{% endif %}
-{% endif %}
 
-{% if templatecfg['netflow']['servers'] != none %}
-{% for server in templatecfg['netflow']['servers'] %}
-nfprobe_receiver[nf_{{ server['address'] }}]: {{ server['address'] }}:{{ server['port'] }}
-nfprobe_version[nf_{{ server['address'] }}]: {{ templatecfg['netflow']['version'] }}
-{% if templatecfg['netflow']['engine-id'] != none %}
-nfprobe_engine[nf_{{ server['address'] }}]: {{ templatecfg['netflow']['engine-id'] }}
-{% endif %}
-{% if templatecfg['netflow']['max-flows'] != none %}
-nfprobe_maxflows[nf_{{ server['address'] }}]: {{ templatecfg['netflow']['max-flows'] }}
-{% endif %}
-{% if templatecfg['netflow']['sampling-rate'] != none %}
-sampling_rate[nf_{{ server['address'] }}]: {{ templatecfg['netflow']['sampling-rate'] }}
+{% set plugin = [] %}
+{% if disable_imt is not defined %}
+{%   set plugin = ['memory'] %}
 {% endif %}
-{% if templatecfg['netflow']['source-ip'] != none %}
-nfprobe_source_ip[nf_{{ server['address'] }}]: {{ templatecfg['netflow']['source-ip'] }}
-{% endif %}
-{% if templatecfg['netflow']['timeout_string'] != '' %}
-nfprobe_timeouts[nf_{{ server['address'] }}]: {{ templatecfg['netflow']['timeout_string'] }}
+{% if netflow is defined and netflow.server is defined and netflow.server is not none %}
+{%   for server in netflow.server %}
+{%     set plugin = plugin.append('nfprobe[nf_' ~ server ~ ']') %}
+{%   endfor %}
 {% endif %}
-{% endfor %}
+{% if sflow is defined and sflow.server is defined and sflow.server is not none %}
+{%   for server in sflow.server %}
+{%     set plugin = plugin.append('sfprobe[sf_' ~ server ~ ']') %}
+{%   endfor %}
 {% endif %}
+plugins: {{ plugin | join(',') }}
 
-{% if templatecfg['sflow']['servers'] != none %}
-{% for server in templatecfg['sflow']['servers'] %}
-sfprobe_receiver[sf_{{ server['address'] }}]: {{ server['address'] }}:{{ server['port'] }}
-sfprobe_agentip[sf_{{ server['address'] }}]: {{ templatecfg['sflow']['agent-address'] }}
-{% if templatecfg['sflow']['sampling-rate'] != none %}
-sampling_rate[sf_{{ server['address'] }}]: {{ templatecfg['sflow']['sampling-rate'] }}
-{% endif %}
-{% if templatecfg['sflow']['source-address'] != none %}
-sfprobe_source_ip[sf_{{ server['address'] }}]: {{ templatecfg['sflow']['source-address'] }}
+{% if netflow is defined and netflow.server is defined and netflow.server is not none %}
+# NetFlow servers
+{%   for server, server_config in netflow.server.items() %}
+nfprobe_receiver[nf_{{ server }}]: {{ server }}:{{ server_config.port }}
+nfprobe_version[nf_{{ server }}]: {{ netflow.version }}
+{%     if netflow.engine_id is defined and netflow.engine_id is not none %}
+nfprobe_engine[nf_{{ server }}]: {{ netflow.engine_id }}
+{%     endif %}
+{%     if netflow.max_flows is defined and netflow.max_flows is not none %}
+nfprobe_maxflows[nf_{{ server }}]: {{ netflow.max_flows }}
+{%     endif %}
+{%     if netflow.sampling_rate is defined and netflow.sampling_rate is not none %}
+sampling_rate[nf_{{ server }}]: {{ netflow.sampling_rate }}
+{%     endif %}
+{%     if netflow.source_address is defined and netflow.source_address is not none %}
+nfprobe_source_ip[nf_{{ server }}]: {{ netflow.source_address }}
+{%     endif %}
+{%     if netflow.timeout is defined and netflow.timeout is not none %}
+nfprobe_timeouts[nf_{{ server }}]: expint={{ netflow.timeout.expiry_interval }}:general={{ netflow.timeout.flow_generic }}:icmp={{ netflow.timeout.icmp }}:maxlife={{ netflow.timeout.max_active_life }}:tcp.fin={{ netflow.timeout.tcp_fin }}:tcp={{ netflow.timeout.tcp_generic }}:tcp.rst={{ netflow.timeout.tcp_rst }}:udp={{ netflow.timeout.udp }}
+{%     endif %}
+{%   endfor %}
 {% endif %}
-{% endfor %}
+
+{% if sflow is defined and sflow.server is defined and sflow.server is not none %}
+# sFlow servers
+{%   for server, server_config in sflow.server.items() %}
+sfprobe_receiver[sf_{{ server }}]: {{ server }}:{{ server_config.port }}
+sfprobe_agentip[sf_{{ server }}]: {{ sflow.agent_address }}
+{%     if sflow.sampling_rate is defined and sflow.sampling_rate is not none %}
+sampling_rate[sf_{{ server }}]: {{ sflow.sampling_rate }}
+{%     endif %}
+{%     if sflow.source_address is defined and sflow.source_address is not none %}
+sfprobe_source_ip[sf_{{ server }}]: {{ sflow.source_address }}
+{%     endif %}
+{%   endfor %}
 {% endif %}
diff --git a/interface-definitions/flow-accounting-conf.xml.in b/interface-definitions/flow-accounting-conf.xml.in
index 9f13c295b..1b57d706c 100644
--- a/interface-definitions/flow-accounting-conf.xml.in
+++ b/interface-definitions/flow-accounting-conf.xml.in
@@ -14,12 +14,26 @@
               <help>Buffer size</help>
               <valueHelp>
                 <format>u32</format>
-                <description>Buffer size in MiB</description>
+                <description>Buffer size in MiB (default: 10)</description>
               </valueHelp>
               <constraint>
                 <validator name="numeric" argument="--range 0-4294967295"/>
               </constraint>
             </properties>
+            <defaultValue>10</defaultValue>
+          </leafNode>
+          <leafNode name="packet-length">
+            <properties>
+              <help>Specifies the maximum number of bytes to capture for each packet</help>
+              <valueHelp>
+                <format>u32:128-750</format>
+                <description>Packet length in bytes (default: 128)</description>
+              </valueHelp>
+              <constraint>
+                <validator name="numeric" argument="--range 128-750"/>
+              </constraint>
+            </properties>
+            <defaultValue>128</defaultValue>
           </leafNode>
           <leafNode name="enable-egress">
             <properties>
@@ -182,23 +196,7 @@
                   </constraint>
                 </properties>
               </leafNode>
-              <leafNode name="source-ip">
-                <properties>
-                  <help>IPv4 or IPv6 source address of NetFlow packets</help>
-                  <valueHelp>
-                    <format>ipv4</format>
-                    <description>IPv4 source address of NetFlow packets</description>
-                  </valueHelp>
-                  <valueHelp>
-                    <format>ipv6</format>
-                    <description>IPv6 source address of NetFlow packets</description>
-                  </valueHelp>
-                  <constraint>
-                    <validator name="ipv4-address"/>
-                    <validator name="ipv6-address"/>
-                  </constraint>
-                </properties>
-              </leafNode>
+              #include <include/source-address-ipv4-ipv6.xml.i>
               <leafNode name="version">
                 <properties>
                   <help>NetFlow version to export</help>
@@ -218,6 +216,7 @@
                     <description>Internet Protocol Flow Information Export (IPFIX)</description>
                   </valueHelp>
                 </properties>
+                <defaultValue>9</defaultValue>
               </leafNode>
               <tagNode name="server">
                 <properties>
@@ -247,6 +246,7 @@
                        <validator name="numeric" argument="--range 1025-65535"/>
                       </constraint>
                     </properties>
+                    <defaultValue>2055</defaultValue>
                   </leafNode>
                 </children>
               </tagNode>
@@ -266,6 +266,7 @@
                         <validator name="numeric" argument="--range 0-2147483647"/>
                       </constraint>
                     </properties>
+                    <defaultValue>60</defaultValue>
                   </leafNode>
                   <leafNode name="flow-generic">
                     <properties>
@@ -278,6 +279,7 @@
                        <validator name="numeric" argument="--range 0-2147483647"/>
                       </constraint>
                     </properties>
+                    <defaultValue>3600</defaultValue>
                   </leafNode>
                   <leafNode name="icmp">
                     <properties>
@@ -290,6 +292,7 @@
                        <validator name="numeric" argument="--range 0-2147483647"/>
                       </constraint>
                     </properties>
+                    <defaultValue>300</defaultValue>
                   </leafNode>
                   <leafNode name="max-active-life">
                     <properties>
@@ -302,6 +305,7 @@
                        <validator name="numeric" argument="--range 0-2147483647"/>
                       </constraint>
                     </properties>
+                    <defaultValue>604800</defaultValue>
                   </leafNode>
                   <leafNode name="tcp-fin">
                     <properties>
@@ -314,6 +318,7 @@
                        <validator name="numeric" argument="--range 0-2147483647"/>
                       </constraint>
                     </properties>
+                    <defaultValue>300</defaultValue>
                   </leafNode>
                   <leafNode name="tcp-generic">
                     <properties>
@@ -326,6 +331,7 @@
                        <validator name="numeric" argument="--range 0-2147483647"/>
                       </constraint>
                     </properties>
+                    <defaultValue>3600</defaultValue>
                   </leafNode>
                   <leafNode name="tcp-rst">
                     <properties>
@@ -338,6 +344,7 @@
                        <validator name="numeric" argument="--range 0-2147483647"/>
                       </constraint>
                     </properties>
+                    <defaultValue>120</defaultValue>
                   </leafNode>
                   <leafNode name="udp">
                     <properties>
@@ -350,6 +357,7 @@
                        <validator name="numeric" argument="--range 0-2147483647"/>
                       </constraint>
                     </properties>
+                    <defaultValue>300</defaultValue>
                   </leafNode>
                 </children>
               </node>
@@ -368,16 +376,11 @@
                     <script>${vyos_completion_dir}/list_local_ips.sh --ipv4</script>
                   </completionHelp>
                   <valueHelp>
-                    <format>auto</format>
-                    <description>auto select sFlow agent-address (default)</description>
-                  </valueHelp>
-                  <valueHelp>
                     <format>ipv4</format>
                     <description>sFlow IPv4 agent address</description>
                   </valueHelp>
                   <constraint>
                     <validator name="ipv4-address"/>
-                    <regex>^auto$</regex>
                   </constraint>
                 </properties>
               </leafNode>
@@ -421,6 +424,7 @@
                        <validator name="numeric" argument="--range 1025-65535"/>
                       </constraint>
                     </properties>
+                    <defaultValue>6343</defaultValue>
                   </leafNode>
                 </children>
               </tagNode>
diff --git a/smoketest/configs/bgp-big-as-cloud b/smoketest/configs/bgp-big-as-cloud
index 694243d1e..10660ec87 100644
--- a/smoketest/configs/bgp-big-as-cloud
+++ b/smoketest/configs/bgp-big-as-cloud
@@ -1819,6 +1819,12 @@ system {
             }
             version 9
         }
+        sflow {
+            agent-address auto
+            server 1.2.3.4 {
+                port 1234
+            }
+        }
         syslog-facility daemon
     }
     host-name vyos
diff --git a/smoketest/scripts/cli/test_system_flow-accounting.py b/smoketest/scripts/cli/test_system_flow-accounting.py
index 57866a198..f3bed635c 100755
--- a/smoketest/scripts/cli/test_system_flow-accounting.py
+++ b/smoketest/scripts/cli/test_system_flow-accounting.py
@@ -201,11 +201,11 @@ class TestSystemFlowAccounting(VyOSUnitTestSHIM.TestCase):
 
         uacctd = read_file(uacctd_conf)
 
-        tmp = 'plugins: '
+        tmp = []
+        tmp.append('memory')
         for server, server_config in netflow_server.items():
-            tmp += f'nfprobe[nf_{server}],'
-        tmp += 'memory'
-        self.assertIn(f'{tmp}', uacctd)
+            tmp.append(f'nfprobe[nf_{server}]')
+        self.assertIn('plugins: ' + ','.join(tmp), uacctd)
 
         for server, server_config in netflow_server.items():
             self.assertIn(f'nfprobe_engine[nf_{server}]: {engine_id}', uacctd)
diff --git a/src/conf_mode/flow_accounting_conf.py b/src/conf_mode/flow_accounting_conf.py
index daad00067..3d3b03e10 100755
--- a/src/conf_mode/flow_accounting_conf.py
+++ b/src/conf_mode/flow_accounting_conf.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2018-2020 VyOS maintainers and contributors
+# Copyright (C) 2018-2021 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -16,65 +16,30 @@
 
 import os
 import re
+
 from sys import exit
 import ipaddress
 
 from ipaddress import ip_address
-from jinja2 import FileSystemLoader, Environment
 
+from vyos.config import Config
+from vyos.configdict import dict_merge
 from vyos.ifconfig import Section
 from vyos.ifconfig import Interface
-from vyos.config import Config
-from vyos import ConfigError
-from vyos.util import cmd
 from vyos.template import render
-
+from vyos.util import cmd
+from vyos.validate import is_addr_assigned
+from vyos.xml import defaults
+from vyos import ConfigError
 from vyos import airbag
 airbag.enable()
 
-# default values
-default_sflow_server_port = 6343
-default_netflow_server_port = 2055
-default_plugin_pipe_size = 10
-default_captured_packet_size = 128
-default_netflow_version = '9'
-default_sflow_agentip = 'auto'
 uacctd_conf_path = '/etc/pmacct/uacctd.conf'
 iptables_nflog_table = 'raw'
 iptables_nflog_chain = 'VYATTA_CT_PREROUTING_HOOK'
 egress_iptables_nflog_table = 'mangle'
 egress_iptables_nflog_chain = 'FORWARD'
 
-# helper functions
-# check if node exists and return True if this is true
-def _node_exists(path):
-    vyos_config = Config()
-    if vyos_config.exists(path):
-        return True
-
-# get sFlow agent-ip if agent-address is "auto" (default behaviour)
-def _sflow_default_agentip(config):
-    # check if any of BGP, OSPF, OSPFv3 protocols are configured and use router-id from there
-    if config.exists('protocols bgp'):
-        bgp_router_id = config.return_value("protocols bgp {} parameters router-id".format(config.list_nodes('protocols bgp')[0]))
-        if bgp_router_id:
-            return bgp_router_id
-    if config.return_value('protocols ospf parameters router-id'):
-        return config.return_value('protocols ospf parameters router-id')
-    if config.return_value('protocols ospfv3 parameters router-id'):
-        return config.return_value('protocols ospfv3 parameters router-id')
-
-    # if router-id was not found, use first available ip of any interface
-    for iface in Section.interfaces():
-        for address in Interface(iface).get_addr():
-            # return an IP, if this is not loopback
-            regex_filter = re.compile('^(?!(127)|(::1)|(fe80))(?P<ipaddr>[a-f\d\.:]+)/\d+$')
-            if regex_filter.search(address):
-                return regex_filter.search(address).group('ipaddr')
-
-    # return nothing by default
-    return None
-
 # get iptables rule dict for chain in table
 def _iptables_get_nflog(chain, table):
     # define list with rules
@@ -99,7 +64,7 @@ def _iptables_get_nflog(chain, table):
     return rules
 
 # modify iptables rules
-def _iptables_config(configured_ifaces, direction):
+def _iptables_config(configured_ifaces, direction, length):
     # define list of iptables commands to modify settings
     iptable_commands = []
     iptables_chain = iptables_nflog_chain
@@ -146,7 +111,7 @@ def _iptables_config(configured_ifaces, direction):
         if direction == "egress":
             iptables_op = "-o"
 
-        rule_definition = f'{iptables_chain} {iptables_op} {iface} -m comment --comment FLOW_ACCOUNTING_RULE -j NFLOG --nflog-group 2 --nflog-size {default_captured_packet_size} --nflog-threshold 100'
+        rule_definition = f'{iptables_chain} {iptables_op} {iface} -m comment --comment FLOW_ACCOUNTING_RULE -j NFLOG --nflog-group 2 --nflog-size {length} --nflog-threshold 100'
         iptable_commands.append(f'{iptables} -t {iptables_table} -I {rule_definition}')
 
     # change iptables
@@ -154,232 +119,154 @@ def _iptables_config(configured_ifaces, direction):
         cmd(command, raising=ConfigError)
 
 
-def get_config():
-    vc = Config()
-    vc.set_level('')
-    # Convert the VyOS config to an abstract internal representation
-    flow_config = {
-        'flow-accounting-configured': vc.exists('system flow-accounting'),
-        'buffer-size': vc.return_value('system flow-accounting buffer-size'),
-        'enable-egress': _node_exists('system flow-accounting enable-egress'),
-        'disable-imt': _node_exists('system flow-accounting disable-imt'),
-        'syslog-facility': vc.return_value('system flow-accounting syslog-facility'),
-        'interfaces': None,
-        'sflow': {
-            'configured': vc.exists('system flow-accounting sflow'),
-            'agent-address': vc.return_value('system flow-accounting sflow agent-address'),
-            'sampling-rate': vc.return_value('system flow-accounting sflow sampling-rate'),
-            'servers': None,
-            'source-address': vc.return_value('system flow-accounting sflow source-address')
-        },
-        'netflow': {
-            'configured': vc.exists('system flow-accounting netflow'),
-            'engine-id': vc.return_value('system flow-accounting netflow engine-id'),
-            'max-flows': vc.return_value('system flow-accounting netflow max-flows'),
-            'sampling-rate': vc.return_value('system flow-accounting netflow sampling-rate'),
-            'source-ip': vc.return_value('system flow-accounting netflow source-ip'),
-            'version': vc.return_value('system flow-accounting netflow version'),
-            'timeout': {
-                'expint': vc.return_value('system flow-accounting netflow timeout expiry-interval'),
-                'general': vc.return_value('system flow-accounting netflow timeout flow-generic'),
-                'icmp': vc.return_value('system flow-accounting netflow timeout icmp'),
-                'maxlife': vc.return_value('system flow-accounting netflow timeout max-active-life'),
-                'tcp.fin': vc.return_value('system flow-accounting netflow timeout tcp-fin'),
-                'tcp': vc.return_value('system flow-accounting netflow timeout tcp-generic'),
-                'tcp.rst': vc.return_value('system flow-accounting netflow timeout tcp-rst'),
-                'udp': vc.return_value('system flow-accounting netflow timeout udp')
-            },
-            'servers': None
-        }
-    }
-
-    # get interfaces list
-    if vc.exists('system flow-accounting interface'):
-        flow_config['interfaces'] = vc.return_values('system flow-accounting interface')
-
-    # get sFlow collectors list
-    if vc.exists('system flow-accounting sflow server'):
-        flow_config['sflow']['servers'] = []
-        sflow_collectors = vc.list_nodes('system flow-accounting sflow server')
-        for collector in sflow_collectors:
-            port = default_sflow_server_port
-            if vc.return_value("system flow-accounting sflow server {} port".format(collector)):
-                port = vc.return_value("system flow-accounting sflow server {} port".format(collector))
-            flow_config['sflow']['servers'].append({ 'address': collector, 'port': port })
-
-    # get NetFlow collectors list
-    if vc.exists('system flow-accounting netflow server'):
-        flow_config['netflow']['servers'] = []
-        netflow_collectors = vc.list_nodes('system flow-accounting netflow server')
-        for collector in netflow_collectors:
-            port = default_netflow_server_port
-            if vc.return_value("system flow-accounting netflow server {} port".format(collector)):
-                port = vc.return_value("system flow-accounting netflow server {} port".format(collector))
-            flow_config['netflow']['servers'].append({ 'address': collector, 'port': port })
-
-    # get sflow agent-id
-    if flow_config['sflow']['agent-address'] == None or flow_config['sflow']['agent-address'] == 'auto':
-        flow_config['sflow']['agent-address'] = _sflow_default_agentip(vc)
-
-    # get NetFlow version
-    if not flow_config['netflow']['version']:
-        flow_config['netflow']['version'] = default_netflow_version
-
-    # convert NetFlow engine-id format, if this is necessary
-    if flow_config['netflow']['engine-id'] and flow_config['netflow']['version'] == '5':
-        regex_filter = re.compile('^\d+$')
-        if regex_filter.search(flow_config['netflow']['engine-id']):
-            flow_config['netflow']['engine-id'] = "{}:0".format(flow_config['netflow']['engine-id'])
-
-    # return dict with flow-accounting configuration
-    return flow_config
-
-def verify(config):
-    # Verify that configuration is valid
-    # skip all checks if flow-accounting was removed
-    if not config['flow-accounting-configured']:
-        return True
+def get_config(config=None):
+    if config:
+        conf = config
+    else:
+        conf = Config()
+    base = ['system', 'flow-accounting']
+    if not conf.exists(base):
+        return None
+
+    flow_accounting = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True)
+
+    # We have gathered the dict representation of the CLI, but there are default
+    # options which we need to update into the dictionary retrived.
+    default_values = defaults(base)
+
+    # delete individual flow type default - should only be added if user uses
+    # this feature
+    for flow_type in ['sflow', 'netflow']:
+        if flow_type in default_values:
+            del default_values[flow_type]
+    flow_accounting = dict_merge(default_values, flow_accounting)
+
+    for flow_type in ['sflow', 'netflow']:
+        if flow_type in flow_accounting:
+            default_values = defaults(base + [flow_type])
+            # we need to merge individual server configurations
+            if 'server' in default_values:
+                del default_values['server']
+            flow_accounting[flow_type] = dict_merge(default_values, flow_accounting[flow_type])
+
+            if 'server' in flow_accounting[flow_type]:
+                default_values = defaults(base + [flow_type, 'server'])
+                for server in flow_accounting[flow_type]['server']:
+                    flow_accounting[flow_type]['server'][server] = dict_merge(
+                        default_values,flow_accounting[flow_type]['server'][server])
+
+    return flow_accounting
+
+def verify(flow_config):
+    if not flow_config:
+        return None
 
     # check if at least one collector is enabled
-    if not (config['sflow']['configured'] or config['netflow']['configured'] or not config['disable-imt']):
-        raise ConfigError("You need to configure at least one sFlow or NetFlow protocol, or not set \"disable-imt\" for flow-accounting")
+    if 'sflow' not in flow_config and 'netflow' not in flow_config and 'disable_imt' in flow_config:
+        raise ConfigError('You need to configure at least sFlow or NetFlow, ' \
+                          'or not set "disable-imt" for flow-accounting!')
 
     # Check if at least one interface is configured
-    if not config['interfaces']:
-        raise ConfigError("You need to configure at least one interface for flow-accounting")
+    if 'interface' not in flow_config:
+        raise ConfigError('Flow accounting requires at least one interface to ' \
+                          'be configured!')
 
     # check that all configured interfaces exists in the system
-    for iface in config['interfaces']:
-        if not iface in Section.interfaces():
-            # chnged from error to warning to allow adding dynamic interfaces and interface templates
-            # raise ConfigError("The {} interface is not presented in the system".format(iface))
-            print("Warning: the {} interface is not presented in the system".format(iface))
+    for interface in flow_config['interface']:
+        if interface not in Section.interfaces():
+            # Changed from error to warning to allow adding dynamic interfaces
+            # and interface templates
+            print(f'Warning: Interface "{interface}" is not presented in the system')
 
     # check sFlow configuration
-    if config['sflow']['configured']:
-        # check if at least one sFlow collector is configured if sFlow configuration is presented
-        if not config['sflow']['servers']:
-            raise ConfigError("You need to configure at least one sFlow server")
+    if 'sflow' in flow_config:
+        # check if at least one sFlow collector is configured
+        if 'server' not in flow_config['sflow']:
+            raise ConfigError('You need to configure at least one sFlow server!')
 
         # check that all sFlow collectors use the same IP protocol version
         sflow_collector_ipver = None
-        for sflow_collector in config['sflow']['servers']:
+        for server in flow_config['sflow']['server']:
             if sflow_collector_ipver:
-                if sflow_collector_ipver != ip_address(sflow_collector['address']).version:
+                if sflow_collector_ipver != ip_address(server).version:
                     raise ConfigError("All sFlow servers must use the same IP protocol")
             else:
-                sflow_collector_ipver = ip_address(sflow_collector['address']).version
-
+                sflow_collector_ipver = ip_address(server).version
 
         # check agent-id for sFlow: we should avoid mixing IPv4 agent-id with IPv6 collectors and vice-versa
-        for sflow_collector in config['sflow']['servers']:
-            if ip_address(sflow_collector['address']).version != ip_address(config['sflow']['agent-address']).version:
-                raise ConfigError("Different IP address versions cannot be mixed in \"sflow agent-address\" and \"sflow server\". You need to set manually the same IP version for \"agent-address\" as for all sFlow servers")
-
-        # check if configured sFlow agent-id exist in the system
-        agent_id_presented = None
-        for iface in Section.interfaces():
-            for address in Interface(iface).get_addr():
-                # check an IP, if this is not loopback
-                regex_filter = re.compile('^(?!(127)|(::1)|(fe80))(?P<ipaddr>[a-f\d\.:]+)/\d+$')
-                if regex_filter.search(address):
-                    if regex_filter.search(address).group('ipaddr') == config['sflow']['agent-address']:
-                        agent_id_presented = True
-                        break
-        if not agent_id_presented:
-            raise ConfigError("Your \"sflow agent-address\" does not exist in the system")
+        for server in flow_config['sflow']['server']:
+            if 'agent_address' in flow_config['sflow']:
+                if ip_address(server).version != ip_address(flow_config['sflow']['agent_address']).version:
+                    raise ConfigError('IPv4 and IPv6 addresses can not be mixed in "sflow agent-address" and "sflow '\
+                                      'server". You need to set the same IP version for both "agent-address" and '\
+                                      'all sFlow servers')
+
+        if 'agent_address' in flow_config['sflow']:
+            if not is_addr_assigned(agent_address):
+                tmp = flow_config['sflow']['agent_address']
+                print(f'Warning: Configured "sflow agent-address {tmp}" does not exist in the system!')
 
     # check NetFlow configuration
-    if config['netflow']['configured']:
+    if 'netflow' in flow_config:
         # check if at least one NetFlow collector is configured if NetFlow configuration is presented
-        if not config['netflow']['servers']:
-            raise ConfigError("You need to configure at least one NetFlow server")
+        if 'server' not in flow_config['netflow']:
+            raise ConfigError('You need to configure at least one NetFlow server!')
 
         # check if configured netflow source-ip exist in the system
-        if config['netflow']['source-ip']:
-            source_ip_presented = None
-            for iface in Section.interfaces():
-                for address in Interface(iface).get_addr():
-                    # check an IP
-                    regex_filter = re.compile('^(?!(127)|(::1)|(fe80))(?P<ipaddr>[a-f\d\.:]+)/\d+$')
-                    if regex_filter.search(address):
-                        if regex_filter.search(address).group('ipaddr') == config['netflow']['source-ip']:
-                            source_ip_presented = True
-                            break
-            if not source_ip_presented:
-                print("Warning: your \"netflow source-ip\" does not exist in the system")
+        if 'source_address' in flow_config['netflow']:
+            if not is_addr_assigned(flow_config['netflow']['source_address']):
+                tmp = flow_config['netflow']['source_address']
+                print(f'Warning: Configured "netflow source-address {tmp}" does not exist on the system!')
 
         # check if engine-id compatible with selected protocol version
-        if config['netflow']['engine-id']:
+        if 'engine_id' in flow_config['netflow']:
             v5_filter = '^(\d|[1-9]\d|1\d{2}|2[0-4]\d|25[0-5]):(\d|[1-9]\d|1\d{2}|2[0-4]\d|25[0-5])$'
             v9v10_filter = '^(\d|[1-9]\d{1,8}|[1-3]\d{9}|4[01]\d{8}|42[0-8]\d{7}|429[0-3]\d{6}|4294[0-8]\d{5}|42949[0-5]\d{4}|429496[0-6]\d{3}|4294967[01]\d{2}|42949672[0-8]\d|429496729[0-5])$'
-            if config['netflow']['version'] == '5':
+            engine_id = flow_config['netflow']['engine_id']
+            version = flow_config['netflow']['version']
+
+            if flow_config['netflow']['version'] == '5':
                 regex_filter = re.compile(v5_filter)
-                if not regex_filter.search(config['netflow']['engine-id']):
-                    raise ConfigError("You cannot use NetFlow engine-id {} together with NetFlow protocol version {}".format(config['netflow']['engine-id'], config['netflow']['version']))
+                if not regex_filter.search(engine_id):
+                    raise ConfigError(f'You cannot use NetFlow engine-id "{engine_id}" '\
+                                      f'together with NetFlow protocol version "{version}"!')
             else:
                 regex_filter = re.compile(v9v10_filter)
-                if not regex_filter.search(config['netflow']['engine-id']):
+                if not regex_filter.search(flow_config['netflow']['engine_id']):
                     raise ConfigError("You cannot use NetFlow engine-id {} together with NetFlow protocol version {}".format(config['netflow']['engine-id'], config['netflow']['version']))
 
     # return True if all checks were passed
     return True
 
-def generate(config):
-    # skip all checks if flow-accounting was removed
-    if not config['flow-accounting-configured']:
-        return True
-
-    # Calculate all necessary values
-    if config['buffer-size']:
-        # circular queue size
-        config['plugin_pipe_size'] = int(config['buffer-size']) * 1024**2
-    else:
-        config['plugin_pipe_size'] = default_plugin_pipe_size * 1024**2
-    # transfer buffer size
-    # recommended value from pmacct developers 1/1000 of pipe size
-    config['plugin_buffer_size'] = int(config['plugin_pipe_size'] / 1000)
-
-    # Prepare a timeouts string
-    timeout_string = ''
-    for timeout_type, timeout_value in config['netflow']['timeout'].items():
-        if timeout_value:
-            if timeout_string == '':
-                timeout_string = "{}{}={}".format(timeout_string, timeout_type, timeout_value)
-            else:
-                timeout_string = "{}:{}={}".format(timeout_string, timeout_type, timeout_value)
-    config['netflow']['timeout_string'] = timeout_string
-
-    render(uacctd_conf_path, 'netflow/uacctd.conf.tmpl', {
-        'templatecfg': config,
-        'snaplen': default_captured_packet_size,
-    })
+def generate(flow_config):
+    if not flow_config:
+        return None
 
+    render(uacctd_conf_path, 'netflow/uacctd.conf.tmpl', flow_config)
 
-def apply(config):
-    # define variables
-    command = None
+def apply(flow_config):
+    action = 'restart'
     # Check if flow-accounting was removed and define command
-    if not config['flow-accounting-configured']:
-        command = 'systemctl stop uacctd.service'
-    else:
-        command = 'systemctl restart uacctd.service'
+    if not flow_config:
+        _iptables_config([], 'ingress', flow_config['packet_length'])
+        _iptables_config([], 'egress', flow_config['packet_length'])
 
-    # run command to start or stop flow-accounting
-    cmd(command, raising=ConfigError, message='Failed to start/stop flow-accounting')
+        # Stop flow-accounting daemon
+        cmd('systemctl stop uacctd.service')
+        return
+
+    # Start/reload flow-accounting daemon
+    cmd(f'systemctl restart uacctd.service')
 
     # configure iptables rules for defined interfaces
-    if config['interfaces']:
-        _iptables_config(config['interfaces'], 'ingress')
+    if 'interface' in flow_config:
+        _iptables_config(flow_config['interface'], 'ingress', flow_config['packet_length'])
 
         # configure egress the same way if configured otherwise remove it
-        if config['enable-egress']:
-            _iptables_config(config['interfaces'], 'egress')
+        if 'enable_egress' in flow_config:
+            _iptables_config(flow_config['interface'], 'egress', flow_config['packet_length'])
         else:
-            _iptables_config([], 'egress')
-    else:
-        _iptables_config([], 'ingress')
-        _iptables_config([], 'egress')
+            _iptables_config([], 'egress', flow_config['packet_length'])
 
 if __name__ == '__main__':
     try:
diff --git a/src/migration-scripts/flow-accounting/0-to-1 b/src/migration-scripts/flow-accounting/0-to-1
new file mode 100755
index 000000000..72cce77b0
--- /dev/null
+++ b/src/migration-scripts/flow-accounting/0-to-1
@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2021 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# T4099: flow-accounting: sync "source-ip" and "source-address" between netflow
+#        and sflow ion CLI
+# T4105: flow-accounting: drop "sflow agent-address auto"
+
+from sys import argv
+from vyos.configtree import ConfigTree
+
+if (len(argv) < 1):
+    print("Must specify file name!")
+    exit(1)
+
+file_name = argv[1]
+
+with open(file_name, 'r') as f:
+    config_file = f.read()
+
+base = ['system', 'flow-accounting']
+config = ConfigTree(config_file)
+
+if not config.exists(base):
+    # Nothing to do
+    exit(0)
+
+# T4099
+tmp = base + ['netflow', 'source-ip']
+if config.exists(tmp):
+    config.rename(tmp, 'source-address')
+
+# T4105
+tmp = base + ['sflow', 'agent-address']
+if config.exists(tmp):
+    value = config.return_value(tmp)
+    if value == 'auto':
+        # delete the "auto"
+        config.delete(tmp)
+
+        # 1) check if BGP router-id is set
+        # 2) check if OSPF router-id is set
+        # 3) check if OSPFv3 router-id is set
+        router_id = None
+        for protocol in ['bgp', 'ospf', 'ospfv3']:
+            if config.exists(['protocols', protocol, 'parameters', 'router-id']):
+                router_id = config.return_value(['protocols', protocol, 'parameters', 'router-id'])
+                break
+        if router_id:
+            config.set(tmp, value=router_id)
+
+try:
+    with open(file_name, 'w') as f:
+        f.write(config.to_string())
+except OSError as e:
+    print("Failed to save the modified config: {}".format(e))
+    exit(1)