Merge pull request #2276 from sarthurdev/conntrack

conntrack: T5571: Refactor conntrack using vyos.configdep

6 files changed, 85 insertions, 161 deletions

diff --git a/src/conf_mode/conntrack.py b/src/conf_mode/conntrack.py
index a0de914bc..21a20ea8d 100755
--- a/src/conf_mode/conntrack.py
+++ b/src/conf_mode/conntrack.py
@@ -20,11 +20,10 @@ import re
 from sys import exit
 
 from vyos.config import Config
-from vyos.firewall import find_nftables_rule
-from vyos.firewall import remove_nftables_rule
 from vyos.utils.process import process_named_running
 from vyos.utils.dict import dict_search
 from vyos.utils.dict import dict_search_args
+from vyos.utils.dict import dict_search_recursive
 from vyos.utils.process import cmd
 from vyos.utils.process import rc_cmd
 from vyos.utils.process import run
@@ -47,8 +46,8 @@ module_map = {
         'ko' : ['nf_nat_h323', 'nf_conntrack_h323'],
     },
     'nfs' : {
-        'nftables' : ['ct helper set "rpc_tcp" tcp dport "{111}" return',
-                      'ct helper set "rpc_udp" udp dport "{111}" return']
+        'nftables' : ['ct helper set "rpc_tcp" tcp dport {111} return',
+                      'ct helper set "rpc_udp" udp dport {111} return']
     },
     'pptp' : {
         'ko' : ['nf_nat_pptp', 'nf_conntrack_pptp'],
@@ -57,7 +56,7 @@ module_map = {
         'ko' : ['nf_nat_sip', 'nf_conntrack_sip'],
      },
     'sqlnet' : {
-        'nftables' : ['ct helper set "tns_tcp" tcp dport "{1521,1525,1536}" return']
+        'nftables' : ['ct helper set "tns_tcp" tcp dport {1521,1525,1536} return']
     },
     'tftp' : {
         'ko' : ['nf_nat_tftp', 'nf_conntrack_tftp'],
@@ -87,10 +86,25 @@ def get_config(config=None):
                                      get_first_key=True,
                                      with_recursive_defaults=True)
 
-    conntrack['firewall_group'] = conf.get_config_dict(['firewall', 'group'], key_mangling=('-', '_'),
+    conntrack['firewall'] = conf.get_config_dict(['firewall'], key_mangling=('-', '_'),
                                                  get_first_key=True,
                                                  no_tag_node_value_mangle=True)
 
+    conntrack['flowtable_enabled'] = False
+    flow_offload = dict_search_args(conntrack['firewall'], 'global_options', 'flow_offload')
+    if flow_offload and 'disable' not in flow_offload:
+        for offload_type in ('software', 'hardware'):
+            if dict_search_args(flow_offload, offload_type, 'interface'):
+                conntrack['flowtable_enabled'] = True
+                break
+
+    conntrack['ipv4_nat_action'] = 'accept' if conf.exists(['nat']) else 'return'
+    conntrack['ipv6_nat_action'] = 'accept' if conf.exists(['nat66']) else 'return'
+    conntrack['wlb_action'] = 'accept' if conf.exists(['load-balancing', 'wan']) else 'return'
+    conntrack['wlb_local_action'] = conf.exists(['load-balancing', 'wan', 'enable-local-traffic'])
+
+    conntrack['module_map'] = module_map
+
     return conntrack
 
 def verify(conntrack):
@@ -104,6 +118,17 @@ def verify(conntrack):
                    if 'protocol' not in rule_config or rule_config['protocol'] not in ['tcp', 'udp']:
                        raise ConfigError(f'Port requires tcp or udp as protocol in rule {rule}')
 
+                tcp_flags = dict_search_args(rule_config, 'tcp', 'flags')
+                if tcp_flags:
+                    if dict_search_args(rule_config, 'protocol') != 'tcp':
+                        raise ConfigError('Protocol must be tcp when specifying tcp flags')
+
+                    not_flags = dict_search_args(rule_config, 'tcp', 'flags', 'not')
+                    if not_flags:
+                        duplicates = [flag for flag in tcp_flags if flag in not_flags]
+                        if duplicates:
+                            raise ConfigError(f'Cannot match a tcp flag as set and not set')
+
                 for side in ['destination', 'source']:
                     if side in rule_config:
                         side_conf = rule_config[side]
@@ -127,7 +152,7 @@ def verify(conntrack):
                                     if inet == 'ipv6':
                                         group = f'ipv6_{group}'
 
-                                    group_obj = dict_search_args(conntrack['firewall_group'], group, group_name)
+                                    group_obj = dict_search_args(conntrack['firewall'], 'group', group, group_name)
 
                                     if group_obj is None:
                                         raise ConfigError(f'Invalid {error_group} "{group_name}" on ignore rule')
@@ -138,22 +163,29 @@ def verify(conntrack):
     return None
 
 def generate(conntrack):
+    if not os.path.exists(nftables_ct_file):
+        conntrack['first_install'] = True
+
+    # Determine if conntrack is needed
+    conntrack['ipv4_firewall_action'] = 'return'
+    conntrack['ipv6_firewall_action'] = 'return'
+
+    if conntrack['flowtable_enabled']:
+        conntrack['ipv4_firewall_action'] = 'accept'
+        conntrack['ipv6_firewall_action'] = 'accept'
+    else:
+        for rules, path in dict_search_recursive(conntrack['firewall'], 'rule'):
+            if any(('state' in rule_conf or 'connection_status' in rule_conf) for rule_conf in rules.values()):
+                if path[0] == 'ipv4':
+                    conntrack['ipv4_firewall_action'] = 'accept'
+                elif path[0] == 'ipv6':
+                    conntrack['ipv6_firewall_action'] = 'accept'
+
     render(conntrack_config, 'conntrack/vyos_nf_conntrack.conf.j2', conntrack)
     render(sysctl_file, 'conntrack/sysctl.conf.j2', conntrack)
     render(nftables_ct_file, 'conntrack/nftables-ct.j2', conntrack)
     return None
 
-def find_nftables_ct_rule(table, chain, rule):
-    helper_search = re.search('ct helper set "(\w+)"', rule)
-    if helper_search:
-        rule = helper_search[1]
-    return find_nftables_rule(table, chain, [rule])
-
-def find_remove_rule(table, chain, rule):
-    handle = find_nftables_ct_rule(table, chain, rule)
-    if handle:
-        remove_nftables_rule(table, chain, handle)
-
 def apply(conntrack):
     # Depending on the enable/disable state of the ALG (Application Layer Gateway)
     # modules we need to either insmod or rmmod the helpers.
@@ -164,21 +196,10 @@ def apply(conntrack):
                     # Only remove the module if it's loaded
                     if os.path.exists(f'/sys/module/{mod}'):
                         cmd(f'rmmod {mod}')
-            if 'nftables' in module_config:
-                for rule in module_config['nftables']:
-                    find_remove_rule('raw', 'VYOS_CT_HELPER', rule)
-                    find_remove_rule('ip6 raw', 'VYOS_CT_HELPER', rule)
         else:
             if 'ko' in module_config:
                 for mod in module_config['ko']:
                     cmd(f'modprobe {mod}')
-            if 'nftables' in module_config:
-                for rule in module_config['nftables']:
-                    if not find_nftables_ct_rule('raw', 'VYOS_CT_HELPER', rule):
-                        cmd(f'nft insert rule raw VYOS_CT_HELPER {rule}')
-
-                    if not find_nftables_ct_rule('ip6 raw', 'VYOS_CT_HELPER', rule):
-                        cmd(f'nft insert rule ip6 raw VYOS_CT_HELPER {rule}')
 
     # Load new nftables ruleset
     install_result, output = rc_cmd(f'nft -f {nftables_ct_file}')
diff --git a/src/conf_mode/firewall.py b/src/conf_mode/firewall.py
index 769cc598f..d999b2a64 100755
--- a/src/conf_mode/firewall.py
+++ b/src/conf_mode/firewall.py
@@ -141,13 +141,7 @@ def get_config(config=None):
 
     fqdn_config_parse(firewall)
 
-    firewall['flowtable_enabled'] = False
-    flow_offload = dict_search_args(firewall, 'global_options', 'flow_offload')
-    if flow_offload and 'disable' not in flow_offload:
-        for offload_type in ('software', 'hardware'):
-            if dict_search_args(flow_offload, offload_type, 'interface'):
-                firewall['flowtable_enabled'] = True
-                break
+    set_dependents('conntrack', conf)
 
     return firewall
 
@@ -350,19 +344,6 @@ def generate(firewall):
     if not os.path.exists(nftables_conf):
         firewall['first_install'] = True
 
-    # Determine if conntrack is needed
-    firewall['ipv4_conntrack_action'] = 'return'
-    firewall['ipv6_conntrack_action'] = 'return'
-    if firewall['flowtable_enabled']:  # Netfilter's flowtable offload requires conntrack
-        firewall['ipv4_conntrack_action'] = 'accept'
-        firewall['ipv6_conntrack_action'] = 'accept'
-    else:  # Check if conntrack is needed by firewall rules
-        for proto in ('ipv4', 'ipv6'):
-            for rules, _ in dict_search_recursive(firewall.get(proto, {}), 'rule'):
-                if any(('state' in rule_conf or 'connection_status' in rule_conf) for rule_conf in rules.values()):
-                    firewall[f'{proto}_conntrack_action'] = 'accept'
-                    break
-
     render(nftables_conf, 'firewall/nftables.j2', firewall)
     return None
 
@@ -392,8 +373,7 @@ def apply(firewall):
 
     apply_sysfs(firewall)
 
-    if firewall['group_resync']:
-        call_dependents()
+    call_dependents()
 
     # T970 Enable a resolver (systemd daemon) that checks
     # domain-group/fqdn addresses and update entries for domains by timeout
diff --git a/src/conf_mode/flow_accounting_conf.py b/src/conf_mode/flow_accounting_conf.py
index 71acd69fa..81ee39df1 100755
--- a/src/conf_mode/flow_accounting_conf.py
+++ b/src/conf_mode/flow_accounting_conf.py
@@ -37,7 +37,7 @@ uacctd_conf_path = '/run/pmacct/uacctd.conf'
 systemd_service = 'uacctd.service'
 systemd_override = f'/run/systemd/system/{systemd_service}.d/override.conf'
 nftables_nflog_table = 'raw'
-nftables_nflog_chain = 'VYOS_CT_PREROUTING_HOOK'
+nftables_nflog_chain = 'VYOS_PREROUTING_HOOK'
 egress_nftables_nflog_table = 'inet mangle'
 egress_nftables_nflog_chain = 'FORWARD'
 
diff --git a/src/conf_mode/load-balancing-wan.py b/src/conf_mode/load-balancing-wan.py
index ad9c80d72..5da0b906b 100755
--- a/src/conf_mode/load-balancing-wan.py
+++ b/src/conf_mode/load-balancing-wan.py
@@ -21,6 +21,7 @@ from shutil import rmtree
 
 from vyos.base import Warning
 from vyos.config import Config
+from vyos.configdep import set_dependents, call_dependents
 from vyos.utils.process import cmd
 from vyos.template import render
 from vyos import ConfigError
@@ -49,6 +50,8 @@ def get_config(config=None):
         if lb.from_defaults(['rule', rule, 'limit']):
             del lb['rule'][rule]['limit']
 
+    set_dependents('conntrack', conf)
+
     return lb
 
 
@@ -132,6 +135,8 @@ def apply(lb):
         cmd('sudo sysctl -w net.netfilter.nf_conntrack_acct=1')
         cmd(f'systemctl restart {systemd_service}')
 
+    call_dependents()
+
     return None
 
 
diff --git a/src/conf_mode/nat.py b/src/conf_mode/nat.py
index e37a7011c..52a7a71fd 100755
--- a/src/conf_mode/nat.py
+++ b/src/conf_mode/nat.py
@@ -18,13 +18,12 @@ import jmespath
 import json
 import os
 
-from distutils.version import LooseVersion
-from platform import release as kernel_version
 from sys import exit
 from netifaces import interfaces
 
 from vyos.base import Warning
 from vyos.config import Config
+from vyos.configdep import set_dependents, call_dependents
 from vyos.template import render
 from vyos.template import is_ip_network
 from vyos.utils.kernel import check_kmod
@@ -38,10 +37,7 @@ from vyos import ConfigError
 from vyos import airbag
 airbag.enable()
 
-if LooseVersion(kernel_version()) > LooseVersion('5.1'):
-    k_mod = ['nft_nat', 'nft_chain_nat']
-else:
-    k_mod = ['nft_nat', 'nft_chain_nat_ipv4']
+k_mod = ['nft_nat', 'nft_chain_nat']
 
 nftables_nat_config = '/run/nftables_nat.conf'
 nftables_static_nat_conf = '/run/nftables_static-nat-rules.nft'
@@ -53,18 +49,27 @@ valid_groups = [
     'port_group'
 ]
 
-def get_handler(json, chain, target):
-    """ Get nftable rule handler number of given chain/target combination.
-    Handler is required when adding NAT/Conntrack helper targets """
-    for x in json:
-        if x['chain'] != chain:
-            continue
-        if x['target'] != target:
-            continue
-        return x['handle']
+def get_config(config=None):
+    if config:
+        conf = config
+    else:
+        conf = Config()
 
-    return None
+    base = ['nat']
+    nat = conf.get_config_dict(base, key_mangling=('-', '_'),
+                               get_first_key=True,
+                               with_recursive_defaults=True)
 
+    set_dependents('conntrack', conf)
+
+    if not conf.exists(base):
+        nat['deleted'] = ''
+        return nat
+
+    nat['firewall_group'] = conf.get_config_dict(['firewall', 'group'], key_mangling=('-', '_'), get_first_key=True,
+                                    no_tag_node_value_mangle=True)
+
+    return nat
 
 def verify_rule(config, err_msg, groups_dict):
     """ Common verify steps used for both source and destination NAT """
@@ -136,62 +141,11 @@ def verify_rule(config, err_msg, groups_dict):
             if count != 100:
                 Warning(f'Sum of weight for nat load balance rule is not 100. You may get unexpected behaviour')
 
-def get_config(config=None):
-    if config:
-        conf = config
-    else:
-        conf = Config()
-
-    base = ['nat']
-    nat = conf.get_config_dict(base, key_mangling=('-', '_'),
-                               get_first_key=True,
-                               with_recursive_defaults=True)
-
-    # read in current nftable (once) for further processing
-    tmp = cmd('nft -j list table raw')
-    nftable_json = json.loads(tmp)
-
-    # condense the full JSON table into a list with only relevand informations
-    pattern = 'nftables[?rule].rule[?expr[].jump].{chain: chain, handle: handle, target: expr[].jump.target | [0]}'
-    condensed_json = jmespath.search(pattern, nftable_json)
-
-    if not conf.exists(base):
-        if get_handler(condensed_json, 'PREROUTING', 'VYOS_CT_HELPER'):
-            nat['helper_functions'] = 'remove'
-
-            # Retrieve current table handler positions
-            nat['pre_ct_ignore'] = get_handler(condensed_json, 'PREROUTING', 'VYOS_CT_HELPER')
-            nat['pre_ct_conntrack'] = get_handler(condensed_json, 'PREROUTING', 'NAT_CONNTRACK')
-            nat['out_ct_ignore'] = get_handler(condensed_json, 'OUTPUT', 'VYOS_CT_HELPER')
-            nat['out_ct_conntrack'] = get_handler(condensed_json, 'OUTPUT', 'NAT_CONNTRACK')
-        nat['deleted'] = ''
-        return nat
-
-    nat['firewall_group'] = conf.get_config_dict(['firewall', 'group'], key_mangling=('-', '_'), get_first_key=True,
-                                    no_tag_node_value_mangle=True)
-
-    # check if NAT connection tracking helpers need to be set up - this has to
-    # be done only once
-    if not get_handler(condensed_json, 'PREROUTING', 'NAT_CONNTRACK'):
-        nat['helper_functions'] = 'add'
-
-        # Retrieve current table handler positions
-        nat['pre_ct_ignore'] = get_handler(condensed_json, 'PREROUTING', 'VYOS_CT_IGNORE')
-        nat['pre_ct_conntrack'] = get_handler(condensed_json, 'PREROUTING', 'VYOS_CT_PREROUTING_HOOK')
-        nat['out_ct_ignore'] = get_handler(condensed_json, 'OUTPUT', 'VYOS_CT_IGNORE')
-        nat['out_ct_conntrack'] = get_handler(condensed_json, 'OUTPUT', 'VYOS_CT_OUTPUT_HOOK')
-
-    return nat
-
 def verify(nat):
     if not nat or 'deleted' in nat:
         # no need to verify the CLI as NAT is going to be deactivated
         return None
 
-    if 'helper_functions' in nat:
-        if not (nat['pre_ct_ignore'] or nat['pre_ct_conntrack'] or nat['out_ct_ignore'] or nat['out_ct_conntrack']):
-            raise Exception('could not determine nftable ruleset handlers')
-
     if dict_search('source.rule', nat):
         for rule, config in dict_search('source.rule', nat).items():
             err_msg = f'Source NAT configuration error in rule {rule}:'
@@ -267,6 +221,8 @@ def apply(nat):
         os.unlink(nftables_nat_config)
         os.unlink(nftables_static_nat_conf)
 
+    call_dependents()
+
     return None
 
 if __name__ == '__main__':
diff --git a/src/conf_mode/nat66.py b/src/conf_mode/nat66.py
index 4c12618bc..46d796bc8 100755
--- a/src/conf_mode/nat66.py
+++ b/src/conf_mode/nat66.py
@@ -23,6 +23,7 @@ from netifaces import interfaces
 
 from vyos.base import Warning
 from vyos.config import Config
+from vyos.configdep import set_dependents, call_dependents
 from vyos.template import render
 from vyos.utils.process import cmd
 from vyos.utils.kernel import check_kmod
@@ -37,18 +38,6 @@ k_mod = ['nft_nat', 'nft_chain_nat']
 nftables_nat66_config = '/run/nftables_nat66.nft'
 ndppd_config = '/run/ndppd/ndppd.conf'
 
-def get_handler(json, chain, target):
-    """ Get nftable rule handler number of given chain/target combination.
-    Handler is required when adding NAT66/Conntrack helper targets """
-    for x in json:
-        if x['chain'] != chain:
-            continue
-        if x['target'] != target:
-            continue
-        return x['handle']
-
-    return None
-
 def get_config(config=None):
     if config:
         conf = config
@@ -58,35 +47,10 @@ def get_config(config=None):
     base = ['nat66']
     nat = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True)
 
-    # read in current nftable (once) for further processing
-    tmp = cmd('nft -j list table ip6 raw')
-    nftable_json = json.loads(tmp)
-
-    # condense the full JSON table into a list with only relevand informations
-    pattern = 'nftables[?rule].rule[?expr[].jump].{chain: chain, handle: handle, target: expr[].jump.target | [0]}'
-    condensed_json = jmespath.search(pattern, nftable_json)
+    set_dependents('conntrack', conf)
 
     if not conf.exists(base):
-        nat['helper_functions'] = 'remove'
-        nat['pre_ct_ignore'] = get_handler(condensed_json, 'PREROUTING', 'VYOS_CT_HELPER')
-        nat['pre_ct_conntrack'] = get_handler(condensed_json, 'PREROUTING', 'NAT_CONNTRACK')
-        nat['out_ct_ignore'] = get_handler(condensed_json, 'OUTPUT', 'VYOS_CT_HELPER')
-        nat['out_ct_conntrack'] = get_handler(condensed_json, 'OUTPUT', 'NAT_CONNTRACK')
         nat['deleted'] = ''
-        return nat
-
-    # check if NAT66 connection tracking helpers need to be set up - this has to
-    # be done only once
-    if not get_handler(condensed_json, 'PREROUTING', 'NAT_CONNTRACK'):
-        nat['helper_functions'] = 'add'
-
-        # Retrieve current table handler positions
-        nat['pre_ct_ignore'] = get_handler(condensed_json, 'PREROUTING', 'VYOS_CT_IGNORE')
-        nat['pre_ct_conntrack'] = get_handler(condensed_json, 'PREROUTING', 'VYOS_CT_PREROUTING_HOOK')
-        nat['out_ct_ignore'] = get_handler(condensed_json, 'OUTPUT', 'VYOS_CT_IGNORE')
-        nat['out_ct_conntrack'] = get_handler(condensed_json, 'OUTPUT', 'VYOS_CT_OUTPUT_HOOK')
-    else:
-        nat['helper_functions'] = 'has'
 
     return nat
 
@@ -95,10 +59,6 @@ def verify(nat):
         # no need to verify the CLI as NAT66 is going to be deactivated
         return None
 
-    if 'helper_functions' in nat and nat['helper_functions'] != 'has':
-        if not (nat['pre_ct_conntrack'] or nat['out_ct_conntrack']):
-            raise Exception('could not determine nftable ruleset handlers')
-
     if dict_search('source.rule', nat):
         for rule, config in dict_search('source.rule', nat).items():
             err_msg = f'Source NAT66 configuration error in rule {rule}:'
@@ -155,6 +115,8 @@ def apply(nat):
     else:
         cmd('systemctl restart ndppd')
 
+    call_dependents()
+
     return None
 
 if __name__ == '__main__':