conntrack: T5376: T5779: backport from current

Backport of the conntrack system from current branch.

(cherry picked from commit fd0bcaf12)
(cherry picked from commit 5acf5aced)
(cherry picked from commit 42ff4d8a7)
(cherry picked from commit 24a1a7059)

4 files changed, 259 insertions, 65 deletions

diff --git a/src/conf_mode/system_conntrack.py b/src/conf_mode/system_conntrack.py
index 9c43640a9..7f6c71440 100755
--- a/src/conf_mode/system_conntrack.py
+++ b/src/conf_mode/system_conntrack.py
@@ -20,11 +20,13 @@ import re
 from sys import exit
 
 from vyos.config import Config
-from vyos.firewall import find_nftables_rule
-from vyos.firewall import remove_nftables_rule
+from vyos.configdep import set_dependents, call_dependents
 from vyos.utils.process import process_named_running
 from vyos.utils.dict import dict_search
+from vyos.utils.dict import dict_search_args
+from vyos.utils.dict import dict_search_recursive
 from vyos.utils.process import cmd
+from vyos.utils.process import rc_cmd
 from vyos.utils.process import run
 from vyos.template import render
 from vyos import ConfigError
@@ -38,34 +40,44 @@ nftables_ct_file = r'/run/nftables-ct.conf'
 # Every ALG (Application Layer Gateway) consists of either a Kernel Object
 # also called a Kernel Module/Driver or some rules present in iptables
 module_map = {
-    'ftp' : {
-        'ko' : ['nf_nat_ftp', 'nf_conntrack_ftp'],
+    'ftp': {
+        'ko': ['nf_nat_ftp', 'nf_conntrack_ftp'],
+        'nftables': ['ct helper set "ftp_tcp" tcp dport {21} return']
     },
-    'h323' : {
-        'ko' : ['nf_nat_h323', 'nf_conntrack_h323'],
+    'h323': {
+        'ko': ['nf_nat_h323', 'nf_conntrack_h323'],
+        'nftables': ['ct helper set "ras_udp" udp dport {1719} return',
+                     'ct helper set "q931_tcp" tcp dport {1720} return']
     },
-    'nfs' : {
-        'nftables' : ['ct helper set "rpc_tcp" tcp dport "{111}" return',
-                      'ct helper set "rpc_udp" udp dport "{111}" return']
+    'nfs': {
+        'nftables': ['ct helper set "rpc_tcp" tcp dport {111} return',
+                     'ct helper set "rpc_udp" udp dport {111} return']
     },
-    'pptp' : {
-        'ko' : ['nf_nat_pptp', 'nf_conntrack_pptp'],
+    'pptp': {
+        'ko': ['nf_nat_pptp', 'nf_conntrack_pptp'],
+        'nftables': ['ct helper set "pptp_tcp" tcp dport {1723} return'],
+        'ipv4': True
      },
-    'sip' : {
-        'ko' : ['nf_nat_sip', 'nf_conntrack_sip'],
+    'sip': {
+        'ko': ['nf_nat_sip', 'nf_conntrack_sip'],
+        'nftables': ['ct helper set "sip_tcp" tcp dport {5060,5061} return',
+                     'ct helper set "sip_udp" udp dport {5060,5061} return']
      },
-    'sqlnet' : {
-        'nftables' : ['ct helper set "tns_tcp" tcp dport "{1521,1525,1536}" return']
+    'sqlnet': {
+        'nftables': ['ct helper set "tns_tcp" tcp dport {1521,1525,1536} return']
     },
-    'tftp' : {
-        'ko' : ['nf_nat_tftp', 'nf_conntrack_tftp'],
+    'tftp': {
+        'ko': ['nf_nat_tftp', 'nf_conntrack_tftp'],
+        'nftables': ['ct helper set "tftp_udp" udp dport {69} return']
      },
 }
 
-def resync_conntrackd():
-    tmp = run('/usr/libexec/vyos/conf_mode/conntrack_sync.py')
-    if tmp > 0:
-        print('ERROR: error restarting conntrackd!')
+valid_groups = [
+    'address_group',
+    'domain_group',
+    'network_group',
+    'port_group'
+]
 
 def get_config(config=None):
     if config:
@@ -78,71 +90,141 @@ def get_config(config=None):
                                      get_first_key=True,
                                      with_recursive_defaults=True)
 
+    conntrack['firewall'] = conf.get_config_dict(['firewall'], key_mangling=('-', '_'),
+                                                 get_first_key=True,
+                                                 no_tag_node_value_mangle=True)
+
+    conntrack['ipv4_nat_action'] = 'accept' if conf.exists(['nat']) else 'return'
+    conntrack['ipv6_nat_action'] = 'accept' if conf.exists(['nat66']) else 'return'
+    conntrack['wlb_action'] = 'accept' if conf.exists(['load-balancing', 'wan']) else 'return'
+    conntrack['wlb_local_action'] = conf.exists(['load-balancing', 'wan', 'enable-local-traffic'])
+
+    conntrack['module_map'] = module_map
+
+    if conf.exists(['service', 'conntrack-sync']):
+        set_dependents('conntrack_sync', conf)
+
     return conntrack
 
 def verify(conntrack):
-    if dict_search('ignore.rule', conntrack) != None:
-        for rule, rule_config in conntrack['ignore']['rule'].items():
-            if dict_search('destination.port', rule_config) or \
-               dict_search('source.port', rule_config):
-               if 'protocol' not in rule_config or rule_config['protocol'] not in ['tcp', 'udp']:
-                   raise ConfigError(f'Port requires tcp or udp as protocol in rule {rule}')
-
+    for inet in ['ipv4', 'ipv6']:
+        if dict_search_args(conntrack, 'ignore', inet, 'rule') != None:
+            for rule, rule_config in conntrack['ignore'][inet]['rule'].items():
+                if dict_search('destination.port', rule_config) or \
+                   dict_search('destination.group.port_group', rule_config) or \
+                   dict_search('source.port', rule_config) or \
+                   dict_search('source.group.port_group', rule_config):
+                   if 'protocol' not in rule_config or rule_config['protocol'] not in ['tcp', 'udp']:
+                       raise ConfigError(f'Port requires tcp or udp as protocol in rule {rule}')
+
+                tcp_flags = dict_search_args(rule_config, 'tcp', 'flags')
+                if tcp_flags:
+                    if dict_search_args(rule_config, 'protocol') != 'tcp':
+                        raise ConfigError('Protocol must be tcp when specifying tcp flags')
+
+                    not_flags = dict_search_args(rule_config, 'tcp', 'flags', 'not')
+                    if not_flags:
+                        duplicates = [flag for flag in tcp_flags if flag in not_flags]
+                        if duplicates:
+                            raise ConfigError(f'Cannot match a tcp flag as set and not set')
+
+                for side in ['destination', 'source']:
+                    if side in rule_config:
+                        side_conf = rule_config[side]
+
+                        if 'group' in side_conf:
+                            if len({'address_group', 'network_group', 'domain_group'} & set(side_conf['group'])) > 1:
+                                raise ConfigError('Only one address-group, network-group or domain-group can be specified')
+
+                            for group in valid_groups:
+                                if group in side_conf['group']:
+                                    group_name = side_conf['group'][group]
+                                    error_group = group.replace("_", "-")
+
+                                    if group in ['address_group', 'network_group', 'domain_group']:
+                                        if 'address' in side_conf:
+                                            raise ConfigError(f'{error_group} and address cannot both be defined')
+
+                                    if group_name and group_name[0] == '!':
+                                        group_name = group_name[1:]
+
+                                    if inet == 'ipv6':
+                                        group = f'ipv6_{group}'
+
+                                    group_obj = dict_search_args(conntrack['firewall'], 'group', group, group_name)
+
+                                    if group_obj is None:
+                                        raise ConfigError(f'Invalid {error_group} "{group_name}" on ignore rule')
+
+                                    if not group_obj:
+                                        Warning(f'{error_group} "{group_name}" has no members!')
+
+        if dict_search_args(conntrack, 'timeout', 'custom', inet, 'rule') != None:
+            for rule, rule_config in conntrack['timeout']['custom'][inet]['rule'].items():
+                if 'protocol' not in rule_config:
+                    raise ConfigError(f'Conntrack custom timeout rule {rule} requires protocol tcp or udp')
+                else:
+                    if 'tcp' in rule_config['protocol'] and 'udp' in rule_config['protocol']:
+                        raise ConfigError(f'conntrack custom timeout rule {rule} - Cant use both tcp and udp protocol')
     return None
 
 def generate(conntrack):
+    if not os.path.exists(nftables_ct_file):
+        conntrack['first_install'] = True
+
+    # Determine if conntrack is needed
+    conntrack['ipv4_firewall_action'] = 'return'
+    conntrack['ipv6_firewall_action'] = 'return'
+
+    for rules, path in dict_search_recursive(conntrack['firewall'], 'rule'):
+        if any(('state' in rule_conf or 'connection_status' in rule_conf or 'offload_target' in rule_conf) for rule_conf in rules.values()):
+            if path[0] == 'ipv4':
+                conntrack['ipv4_firewall_action'] = 'accept'
+            elif path[0] == 'ipv6':
+                conntrack['ipv6_firewall_action'] = 'accept'
+
     render(conntrack_config, 'conntrack/vyos_nf_conntrack.conf.j2', conntrack)
     render(sysctl_file, 'conntrack/sysctl.conf.j2', conntrack)
     render(nftables_ct_file, 'conntrack/nftables-ct.j2', conntrack)
-
-    # dry-run newly generated configuration
-    tmp = run(f'nft -c -f {nftables_ct_file}')
-    if tmp > 0:
-        if os.path.exists(nftables_ct_file):
-            os.unlink(nftables_ct_file)
-        raise ConfigError('Configuration file errors encountered!')
-
     return None
 
-def find_nftables_ct_rule(rule):
-    helper_search = re.search('ct helper set "(\w+)"', rule)
-    if helper_search:
-        rule = helper_search[1]
-    return find_nftables_rule('raw', 'VYOS_CT_HELPER', [rule])
-
-def find_remove_rule(rule):
-    handle = find_nftables_ct_rule(rule)
-    if handle:
-        remove_nftables_rule('raw', 'VYOS_CT_HELPER', handle)
-
 def apply(conntrack):
     # Depending on the enable/disable state of the ALG (Application Layer Gateway)
     # modules we need to either insmod or rmmod the helpers.
+    
+    add_modules = []
+    rm_modules = []
+
     for module, module_config in module_map.items():
-        if dict_search(f'modules.{module}', conntrack) is None:
+        if dict_search_args(conntrack, 'modules', module) is None:
             if 'ko' in module_config:
-                for mod in module_config['ko']:
-                    # Only remove the module if it's loaded
-                    if os.path.exists(f'/sys/module/{mod}'):
-                        cmd(f'rmmod {mod}')
-            if 'nftables' in module_config:
-                for rule in module_config['nftables']:
-                    find_remove_rule(rule)
+                unloaded = [mod for mod in module_config['ko'] if os.path.exists(f'/sys/module/{mod}')]
+                rm_modules.extend(unloaded)
         else:
             if 'ko' in module_config:
-                for mod in module_config['ko']:
-                    cmd(f'modprobe {mod}')
-            if 'nftables' in module_config:
-                for rule in module_config['nftables']:
-                    if not find_nftables_ct_rule(rule):
-                        cmd(f'nft insert rule ip raw VYOS_CT_HELPER {rule}')
+                add_modules.extend(module_config['ko'])
+
+    # Add modules before nftables uses them
+    if add_modules:
+        module_str = ' '.join(add_modules)
+        cmd(f'modprobe -a {module_str}')
 
     # Load new nftables ruleset
-    cmd(f'nft -f {nftables_ct_file}')
+    install_result, output = rc_cmd(f'nft -f {nftables_ct_file}')
+    if install_result == 1:
+        raise ConfigError(f'Failed to apply configuration: {output}')
 
-    if process_named_running('conntrackd'):
-        # Reload conntrack-sync daemon to fetch new sysctl values
-        resync_conntrackd()
+    # Remove modules after nftables stops using them
+    if rm_modules:
+        module_str = ' '.join(rm_modules)
+        cmd(f'rmmod {module_str}')
+
+    try:
+        call_dependents()
+    except ConfigError:
+        # Ignore config errors on dependent due to being called too early. Example:
+        # ConfigError("ConfigError('Interface ethN requires an IP address!')")
+        pass
 
     # We silently ignore all errors
     # See: https://bugzilla.redhat.com/show_bug.cgi?id=1264080
diff --git a/src/init/vyos-router b/src/init/vyos-router
index ff95be994..aaecbf2a1 100755
--- a/src/init/vyos-router
+++ b/src/init/vyos-router
@@ -364,6 +364,9 @@ start ()
     nfct helper add rpc inet tcp
     nfct helper add rpc inet udp
     nfct helper add tns inet tcp
+    nfct helper add rpc inet6 tcp
+    nfct helper add rpc inet6 udp
+    nfct helper add tns inet6 tcp
     nft -f /usr/share/vyos/vyos-firewall-init.conf || log_failure_msg "could not initiate firewall rules"
 
     # As VyOS does not execute commands that are not present in the CLI we call
diff --git a/src/migration-scripts/conntrack/3-to-4 b/src/migration-scripts/conntrack/3-to-4
new file mode 100755
index 000000000..e90c383af
--- /dev/null
+++ b/src/migration-scripts/conntrack/3-to-4
@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2023 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Add support for IPv6 conntrack ignore, move existing nodes to `system conntrack ignore ipv4`
+
+from sys import argv
+from sys import exit
+
+from vyos.configtree import ConfigTree
+
+if len(argv) < 2:
+    print("Must specify file name!")
+    exit(1)
+
+file_name = argv[1]
+
+with open(file_name, 'r') as f:
+    config_file = f.read()
+
+base = ['system', 'conntrack']
+config = ConfigTree(config_file)
+
+if not config.exists(base):
+    # Nothing to do
+    exit(0)
+
+if config.exists(base + ['ignore', 'rule']):
+    config.set(base + ['ignore', 'ipv4'])
+    config.copy(base + ['ignore', 'rule'], base + ['ignore', 'ipv4', 'rule'])
+    config.delete(base + ['ignore', 'rule'])
+
+try:
+    with open(file_name, 'w') as f:
+        f.write(config.to_string())
+except OSError as e:
+    print("Failed to save the modified config: {}".format(e))
+    exit(1)
diff --git a/src/migration-scripts/conntrack/4-to-5 b/src/migration-scripts/conntrack/4-to-5
new file mode 100755
index 000000000..d2e5fc5fa
--- /dev/null
+++ b/src/migration-scripts/conntrack/4-to-5
@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2023 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# T5779: system conntrack timeout custom
+# Before:
+#   Protocols tcp, udp and icmp allowed. When using udp it did not work
+#   Only ipv4 custom timeout rules
+# Now:
+#   Valid protocols are only tcp or udp.
+#   Extend functionality to ipv6 and move ipv4 custom rules to new node:
+#       set system conntrack timeout custom [ipv4 | ipv6] rule <rule> ...
+
+from sys import argv
+from sys import exit
+
+from vyos.configtree import ConfigTree
+
+if len(argv) < 2:
+    print("Must specify file name!")
+    exit(1)
+
+file_name = argv[1]
+
+with open(file_name, 'r') as f:
+    config_file = f.read()
+
+base = ['system', 'conntrack']
+config = ConfigTree(config_file)
+
+if not config.exists(base):
+    # Nothing to do
+    exit(0)
+
+if config.exists(base + ['timeout', 'custom', 'rule']):
+    for rule in config.list_nodes(base + ['timeout', 'custom', 'rule']):
+        if config.exists(base + ['timeout', 'custom', 'rule', rule, 'protocol', 'tcp']):
+            config.set(base + ['timeout', 'custom', 'ipv4', 'rule'])
+            config.copy(base + ['timeout', 'custom', 'rule', rule], base + ['timeout', 'custom', 'ipv4', 'rule', rule])
+    config.delete(base + ['timeout', 'custom', 'rule'])
+
+try:
+    with open(file_name, 'w') as f:
+        f.write(config.to_string())
+except OSError as e:
+    print("Failed to save the modified config: {}".format(e))
+    exit(1)