T4502: firewall: Add software flow offload using flowtable

The following commands will enable nftables flowtable offload on interfaces eth0 eth1:

```
set firewall global-options flow-offload software interface <name>
set firewall global-options flow-offload hardware interface <name>
```

Generated nftables rules:

```
table inet vyos_offload {
    flowtable VYOS_FLOWTABLE_software {
        hook ingress priority filter - 1; devices = { eth0, eth1, eth2, eth3 };
        counter
    }

    chain VYOS_OFFLOAD_software {
        type filter hook forward priority filter - 1; policy accept;
        ct state { established, related } meta l4proto { tcp, udp } flow add @VYOS_FLOWTABLE_software
    }
}
```

Use this option to count packets and bytes for each offloaded flow:
```
set system conntrack flow-accounting
```

To verify a connection is offloaded, run

```
cat /proc/net/nf_conntrack|grep OFFLOAD
```

This PR follows firewalld's implementation: https://github.com/firewalld/firewalld/blob/e748b97787d685d0ca93f58e8d4292e87d3f0da6/src/firewall/core/nftables.py#L590

A good introduction to nftables flowtable: https://thermalcircle.de/doku.php?id=blog:linux:flowtables_1_a_netfilter_nftables_fastpath

8 files changed, 130 insertions, 9 deletions

diff --git a/data/templates/conntrack/sysctl.conf.j2 b/data/templates/conntrack/sysctl.conf.j2
index 075402c04..3d6fc43f2 100644
--- a/data/templates/conntrack/sysctl.conf.j2
+++ b/data/templates/conntrack/sysctl.conf.j2
@@ -24,3 +24,4 @@ net.netfilter.nf_conntrack_tcp_timeout_time_wait = {{ timeout.tcp.time_wait }}
 net.netfilter.nf_conntrack_udp_timeout = {{ timeout.udp.other }}
 net.netfilter.nf_conntrack_udp_timeout_stream = {{ timeout.udp.stream }}
 
+net.netfilter.nf_conntrack_acct = {{ '1' if flow_accounting is vyos_defined else '0' }}
diff --git a/data/templates/firewall/nftables-offload.j2 b/data/templates/firewall/nftables-offload.j2
new file mode 100644
index 000000000..6afcd79f7
--- /dev/null
+++ b/data/templates/firewall/nftables-offload.j2
@@ -0,0 +1,11 @@
+{% macro render_flowtable(name, devices, priority='filter', hardware_offload=false, with_counter=true) %}
+flowtable {{ name }} {
+    hook ingress priority {{ priority }}; devices = { {{ devices | join(', ') }} };
+{% if hardware_offload %}
+    flags offload;
+{% endif %}
+{% if with_counter %}
+    counter
+{% endif %}
+}
+{% endmacro %}
diff --git a/data/templates/firewall/nftables.j2 b/data/templates/firewall/nftables.j2
index 87630940b..1b764c9da 100644
--- a/data/templates/firewall/nftables.j2
+++ b/data/templates/firewall/nftables.j2
@@ -2,6 +2,7 @@
 
 {% import 'firewall/nftables-defines.j2' as group_tmpl %}
 {% import 'firewall/nftables-bridge.j2' as bridge_tmpl %}
+{% import 'firewall/nftables-offload.j2' as offload %}
 
 flush chain raw FW_CONNTRACK
 flush chain ip6 raw FW_CONNTRACK
@@ -271,3 +272,26 @@ table bridge vyos_filter {
 {{ group_tmpl.groups(group, False, False) }}
 }
 {% endif %}
+{{ group_tmpl.groups(group, True) }}
+}
+
+table inet vyos_offload
+delete table inet vyos_offload
+table inet vyos_offload {
+{% if flowtable_enabled %}
+{%     if global_options.flow_offload.hardware.interface is vyos_defined %}
+    {{- offload.render_flowtable('VYOS_FLOWTABLE_hardware', global_options.flow_offload.hardware.interface | list, priority='filter - 2', hardware_offload=true) }}
+    chain VYOS_OFFLOAD_hardware {
+        type filter hook forward priority filter - 2; policy accept;
+        ct state { established, related } meta l4proto { tcp, udp } flow add @VYOS_FLOWTABLE_hardware
+    }
+{%     endif %}
+{%     if global_options.flow_offload.software.interface is vyos_defined %}
+    {{- offload.render_flowtable('VYOS_FLOWTABLE_software', global_options.flow_offload.software.interface | list, priority='filter - 1') }}
+    chain VYOS_OFFLOAD_software {
+        type filter hook forward priority filter - 1; policy accept;
+        ct state { established, related } meta l4proto { tcp, udp } flow add @VYOS_FLOWTABLE_software
+    }
+{%     endif %}
+{% endif %}
+}
diff --git a/interface-definitions/include/firewall/flow-offload.xml.i b/interface-definitions/include/firewall/flow-offload.xml.i
new file mode 100644
index 000000000..706836362
--- /dev/null
+++ b/interface-definitions/include/firewall/flow-offload.xml.i
@@ -0,0 +1,47 @@
+<!-- include start from firewall/flow-offload.xml.i -->
+<node name="flow-offload">
+  <properties>
+    <help>Configurable flow offload options</help>
+  </properties>
+  <children>
+    <leafNode name="disable">
+      <properties>
+        <help>Disable flow offload</help>
+        <valueless/>
+      </properties>
+    </leafNode>
+    <node name="software">
+      <properties>
+        <help>Software offload</help>
+      </properties>
+      <children>
+        <leafNode name="interface">
+          <properties>
+            <help>Interfaces to enable</help>
+            <completionHelp>
+              <script>${vyos_completion_dir}/list_interfaces</script>
+            </completionHelp>
+            <multi/>
+          </properties>
+        </leafNode>
+      </children>
+    </node>
+    <node name="hardware">
+      <properties>
+        <help>Hardware offload</help>
+      </properties>
+      <children>
+        <leafNode name="interface">
+          <properties>
+            <help>Interfaces to enable</help>
+            <completionHelp>
+              <script>${vyos_completion_dir}/list_interfaces</script>
+            </completionHelp>
+            <multi/>
+          </properties>
+        </leafNode>
+      </children>
+    </node>
+  </children>
+</node>
+<!-- include end -->
diff --git a/interface-definitions/include/firewall/global-options.xml.i b/interface-definitions/include/firewall/global-options.xml.i
index e655cd6ac..03c07e657 100644
--- a/interface-definitions/include/firewall/global-options.xml.i
+++ b/interface-definitions/include/firewall/global-options.xml.i
@@ -271,6 +271,7 @@
       </properties>
       <defaultValue>disable</defaultValue>
     </leafNode>
+    #include <include/firewall/flow-offload.xml.i>
   </children>
 </node>
 <!-- include end -->
diff --git a/interface-definitions/system-conntrack.xml.in b/interface-definitions/system-conntrack.xml.in
index 3abf9bbf0..78d19090c 100644
--- a/interface-definitions/system-conntrack.xml.in
+++ b/interface-definitions/system-conntrack.xml.in
@@ -9,6 +9,12 @@
           <priority>218</priority>
         </properties>
         <children>
+          <leafNode name="flow-accounting">
+            <properties>
+              <help>Enable connection tracking flow accounting</help>
+              <valueless/>
+            </properties>
+          </leafNode>
           <leafNode name="expect-table-size">
             <properties>
               <help>Size of connection tracking expect table</help>
diff --git a/smoketest/scripts/cli/test_firewall.py b/smoketest/scripts/cli/test_firewall.py
index f74ce4b72..391ef03ff 100755
--- a/smoketest/scripts/cli/test_firewall.py
+++ b/smoketest/scripts/cli/test_firewall.py
@@ -603,5 +603,17 @@ class TestFirewall(VyOSUnitTestSHIM.TestCase):
                 with open(path, 'r') as f:
                     self.assertNotEqual(f.read().strip(), conf['default'], msg=path)
 
+    def test_flow_offload_software(self):
+        self.cli_set(['firewall', 'global-options', 'flow-offload', 'software', 'interface', 'eth0'])
+        self.cli_commit()
+        nftables_search = [
+            ['flowtable VYOS_FLOWTABLE_software'],
+            ['hook ingress priority filter - 1'],
+            ['devices = { eth0 }'],
+            ['flow add @VYOS_FLOWTABLE_software'],
+        ]
+        self.verify_nftables(nftables_search, 'inet vyos_offload')
+
+
 if __name__ == '__main__':
     unittest.main(verbosity=2)
diff --git a/src/conf_mode/firewall.py b/src/conf_mode/firewall.py
index c3b1ee015..769cc598f 100755
--- a/src/conf_mode/firewall.py
+++ b/src/conf_mode/firewall.py
@@ -26,7 +26,7 @@ from vyos.config import Config
 from vyos.configdict import node_changed
 from vyos.configdiff import get_config_diff, Diff
 from vyos.configdep import set_dependents, call_dependents
-# from vyos.configverify import verify_interface_exists
+from vyos.configverify import verify_interface_exists
 from vyos.firewall import fqdn_config_parse
 from vyos.firewall import geoip_update
 from vyos.template import render
@@ -38,6 +38,7 @@ from vyos.utils.process import process_named_running
 from vyos.utils.process import rc_cmd
 from vyos import ConfigError
 from vyos import airbag
+
 airbag.enable()
 
 nat_conf_script = 'nat.py'
@@ -100,7 +101,7 @@ def geoip_updated(conf, firewall):
         elif (path[0] == 'ipv6'):
             set_name = f'GEOIP_CC6_{path[1]}_{path[2]}_{path[4]}'
             out['ipv6_name'].append(set_name)
-            
+
         updated = True
 
     if 'delete' in node_diff:
@@ -140,6 +141,14 @@ def get_config(config=None):
 
     fqdn_config_parse(firewall)
 
+    firewall['flowtable_enabled'] = False
+    flow_offload = dict_search_args(firewall, 'global_options', 'flow_offload')
+    if flow_offload and 'disable' not in flow_offload:
+        for offload_type in ('software', 'hardware'):
+            if dict_search_args(flow_offload, offload_type, 'interface'):
+                firewall['flowtable_enabled'] = True
+                break
+
     return firewall
 
 def verify_rule(firewall, rule_conf, ipv6):
@@ -327,6 +336,14 @@ def verify(firewall):
                         for rule_id, rule_conf in name_conf['rule'].items():
                             verify_rule(firewall, rule_conf, True)
 
+    # Verify flow offload options
+    flow_offload = dict_search_args(firewall, 'global_options', 'flow_offload')
+    for offload_type in ('software', 'hardware'):
+        interfaces = dict_search_args(flow_offload, offload_type, 'interface') or []
+        for interface in interfaces:
+            # nft will raise an error when adding a non-existent interface to a flowtable
+            verify_interface_exists(interface)
+
     return None
 
 def generate(firewall):
@@ -336,13 +353,15 @@ def generate(firewall):
     # Determine if conntrack is needed
     firewall['ipv4_conntrack_action'] = 'return'
     firewall['ipv6_conntrack_action'] = 'return'
-
-    for rules, path in dict_search_recursive(firewall, 'rule'):
-        if any(('state' in rule_conf or 'connection_status' in rule_conf) for rule_conf in rules.values()):
-            if path[0] == 'ipv4':
-                firewall['ipv4_conntrack_action'] = 'accept'
-            elif path[0] == 'ipv6':
-                firewall['ipv6_conntrack_action'] = 'accept'
+    if firewall['flowtable_enabled']:  # Netfilter's flowtable offload requires conntrack
+        firewall['ipv4_conntrack_action'] = 'accept'
+        firewall['ipv6_conntrack_action'] = 'accept'
+    else:  # Check if conntrack is needed by firewall rules
+        for proto in ('ipv4', 'ipv6'):
+            for rules, _ in dict_search_recursive(firewall.get(proto, {}), 'rule'):
+                if any(('state' in rule_conf or 'connection_status' in rule_conf) for rule_conf in rules.values()):
+                    firewall[f'{proto}_conntrack_action'] = 'accept'
+                    break
 
     render(nftables_conf, 'firewall/nftables.j2', firewall)
     return None