firewall: T4502: Update to flowtable CLI

`set firewall flowtable <name> interface <ifname>`
`set firewall flowtable <name> offload [software|hardware]`

`set firewall [ipv4|ipv6] forward filter rule N action offload`
`set firewall [ipv4|ipv6] forward filter rule N offload-target <name>`

12 files changed, 164 insertions, 94 deletions

diff --git a/data/templates/firewall/nftables-offload.j2 b/data/templates/firewall/nftables-offload.j2
index 6afcd79f7..a893e05b2 100644
--- a/data/templates/firewall/nftables-offload.j2
+++ b/data/templates/firewall/nftables-offload.j2
@@ -1,11 +1,9 @@
-{% macro render_flowtable(name, devices, priority='filter', hardware_offload=false, with_counter=true) %}
-flowtable {{ name }} {
-    hook ingress priority {{ priority }}; devices = { {{ devices | join(', ') }} };
-{% if hardware_offload %}
-    flags offload;
+{% macro flowtable(name, config) %}
+    flowtable VYOS_FLOWTABLE_{{ name }} {
+        hook ingress priority 0; devices = { {{ config.interface | join(', ') }} };
+{% if config.offload is vyos_defined('hardware') %}
+        flags offload;
 {% endif %}
-{% if with_counter %}
-    counter
-{% endif %}
-}
+        counter
+    }
 {% endmacro %}
diff --git a/data/templates/firewall/nftables.j2 b/data/templates/firewall/nftables.j2
index 1564b3ef8..b74522109 100644
--- a/data/templates/firewall/nftables.j2
+++ b/data/templates/firewall/nftables.j2
@@ -2,7 +2,7 @@
 
 {% import 'firewall/nftables-defines.j2' as group_tmpl %}
 {% import 'firewall/nftables-bridge.j2' as bridge_tmpl %}
-{% import 'firewall/nftables-offload.j2' as offload %}
+{% import 'firewall/nftables-offload.j2' as offload_tmpl %}
 
 flush chain raw vyos_global_rpfilter
 flush chain ip6 raw vyos_global_rpfilter
@@ -34,6 +34,12 @@ delete table ip vyos_filter
 {% endif %}
 table ip vyos_filter {
 {% if ipv4 is vyos_defined %}
+{%     if flowtable is vyos_defined %}
+{%         for name, flowtable_conf in flowtable.items() %}
+{{ offload_tmpl.flowtable(name, flowtable_conf) }}
+{%         endfor %}
+{%     endif %}
+
 {%     set ns = namespace(sets=[]) %}
 {%     if ipv4.forward is vyos_defined %}
 {%         for prior, conf in ipv4.forward.items() %}
@@ -153,6 +159,12 @@ delete table ip6 vyos_filter
 {% endif %}
 table ip6 vyos_filter {
 {% if ipv6 is vyos_defined %}
+{%     if flowtable is vyos_defined %}
+{%         for name, flowtable_conf in flowtable.items() %}
+{{ offload_tmpl.flowtable(name, flowtable_conf) }}
+{%         endfor %}
+{%     endif %}
+
 {%     set ns = namespace(sets=[]) %}
 {%     if ipv6.forward is vyos_defined %}
 {%         for prior, conf in ipv6.forward.items() %}
@@ -261,23 +273,3 @@ table bridge vyos_filter {
 {{ group_tmpl.groups(group, False, False) }}
 }
 {% endif %}
-
-{% if first_install is not vyos_defined %}
-delete table inet vyos_offload
-{% endif %}
-table inet vyos_offload {
-{% if global_options.flow_offload.hardware.interface is vyos_defined %}
-    {{- offload.render_flowtable('VYOS_FLOWTABLE_hardware', global_options.flow_offload.hardware.interface | list, priority='filter - 2', hardware_offload=true) }}
-    chain VYOS_OFFLOAD_hardware {
-        type filter hook forward priority filter - 2; policy accept;
-        ct state { established, related } meta l4proto { tcp, udp } flow add @VYOS_FLOWTABLE_hardware
-    }
-{% endif %}
-{% if global_options.flow_offload.software.interface is vyos_defined %}
-    {{- offload.render_flowtable('VYOS_FLOWTABLE_software', global_options.flow_offload.software.interface | list, priority='filter - 1') }}
-    chain VYOS_OFFLOAD_software {
-        type filter hook forward priority filter - 1; policy accept;
-        ct state { established, related } meta l4proto { tcp, udp } flow add @VYOS_FLOWTABLE_software
-    }
-{% endif %}
-}
diff --git a/interface-definitions/firewall.xml.in b/interface-definitions/firewall.xml.in
index 8e462f3eb..81e6b89ea 100644
--- a/interface-definitions/firewall.xml.in
+++ b/interface-definitions/firewall.xml.in
@@ -7,6 +7,46 @@
     </properties>
     <children>
       #include <include/firewall/global-options.xml.i>
+      <tagNode name="flowtable">
+        <properties>
+          <help>Flowtable</help>
+          <constraint>
+            <regex>[a-zA-Z0-9][\w\-\.]*</regex>
+          </constraint>
+        </properties>
+        <children>
+          #include <include/generic-description.xml.i>
+          <leafNode name="interface">
+            <properties>
+              <help>Interfaces to use this flowtable</help>
+              <completionHelp>
+                <script>${vyos_completion_dir}/list_interfaces</script>
+              </completionHelp>
+              <multi/>
+            </properties>
+          </leafNode>
+          <leafNode name="offload">
+            <properties>
+              <help>Offloading method</help>
+              <completionHelp>
+                <list>hardware software</list>
+              </completionHelp>
+              <valueHelp>
+                <format>hardware</format>
+                <description>Hardware offload</description>
+              </valueHelp>
+              <valueHelp>
+                <format>software</format>
+                <description>Software offload</description>
+              </valueHelp>
+              <constraint>
+                <regex>(hardware|software)</regex>
+              </constraint>
+            </properties>
+            <defaultValue>software</defaultValue>
+          </leafNode>
+        </children>
+      </tagNode>
       <node name="group">
         <properties>
           <help>Firewall group</help>
diff --git a/interface-definitions/include/firewall/action-forward.xml.i b/interface-definitions/include/firewall/action-forward.xml.i
new file mode 100644
index 000000000..f61e51887
--- /dev/null
+++ b/interface-definitions/include/firewall/action-forward.xml.i
@@ -0,0 +1,45 @@
+<!-- include start from firewall/action-forward.xml.i -->
+<leafNode name="action">
+  <properties>
+    <help>Rule action</help>
+    <completionHelp>
+      <list>accept continue jump reject return drop queue offload</list>
+    </completionHelp>
+    <valueHelp>
+      <format>accept</format>
+      <description>Accept matching entries</description>
+    </valueHelp>
+    <valueHelp>
+      <format>continue</format>
+      <description>Continue parsing next rule</description>
+    </valueHelp>
+    <valueHelp>
+      <format>jump</format>
+      <description>Jump to another chain</description>
+    </valueHelp>
+    <valueHelp>
+      <format>reject</format>
+      <description>Reject matching entries</description>
+    </valueHelp>
+    <valueHelp>
+      <format>return</format>
+      <description>Return from the current chain and continue at the next rule of the last chain</description>
+    </valueHelp>
+    <valueHelp>
+      <format>drop</format>
+      <description>Drop matching entries</description>
+    </valueHelp>
+    <valueHelp>
+      <format>queue</format>
+      <description>Enqueue packet to userspace</description>
+    </valueHelp>
+    <valueHelp>
+      <format>offload</format>
+      <description>Offload packet via flowtable</description>
+    </valueHelp>
+    <constraint>
+      <regex>(accept|continue|jump|reject|return|drop|queue|offload)</regex>
+    </constraint>
+  </properties>
+</leafNode>
+<!-- include end -->
diff --git a/interface-definitions/include/firewall/flow-offload.xml.i b/interface-definitions/include/firewall/flow-offload.xml.i
deleted file mode 100644
index 706836362..000000000
--- a/interface-definitions/include/firewall/flow-offload.xml.i
+++ /dev/null
@@ -1,47 +0,0 @@
-<!-- include start from firewall/flow-offload.xml.i -->
-<node name="flow-offload">
-  <properties>
-    <help>Configurable flow offload options</help>
-  </properties>
-  <children>
-    <leafNode name="disable">
-      <properties>
-        <help>Disable flow offload</help>
-        <valueless/>
-      </properties>
-    </leafNode>
-    <node name="software">
-      <properties>
-        <help>Software offload</help>
-      </properties>
-      <children>
-        <leafNode name="interface">
-          <properties>
-            <help>Interfaces to enable</help>
-            <completionHelp>
-              <script>${vyos_completion_dir}/list_interfaces</script>
-            </completionHelp>
-            <multi/>
-          </properties>
-        </leafNode>
-      </children>
-    </node>
-    <node name="hardware">
-      <properties>
-        <help>Hardware offload</help>
-      </properties>
-      <children>
-        <leafNode name="interface">
-          <properties>
-            <help>Interfaces to enable</help>
-            <completionHelp>
-              <script>${vyos_completion_dir}/list_interfaces</script>
-            </completionHelp>
-            <multi/>
-          </properties>
-        </leafNode>
-      </children>
-    </node>
-  </children>
-</node>
-<!-- include end -->
diff --git a/interface-definitions/include/firewall/global-options.xml.i b/interface-definitions/include/firewall/global-options.xml.i
index 03c07e657..e655cd6ac 100644
--- a/interface-definitions/include/firewall/global-options.xml.i
+++ b/interface-definitions/include/firewall/global-options.xml.i
@@ -271,7 +271,6 @@
       </properties>
       <defaultValue>disable</defaultValue>
     </leafNode>
-    #include <include/firewall/flow-offload.xml.i>
   </children>
 </node>
 <!-- include end -->
diff --git a/interface-definitions/include/firewall/ipv4-hook-forward.xml.i b/interface-definitions/include/firewall/ipv4-hook-forward.xml.i
index 08ee96419..70c0adb77 100644
--- a/interface-definitions/include/firewall/ipv4-hook-forward.xml.i
+++ b/interface-definitions/include/firewall/ipv4-hook-forward.xml.i
@@ -24,8 +24,10 @@
             <constraintErrorMessage>Firewall rule number must be between 1 and 999999</constraintErrorMessage>
           </properties>
           <children>
+            #include <include/firewall/action-forward.xml.i>
             #include <include/firewall/common-rule-ipv4.xml.i>
             #include <include/firewall/inbound-interface.xml.i>
+            #include <include/firewall/offload-target.xml.i>
             #include <include/firewall/outbound-interface.xml.i>
           </children>
         </tagNode>
diff --git a/interface-definitions/include/firewall/ipv6-hook-forward.xml.i b/interface-definitions/include/firewall/ipv6-hook-forward.xml.i
index 20ab8dbe8..d83827161 100644
--- a/interface-definitions/include/firewall/ipv6-hook-forward.xml.i
+++ b/interface-definitions/include/firewall/ipv6-hook-forward.xml.i
@@ -24,8 +24,10 @@
             <constraintErrorMessage>Firewall rule number must be between 1 and 999999</constraintErrorMessage>
           </properties>
           <children>
+            #include <include/firewall/action-forward.xml.i>
             #include <include/firewall/common-rule-ipv6.xml.i>
             #include <include/firewall/inbound-interface.xml.i>
+            #include <include/firewall/offload-target.xml.i>
             #include <include/firewall/outbound-interface.xml.i>
           </children>
         </tagNode>
diff --git a/interface-definitions/include/firewall/offload-target.xml.i b/interface-definitions/include/firewall/offload-target.xml.i
new file mode 100644
index 000000000..940ed8091
--- /dev/null
+++ b/interface-definitions/include/firewall/offload-target.xml.i
@@ -0,0 +1,10 @@
+<!-- include start from firewall/offload-target.xml.i -->
+<leafNode name="offload-target">
+  <properties>
+    <help>Set flowtable offload target. Action offload must be defined to use this setting</help>
+    <completionHelp>
+      <path>firewall flowtable</path>
+    </completionHelp>
+  </properties>
+</leafNode>
+<!-- include end -->
diff --git a/python/vyos/firewall.py b/python/vyos/firewall.py
index 69ad11d1d..3ca7a25b9 100644
--- a/python/vyos/firewall.py
+++ b/python/vyos/firewall.py
@@ -401,20 +401,24 @@ def parse_rule(rule_conf, hook, fw_name, rule_id, ip_name):
     if 'action' in rule_conf:
         # Change action=return to action=action
         # #output.append(nft_action(rule_conf['action']))
-        output.append(f'{rule_conf["action"]}')
-        if 'jump' in rule_conf['action']:
-            target = rule_conf['jump_target']
-            output.append(f'NAME{def_suffix}_{target}')
+        if rule_conf['action'] == 'offload':
+            offload_target = rule_conf['offload_target']
+            output.append(f'flow add @VYOS_FLOWTABLE_{offload_target}')
+        else:
+            output.append(f'{rule_conf["action"]}')
 
-        if 'queue' in rule_conf['action']:
-            if 'queue' in rule_conf:
-                target = rule_conf['queue']
-                output.append(f'num {target}')
+            if 'jump' in rule_conf['action']:
+                target = rule_conf['jump_target']
+                output.append(f'NAME{def_suffix}_{target}')
 
-            if 'queue_options' in rule_conf:
-                queue_opts = ','.join(rule_conf['queue_options'])
-                output.append(f'{queue_opts}')
+            if 'queue' in rule_conf['action']:
+                if 'queue' in rule_conf:
+                    target = rule_conf['queue']
+                    output.append(f'num {target}')
 
+                if 'queue_options' in rule_conf:
+                    queue_opts = ','.join(rule_conf['queue_options'])
+                    output.append(f'{queue_opts}')
     else:
         output.append('return')
 
diff --git a/smoketest/scripts/cli/test_firewall.py b/smoketest/scripts/cli/test_firewall.py
index 75d6e9bb1..75fdec207 100755
--- a/smoketest/scripts/cli/test_firewall.py
+++ b/smoketest/scripts/cli/test_firewall.py
@@ -604,16 +604,32 @@ class TestFirewall(VyOSUnitTestSHIM.TestCase):
                     self.assertNotEqual(f.read().strip(), conf['default'], msg=path)
 
     def test_flow_offload_software(self):
-        self.cli_set(['firewall', 'global-options', 'flow-offload', 'software', 'interface', 'eth0'])
+        self.cli_set(['firewall', 'flowtable', 'smoketest', 'interface', 'eth0'])
+        self.cli_set(['firewall', 'flowtable', 'smoketest', 'offload', 'software'])
+
+        self.cli_set(['firewall', 'ipv4', 'forward', 'filter', 'rule', '1', 'action', 'offload'])
+        self.cli_set(['firewall', 'ipv4', 'forward', 'filter', 'rule', '1', 'offload-target', 'smoketest'])
+        self.cli_set(['firewall', 'ipv4', 'forward', 'filter', 'rule', '1', 'protocol', 'tcp_udp'])
+        self.cli_set(['firewall', 'ipv4', 'forward', 'filter', 'rule', '1', 'state', 'established', 'enable'])
+        self.cli_set(['firewall', 'ipv4', 'forward', 'filter', 'rule', '1', 'state', 'related', 'enable'])
+
+        self.cli_set(['firewall', 'ipv6', 'forward', 'filter', 'rule', '1', 'action', 'offload'])
+        self.cli_set(['firewall', 'ipv6', 'forward', 'filter', 'rule', '1', 'offload-target', 'smoketest'])
+        self.cli_set(['firewall', 'ipv6', 'forward', 'filter', 'rule', '1', 'protocol', 'tcp_udp'])
+        self.cli_set(['firewall', 'ipv6', 'forward', 'filter', 'rule', '1', 'state', 'established', 'enable'])
+        self.cli_set(['firewall', 'ipv6', 'forward', 'filter', 'rule', '1', 'state', 'related', 'enable'])
+
         self.cli_commit()
+
         nftables_search = [
-            ['flowtable VYOS_FLOWTABLE_software'],
-            ['hook ingress priority filter - 1'],
+            ['flowtable VYOS_FLOWTABLE_smoketest'],
+            ['hook ingress priority filter'],
             ['devices = { eth0 }'],
-            ['flow add @VYOS_FLOWTABLE_software'],
+            ['ct state { established, related }', 'meta l4proto { tcp, udp }', 'flow add @VYOS_FLOWTABLE_smoketest'],
         ]
-        self.verify_nftables(nftables_search, 'inet vyos_offload')
 
+        self.verify_nftables(nftables_search, 'ip vyos_filter')
+        self.verify_nftables(nftables_search, 'ip6 vyos_filter')
 
 if __name__ == '__main__':
     unittest.main(verbosity=2)
diff --git a/src/conf_mode/firewall.py b/src/conf_mode/firewall.py
index d999b2a64..eba3ebf59 100755
--- a/src/conf_mode/firewall.py
+++ b/src/conf_mode/firewall.py
@@ -163,6 +163,15 @@ def verify_rule(firewall, rule_conf, ipv6):
             if target not in dict_search_args(firewall, 'ipv6', 'name'):
                 raise ConfigError(f'Invalid jump-target. Firewall ipv6 name {target} does not exist on the system')
 
+    if rule_conf['action'] == 'offload':
+        if 'offload_target' not in rule_conf:
+            raise ConfigError('Action set to offload, but no offload-target specified')
+
+        offload_target = rule_conf['offload_target']
+
+        if not dict_search_args(firewall, 'flowtable', offload_target):
+            raise ConfigError(f'Invalid offload-target. Flowtable "{offload_target}" does not exist on the system')
+
     if 'queue_options' in rule_conf:
         if 'queue' not in rule_conf['action']:
             raise ConfigError('queue-options defined, but action queue needed and it is not defined')