T5729: T5590: T5616: backport to sagita fwall marks, fix on firewall logs parsing, and migration to valueless node for log and state matchers

2 files changed, 30 insertions, 24 deletions

diff --git a/smoketest/scripts/cli/test_firewall.py b/smoketest/scripts/cli/test_firewall.py
index 5f842727d..c51592ec2 100755
--- a/smoketest/scripts/cli/test_firewall.py
+++ b/smoketest/scripts/cli/test_firewall.py
@@ -213,13 +213,13 @@ class TestFirewall(VyOSUnitTestSHIM.TestCase):
         self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '1', 'action', 'accept'])
         self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '1', 'source', 'address', '172.16.20.10'])
         self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '1', 'destination', 'address', '172.16.10.10'])
-        self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '1', 'log', 'enable'])
+        self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '1', 'log'])
         self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '1', 'log-options', 'level', 'debug'])
         self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '1', 'ttl', 'eq', '15'])
         self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '2', 'action', 'reject'])
         self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '2', 'protocol', 'tcp'])
         self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '2', 'destination', 'port', '8888'])
-        self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '2', 'log', 'enable'])
+        self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '2', 'log'])
         self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '2', 'log-options', 'level', 'err'])
         self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '2', 'tcp', 'flags', 'syn'])
         self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '2', 'tcp', 'flags', 'not', 'ack'])
@@ -231,7 +231,7 @@ class TestFirewall(VyOSUnitTestSHIM.TestCase):
         self.cli_set(['firewall', 'ipv4', 'forward', 'filter', 'rule', '3', 'protocol', 'tcp'])
         self.cli_set(['firewall', 'ipv4', 'forward', 'filter', 'rule', '3', 'destination', 'port', '22'])
         self.cli_set(['firewall', 'ipv4', 'forward', 'filter', 'rule', '3', 'limit', 'rate', '5/minute'])
-        self.cli_set(['firewall', 'ipv4', 'forward', 'filter', 'rule', '3', 'log', 'disable'])
+        self.cli_set(['firewall', 'ipv4', 'forward', 'filter', 'rule', '3', 'log'])
         self.cli_set(['firewall', 'ipv4', 'forward', 'filter', 'rule', '4', 'action', 'drop'])
         self.cli_set(['firewall', 'ipv4', 'forward', 'filter', 'rule', '4', 'protocol', 'tcp'])
         self.cli_set(['firewall', 'ipv4', 'forward', 'filter', 'rule', '4', 'destination', 'port', '22'])
@@ -299,7 +299,7 @@ class TestFirewall(VyOSUnitTestSHIM.TestCase):
         self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '6', 'packet-length', '1024'])
         self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '6', 'dscp', '17'])
         self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '6', 'dscp', '52'])
-        self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '6', 'log', 'enable'])
+        self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '6', 'log'])
         self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '6', 'log-options', 'group', '66'])
         self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '6', 'log-options', 'snapshot-length', '6666'])
         self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '6', 'log-options', 'queue-threshold','32000'])
@@ -312,10 +312,12 @@ class TestFirewall(VyOSUnitTestSHIM.TestCase):
 
         self.cli_set(['firewall', 'ipv4', 'forward', 'filter', 'default-action', 'drop'])
         self.cli_set(['firewall', 'ipv4', 'forward', 'filter', 'rule', '1', 'source', 'address', '198.51.100.1'])
+        self.cli_set(['firewall', 'ipv4', 'forward', 'filter', 'rule', '1', 'mark', '1010'])
         self.cli_set(['firewall', 'ipv4', 'forward', 'filter', 'rule', '1', 'action', 'jump'])
         self.cli_set(['firewall', 'ipv4', 'forward', 'filter', 'rule', '1', 'jump-target', name])
 
         self.cli_set(['firewall', 'ipv4', 'input', 'filter', 'rule', '2', 'protocol', 'tcp'])
+        self.cli_set(['firewall', 'ipv4', 'input', 'filter', 'rule', '2', 'mark', '!98765'])
         self.cli_set(['firewall', 'ipv4', 'input', 'filter', 'rule', '2', 'action', 'queue'])
         self.cli_set(['firewall', 'ipv4', 'input', 'filter', 'rule', '2', 'queue', '3'])
         self.cli_set(['firewall', 'ipv4', 'input', 'filter', 'rule', '3', 'protocol', 'udp'])
@@ -329,11 +331,11 @@ class TestFirewall(VyOSUnitTestSHIM.TestCase):
         nftables_search = [
             ['chain VYOS_FORWARD_filter'],
             ['type filter hook forward priority filter; policy accept;'],
-            ['ip saddr 198.51.100.1', f'jump NAME_{name}'],
+            ['ip saddr 198.51.100.1', 'meta mark 0x000003f2', f'jump NAME_{name}'],
             ['FWD-filter default-action drop', 'drop'],
             ['chain VYOS_INPUT_filter'],
             ['type filter hook input priority filter; policy accept;'],
-            [f'meta l4proto tcp','queue to 3'],
+            ['meta mark != 0x000181cd', 'meta l4proto tcp','queue to 3'],
             ['meta l4proto udp','queue flags bypass,fanout to 0-15'],
             ['INP-filter default-action accept', 'accept'],
             [f'chain NAME_{name}'],
@@ -386,7 +388,7 @@ class TestFirewall(VyOSUnitTestSHIM.TestCase):
         self.cli_set(['firewall', 'ipv6', 'name', name, 'rule', '1', 'action', 'accept'])
         self.cli_set(['firewall', 'ipv6', 'name', name, 'rule', '1', 'source', 'address', '2002::1'])
         self.cli_set(['firewall', 'ipv6', 'name', name, 'rule', '1', 'destination', 'address', '2002::1:1'])
-        self.cli_set(['firewall', 'ipv6', 'name', name, 'rule', '1', 'log', 'enable'])
+        self.cli_set(['firewall', 'ipv6', 'name', name, 'rule', '1', 'log'])
         self.cli_set(['firewall', 'ipv6', 'name', name, 'rule', '1', 'log-options', 'level', 'crit'])
 
         self.cli_set(['firewall', 'ipv6', 'forward', 'filter', 'default-action', 'accept'])
@@ -508,19 +510,19 @@ class TestFirewall(VyOSUnitTestSHIM.TestCase):
 
         self.cli_set(['firewall', 'ipv4', 'name', name, 'default-action', 'drop'])
         self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '1', 'action', 'accept'])
-        self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '1', 'state', 'established', 'enable'])
-        self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '1', 'state', 'related', 'enable'])
+        self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '1', 'state', 'established'])
+        self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '1', 'state', 'related'])
         self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '2', 'action', 'reject'])
-        self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '2', 'state', 'invalid', 'enable'])
+        self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '2', 'state', 'invalid'])
         self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '3', 'action', 'accept'])
-        self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '3', 'state', 'new', 'enable'])
+        self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '3', 'state', 'new'])
         self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '3', 'connection-status', 'nat', 'destination'])
         self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '4', 'action', 'accept'])
-        self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '4', 'state', 'new', 'enable'])
-        self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '4', 'state', 'established', 'enable'])
+        self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '4', 'state', 'new'])
+        self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '4', 'state', 'established'])
         self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '4', 'connection-status', 'nat', 'source'])
         self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '5', 'action', 'accept'])
-        self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '5', 'state', 'related', 'enable'])
+        self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '5', 'state', 'related'])
         self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '5', 'conntrack-helper', 'ftp'])
         self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '5', 'conntrack-helper', 'pptp'])
 
diff --git a/smoketest/scripts/cli/test_policy_route.py b/smoketest/scripts/cli/test_policy_route.py
index 118b1d3a2..40fff03e6 100755
--- a/smoketest/scripts/cli/test_policy_route.py
+++ b/smoketest/scripts/cli/test_policy_route.py
@@ -191,25 +191,29 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase):
     def test_pbr_matching_criteria(self):
         self.cli_set(['policy', 'route', 'smoketest', 'rule', '1', 'protocol', 'udp'])
         self.cli_set(['policy', 'route', 'smoketest', 'rule', '1', 'action', 'drop'])
+        self.cli_set(['policy', 'route', 'smoketest', 'rule', '1', 'mark', '2020'])
         self.cli_set(['policy', 'route', 'smoketest', 'rule', '2', 'protocol', 'tcp'])
         self.cli_set(['policy', 'route', 'smoketest', 'rule', '2', 'tcp', 'flags', 'syn'])
         self.cli_set(['policy', 'route', 'smoketest', 'rule', '2', 'tcp', 'flags', 'not', 'ack'])
+        self.cli_set(['policy', 'route', 'smoketest', 'rule', '2', 'mark', '2-3000'])
         self.cli_set(['policy', 'route', 'smoketest', 'rule', '2', 'set', 'table', table_id])
         self.cli_set(['policy', 'route', 'smoketest', 'rule', '3', 'source', 'address', '198.51.100.0/24'])
         self.cli_set(['policy', 'route', 'smoketest', 'rule', '3', 'protocol', 'tcp'])
         self.cli_set(['policy', 'route', 'smoketest', 'rule', '3', 'destination', 'port', '22'])
-        self.cli_set(['policy', 'route', 'smoketest', 'rule', '3', 'state', 'new', 'enable'])
+        self.cli_set(['policy', 'route', 'smoketest', 'rule', '3', 'state', 'new'])
         self.cli_set(['policy', 'route', 'smoketest', 'rule', '3', 'ttl', 'gt', '2'])
+        self.cli_set(['policy', 'route', 'smoketest', 'rule', '3', 'mark', '!456'])
         self.cli_set(['policy', 'route', 'smoketest', 'rule', '3', 'set', 'table', table_id])
         self.cli_set(['policy', 'route', 'smoketest', 'rule', '4', 'protocol', 'icmp'])
         self.cli_set(['policy', 'route', 'smoketest', 'rule', '4', 'icmp', 'type-name', 'echo-request'])
         self.cli_set(['policy', 'route', 'smoketest', 'rule', '4', 'packet-length', '128'])
         self.cli_set(['policy', 'route', 'smoketest', 'rule', '4', 'packet-length', '1024-2048'])
         self.cli_set(['policy', 'route', 'smoketest', 'rule', '4', 'packet-type', 'other'])
-        self.cli_set(['policy', 'route', 'smoketest', 'rule', '4', 'log', 'enable'])
+        self.cli_set(['policy', 'route', 'smoketest', 'rule', '4', 'log'])
         self.cli_set(['policy', 'route', 'smoketest', 'rule', '4', 'set', 'table', table_id])
         self.cli_set(['policy', 'route', 'smoketest', 'rule', '5', 'dscp', '41'])
         self.cli_set(['policy', 'route', 'smoketest', 'rule', '5', 'dscp', '57-59'])
+        self.cli_set(['policy', 'route', 'smoketest', 'rule', '5', 'mark', '!456-500'])
         self.cli_set(['policy', 'route', 'smoketest', 'rule', '5', 'set', 'table', table_id])
 
         self.cli_set(['policy', 'route6', 'smoketest6', 'rule', '1', 'protocol', 'udp'])
@@ -221,7 +225,7 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase):
         self.cli_set(['policy', 'route6', 'smoketest6', 'rule', '3', 'source', 'address', '2001:db8::0/64'])
         self.cli_set(['policy', 'route6', 'smoketest6', 'rule', '3', 'protocol', 'tcp'])
         self.cli_set(['policy', 'route6', 'smoketest6', 'rule', '3', 'destination', 'port', '22'])
-        self.cli_set(['policy', 'route6', 'smoketest6', 'rule', '3', 'state', 'new', 'enable'])
+        self.cli_set(['policy', 'route6', 'smoketest6', 'rule', '3', 'state', 'new'])
         self.cli_set(['policy', 'route6', 'smoketest6', 'rule', '3', 'hop-limit', 'gt', '2'])
         self.cli_set(['policy', 'route6', 'smoketest6', 'rule', '3', 'set', 'table', table_id])
         self.cli_set(['policy', 'route6', 'smoketest6', 'rule', '4', 'protocol', 'icmpv6'])
@@ -230,7 +234,7 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase):
         self.cli_set(['policy', 'route6', 'smoketest6', 'rule', '4', 'packet-length-exclude', '1024-2048'])
         self.cli_set(['policy', 'route6', 'smoketest6', 'rule', '4', 'packet-type', 'multicast'])
 
-        self.cli_set(['policy', 'route6', 'smoketest6', 'rule', '4', 'log', 'enable'])
+        self.cli_set(['policy', 'route6', 'smoketest6', 'rule', '4', 'log'])
         self.cli_set(['policy', 'route6', 'smoketest6', 'rule', '4', 'set', 'table', table_id])
         self.cli_set(['policy', 'route6', 'smoketest6', 'rule', '5', 'dscp-exclude', '61'])
         self.cli_set(['policy', 'route6', 'smoketest6', 'rule', '5', 'dscp-exclude', '14-19'])
@@ -247,11 +251,11 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase):
         # IPv4
         nftables_search = [
             ['iifname { "' + interface + '", "' + interface_wc + '" }', 'jump VYOS_PBR_UD_smoketest'],
-            ['meta l4proto udp', 'drop'],
-            ['tcp flags syn / syn,ack', 'meta mark set ' + mark_hex],
-            ['ct state new', 'tcp dport 22', 'ip saddr 198.51.100.0/24', 'ip ttl > 2', 'meta mark set ' + mark_hex],
-            ['meta l4proto icmp', 'log prefix "[ipv4-route-smoketest-4-A]"', 'icmp type echo-request', 'ip length { 128, 1024-2048 }', 'meta pkttype other', 'meta mark set ' + mark_hex],
-            ['ip dscp { 0x29, 0x39-0x3b }', 'meta mark set ' + mark_hex]
+            ['meta l4proto udp', 'meta mark 0x000007e4', 'drop'],
+            ['tcp flags syn / syn,ack', 'meta mark 0x00000002-0x00000bb8', 'meta mark set ' + mark_hex],
+            ['ct state new', 'tcp dport 22', 'ip saddr 198.51.100.0/24', 'ip ttl > 2', 'meta mark != 0x000001c8', 'meta mark set ' + mark_hex],
+            ['log prefix "[ipv4-route-smoketest-4-A]"', 'icmp type echo-request', 'ip length { 128, 1024-2048 }', 'meta pkttype other', 'meta mark set ' + mark_hex],
+            ['ip dscp { 0x29, 0x39-0x3b }', 'meta mark != 0x000001c8-0x000001f4', 'meta mark set ' + mark_hex]
         ]
 
         self.verify_nftables(nftables_search, 'ip vyos_mangle')
@@ -262,7 +266,7 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase):
             ['meta l4proto udp', 'drop'],
             ['tcp flags syn / syn,ack', 'meta mark set ' + mark_hex],
             ['ct state new', 'tcp dport 22', 'ip6 saddr 2001:db8::/64', 'ip6 hoplimit > 2', 'meta mark set ' + mark_hex],
-            ['meta l4proto ipv6-icmp', 'log prefix "[ipv6-route6-smoketest6-4-A]"', 'icmpv6 type echo-request', 'ip6 length != { 128, 1024-2048 }', 'meta pkttype multicast', 'meta mark set ' + mark_hex],
+            ['log prefix "[ipv6-route6-smoketest6-4-A]"', 'icmpv6 type echo-request', 'ip6 length != { 128, 1024-2048 }', 'meta pkttype multicast', 'meta mark set ' + mark_hex],            
             ['ip6 dscp != { 0x0e-0x13, 0x3d }', 'meta mark set ' + mark_hex]
         ]