policy: T2199: T4605: Migrate policy route interface to `policy route|route6 <name> interface <ifname>`

* Include refactor to policy route to allow for deletion of mangle table instead of complex cleanup
* T4605: Rename mangle table to vyos_mangle

5 files changed, 98 insertions, 278 deletions

diff --git a/src/conf_mode/policy-route-interface.py b/src/conf_mode/policy-route-interface.py
deleted file mode 100755
index 58c5fd93d..000000000
--- a/src/conf_mode/policy-route-interface.py
+++ /dev/null
@@ -1,132 +0,0 @@
-#!/usr/bin/env python3
-#
-# Copyright (C) 2021 VyOS maintainers and contributors
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 or later as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import os
-import re
-
-from sys import argv
-from sys import exit
-
-from vyos.config import Config
-from vyos.ifconfig import Section
-from vyos.template import render
-from vyos.util import cmd
-from vyos.util import run
-from vyos import ConfigError
-from vyos import airbag
-airbag.enable()
-
-def get_config(config=None):
-    if config:
-        conf = config
-    else:
-        conf = Config()
-
-    ifname = argv[1]
-    ifpath = Section.get_config_path(ifname)
-    if_policy_path = f'interfaces {ifpath} policy'
-
-    if_policy = conf.get_config_dict(if_policy_path, key_mangling=('-', '_'), get_first_key=True,
-                                    no_tag_node_value_mangle=True)
-
-    if_policy['ifname'] = ifname
-    if_policy['policy'] = conf.get_config_dict(['policy'], key_mangling=('-', '_'), get_first_key=True,
-                                    no_tag_node_value_mangle=True)
-
-    return if_policy
-
-def verify_chain(table, chain):
-    # Verify policy route applied
-    code = run(f'nft list chain {table} {chain}')
-    return code == 0
-
-def verify(if_policy):
-    # bail out early - looks like removal from running config
-    if not if_policy:
-        return None
-
-    for route in ['route', 'route6']:
-        if route in if_policy:
-            if route not in if_policy['policy']:
-                raise ConfigError('Policy route not configured')
-
-            route_name = if_policy[route]
-
-            if route_name not in if_policy['policy'][route]:
-                raise ConfigError(f'Invalid policy route name "{name}"')
-
-            nft_prefix = 'VYOS_PBR6_' if route == 'route6' else 'VYOS_PBR_'
-            nft_table = 'ip6 mangle' if route == 'route6' else 'ip mangle'
-
-            if not verify_chain(nft_table, nft_prefix + route_name):
-                raise ConfigError('Policy route did not apply')
-
-    return None
-
-def generate(if_policy):
-    return None
-
-def cleanup_rule(table, chain, ifname, new_name=None):
-    results = cmd(f'nft -a list chain {table} {chain}').split("\n")
-    retval = None
-    for line in results:
-        if f'ifname "{ifname}"' in line:
-            if new_name and f'jump {new_name}' in line:
-                # new_name is used to clear rules for any previously referenced chains
-                # returns true when rule exists and doesn't need to be created
-                retval = True
-                continue
-
-            handle_search = re.search('handle (\d+)', line)
-            if handle_search:
-                cmd(f'nft delete rule {table} {chain} handle {handle_search[1]}')
-    return retval
-
-def apply(if_policy):
-    ifname = if_policy['ifname']
-
-    route_chain = 'VYOS_PBR_PREROUTING'
-    ipv6_route_chain = 'VYOS_PBR6_PREROUTING'
-
-    if 'route' in if_policy:
-        name = 'VYOS_PBR_' + if_policy['route']
-        rule_exists = cleanup_rule('ip mangle', route_chain, ifname, name)
-
-        if not rule_exists:
-            cmd(f'nft insert rule ip mangle {route_chain} iifname {ifname} counter jump {name}')
-    else:
-        cleanup_rule('ip mangle', route_chain, ifname)
-
-    if 'route6' in if_policy:
-        name = 'VYOS_PBR6_' + if_policy['route6']
-        rule_exists = cleanup_rule('ip6 mangle', ipv6_route_chain, ifname, name)
-
-        if not rule_exists:
-            cmd(f'nft insert rule ip6 mangle {ipv6_route_chain} iifname {ifname} counter jump {name}')
-    else:
-        cleanup_rule('ip6 mangle', ipv6_route_chain, ifname)
-
-    return None
-
-if __name__ == '__main__':
-    try:
-        c = get_config()
-        verify(c)
-        generate(c)
-        apply(c)
-    except ConfigError as e:
-        print(e)
-        exit(1)
diff --git a/src/conf_mode/policy-route.py b/src/conf_mode/policy-route.py
index 00539b9c7..1d016695e 100755
--- a/src/conf_mode/policy-route.py
+++ b/src/conf_mode/policy-route.py
@@ -15,7 +15,6 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
-import re
 
 from json import loads
 from sys import exit
@@ -25,7 +24,6 @@ from vyos.config import Config
 from vyos.template import render
 from vyos.util import cmd
 from vyos.util import dict_search_args
-from vyos.util import dict_search_recursive
 from vyos.util import run
 from vyos import ConfigError
 from vyos import airbag
@@ -34,48 +32,13 @@ airbag.enable()
 mark_offset = 0x7FFFFFFF
 nftables_conf = '/run/nftables_policy.conf'
 
-ROUTE_PREFIX = 'VYOS_PBR_'
-ROUTE6_PREFIX = 'VYOS_PBR6_'
-
-preserve_chains = [
-    'VYOS_PBR_PREROUTING',
-    'VYOS_PBR_POSTROUTING',
-    'VYOS_PBR6_PREROUTING',
-    'VYOS_PBR6_POSTROUTING'
-]
-
 valid_groups = [
     'address_group',
+    'domain_group',
     'network_group',
     'port_group'
 ]
 
-group_set_prefix = {
-    'A_': 'address_group',
-    'A6_': 'ipv6_address_group',
-#    'D_': 'domain_group',
-    'M_': 'mac_group',
-    'N_': 'network_group',
-    'N6_': 'ipv6_network_group',
-    'P_': 'port_group'
-}
-
-def get_policy_interfaces(conf):
-    out = {}
-    interfaces = conf.get_config_dict(['interfaces'], key_mangling=('-', '_'), get_first_key=True,
-                                    no_tag_node_value_mangle=True)
-    def find_interfaces(iftype_conf, output={}, prefix=''):
-        for ifname, if_conf in iftype_conf.items():
-            if 'policy' in if_conf:
-                output[prefix + ifname] = if_conf['policy']
-            for vif in ['vif', 'vif_s', 'vif_c']:
-                if vif in if_conf:
-                    output.update(find_interfaces(if_conf[vif], output, f'{prefix}{ifname}.'))
-        return output
-    for iftype, iftype_conf in interfaces.items():
-        out.update(find_interfaces(iftype_conf))
-    return out
-
 def get_config(config=None):
     if config:
         conf = config
@@ -88,7 +51,6 @@ def get_config(config=None):
 
     policy['firewall_group'] = conf.get_config_dict(['firewall', 'group'], key_mangling=('-', '_'), get_first_key=True,
                                     no_tag_node_value_mangle=True)
-    policy['interfaces'] = get_policy_interfaces(conf)
 
     return policy
 
@@ -132,8 +94,8 @@ def verify_rule(policy, name, rule_conf, ipv6, rule_id):
             side_conf = rule_conf[side]
 
             if 'group' in side_conf:
-                if {'address_group', 'network_group'} <= set(side_conf['group']):
-                    raise ConfigError('Only one address-group or network-group can be specified')
+                if len({'address_group', 'domain_group', 'network_group'} & set(side_conf['group'])) > 1:
+                    raise ConfigError('Only one address-group, domain-group or network-group can be specified')
 
                 for group in valid_groups:
                     if group in side_conf['group']:
@@ -168,73 +130,11 @@ def verify(policy):
                     for rule_id, rule_conf in pol_conf['rule'].items():
                         verify_rule(policy, name, rule_conf, ipv6, rule_id)
 
-    for ifname, if_policy in policy['interfaces'].items():
-        name = dict_search_args(if_policy, 'route')
-        ipv6_name = dict_search_args(if_policy, 'route6')
-
-        if name and not dict_search_args(policy, 'route', name):
-            raise ConfigError(f'Policy route "{name}" is still referenced on interface {ifname}')
-
-        if ipv6_name and not dict_search_args(policy, 'route6', ipv6_name):
-            raise ConfigError(f'Policy route6 "{ipv6_name}" is still referenced on interface {ifname}')
-
     return None
 
-def cleanup_commands(policy):
-    commands = []
-    commands_chains = []
-    commands_sets = []
-    for table in ['ip mangle', 'ip6 mangle']:
-        route_node = 'route' if table == 'ip mangle' else 'route6'
-        chain_prefix = ROUTE_PREFIX if table == 'ip mangle' else ROUTE6_PREFIX
-
-        json_str = cmd(f'nft -t -j list table {table}')
-        obj = loads(json_str)
-        if 'nftables' not in obj:
-            continue
-        for item in obj['nftables']:
-            if 'chain' in item:
-                chain = item['chain']['name']
-                if chain in preserve_chains or not chain.startswith("VYOS_PBR"):
-                    continue
-
-                if dict_search_args(policy, route_node, chain.replace(chain_prefix, "", 1)) != None:
-                    commands.append(f'flush chain {table} {chain}')
-                else:
-                    commands_chains.append(f'delete chain {table} {chain}')
-
-            if 'rule' in item:
-                rule = item['rule']
-                chain = rule['chain']
-                handle = rule['handle']
-
-                if chain not in preserve_chains:
-                    continue
-
-                target, _ = next(dict_search_recursive(rule['expr'], 'target'))
-
-                if target.startswith(chain_prefix):
-                    if dict_search_args(policy, route_node, target.replace(chain_prefix, "", 1)) == None:
-                        commands.append(f'delete rule {table} {chain} handle {handle}')
-
-            if 'set' in item:
-                set_name = item['set']['name']
-
-                for prefix, group_type in group_set_prefix.items():
-                    if set_name.startswith(prefix):
-                        group_name = set_name.replace(prefix, "", 1)
-                        if dict_search_args(policy, 'firewall_group', group_type, group_name) != None:
-                            commands_sets.append(f'flush set {table} {set_name}')
-                        else:
-                            commands_sets.append(f'delete set {table} {set_name}')
-
-    return commands + commands_chains + commands_sets
-
 def generate(policy):
     if not os.path.exists(nftables_conf):
         policy['first_install'] = True
-    else:
-        policy['cleanup_commands'] = cleanup_commands(policy)
 
     render(nftables_conf, 'firewall/nftables-policy.j2', policy)
     return None
diff --git a/src/helpers/vyos-domain-resolver.py b/src/helpers/vyos-domain-resolver.py
index 035c208b2..e31d9238e 100755
--- a/src/helpers/vyos-domain-resolver.py
+++ b/src/helpers/vyos-domain-resolver.py
@@ -35,13 +35,13 @@ cache = False
 domain_state = {}
 
 ipv4_tables = {
-    'ip mangle',
+    'ip vyos_mangle',
     'ip vyos_filter',
     'ip vyos_nat'
 }
 
 ipv6_tables = {
-    'ip6 mangle',
+    'ip6 vyos_mangle',
     'ip6 vyos_filter'
 }
 
diff --git a/src/migration-scripts/policy/4-to-5 b/src/migration-scripts/policy/4-to-5
new file mode 100755
index 000000000..33c9e6ade
--- /dev/null
+++ b/src/migration-scripts/policy/4-to-5
@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2022 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# T2199: Migrate interface policy nodes to policy route <name> interface <ifname>
+
+import re
+
+from sys import argv
+from sys import exit
+
+from vyos.configtree import ConfigTree
+from vyos.ifconfig import Section
+
+if (len(argv) < 1):
+    print("Must specify file name!")
+    exit(1)
+
+file_name = argv[1]
+
+with open(file_name, 'r') as f:
+    config_file = f.read()
+
+base4 = ['policy', 'route']
+base6 = ['policy', 'route6']
+config = ConfigTree(config_file)
+
+if not config.exists(base4) and not config.exists(base6):
+    # Nothing to do
+    exit(0)
+
+def migrate_interface(config, iftype, ifname, vif=None, vifs=None, vifc=None):
+    if_path = ['interfaces', iftype, ifname]
+    ifname_full = ifname
+
+    if vif:
+        if_path += ['vif', vif]
+        ifname_full = f'{ifname}.{vif}'
+    elif vifs:
+        if_path += ['vif-s', vifs]
+        ifname_full = f'{ifname}.{vifs}'
+        if vifc:
+            if_path += ['vif-c', vifc]
+            ifname_full = f'{ifname}.{vifs}.{vifc}'
+
+    if not config.exists(if_path + ['policy']):
+        return
+
+    if config.exists(if_path + ['policy', 'route']):
+        route_name = config.return_value(if_path + ['policy', 'route'])
+        config.set(base4 + [route_name, 'interface'], value=ifname_full, replace=False)
+
+    if config.exists(if_path + ['policy', 'route6']):
+        route_name = config.return_value(if_path + ['policy', 'route6'])
+        config.set(base6 + [route_name, 'interface'], value=ifname_full, replace=False)
+
+    config.delete(if_path + ['policy'])
+
+for iftype in config.list_nodes(['interfaces']):
+    for ifname in config.list_nodes(['interfaces', iftype]):
+        migrate_interface(config, iftype, ifname)
+
+        if config.exists(['interfaces', iftype, ifname, 'vif']):
+            for vif in config.list_nodes(['interfaces', iftype, ifname, 'vif']):
+                migrate_interface(config, iftype, ifname, vif=vif)
+
+        if config.exists(['interfaces', iftype, ifname, 'vif-s']):
+            for vifs in config.list_nodes(['interfaces', iftype, ifname, 'vif-s']):
+                migrate_interface(config, iftype, ifname, vifs=vifs)
+
+                if config.exists(['interfaces', iftype, ifname, 'vif-s', vifs, 'vif-c']):
+                    for vifc in config.list_nodes(['interfaces', iftype, ifname, 'vif-s', vifs, 'vif-c']):
+                        migrate_interface(config, iftype, ifname, vifs=vifs, vifc=vifc)
+
+try:
+    with open(file_name, 'w') as f:
+        f.write(config.to_string())
+except OSError as e:
+    print("Failed to save the modified config: {}".format(e))
+    exit(1)
diff --git a/src/op_mode/policy_route.py b/src/op_mode/policy_route.py
index 5be40082f..5953786f3 100755
--- a/src/op_mode/policy_route.py
+++ b/src/op_mode/policy_route.py
@@ -22,53 +22,13 @@ from vyos.config import Config
 from vyos.util import cmd
 from vyos.util import dict_search_args
 
-def get_policy_interfaces(conf, policy, name=None, ipv6=False):
-    interfaces = conf.get_config_dict(['interfaces'], key_mangling=('-', '_'),
-                                      get_first_key=True, no_tag_node_value_mangle=True)
-
-    routes = ['route', 'route6']
-
-    def parse_if(ifname, if_conf):
-        if 'policy' in if_conf:
-            for route in routes:
-                if route in if_conf['policy']:
-                    route_name = if_conf['policy'][route]
-                    name_str = f'({ifname},{route})'
-
-                    if not name:
-                        policy[route][route_name]['interface'].append(name_str)
-                    elif not ipv6 and name == route_name:
-                        policy['interface'].append(name_str)
-
-        for iftype in ['vif', 'vif_s', 'vif_c']:
-            if iftype in if_conf:
-                for vifname, vif_conf in if_conf[iftype].items():
-                    parse_if(f'{ifname}.{vifname}', vif_conf)
-
-    for iftype, iftype_conf in interfaces.items():
-        for ifname, if_conf in iftype_conf.items():
-            parse_if(ifname, if_conf)
-
-def get_config_policy(conf, name=None, ipv6=False, interfaces=True):
+def get_config_policy(conf, name=None, ipv6=False):
     config_path = ['policy']
     if name:
         config_path += ['route6' if ipv6 else 'route', name]
 
     policy = conf.get_config_dict(config_path, key_mangling=('-', '_'),
                                 get_first_key=True, no_tag_node_value_mangle=True)
-    if policy and interfaces:
-        if name:
-            policy['interface'] = []
-        else:
-            if 'route' in policy:
-                for route_name, route_conf in policy['route'].items():
-                    route_conf['interface'] = []
-
-            if 'route6' in policy:
-                for route_name, route_conf in policy['route6'].items():
-                    route_conf['interface'] = []
-
-        get_policy_interfaces(conf, policy, name, ipv6)
 
     return policy