Merge branch 't2450-vrf' of github.com:c-po/vyos-1x into current

* 't2450-vrf' of github.com:c-po/vyos-1x:
  xml: add new common "interface-name" validator
  xml: include: accel: add missing file include comments
  smoketest: static: improve static route testing
  route: static: T2450: provide full protocol support in XML and Python with new CLI
  vrf: T2450: provide full protocol support in XML and Python with new CLI
  migrator: ospf: T3267: cleanup empty interface "ip" nodes

7 files changed, 485 insertions, 3 deletions

diff --git a/src/conf_mode/protocols_static.py b/src/conf_mode/protocols_static.py
new file mode 100755
index 000000000..62a3fecd7
--- /dev/null
+++ b/src/conf_mode/protocols_static.py
@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2021 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+
+from sys import exit
+
+from vyos.config import Config
+from vyos.template import render
+from vyos.template import render_to_string
+from vyos.util import call
+from vyos.configverify import verify_route_maps
+from vyos import ConfigError
+from vyos import frr
+from vyos import airbag
+airbag.enable()
+
+config_file = r'/tmp/static.frr'
+frr_daemon = 'staticd'
+
+DEBUG = os.path.exists('/tmp/static.debug')
+if DEBUG:
+    import logging
+    lg = logging.getLogger("vyos.frr")
+    lg.setLevel(logging.DEBUG)
+    ch = logging.StreamHandler()
+    lg.addHandler(ch)
+
+def get_config(config=None):
+    if config:
+        conf = config
+    else:
+        conf = Config()
+    base = ['protocols', 'static']
+    static = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True)
+    return static
+
+def verify(static):
+    verify_route_maps(static)
+    return None
+
+def generate(static):
+    # render(config) not needed, its only for debug
+    render(config_file, 'frr/static.frr.tmpl', static)
+    static['new_frr_config'] = render_to_string('frr/static.frr.tmpl', static)
+
+    return None
+
+def apply(static):
+    # Save original configuration prior to starting any commit actions
+    frr_cfg = frr.FRRConfig()
+    frr_cfg.load_configuration(frr_daemon)
+    frr_cfg.modify_section(r'^ip route .*', '')
+    frr_cfg.modify_section(r'^ipv6 route .*', '')
+    frr_cfg.add_before(r'(interface .*|line vty)', static['new_frr_config'])
+
+    # Debugging
+    if DEBUG:
+        from pprint import pprint
+        print('')
+        print('--------- DEBUGGING ----------')
+        pprint(dir(frr_cfg))
+        print('Existing config:\n')
+        for line in frr_cfg.original_config:
+            print(line)
+        print(f'Replacement config:\n')
+        print(f'{static["new_frr_config"]}')
+        print(f'Modified config:\n')
+        print(f'{frr_cfg}')
+
+    frr_cfg.commit_configuration(frr_daemon)
+
+    # If FRR config is blank, rerun the blank commit x times due to frr-reload
+    # behavior/bug not properly clearing out on one commit.
+    if static['new_frr_config'] == '':
+        for a in range(5):
+            frr_cfg.commit_configuration(frr_daemon)
+
+    return None
+
+if __name__ == '__main__':
+    try:
+        c = get_config()
+        verify(c)
+        generate(c)
+        apply(c)
+    except ConfigError as e:
+        print(e)
+        exit(1)
diff --git a/src/conf_mode/protocols_vrf.py b/src/conf_mode/protocols_vrf.py
new file mode 100755
index 000000000..7c32c7013
--- /dev/null
+++ b/src/conf_mode/protocols_vrf.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2021 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+
+from sys import exit
+
+from vyos.config import Config
+from vyos.template import render
+from vyos.template import render_to_string
+from vyos.util import call
+from vyos import ConfigError
+from vyos import frr
+from vyos import airbag
+airbag.enable()
+
+config_file = r'/tmp/vrf.frr'
+frr_daemon = 'staticd'
+
+DEBUG = os.path.exists('/tmp/vrf.debug')
+if DEBUG:
+    import logging
+    lg = logging.getLogger("vyos.frr")
+    lg.setLevel(logging.DEBUG)
+    ch = logging.StreamHandler()
+    lg.addHandler(ch)
+
+def get_config(config=None):
+    if config:
+        conf = config
+    else:
+        conf = Config()
+    base = ['protocols', 'vrf']
+    vrf = conf.get_config_dict(base, key_mangling=('-', '_'))
+    return vrf
+
+def verify(vrf):
+
+    return None
+
+def generate(vrf):
+    # render(config) not needed, its only for debug
+    render(config_file, 'frr/vrf.frr.tmpl', vrf)
+    vrf['new_frr_config'] = render_to_string('frr/vrf.frr.tmpl', vrf)
+
+    return None
+
+def apply(vrf):
+    # Save original configuration prior to starting any commit actions
+    frr_cfg = frr.FRRConfig()
+    frr_cfg.load_configuration(frr_daemon)
+    frr_cfg.modify_section(r'vrf \S+', '')
+    frr_cfg.add_before(r'(ip prefix-list .*|route-map .*|line vty)', vrf['new_frr_config'])
+
+    # Debugging
+    if DEBUG:
+        from pprint import pprint
+        print('')
+        print('--------- DEBUGGING ----------')
+        pprint(dir(frr_cfg))
+        print('Existing config:\n')
+        for line in frr_cfg.original_config:
+            print(line)
+        print(f'Replacement config:\n')
+        print(f'{vrf["new_frr_config"]}')
+        print(f'Modified config:\n')
+        print(f'{frr_cfg}')
+
+    frr_cfg.commit_configuration(frr_daemon)
+
+    # If FRR config is blank, rerun the blank commit x times due to frr-reload
+    # behavior/bug not properly clearing out on one commit.
+    if vrf['new_frr_config'] == '':
+        for a in range(5):
+            frr_cfg.commit_configuration(frr_daemon)
+
+    return None
+
+if __name__ == '__main__':
+    try:
+        c = get_config()
+        verify(c)
+        generate(c)
+        apply(c)
+    except ConfigError as e:
+        print(e)
+        exit(1)
diff --git a/src/migration-scripts/interfaces/18-to-19 b/src/migration-scripts/interfaces/18-to-19
index e24421c90..965b76a04 100755
--- a/src/migration-scripts/interfaces/18-to-19
+++ b/src/migration-scripts/interfaces/18-to-19
@@ -14,7 +14,8 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from sys import exit, argv
+from sys import argv
+from sys import exit
 from vyos.configtree import ConfigTree
 
 if __name__ == '__main__':
@@ -41,6 +42,11 @@ if __name__ == '__main__':
                 config.copy(ip_ospf, ['protocols', 'ospf', 'interface', interface])
                 config.delete(ip_ospf)
 
+                # if "ip ospf" was the only setting, we can clean out the empty
+                # ip node afterwards
+                if len(config.list_nodes(ip_ospf[:-1])) == 0:
+                    config.delete(ip_ospf[:-1])
+
             vif_path = ['interfaces', type, interface, 'vif']
             if config.exists(vif_path):
                 for vif in config.list_nodes(vif_path):
@@ -51,6 +57,11 @@ if __name__ == '__main__':
                         config.copy(vif_ospf_path, ['protocols', 'ospf', 'interface', f'{interface}.{vif}'])
                         config.delete(vif_ospf_path)
 
+                        # if "ip ospf" was the only setting, we can clean out the empty
+                        # ip node afterwards
+                        if len(config.list_nodes(vif_ospf_path[:-1])) == 0:
+                            config.delete(vif_ospf_path[:-1])
+
             vif_s_path = ['interfaces', type, interface, 'vif-s']
             if config.exists(vif_s_path):
                 for vif_s in config.list_nodes(vif_s_path):
@@ -70,8 +81,18 @@ if __name__ == '__main__':
                                     config.copy(vif_c_ospf_path, ['protocols', 'ospf', 'interface', f'{interface}.{vif_s}.{vif_c}'])
                                     config.delete(vif_c_ospf_path)
 
+                                    # if "ip ospf" was the only setting, we can clean out the empty
+                                    # ip node afterwards
+                                    if len(config.list_nodes(vif_c_ospf_path[:-1])) == 0:
+                                        config.delete(vif_c_ospf_path[:-1])
+
                         config.delete(vif_s_ospf_path)
 
+                        # if "ip ospf" was the only setting, we can clean out the empty
+                        # ip node afterwards
+                        if len(config.list_nodes(vif_s_ospf_path[:-1])) == 0:
+                            config.delete(vif_s_ospf_path[:-1])
+
     try:
         with open(file_name, 'w') as f:
             f.write(config.to_string())
diff --git a/src/migration-scripts/quagga/7-to-8 b/src/migration-scripts/quagga/7-to-8
new file mode 100755
index 000000000..9c277a6f1
--- /dev/null
+++ b/src/migration-scripts/quagga/7-to-8
@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2021 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# - T2450: drop interface-route and interface-route6 from "protocols static"
+
+from sys import argv
+from sys import exit
+
+from vyos.configtree import ConfigTree
+
+def migrate_interface_route(config, base, path, route_route6):
+    """ Generic migration function which can be called on every instance of
+    interface-route, beeing it ipv4, ipv6 or nested under the "static table" nodes.
+
+    What we do?
+      - Drop 'interface-route' or 'interface-route6' and migrate the route unter the
+        'route' or 'route6' tag node.
+    """
+    if config.exists(base + path):
+        for route in config.list_nodes(base + path):
+            interface = config.list_nodes(base + path + [route, 'next-hop-interface'])
+
+            tmp = base + path + [route, 'next-hop-interface']
+            for interface in config.list_nodes(tmp):
+                new_base = base + [route_route6, route, 'interface']
+                config.set(new_base)
+                config.set_tag(base + [route_route6])
+                config.set_tag(new_base)
+                config.copy(tmp + [interface], new_base + [interface])
+
+        config.delete(base + path)
+
+def migrate_route(config, base, path, route_route6):
+    """ Generic migration function which can be called on every instance of
+    route, beeing it ipv4, ipv6 or even nested under the static table nodes.
+
+    What we do?
+      - for consistency reasons rename next-hop-interface to interface
+      - for consistency reasons rename next-hop-vrf to vrf
+    """
+    if config.exists(base + path):
+        for route in config.list_nodes(base + path):
+            next_hop = base + path + [route, 'next-hop']
+            if config.exists(next_hop):
+                for gateway in config.list_nodes(next_hop):
+                    # IPv4 routes calls it next-hop-interface, rename this to
+                    # interface instead so it's consitent with IPv6
+                    interface_path = next_hop + [gateway, 'next-hop-interface']
+                    if config.exists(interface_path):
+                        config.rename(interface_path, 'interface')
+
+                    # When VRFs got introduced, I (c-po) named it next-hop-vrf,
+                    # we can also call it vrf which is simply shorter.
+                    vrf_path = next_hop + [gateway, 'next-hop-vrf']
+                    if config.exists(vrf_path):
+                        config.rename(vrf_path, 'vrf')
+
+
+if (len(argv) < 2):
+    print("Must specify file name!")
+    exit(1)
+
+file_name = argv[1]
+
+with open(file_name, 'r') as f:
+    config_file = f.read()
+
+base = ['protocols', 'static']
+
+config = ConfigTree(config_file)
+if not config.exists(base):
+    # Nothing to do
+    exit(0)
+
+# Migrate interface-route into route
+migrate_interface_route(config, base, ['interface-route'], 'route')
+
+# Migrate interface-route6 into route6
+migrate_interface_route(config, base, ['interface-route6'], 'route6')
+
+# Cleanup nodes inside route
+migrate_route(config, base, ['route'], 'route')
+
+# Cleanup nodes inside route6
+migrate_route(config, base, ['route6'], 'route6')
+
+#
+# PBR table cleanup
+table_path = base + ['table']
+if config.exists(table_path):
+    for table in config.list_nodes(table_path):
+        # Migrate interface-route into route
+        migrate_interface_route(config, table_path + [table], ['interface-route'], 'route')
+
+        # Migrate interface-route6 into route6
+        migrate_interface_route(config, table_path + [table], ['interface-route6'], 'route6')
+
+        # Cleanup nodes inside route
+        migrate_route(config, table_path + [table], ['route'], 'route')
+
+        # Cleanup nodes inside route6
+        migrate_route(config, table_path + [table], ['route6'], 'route6')
+
+try:
+    with open(file_name, 'w') as f:
+        f.write(config.to_string())
+except OSError as e:
+    print("Failed to save the modified config: {}".format(e))
+    exit(1)
diff --git a/src/migration-scripts/vrf/0-to-1 b/src/migration-scripts/vrf/0-to-1
new file mode 100755
index 000000000..29b2fab74
--- /dev/null
+++ b/src/migration-scripts/vrf/0-to-1
@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2021 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# - T2450: drop interface-route and interface-route6 from "protocols vrf"
+
+from sys import argv
+from sys import exit
+from vyos.configtree import ConfigTree
+
+if (len(argv) < 2):
+    print("Must specify file name!")
+    exit(1)
+
+file_name = argv[1]
+
+with open(file_name, 'r') as f:
+    config_file = f.read()
+
+base = ['protocols', 'vrf']
+config = ConfigTree(config_file)
+
+if not config.exists(base):
+    # Nothing to do
+    exit(0)
+
+for vrf in config.list_nodes(base):
+    static_base = base + [vrf, 'static']
+    if not config.exists(static_base):
+        continue
+
+    #
+    # Migrate interface-route into route
+    #
+    interface_route_path = static_base + ['interface-route']
+    if config.exists(interface_route_path):
+        for route in config.list_nodes(interface_route_path):
+            interface = config.list_nodes(interface_route_path + [route, 'next-hop-interface'])
+
+            tmp = interface_route_path + [route, 'next-hop-interface']
+            for interface in config.list_nodes(tmp):
+                new_base = static_base + ['route', route, 'interface']
+                config.set(new_base)
+                config.set_tag(new_base)
+                config.copy(tmp + [interface], new_base + [interface])
+
+        config.delete(interface_route_path)
+
+    #
+    # Migrate interface-route6 into route6
+    #
+    interface_route_path = static_base + ['interface-route6']
+    if config.exists(interface_route_path):
+        for route in config.list_nodes(interface_route_path):
+            interface = config.list_nodes(interface_route_path + [route, 'next-hop-interface'])
+
+            tmp = interface_route_path + [route, 'next-hop-interface']
+            for interface in config.list_nodes(tmp):
+                new_base = static_base + ['route6', route, 'interface']
+                config.set(new_base)
+                config.set_tag(new_base)
+                config.copy(tmp + [interface], new_base + [interface])
+
+        config.delete(interface_route_path)
+
+    #
+    # Cleanup nodes inside route
+    #
+    route_path = static_base + ['route']
+    if config.exists(route_path):
+        for route in config.list_nodes(route_path):
+            next_hop = route_path + [route, 'next-hop']
+            if config.exists(next_hop):
+                for gateway in config.list_nodes(next_hop):
+                    interface_path = next_hop + [gateway, 'next-hop-interface']
+                    if config.exists(interface_path):
+                        config.rename(interface_path, 'interface')
+                    vrf_path = next_hop + [gateway, 'next-hop-vrf']
+                    if config.exists(vrf_path):
+                        config.rename(vrf_path, 'vrf')
+
+    #
+    # Cleanup nodes inside route6
+    #
+    route_path = static_base + ['route6']
+    if config.exists(route_path):
+        for route in config.list_nodes(route_path):
+            next_hop = route_path + [route, 'next-hop']
+            if config.exists(next_hop):
+                for gateway in config.list_nodes(next_hop):
+                    vrf_path = next_hop + [gateway, 'next-hop-vrf']
+                    if config.exists(vrf_path):
+                        config.rename(vrf_path, 'vrf')
+
+try:
+    with open(file_name, 'w') as f:
+        f.write(config.to_string())
+except OSError as e:
+    print("Failed to save the modified config: {}".format(e))
+    exit(1)
diff --git a/src/validators/fqdn b/src/validators/fqdn
index 347ffda42..66276c093 100755
--- a/src/validators/fqdn
+++ b/src/validators/fqdn
@@ -17,11 +17,9 @@
 import re
 import sys
 
-
 # pattern copied from: https://www.regextester.com/103452
 pattern = "(?=^.{4,253}$)(^((?!-)[a-zA-Z0-9-]{0,62}[a-zA-Z0-9]\.)+[a-zA-Z]{2,63}$)"
 
-
 if __name__ == '__main__':
     if len(sys.argv) != 2:
         sys.exit(1)
diff --git a/src/validators/interface-name b/src/validators/interface-name
new file mode 100755
index 000000000..32cd42fbd
--- /dev/null
+++ b/src/validators/interface-name
@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2021 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import re
+import sys
+
+pattern = '^(br|bond|dum|en|eth|gnv|peth|pppoe|tun|vti|vtun|vxlan|wg|wlan)[0-9]+|lo$'
+
+if __name__ == '__main__':
+    if len(sys.argv) != 2:
+        sys.exit(1)
+    if not re.match(pattern, sys.argv[1]):
+        sys.exit(1)
+    sys.exit(0)