Merge pull request #4749 from hedrok/T7803-failover-vrf

T7803: Make failover route vrf-aware

1 files changed, 507 insertions, 0 deletions

diff --git a/smoketest/scripts/cli/test_protocols_failover.py b/smoketest/scripts/cli/test_protocols_failover.py
new file mode 100755
index 000000000..26f5e87b5
--- /dev/null
+++ b/smoketest/scripts/cli/test_protocols_failover.py
@@ -0,0 +1,507 @@
+#!/usr/bin/env python3
+#
+# Copyright VyOS maintainers and contributors <maintainers@vyos.io>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import shutil
+import unittest
+
+from math import ceil
+from time import sleep
+
+from base_vyostest_shim import VyOSUnitTestSHIM
+
+from vyos.utils.process import ip_cmd
+
+vrf_base_path = ['vrf', 'name']
+red = 'red-309aba83'
+blue = 'blue-46b27cb'
+used_vrf_names = [red, blue]
+base_path = ['protocols', 'failover']
+
+config_dir_root = '/run/vyos-failover.conf.d'
+
+check_timeout = 1
+wait_timeout = 5
+
+# Use numeric value to not get ip errors while
+# /etc/iproute2/rt_protos.d/failover.conf is not installed yet
+failover_protocol_value = 111
+
+dummy_if1 = 'dum3711'
+dummy_if2 = 'dum3712'
+dummy_if3 = 'dum3713'
+
+route_prefix = '203.0.113.0/24'
+route2_prefix = '172.16.0.0/24'
+route_base_path = base_path + ['route', route_prefix]
+
+dummy_if1_addr = '192.168.30.1'
+dummy_if2_addr = '10.0.70.1'
+dummy_if3_addr = '10.20.0.1'
+
+
+class RoutesChecker:
+    def __init__(self, required_routes, allow_extra=False):
+        self.required_routes = required_routes
+        self.allow_extra = allow_extra
+        self.error = ''
+
+    def __call__(self, got_routes):
+        self.error = ''
+        if len(got_routes) < len(self.required_routes):
+            self.error = f"Not enough routes: expected {len(self.required_routes)}, got {len(got_routes)}: {got_routes}"
+            return False
+
+        if not self.allow_extra and len(got_routes) != len(self.required_routes):
+            self.error = f"Extra routes: expected {len(self.required_routes)}, got {len(got_routes)}: {got_routes}"
+
+        for route in self.required_routes:
+            found = False
+            for got_route in got_routes:
+                mismatch = False
+                for key in route:
+                    if route[key] is None:
+                        if key in got_route:
+                            mismatch = True
+                            break
+                    elif key not in got_route or got_route[key] != route[key]:
+                        mismatch = True
+                        break
+                if not mismatch:
+                    found = True
+                    break
+            if not found:
+                self.error = f"Couldn't find required route {route} among received routes {got_routes}"
+                return False
+        return True
+
+
+class TestProtocolsFailover(VyOSUnitTestSHIM.TestCase):
+    def clean_and_stop_daemon(self):
+        self.cli_delete(base_path)
+        for vrf in used_vrf_names:
+            self.cli_delete(vrf_base_path + [vrf] + base_path)
+        self.cli_commit()
+        shutil.rmtree(config_dir_root, ignore_errors=True)
+
+    def setUp(self):
+        # Needed dummy interfaces
+        self.cli_set(['interfaces', 'dummy', dummy_if1])
+        self.cli_set(['interfaces', 'dummy', dummy_if2])
+        self.cli_set(['interfaces', 'dummy', dummy_if3])
+
+        self.clean_and_stop_daemon()
+
+    def tearDown(self):
+        self.cli_delete(['interfaces', 'dummy', dummy_if3])
+        self.cli_delete(['interfaces', 'dummy', dummy_if2])
+        self.cli_delete(['interfaces', 'dummy', dummy_if1])
+
+        self.clean_and_stop_daemon()
+
+        failover_routes = ip_cmd(
+            f'route show proto {failover_protocol_value} table all'
+        )
+        self.assertEqual(failover_routes, [], "Some failover IPv4 routes left")
+        failover_routes = ip_cmd(
+            f'-6 route show proto {failover_protocol_value} table all'
+        )
+        self.assertEqual(failover_routes, [], "Some failover IPv6 routes left")
+
+    def wait_for_ip_output(self, ip_command_args, check, pause=0.1, timeout=3):
+        tries = ceil(timeout / pause)
+        result = None
+        for i in range(tries):
+            result = ip_cmd(ip_command_args)
+            if callable(check):
+                if check(result):
+                    return True, result
+            elif result == check:
+                return True, result
+
+            sleep(pause)
+
+        return False, result
+
+    def test_01_basic(self):
+        res, output = self.wait_for_ip_output(
+            f'route show proto {failover_protocol_value}',
+            [],
+            timeout=wait_timeout,
+        )
+        self.assertTrue(
+            res, f"No failover routes must exist before test, last result: {output}"
+        )
+
+        self.cli_set(
+            ['interfaces', 'dummy', dummy_if1, 'address', dummy_if1_addr + '/24']
+        )
+        self.cli_set(
+            ['interfaces', 'dummy', dummy_if2, 'address', dummy_if2_addr + '/24']
+        )
+        self.cli_set(
+            route_base_path + ['next-hop', dummy_if2_addr, 'interface', dummy_if2]
+        )
+        self.cli_set(route_base_path + ['next-hop', dummy_if2_addr, 'metric', '30'])
+        self.cli_set(
+            route_base_path
+            + [
+                'next-hop',
+                dummy_if2_addr,
+                'check',
+                'target',
+                dummy_if1_addr,
+                'interface',
+                dummy_if1,
+            ]
+        )
+        self.cli_set(
+            route_base_path
+            + ['next-hop', dummy_if2_addr, 'check', 'timeout', str(check_timeout)]
+        )
+        self.cli_commit()
+
+        # Now vyos-failover must be launched, it should create route, waiting for it...
+        checker = RoutesChecker([{'dst': route_prefix}])
+        res, output = self.wait_for_ip_output(
+            f"route show proto {failover_protocol_value}",
+            checker,
+            timeout=wait_timeout,
+        )
+        self.assertTrue(res, f"Route must have been created, last result: {output}")
+
+        self.cli_delete(['interfaces', 'dummy', dummy_if1, 'address'])
+        self.cli_commit()
+
+        res, output = self.wait_for_ip_output(
+            f'route show proto {failover_protocol_value}',
+            [],
+            timeout=wait_timeout,
+        )
+        self.assertTrue(res, f"Route must have been deleted, last result: {output}")
+
+    def test_02_vrf(self):
+        # route 1 default VRF, check red
+        # route 2 red, check blue
+        # route 3 red, check red
+        # route 1 and route 3 with same destination
+
+        res, output = self.wait_for_ip_output(
+            f'route show proto {failover_protocol_value} table all',
+            [],
+            timeout=wait_timeout,
+        )
+        self.assertTrue(
+            res, f"No failover routes must exist before test, last result: {output}"
+        )
+
+        self.cli_set(['vrf', 'name', red, 'table', '43310'])
+        self.cli_set(['vrf', 'name', blue, 'table', '43311'])
+
+        self.cli_set(
+            ['interfaces', 'dummy', dummy_if1, 'address', dummy_if1_addr + '/24']
+        )
+        self.cli_set(
+            ['interfaces', 'dummy', dummy_if2, 'address', dummy_if2_addr + '/24']
+        )
+        self.cli_set(
+            ['interfaces', 'dummy', dummy_if3, 'address', dummy_if3_addr + '/24']
+        )
+        self.cli_set(['interfaces', 'dummy', dummy_if2, 'vrf', red])
+        self.cli_set(['interfaces', 'dummy', dummy_if3, 'vrf', blue])
+
+        route_1_base = route_base_path
+        route_2_base = vrf_base_path + [red] + base_path + ['route', route2_prefix]
+        route_3_base = vrf_base_path + [red] + base_path + ['route', route_prefix]
+
+        self.cli_set(
+            route_1_base + ['next-hop', dummy_if1_addr, 'interface', dummy_if1]
+        )
+        self.cli_set(
+            route_1_base
+            + [
+                'next-hop',
+                dummy_if1_addr,
+                'check',
+                'target',
+                dummy_if2_addr,
+                'vrf',
+                red,
+            ]
+        )
+        self.cli_set(
+            route_1_base
+            + ['next-hop', dummy_if1_addr, 'check', 'timeout', str(check_timeout)]
+        )
+
+        self.cli_set(
+            route_2_base + ['next-hop', dummy_if2_addr, 'interface', dummy_if2]
+        )
+        self.cli_set(
+            route_2_base
+            + [
+                'next-hop',
+                dummy_if2_addr,
+                'check',
+                'target',
+                dummy_if3_addr,
+                'vrf',
+                blue,
+            ]
+        )
+        self.cli_set(
+            route_2_base
+            + ['next-hop', dummy_if2_addr, 'check', 'timeout', str(check_timeout)]
+        )
+
+        self.cli_set(
+            route_3_base + ['next-hop', dummy_if2_addr, 'interface', dummy_if2]
+        )
+        self.cli_set(
+            route_3_base
+            + [
+                'next-hop',
+                dummy_if2_addr,
+                'check',
+                'target',
+                dummy_if2_addr,
+            ]
+        )
+        self.cli_set(
+            route_3_base
+            + ['next-hop', dummy_if2_addr, 'check', 'timeout', str(check_timeout)]
+        )
+
+        self.cli_commit()
+
+        route1_fields = {'dst': route_prefix, 'gateway': dummy_if1_addr, 'table': None}
+        route2_fields = {
+            'dst': route2_prefix,
+            'gateway': dummy_if2_addr,
+            'table': red,
+        }
+        route3_fields = {'dst': route_prefix, 'gateway': dummy_if2_addr, 'table': red}
+
+        # All three routes must be created
+        checker1 = RoutesChecker([route1_fields, route2_fields, route3_fields])
+        res, output = self.wait_for_ip_output(
+            f"route show proto {failover_protocol_value} table all",
+            checker1,
+            timeout=wait_timeout * 3,
+        )
+        self.assertTrue(
+            res, f"Routes must have been created, checker error: {checker1.error}"
+        )
+
+        # Delete dummy_if3, route2 must be deleted
+        self.cli_delete(['interfaces', 'dummy', dummy_if3, 'address'])
+        self.cli_commit()
+
+        checker2 = RoutesChecker([route1_fields, route3_fields])
+        res, output = self.wait_for_ip_output(
+            f'route show proto {failover_protocol_value} table all',
+            checker2,
+            timeout=wait_timeout * 3,
+        )
+        self.assertTrue(
+            res,
+            f"Only route1 and route3 must have been left, checker error: {checker2.error}",
+        )
+
+        # Delete dummy_if2, all routes must be deleted
+        self.cli_delete(['interfaces', 'dummy', dummy_if2, 'address'])
+        self.cli_commit()
+        res, output = self.wait_for_ip_output(
+            f'route show proto {failover_protocol_value} table all',
+            [],
+            timeout=wait_timeout * 3,
+        )
+        self.assertTrue(res, f"No routes should have been left, got: {output}")
+
+    def test_03_config(self):
+        # Test how daemon reacts to routes add/delete, files add/delete
+        # All checks in this test are always true, configuration is added/deleted only
+
+        # route 1 default VRF
+        # route 2 default VRF
+        # route 3 red
+        route1_fields = {'dst': route_prefix, 'gateway': dummy_if1_addr, 'table': None}
+        route2_fields = {'dst': route2_prefix, 'gateway': dummy_if1_addr, 'table': None}
+        route3_fields = {'dst': route_prefix, 'gateway': dummy_if2_addr, 'table': red}
+
+        res, output = self.wait_for_ip_output(
+            f'route show proto {failover_protocol_value} table all',
+            [],
+            timeout=wait_timeout,
+        )
+        self.assertTrue(
+            res, f"No failover routes must exist before test, last result: {output}"
+        )
+
+        self.cli_set(['vrf', 'name', red, 'table', '43310'])
+        self.cli_set(['vrf', 'name', blue, 'table', '43311'])
+
+        self.cli_set(
+            ['interfaces', 'dummy', dummy_if1, 'address', dummy_if1_addr + '/24']
+        )
+        self.cli_set(
+            ['interfaces', 'dummy', dummy_if2, 'address', dummy_if2_addr + '/24']
+        )
+        self.cli_set(['interfaces', 'dummy', dummy_if2, 'vrf', red])
+
+        route_1_base = route_base_path
+        route_2_base = base_path + ['route', route2_prefix]
+        route_3_base = vrf_base_path + [red] + base_path + ['route', route_prefix]
+
+        self.cli_set(
+            route_1_base + ['next-hop', dummy_if1_addr, 'interface', dummy_if1]
+        )
+        self.cli_set(
+            route_1_base
+            + [
+                'next-hop',
+                dummy_if1_addr,
+                'check',
+                'target',
+                dummy_if1_addr,
+            ]
+        )
+        self.cli_set(
+            route_1_base
+            + ['next-hop', dummy_if1_addr, 'check', 'timeout', str(check_timeout)]
+        )
+        self.cli_commit()
+
+        # Adding only route1
+        checker1 = RoutesChecker([route1_fields])
+        res, output = self.wait_for_ip_output(
+            f"route show proto {failover_protocol_value} table all",
+            checker1,
+            timeout=wait_timeout * 3,
+        )
+        self.assertTrue(
+            res, f"Route 1 must have been created, checker error: {checker1.error}"
+        )
+
+        # adding route 3 - new file
+        self.cli_set(
+            route_3_base + ['next-hop', dummy_if2_addr, 'interface', dummy_if2]
+        )
+        self.cli_set(
+            route_3_base
+            + [
+                'next-hop',
+                dummy_if2_addr,
+                'check',
+                'target',
+                dummy_if2_addr,
+            ]
+        )
+        self.cli_set(
+            route_3_base
+            + ['next-hop', dummy_if2_addr, 'check', 'timeout', str(check_timeout)]
+        )
+        self.cli_commit()
+
+        # Now route1 and route3 must be active
+        checker13 = RoutesChecker([route1_fields, route3_fields])
+        res, output = self.wait_for_ip_output(
+            f"route show proto {failover_protocol_value} table all",
+            checker13,
+            timeout=wait_timeout * 3,
+        )
+        self.assertTrue(
+            res,
+            f"Route 1 and route 3 must have been created, checker error: {checker13.error}",
+        )
+
+        # Now add route2 (add of route to file)
+        self.cli_set(
+            route_2_base + ['next-hop', dummy_if1_addr, 'interface', dummy_if1]
+        )
+        self.cli_set(
+            route_2_base
+            + [
+                'next-hop',
+                dummy_if1_addr,
+                'check',
+                'target',
+                dummy_if1_addr,
+            ]
+        )
+        self.cli_set(
+            route_2_base
+            + ['next-hop', dummy_if1_addr, 'check', 'timeout', str(check_timeout)]
+        )
+        self.cli_commit()
+
+        # All three routes must be created
+        checker123 = RoutesChecker([route1_fields, route2_fields, route3_fields])
+        res, output = self.wait_for_ip_output(
+            f'route show proto {failover_protocol_value} table all',
+            checker123,
+            timeout=wait_timeout * 3,
+        )
+        self.assertTrue(
+            res,
+            f"All three routes must have been created, checker error: {checker123.error}",
+        )
+
+        # Delete route1
+        self.cli_delete(route_1_base)
+        self.cli_commit()
+
+        # Now route2 and route3 must be active
+        checker23 = RoutesChecker([route2_fields, route3_fields])
+        res, output = self.wait_for_ip_output(
+            f"route show proto {failover_protocol_value} table all",
+            checker23,
+            timeout=wait_timeout * 3,
+        )
+        self.assertTrue(
+            res,
+            f"Route 1 must have been deleted, routes 2 and 3 active. Checker error: {checker23.error}",
+        )
+
+        # Delete route2 - file deletion
+        self.cli_delete(route_2_base)
+        self.cli_commit()
+
+        # Now only route3 must be active
+        checker3 = RoutesChecker([route3_fields])
+        res, output = self.wait_for_ip_output(
+            f"route show proto {failover_protocol_value} table all",
+            checker3,
+            timeout=wait_timeout * 3,
+        )
+        self.assertTrue(
+            res,
+            f"Route 2 must have been deleted, only routes 3 should be active. Checker error: {checker3.error}",
+        )
+
+        # Deleting last route
+        self.cli_delete(route_3_base)
+        self.cli_commit()
+
+        res, output = self.wait_for_ip_output(
+            f'route show proto {failover_protocol_value} table all',
+            [],
+            timeout=wait_timeout * 3,
+        )
+        self.assertTrue(res, f"No routes should have been left, got: {output}")
+
+
+if __name__ == '__main__':
+    unittest.main(verbosity=2)