Merge pull request #4749 from hedrok/T7803-failover-vrf

T7803: Make failover route vrf-aware

7 files changed, 1115 insertions, 268 deletions

diff --git a/data/templates/protocols/systemd_vyos_failover_service.j2 b/data/templates/protocols/systemd_vyos_failover_service.j2
index e6501e0f5..59189b1b8 100644
--- a/data/templates/protocols/systemd_vyos_failover_service.j2
+++ b/data/templates/protocols/systemd_vyos_failover_service.j2
@@ -5,7 +5,7 @@ After=vyos-router.service
 [Service]
 Type=simple
 Restart=always
-ExecStart=/usr/bin/python3 /usr/libexec/vyos/vyos-failover.py --config /run/vyos-failover.conf
+ExecStart=/usr/bin/python3 /usr/libexec/vyos/vyos-failover.py --config-dir {{ config_dir }}
 
 [Install]
 WantedBy=multi-user.target
diff --git a/interface-definitions/include/failover/protocol-common-config.xml.i b/interface-definitions/include/failover/protocol-common-config.xml.i
new file mode 100644
index 000000000..a106fdc74
--- /dev/null
+++ b/interface-definitions/include/failover/protocol-common-config.xml.i
@@ -0,0 +1,131 @@
+<!-- include start from failover/protocol-common-config.xml.i -->
+<tagNode name="route">
+  <properties>
+    <help>Failover IPv4 route</help>
+    <valueHelp>
+      <format>ipv4net</format>
+      <description>IPv4 failover route</description>
+    </valueHelp>
+    <constraint>
+      <validator name="ipv4-prefix"/>
+    </constraint>
+  </properties>
+  <children>
+    <tagNode name="next-hop">
+      <properties>
+        <help>Next-hop IPv4 router address</help>
+        <valueHelp>
+          <format>ipv4</format>
+          <description>Next-hop router address</description>
+        </valueHelp>
+        <constraint>
+          <validator name="ipv4-address"/>
+        </constraint>
+      </properties>
+      <children>
+        <node name="check">
+          <properties>
+            <help>Check target options</help>
+          </properties>
+          <children>
+            <leafNode name="policy">
+              <properties>
+                <help>Policy for check targets</help>
+                <completionHelp>
+                  <list>any-available all-available</list>
+                </completionHelp>
+                <valueHelp>
+                  <format>all-available</format>
+                  <description>All targets must be alive</description>
+                </valueHelp>
+                <valueHelp>
+                  <format>any-available</format>
+                  <description>Any target must be alive</description>
+                </valueHelp>
+                <constraint>
+                  <regex>(all-available|any-available)</regex>
+                </constraint>
+              </properties>
+              <defaultValue>any-available</defaultValue>
+            </leafNode>
+            #include <include/port-number.xml.i>
+            <tagNode name="target">
+              <properties>
+                <help>Check target address</help>
+                <valueHelp>
+                  <format>ipv4</format>
+                  <description>Address to check</description>
+                </valueHelp>
+                <constraint>
+                  <validator name="ipv4-address"/>
+                </constraint>
+              </properties>
+              <children>
+                #include <include/interface/vrf.xml.i>
+                #include <include/generic-interface.xml.i>
+              </children>
+            </tagNode>
+            <leafNode name="timeout">
+              <properties>
+                <help>Timeout between checks</help>
+                <valueHelp>
+                  <format>u32:1-300</format>
+                  <description>Timeout in seconds between checks</description>
+                </valueHelp>
+                <constraint>
+                  <validator name="numeric" argument="--range 1-255"/>
+                </constraint>
+              </properties>
+              <defaultValue>10</defaultValue>
+            </leafNode>
+            <leafNode name="type">
+              <properties>
+                <help>Check type</help>
+                <completionHelp>
+                  <list>arp icmp tcp</list>
+                </completionHelp>
+                <valueHelp>
+                  <format>arp</format>
+                  <description>Check target by ARP</description>
+                </valueHelp>
+                <valueHelp>
+                  <format>icmp</format>
+                  <description>Check target by ICMP</description>
+                </valueHelp>
+                <valueHelp>
+                  <format>tcp</format>
+                  <description>Check target by TCP</description>
+                </valueHelp>
+                <constraint>
+                  <regex>(arp|icmp|tcp)</regex>
+                </constraint>
+              </properties>
+              <defaultValue>icmp</defaultValue>
+            </leafNode>
+          </children>
+        </node>
+        #include <include/generic-interface.xml.i>
+        <leafNode name="metric">
+          <properties>
+            <help>Route metric for this gateway</help>
+            <valueHelp>
+              <format>u32:1-255</format>
+              <description>Route metric</description>
+            </valueHelp>
+            <constraint>
+              <validator name="numeric" argument="--range 1-255"/>
+            </constraint>
+          </properties>
+          <defaultValue>1</defaultValue>
+        </leafNode>
+        <leafNode name="onlink">
+          <properties>
+            <help>The next hop is directly connected to the interface, even if it does not match interface prefix</help>
+            <valueless/>
+          </properties>
+        </leafNode>
+      </children>
+    </tagNode>
+  </children>
+</tagNode>
+<!-- include end -->
diff --git a/interface-definitions/protocols_failover.xml.in b/interface-definitions/protocols_failover.xml.in
index fae9be76a..0f8a3b2ad 100644
--- a/interface-definitions/protocols_failover.xml.in
+++ b/interface-definitions/protocols_failover.xml.in
@@ -8,132 +8,7 @@
           <priority>490</priority>
         </properties>
         <children>
-          <tagNode name="route">
-            <properties>
-              <help>Failover IPv4 route</help>
-              <valueHelp>
-                <format>ipv4net</format>
-                <description>IPv4 failover route</description>
-              </valueHelp>
-              <constraint>
-                <validator name="ipv4-prefix"/>
-              </constraint>
-            </properties>
-            <children>
-              <tagNode name="next-hop">
-                <properties>
-                  <help>Next-hop IPv4 router address</help>
-                  <valueHelp>
-                    <format>ipv4</format>
-                    <description>Next-hop router address</description>
-                  </valueHelp>
-                  <constraint>
-                    <validator name="ipv4-address"/>
-                  </constraint>
-                </properties>
-                <children>
-                  <node name="check">
-                    <properties>
-                      <help>Check target options</help>
-                    </properties>
-                    <children>
-                      <leafNode name="policy">
-                        <properties>
-                          <help>Policy for check targets</help>
-                          <completionHelp>
-                            <list>any-available all-available</list>
-                          </completionHelp>
-                          <valueHelp>
-                            <format>all-available</format>
-                            <description>All targets must be alive</description>
-                          </valueHelp>
-                          <valueHelp>
-                            <format>any-available</format>
-                            <description>Any target must be alive</description>
-                          </valueHelp>
-                          <constraint>
-                            <regex>(all-available|any-available)</regex>
-                          </constraint>
-                        </properties>
-                        <defaultValue>any-available</defaultValue>
-                      </leafNode>
-                      #include <include/port-number.xml.i>
-                      <leafNode name="target">
-                        <properties>
-                          <help>Check target address</help>
-                          <valueHelp>
-                            <format>ipv4</format>
-                            <description>Address to check</description>
-                          </valueHelp>
-                          <constraint>
-                            <validator name="ipv4-address"/>
-                          </constraint>
-                          <multi/>
-                        </properties>
-                      </leafNode>
-                      <leafNode name="timeout">
-                        <properties>
-                          <help>Timeout between checks</help>
-                          <valueHelp>
-                            <format>u32:1-300</format>
-                            <description>Timeout in seconds between checks</description>
-                          </valueHelp>
-                          <constraint>
-                            <validator name="numeric" argument="--range 1-255"/>
-                          </constraint>
-                        </properties>
-                        <defaultValue>10</defaultValue>
-                      </leafNode>
-                      <leafNode name="type">
-                        <properties>
-                          <help>Check type</help>
-                          <completionHelp>
-                            <list>arp icmp tcp</list>
-                          </completionHelp>
-                          <valueHelp>
-                            <format>arp</format>
-                            <description>Check target by ARP</description>
-                          </valueHelp>
-                          <valueHelp>
-                            <format>icmp</format>
-                            <description>Check target by ICMP</description>
-                          </valueHelp>
-                          <valueHelp>
-                            <format>tcp</format>
-                            <description>Check target by TCP</description>
-                          </valueHelp>
-                          <constraint>
-                            <regex>(arp|icmp|tcp)</regex>
-                          </constraint>
-                        </properties>
-                        <defaultValue>icmp</defaultValue>
-                      </leafNode>
-                    </children>
-                  </node>
-                  #include <include/generic-interface.xml.i>
-                  <leafNode name="metric">
-                    <properties>
-                      <help>Route metric for this gateway</help>
-                      <valueHelp>
-                        <format>u32:1-255</format>
-                        <description>Route metric</description>
-                      </valueHelp>
-                      <constraint>
-                        <validator name="numeric" argument="--range 1-255"/>
-                      </constraint>
-                    </properties>
-                    <defaultValue>1</defaultValue>
-                  </leafNode>
-                  <leafNode name="onlink">
-                    <properties>
-                      <help>The next hop is directly connected to the interface, even if it does not match interface prefix</help>
-                      <valueless/>
-                    </properties>
-                  </leafNode>
-                </children>
-              </tagNode>
-            </children>
-          </tagNode>
+          #include <include/failover/protocol-common-config.xml.i>
         </children>
       </node>
     </children>
diff --git a/interface-definitions/vrf.xml.in b/interface-definitions/vrf.xml.in
index 3fa95076e..0adbeadd9 100644
--- a/interface-definitions/vrf.xml.in
+++ b/interface-definitions/vrf.xml.in
@@ -114,6 +114,15 @@
                   #include <include/static/static-route6.xml.i>
                 </children>
               </node>
+              <node name="failover" owner="${vyos_conf_scripts_dir}/protocols_failover.py $VAR(../../@)">
+                <properties>
+                  <help>Failover Routing</help>
+                  <priority>491</priority>
+                </properties>
+                <children>
+                  #include <include/failover/protocol-common-config.xml.i>
+                </children>
+              </node>
             </children>
           </node>
           <node name="service">
diff --git a/smoketest/scripts/cli/test_protocols_failover.py b/smoketest/scripts/cli/test_protocols_failover.py
new file mode 100755
index 000000000..26f5e87b5
--- /dev/null
+++ b/smoketest/scripts/cli/test_protocols_failover.py
@@ -0,0 +1,507 @@
+#!/usr/bin/env python3
+#
+# Copyright VyOS maintainers and contributors <maintainers@vyos.io>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import shutil
+import unittest
+
+from math import ceil
+from time import sleep
+
+from base_vyostest_shim import VyOSUnitTestSHIM
+
+from vyos.utils.process import ip_cmd
+
+vrf_base_path = ['vrf', 'name']
+red = 'red-309aba83'
+blue = 'blue-46b27cb'
+used_vrf_names = [red, blue]
+base_path = ['protocols', 'failover']
+
+config_dir_root = '/run/vyos-failover.conf.d'
+
+check_timeout = 1
+wait_timeout = 5
+
+# Use numeric value to not get ip errors while
+# /etc/iproute2/rt_protos.d/failover.conf is not installed yet
+failover_protocol_value = 111
+
+dummy_if1 = 'dum3711'
+dummy_if2 = 'dum3712'
+dummy_if3 = 'dum3713'
+
+route_prefix = '203.0.113.0/24'
+route2_prefix = '172.16.0.0/24'
+route_base_path = base_path + ['route', route_prefix]
+
+dummy_if1_addr = '192.168.30.1'
+dummy_if2_addr = '10.0.70.1'
+dummy_if3_addr = '10.20.0.1'
+
+
+class RoutesChecker:
+    def __init__(self, required_routes, allow_extra=False):
+        self.required_routes = required_routes
+        self.allow_extra = allow_extra
+        self.error = ''
+
+    def __call__(self, got_routes):
+        self.error = ''
+        if len(got_routes) < len(self.required_routes):
+            self.error = f"Not enough routes: expected {len(self.required_routes)}, got {len(got_routes)}: {got_routes}"
+            return False
+
+        if not self.allow_extra and len(got_routes) != len(self.required_routes):
+            self.error = f"Extra routes: expected {len(self.required_routes)}, got {len(got_routes)}: {got_routes}"
+
+        for route in self.required_routes:
+            found = False
+            for got_route in got_routes:
+                mismatch = False
+                for key in route:
+                    if route[key] is None:
+                        if key in got_route:
+                            mismatch = True
+                            break
+                    elif key not in got_route or got_route[key] != route[key]:
+                        mismatch = True
+                        break
+                if not mismatch:
+                    found = True
+                    break
+            if not found:
+                self.error = f"Couldn't find required route {route} among received routes {got_routes}"
+                return False
+        return True
+
+
+class TestProtocolsFailover(VyOSUnitTestSHIM.TestCase):
+    def clean_and_stop_daemon(self):
+        self.cli_delete(base_path)
+        for vrf in used_vrf_names:
+            self.cli_delete(vrf_base_path + [vrf] + base_path)
+        self.cli_commit()
+        shutil.rmtree(config_dir_root, ignore_errors=True)
+
+    def setUp(self):
+        # Needed dummy interfaces
+        self.cli_set(['interfaces', 'dummy', dummy_if1])
+        self.cli_set(['interfaces', 'dummy', dummy_if2])
+        self.cli_set(['interfaces', 'dummy', dummy_if3])
+
+        self.clean_and_stop_daemon()
+
+    def tearDown(self):
+        self.cli_delete(['interfaces', 'dummy', dummy_if3])
+        self.cli_delete(['interfaces', 'dummy', dummy_if2])
+        self.cli_delete(['interfaces', 'dummy', dummy_if1])
+
+        self.clean_and_stop_daemon()
+
+        failover_routes = ip_cmd(
+            f'route show proto {failover_protocol_value} table all'
+        )
+        self.assertEqual(failover_routes, [], "Some failover IPv4 routes left")
+        failover_routes = ip_cmd(
+            f'-6 route show proto {failover_protocol_value} table all'
+        )
+        self.assertEqual(failover_routes, [], "Some failover IPv6 routes left")
+
+    def wait_for_ip_output(self, ip_command_args, check, pause=0.1, timeout=3):
+        tries = ceil(timeout / pause)
+        result = None
+        for i in range(tries):
+            result = ip_cmd(ip_command_args)
+            if callable(check):
+                if check(result):
+                    return True, result
+            elif result == check:
+                return True, result
+
+            sleep(pause)
+
+        return False, result
+
+    def test_01_basic(self):
+        res, output = self.wait_for_ip_output(
+            f'route show proto {failover_protocol_value}',
+            [],
+            timeout=wait_timeout,
+        )
+        self.assertTrue(
+            res, f"No failover routes must exist before test, last result: {output}"
+        )
+
+        self.cli_set(
+            ['interfaces', 'dummy', dummy_if1, 'address', dummy_if1_addr + '/24']
+        )
+        self.cli_set(
+            ['interfaces', 'dummy', dummy_if2, 'address', dummy_if2_addr + '/24']
+        )
+        self.cli_set(
+            route_base_path + ['next-hop', dummy_if2_addr, 'interface', dummy_if2]
+        )
+        self.cli_set(route_base_path + ['next-hop', dummy_if2_addr, 'metric', '30'])
+        self.cli_set(
+            route_base_path
+            + [
+                'next-hop',
+                dummy_if2_addr,
+                'check',
+                'target',
+                dummy_if1_addr,
+                'interface',
+                dummy_if1,
+            ]
+        )
+        self.cli_set(
+            route_base_path
+            + ['next-hop', dummy_if2_addr, 'check', 'timeout', str(check_timeout)]
+        )
+        self.cli_commit()
+
+        # Now vyos-failover must be launched, it should create route, waiting for it...
+        checker = RoutesChecker([{'dst': route_prefix}])
+        res, output = self.wait_for_ip_output(
+            f"route show proto {failover_protocol_value}",
+            checker,
+            timeout=wait_timeout,
+        )
+        self.assertTrue(res, f"Route must have been created, last result: {output}")
+
+        self.cli_delete(['interfaces', 'dummy', dummy_if1, 'address'])
+        self.cli_commit()
+
+        res, output = self.wait_for_ip_output(
+            f'route show proto {failover_protocol_value}',
+            [],
+            timeout=wait_timeout,
+        )
+        self.assertTrue(res, f"Route must have been deleted, last result: {output}")
+
+    def test_02_vrf(self):
+        # route 1 default VRF, check red
+        # route 2 red, check blue
+        # route 3 red, check red
+        # route 1 and route 3 with same destination
+
+        res, output = self.wait_for_ip_output(
+            f'route show proto {failover_protocol_value} table all',
+            [],
+            timeout=wait_timeout,
+        )
+        self.assertTrue(
+            res, f"No failover routes must exist before test, last result: {output}"
+        )
+
+        self.cli_set(['vrf', 'name', red, 'table', '43310'])
+        self.cli_set(['vrf', 'name', blue, 'table', '43311'])
+
+        self.cli_set(
+            ['interfaces', 'dummy', dummy_if1, 'address', dummy_if1_addr + '/24']
+        )
+        self.cli_set(
+            ['interfaces', 'dummy', dummy_if2, 'address', dummy_if2_addr + '/24']
+        )
+        self.cli_set(
+            ['interfaces', 'dummy', dummy_if3, 'address', dummy_if3_addr + '/24']
+        )
+        self.cli_set(['interfaces', 'dummy', dummy_if2, 'vrf', red])
+        self.cli_set(['interfaces', 'dummy', dummy_if3, 'vrf', blue])
+
+        route_1_base = route_base_path
+        route_2_base = vrf_base_path + [red] + base_path + ['route', route2_prefix]
+        route_3_base = vrf_base_path + [red] + base_path + ['route', route_prefix]
+
+        self.cli_set(
+            route_1_base + ['next-hop', dummy_if1_addr, 'interface', dummy_if1]
+        )
+        self.cli_set(
+            route_1_base
+            + [
+                'next-hop',
+                dummy_if1_addr,
+                'check',
+                'target',
+                dummy_if2_addr,
+                'vrf',
+                red,
+            ]
+        )
+        self.cli_set(
+            route_1_base
+            + ['next-hop', dummy_if1_addr, 'check', 'timeout', str(check_timeout)]
+        )
+
+        self.cli_set(
+            route_2_base + ['next-hop', dummy_if2_addr, 'interface', dummy_if2]
+        )
+        self.cli_set(
+            route_2_base
+            + [
+                'next-hop',
+                dummy_if2_addr,
+                'check',
+                'target',
+                dummy_if3_addr,
+                'vrf',
+                blue,
+            ]
+        )
+        self.cli_set(
+            route_2_base
+            + ['next-hop', dummy_if2_addr, 'check', 'timeout', str(check_timeout)]
+        )
+
+        self.cli_set(
+            route_3_base + ['next-hop', dummy_if2_addr, 'interface', dummy_if2]
+        )
+        self.cli_set(
+            route_3_base
+            + [
+                'next-hop',
+                dummy_if2_addr,
+                'check',
+                'target',
+                dummy_if2_addr,
+            ]
+        )
+        self.cli_set(
+            route_3_base
+            + ['next-hop', dummy_if2_addr, 'check', 'timeout', str(check_timeout)]
+        )
+
+        self.cli_commit()
+
+        route1_fields = {'dst': route_prefix, 'gateway': dummy_if1_addr, 'table': None}
+        route2_fields = {
+            'dst': route2_prefix,
+            'gateway': dummy_if2_addr,
+            'table': red,
+        }
+        route3_fields = {'dst': route_prefix, 'gateway': dummy_if2_addr, 'table': red}
+
+        # All three routes must be created
+        checker1 = RoutesChecker([route1_fields, route2_fields, route3_fields])
+        res, output = self.wait_for_ip_output(
+            f"route show proto {failover_protocol_value} table all",
+            checker1,
+            timeout=wait_timeout * 3,
+        )
+        self.assertTrue(
+            res, f"Routes must have been created, checker error: {checker1.error}"
+        )
+
+        # Delete dummy_if3, route2 must be deleted
+        self.cli_delete(['interfaces', 'dummy', dummy_if3, 'address'])
+        self.cli_commit()
+
+        checker2 = RoutesChecker([route1_fields, route3_fields])
+        res, output = self.wait_for_ip_output(
+            f'route show proto {failover_protocol_value} table all',
+            checker2,
+            timeout=wait_timeout * 3,
+        )
+        self.assertTrue(
+            res,
+            f"Only route1 and route3 must have been left, checker error: {checker2.error}",
+        )
+
+        # Delete dummy_if2, all routes must be deleted
+        self.cli_delete(['interfaces', 'dummy', dummy_if2, 'address'])
+        self.cli_commit()
+        res, output = self.wait_for_ip_output(
+            f'route show proto {failover_protocol_value} table all',
+            [],
+            timeout=wait_timeout * 3,
+        )
+        self.assertTrue(res, f"No routes should have been left, got: {output}")
+
+    def test_03_config(self):
+        # Test how daemon reacts to routes add/delete, files add/delete
+        # All checks in this test are always true, configuration is added/deleted only
+
+        # route 1 default VRF
+        # route 2 default VRF
+        # route 3 red
+        route1_fields = {'dst': route_prefix, 'gateway': dummy_if1_addr, 'table': None}
+        route2_fields = {'dst': route2_prefix, 'gateway': dummy_if1_addr, 'table': None}
+        route3_fields = {'dst': route_prefix, 'gateway': dummy_if2_addr, 'table': red}
+
+        res, output = self.wait_for_ip_output(
+            f'route show proto {failover_protocol_value} table all',
+            [],
+            timeout=wait_timeout,
+        )
+        self.assertTrue(
+            res, f"No failover routes must exist before test, last result: {output}"
+        )
+
+        self.cli_set(['vrf', 'name', red, 'table', '43310'])
+        self.cli_set(['vrf', 'name', blue, 'table', '43311'])
+
+        self.cli_set(
+            ['interfaces', 'dummy', dummy_if1, 'address', dummy_if1_addr + '/24']
+        )
+        self.cli_set(
+            ['interfaces', 'dummy', dummy_if2, 'address', dummy_if2_addr + '/24']
+        )
+        self.cli_set(['interfaces', 'dummy', dummy_if2, 'vrf', red])
+
+        route_1_base = route_base_path
+        route_2_base = base_path + ['route', route2_prefix]
+        route_3_base = vrf_base_path + [red] + base_path + ['route', route_prefix]
+
+        self.cli_set(
+            route_1_base + ['next-hop', dummy_if1_addr, 'interface', dummy_if1]
+        )
+        self.cli_set(
+            route_1_base
+            + [
+                'next-hop',
+                dummy_if1_addr,
+                'check',
+                'target',
+                dummy_if1_addr,
+            ]
+        )
+        self.cli_set(
+            route_1_base
+            + ['next-hop', dummy_if1_addr, 'check', 'timeout', str(check_timeout)]
+        )
+        self.cli_commit()
+
+        # Adding only route1
+        checker1 = RoutesChecker([route1_fields])
+        res, output = self.wait_for_ip_output(
+            f"route show proto {failover_protocol_value} table all",
+            checker1,
+            timeout=wait_timeout * 3,
+        )
+        self.assertTrue(
+            res, f"Route 1 must have been created, checker error: {checker1.error}"
+        )
+
+        # adding route 3 - new file
+        self.cli_set(
+            route_3_base + ['next-hop', dummy_if2_addr, 'interface', dummy_if2]
+        )
+        self.cli_set(
+            route_3_base
+            + [
+                'next-hop',
+                dummy_if2_addr,
+                'check',
+                'target',
+                dummy_if2_addr,
+            ]
+        )
+        self.cli_set(
+            route_3_base
+            + ['next-hop', dummy_if2_addr, 'check', 'timeout', str(check_timeout)]
+        )
+        self.cli_commit()
+
+        # Now route1 and route3 must be active
+        checker13 = RoutesChecker([route1_fields, route3_fields])
+        res, output = self.wait_for_ip_output(
+            f"route show proto {failover_protocol_value} table all",
+            checker13,
+            timeout=wait_timeout * 3,
+        )
+        self.assertTrue(
+            res,
+            f"Route 1 and route 3 must have been created, checker error: {checker13.error}",
+        )
+
+        # Now add route2 (add of route to file)
+        self.cli_set(
+            route_2_base + ['next-hop', dummy_if1_addr, 'interface', dummy_if1]
+        )
+        self.cli_set(
+            route_2_base
+            + [
+                'next-hop',
+                dummy_if1_addr,
+                'check',
+                'target',
+                dummy_if1_addr,
+            ]
+        )
+        self.cli_set(
+            route_2_base
+            + ['next-hop', dummy_if1_addr, 'check', 'timeout', str(check_timeout)]
+        )
+        self.cli_commit()
+
+        # All three routes must be created
+        checker123 = RoutesChecker([route1_fields, route2_fields, route3_fields])
+        res, output = self.wait_for_ip_output(
+            f'route show proto {failover_protocol_value} table all',
+            checker123,
+            timeout=wait_timeout * 3,
+        )
+        self.assertTrue(
+            res,
+            f"All three routes must have been created, checker error: {checker123.error}",
+        )
+
+        # Delete route1
+        self.cli_delete(route_1_base)
+        self.cli_commit()
+
+        # Now route2 and route3 must be active
+        checker23 = RoutesChecker([route2_fields, route3_fields])
+        res, output = self.wait_for_ip_output(
+            f"route show proto {failover_protocol_value} table all",
+            checker23,
+            timeout=wait_timeout * 3,
+        )
+        self.assertTrue(
+            res,
+            f"Route 1 must have been deleted, routes 2 and 3 active. Checker error: {checker23.error}",
+        )
+
+        # Delete route2 - file deletion
+        self.cli_delete(route_2_base)
+        self.cli_commit()
+
+        # Now only route3 must be active
+        checker3 = RoutesChecker([route3_fields])
+        res, output = self.wait_for_ip_output(
+            f"route show proto {failover_protocol_value} table all",
+            checker3,
+            timeout=wait_timeout * 3,
+        )
+        self.assertTrue(
+            res,
+            f"Route 2 must have been deleted, only routes 3 should be active. Checker error: {checker3.error}",
+        )
+
+        # Deleting last route
+        self.cli_delete(route_3_base)
+        self.cli_commit()
+
+        res, output = self.wait_for_ip_output(
+            f'route show proto {failover_protocol_value} table all',
+            [],
+            timeout=wait_timeout * 3,
+        )
+        self.assertTrue(res, f"No routes should have been left, got: {output}")
+
+
+if __name__ == '__main__':
+    unittest.main(verbosity=2)
diff --git a/src/conf_mode/protocols_failover.py b/src/conf_mode/protocols_failover.py
index 9b50cc1ed..956c060f4 100755
--- a/src/conf_mode/protocols_failover.py
+++ b/src/conf_mode/protocols_failover.py
@@ -15,12 +15,15 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import json
+import os
 
 from pathlib import Path
+from sys import argv
 
 from vyos.config import Config
 from vyos.template import render
 from vyos.utils.process import call
+from vyos.utils.process import is_systemd_service_running
 from vyos import ConfigError
 from vyos import airbag
 
@@ -28,9 +31,24 @@ airbag.enable()
 
 
 service_name = 'vyos-failover'
-service_conf = Path(f'/run/{service_name}.conf')
+service_conf_dir = Path(f'/run/{service_name}.conf.d/')
 systemd_service = '/run/systemd/system/vyos-failover.service'
-rt_proto_failover = '/etc/iproute2/rt_protos.d/failover.conf'
+rt_proto_failover = Path('/etc/iproute2/rt_protos.d/failover.conf')
+
+
+def get_vrf_name():
+    if argv and len(argv) > 1:
+        return argv[1]
+    return None
+
+
+def get_service_conf_path():
+    vrf_name = get_vrf_name()
+    if vrf_name:
+        filename = f'vrf-{vrf_name}.conf'
+    else:
+        filename = 'default.conf'
+    return service_conf_dir / filename
 
 
 def get_config(config=None):
@@ -39,7 +57,14 @@ def get_config(config=None):
     else:
         conf = Config()
 
-    base = ['protocols', 'failover']
+    vrf_name = get_vrf_name()
+    if vrf_name:
+        base = ['vrf', 'name', vrf_name]
+    else:
+        base = []
+
+    base += ['protocols', 'failover']
+
     failover = conf.get_config_dict(base, key_mangling=('-', '_'),
                                     get_first_key=True)
 
@@ -47,6 +72,9 @@ def get_config(config=None):
     if failover.get('route') is not None:
         failover = conf.merge_defaults(failover, recursive=True)
 
+    if failover:
+        failover['vrf_context'] = vrf_name
+
     return failover
 
 def verify(failover):
@@ -75,33 +103,66 @@ def verify(failover):
             if check_type == 'tcp' and 'port' not in next_hop_config['check']:
                 raise ConfigError(f'Check port for next-hop "{next_hop}" and type TCP is mandatory!')
 
+            errors = {
+                'icmp': {},
+                'tcp': {
+                    'interface': 'Check target "interface" option does nothing for type TCP. Use "vrf" if needed',
+                },
+                'arp': {
+                    'vrf': 'Check target "vrf" option is incompatible with type ARP, use "interface" option if needed',
+                },
+            }
+
+            for target, target_config in next_hop_config['check']['target'].items():
+                for key, msg in errors[check_type].items():
+                    if key in target_config:
+                        raise ConfigError(msg)
+
     return None
 
+
 def generate(failover):
+    service_conf = get_service_conf_path()
     if not failover:
         service_conf.unlink(missing_ok=True)
+        try:
+            os.rmdir(service_conf_dir)
+        # Ignore if directory doesn't exist
+        # or not empty (probably configs for other VRFs are there)
+        except (FileNotFoundError, OSError):
+            pass
         return None
 
     # Add own rt_proto 'failover'
     # Helps to detect all own routes 'proto failover'
-    with open(rt_proto_failover, 'w') as f:
-        f.write('111  failover\n')
+    rt_proto_failover.write_text('111  failover\n')
+
+    service_conf_dir.mkdir(exist_ok=True)
 
     # Write configuration file
     conf_json = json.dumps(failover, indent=4)
     service_conf.write_text(conf_json)
-    render(systemd_service, 'protocols/systemd_vyos_failover_service.j2', failover)
+    render(
+        systemd_service,
+        'protocols/systemd_vyos_failover_service.j2',
+        {'config_dir': str(service_conf_dir)},
+    )
 
     return None
 
 def apply(failover):
-    if not failover:
+    # If directory is removed - we can stop the service
+    if not service_conf_dir.is_dir():
         call(f'systemctl stop {service_name}.service')
-        call('ip route flush protocol failover')
-    else:
         call('systemctl daemon-reload')
-        call(f'systemctl restart {service_name}.service')
-        call(f'ip route flush protocol failover')
+    # Otherwise even if `failover` is False, service is
+    # still needed for other VRFs.
+    else:
+        # Daemon watches for configuration updates, so we need only
+        # to start it if it is not started yet
+        if not is_systemd_service_running(service_name):
+            call('systemctl daemon-reload')
+            call(f'systemctl start {service_name}.service')
 
     return None
 
diff --git a/src/helpers/vyos-failover.py b/src/helpers/vyos-failover.py
index 96db947e1..d255f2303 100755
--- a/src/helpers/vyos-failover.py
+++ b/src/helpers/vyos-failover.py
@@ -15,22 +15,49 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import argparse
+import atexit
 import json
-import socket
+import signal
 import time
 
+from collections import namedtuple
 from vyos.utils.process import rc_cmd
+from vyos.utils.process import run
 from pathlib import Path
 from systemd import journal
 
 
 my_name = Path(__file__).stem
 
+# Timeout between configuration reading
+# When no checks timeouts worked (e.g. no config files)
+config_timeout = 1
 
-def is_route_exists(route, gateway, interface, metric):
+# Useful debug info to console, use debug = True
+# sudo systemctl stop vyos-failover.service
+# sudo /usr/libexec/vyos/vyos-failover.py --config /run/vyos-failover.conf
+debug = False
+debug_output_journal = False
+debug_output_print = True
+
+
+def print_debug(*args, **kwargs):
+    if debug:
+        if debug_output_print:
+            print(*args, **kwargs)
+        if debug_output_journal:
+            journal.send(*args, **kwargs, SYSLOG_IDENTIFIER=my_name)
+
+
+def wrap_vrf(command, vrf):
+    if not vrf:
+        return command
+    return f"sudo ip vrf exec {vrf} {command}"
+
+
+def is_route_exists(ip_args):
     """Check if route with expected gateway, dev and metric exists"""
-    rc, data = rc_cmd(f'ip --json route show protocol failover {route} '
-                      f'via {gateway} dev {interface} metric {metric}')
+    rc, data = rc_cmd(f'ip --json route show {ip_args}')
     if rc == 0:
         data = json.loads(data)
         if len(data) > 0:
@@ -38,41 +65,7 @@ def is_route_exists(route, gateway, interface, metric):
     return False
 
 
-def get_best_route_options(route, debug=False):
-    """
-    Return current best route ('gateway, interface, metric)
-
-    % get_best_route_options('203.0.113.1')
-    ('192.168.0.1', 'eth1', 1)
-
-    % get_best_route_options('203.0.113.254')
-    (None, None, None)
-    """
-    rc, data = rc_cmd(f'ip --detail --json route show protocol failover {route}')
-    if rc == 0:
-        data = json.loads(data)
-        if len(data) == 0:
-            print(f'\nRoute {route} for protocol failover was not found')
-            return None, None, None
-        # Fake metric 999 by default
-        # Search route with the lowest metric
-        best_metric = 999
-        for entry in data:
-            if debug: print('\n', entry)
-            metric = entry.get('metric')
-            gateway = entry.get('gateway')
-            iface = entry.get('dev')
-            if metric < best_metric:
-                best_metric = metric
-                best_gateway = gateway
-                best_interface = iface
-        if debug:
-            print(f'### Best_route exists: {route}, best_gateway: {best_gateway}, '
-                  f'best_metric: {best_metric}, best_iface: {best_interface}')
-        return best_gateway, best_interface, best_metric
-
-
-def is_port_open(ip, port):
+def is_port_open(ip, port, vrf=''):
     """
     Check connection to remote host and port
     Return True if host alive
@@ -80,33 +73,29 @@ def is_port_open(ip, port):
     % is_port_open('example.com', 8080)
     True
     """
-    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP)
-    s.settimeout(2)
-    try:
-        s.connect((ip, int(port)))
-        s.shutdown(socket.SHUT_RDWR)
-        return True
-    except:
-        return False
-    finally:
-        s.close()
-
-
-def is_target_alive(target_list=None,
-                    iface='',
-                    proto='icmp',
-                    port=None,
-                    debug=False,
-                    policy='any-available') -> bool:
-    """Check the availability of each target in the target_list using
+    cmd = wrap_vrf(f"nc -w2 -z {ip} {port}", vrf)
+    rc, data = rc_cmd(cmd)
+    return rc == 0
+
+
+def is_target_alive(
+    targets=None,
+    iface='',
+    proto='icmp',
+    port=None,
+    nexthop_vrf='',
+    policy='any-available',
+) -> bool:
+    """Check the availability of each target in the target_dict using
     the specified protocol ICMP, ARP, TCP
 
     Args:
-        target_list (list): A list of IP addresses or hostnames to check.
+        targets (tuple of TargetNamedTuple): A dict: keys are IP addresses to check, values - dicts with options.
+            Possible keys (all optional): 'vrf' and 'interface'.
         iface (str): The name of the network interface to use for the check.
         proto (str): The protocol to use for the check. Options are 'icmp', 'arp', or 'tcp'.
         port (int): The port number to use for the TCP check. Only applicable if proto is 'tcp'.
-        debug (bool): If True, print debug information during the check.
+        nexthop_vrf (str): Nexthop VRF name - if specific vrf is not given for target, use this one
         policy (str): The policy to use for the check. Options are 'any-available' or 'all-available'.
 
     Returns:
@@ -120,30 +109,41 @@ def is_target_alive(target_list=None,
         iface = f'-I {iface}'
 
     num_reachable_targets = 0
-    for target in target_list:
+    for options in targets:
+        target = options.target
+        vrf = options.vrf if options.vrf else nexthop_vrf
+        # don't use nexthop interface if 'vrf' is given
+        iface_opt = iface if vrf == nexthop_vrf else ''
+        # in any case if 'interface' is given, use it
+        if options.interface:
+            iface_opt = f'-I {options.interface}'
         match proto:
             case 'icmp':
-                command = f'/usr/bin/ping -q {target} {iface} -n -c 2 -W 1'
+                command = f'/usr/bin/ping -q {target} {iface_opt} -n -c 2 -W 1'
+                command = wrap_vrf(command, vrf)
                 rc, response = rc_cmd(command)
-                if debug:
-                    print(f'    [ CHECK-TARGET ]: [{command}] -- return-code [RC: {rc}]')
+                print_debug(
+                    f'    [ CHECK-TARGET ]: [{command}] -- return-code [RC: {rc}]'
+                )
                 if rc == 0:
                     num_reachable_targets += 1
                     if policy == 'any-available':
                         return True
 
             case 'arp':
-                command = f'/usr/bin/arping -b -c 2 -f -w 1 -i 1 {iface} {target}'
+                command = f'/usr/bin/arping -b -c 2 -f -w 1 -i 1 {iface_opt} {target}'
+                command = wrap_vrf(command, vrf)
                 rc, response = rc_cmd(command)
-                if debug:
-                    print(f'    [ CHECK-TARGET ]: [{command}] -- return-code [RC: {rc}]')
+                print_debug(
+                    f'    [ CHECK-TARGET ]: [{command}] -- return-code [RC: {rc}]'
+                )
                 if rc == 0:
                     num_reachable_targets += 1
                     if policy == 'any-available':
                         return True
 
             case _ if proto == 'tcp' and port is not None:
-                if is_port_open(target, port):
+                if is_port_open(target, port, vrf=vrf):
                     num_reachable_targets += 1
                     if policy == 'any-available':
                         return True
@@ -151,85 +151,349 @@ def is_target_alive(target_list=None,
             case _:
                 return False
 
-        if policy == 'all-available' and num_reachable_targets == len(target_list):
+        if policy == 'all-available' and num_reachable_targets == len(targets):
             return True
 
     return False
 
 
-if __name__ == '__main__':
-    # Parse command arguments and get config
-    parser = argparse.ArgumentParser()
-    parser.add_argument('-c',
-                        '--config',
-                        action='store',
-                        help='Path to protocols failover configuration',
-                        required=True,
-                        type=Path)
+TargetNamedTuple = namedtuple(
+    'TargetConfig',
+    [
+        'target',
+        'vrf',
+        'interface',
+    ],
+)
+
+NextHopNamedTuple = namedtuple(
+    'NextHopConfig',
+    [
+        'route',
+        'next_hop',
+        'vrf',
+        'vrf_opt',
+        'conf_iface',
+        'conf_metric',
+        'port',
+        'port_opt',
+        'policy',
+        'proto',
+        'targets',
+        'pretty_targets',
+        'timeout',
+        'onlink',
+    ],
+)
+
+
+def get_nexthop_config_vars(destination, vrf, vrf_opt, nexthop_config, next_hop):
+    port = nexthop_config.get('check').get('port')
+
+    targets = tuple(
+        TargetNamedTuple(
+            target=target,
+            vrf=target_config.get('vrf', None),
+            interface=target_config.get('interface', None),
+        )
+        for target, target_config in nexthop_config.get('check').get('target').items()
+    )
+
+    # for print to journal and debug
+    pretty_targets = []
+    for target in targets:
+        p = target.target
+        options = []
+        if target.vrf:
+            options.append(f"vrf: {target.vrf}")
+        if target.interface:
+            options.append(f"interface: {target.interface}")
+        if options:
+            p += ' (' + ', '.join(options) + ')'
+        pretty_targets.append(p)
+    pretty_targets = ', '.join(pretty_targets)
+
+    return NextHopNamedTuple(
+        route=destination,
+        next_hop=next_hop,
+        vrf=vrf,
+        vrf_opt=vrf_opt,
+        conf_iface=nexthop_config.get('interface'),
+        conf_metric=int(nexthop_config.get('metric')),
+        port=port,
+        port_opt=f'port {port}' if port else '',
+        policy=nexthop_config.get('check').get('policy'),
+        proto=nexthop_config.get('check').get('type'),
+        targets=targets,
+        pretty_targets=pretty_targets,
+        timeout=nexthop_config.get('check').get('timeout'),
+        onlink='onlink' if 'onlink' in nexthop_config else '',
+    )
+
+
+RouteNamedTuple = namedtuple(
+    'RouteConfig',
+    [
+        'destination',
+        'vrf',
+        'vrf_opt',
+        'config_path',
+        'nexthops',
+    ],
+)
+
+
+def get_route_config(route, route_config, config_path, vrf):
+    vrf_opt = f'vrf {vrf}' if vrf else ''
+    nexthops = tuple(
+        get_nexthop_config_vars(route, vrf, vrf_opt, nexthop_config, next_hop)
+        for next_hop, nexthop_config in route_config.get('next_hop').items()
+    )
+    return RouteNamedTuple(
+        destination=route,
+        vrf=vrf,
+        vrf_opt=vrf_opt,
+        config_path=config_path,
+        nexthops=nexthops,
+    )
+
+
+def parse_config(config, path):
+    parsed = []
+    vrf = config.get('vrf_context', '')
+    for route, route_config in config.get('route').items():
+        parsed.append(get_route_config(route, route_config, path, vrf))
+    return parsed
+
+
+def flush_all_routes():
+    print_debug("flush_all_routes called")
+    flush_cmd = 'ip route flush protocol failover table all'
+    run(flush_cmd)
+    journal.send(
+        flush_cmd,
+        SYSLOG_IDENTIFIER=my_name,
+    )
+
+
+kill_called = False
+
+
+def kill_handler(*args):
+    global kill_called
+    if kill_called:
+        return
+    kill_called = True
+    print_debug(f"kill_handler called for signal {args[0]}")
+
+
+def get_ip_command_args(nhc):
+    return (
+        f'{nhc.route} via {nhc.next_hop} dev {nhc.conf_iface} '
+        f'{nhc.onlink} metric {nhc.conf_metric} {nhc.vrf_opt} proto failover'
+    )
+
+
+def delete_route(ip_args):
+    print_debug(f'    [ DEL ] -- ip route del {ip_args} [DELETE]')
+    rc_cmd(f'ip route del {ip_args}')
+    journal.send(
+        f'ip route del {ip_args}',
+        SYSLOG_IDENTIFIER=my_name,
+    )
+
+
+def update_configuration(last_modification_times, all_routes, config_dir):
+    """
+    Updates configuration:
+        rechecks config_dir for new/updated files,
+        deletes routes that were deleted from configuration
+
+    Args:
+        last_modification_times(dict): keys: relative path to file, value: last modification time.
+            Is updated.
+        all_routes(list): list of routes that were configured in previous call.
+            Is updated.
+        config_dir(Path): path to configuration directory
+    """
 
-    args = parser.parse_args()
     try:
-        config_path = Path(args.config)
-        config = json.loads(config_path.read_text())
-    except Exception as err:
-        print(
-            f'Configuration file "{config_path}" does not exist or malformed: {err}'
-        )
+        # First check if there are any changes at all
+        have_changes = False
+        for child in config_dir.iterdir():
+            file_key = str(child)
+            if file_key not in last_modification_times:
+                have_changes = True
+                print_debug(f"New file '{child}', have changes, rereading all")
+                break
+            modtime = child.stat().st_mtime_ns
+            if modtime != last_modification_times[file_key]:
+                have_changes = True
+                print_debug(f"File '{child} modified, have changes, rereading all...")
+                break
+
+        if not have_changes:
+            print_debug("No changes in configuration detected.")
+            return
+
+        last_modification_times.clear()
+        new_routes = []
+
+        # It is important that in configuration directory there MUST be
+        # only files generated by conf_mode/protocols_failover.py - otherwise
+        # the script won't be able to detect when all VRFs are disabled and
+        # won't be able to stop the service gracefully
+        for child in config_dir.iterdir():
+            if not child.is_file():
+                print(
+                    f"Path {child} under configuration dir is not a file! Please clean configuration directory {config_dir}."
+                )
+                exit(1)
+
+            modtime = child.stat().st_mtime_ns
+            file_key = str(child)
+            last_modification_times[file_key] = modtime
+
+            try:
+                config = json.loads(child.read_text())
+                print_debug(f"Config from '{child}': {config}")
+            except OSError as err:
+                print(f'Configuration file "{child}" could not be read: {err}')
+                exit(1)
+            except json.JSONDecodeError as err:
+                print(
+                    f'Configuration file "{child}" could not be parsed as JSON: {err}'
+                )
+                exit(1)
+            except UnicodeDecodeError as err:
+                print(f'Configuration file "{child}" has Unicode errors: {err}')
+                exit(1)
+
+            parsed_config = parse_config(config, file_key)
+            new_routes.extend(parsed_config)
+    except OSError as err:
+        print(f'Configuration dir "{config_dir}" does not exist or not readable: {err}')
         exit(1)
 
-    # Useful debug info to console, use debug = True
-    # sudo systemctl stop vyos-failover.service
-    # sudo /usr/libexec/vyos/vyos-failover.py --config /run/vyos-failover.conf
-    debug = False
+    old_routes_set = set(all_routes)
+    new_routes_set = set(new_routes)
+
+    delete_routes = old_routes_set - new_routes_set
+    add_routes = new_routes_set - old_routes_set
+
+    # Delete not needed routes
+    for route_config in delete_routes:
+        print_debug(
+            f"Deleting route {route_config}, not present in updated configuration"
+        )
+        for nhc in route_config.nexthops:
+            ip_args = get_ip_command_args(nhc)
+            if is_route_exists(ip_args):
+                delete_route(ip_args)
+        all_routes.remove(route_config)
+
+    # Add new routes
+    print_debug(f"Adding routes {add_routes}, new in updated configuration")
+    all_routes.extend(add_routes)
+
+    print_debug(f"All routes: {all_routes}")
 
-    while(True):
 
-        for route, route_config in config.get('route').items():
+if __name__ == '__main__':
+    print_debug(f"{my_name} started")
 
-            exists_gateway, exists_iface, exists_metric =  get_best_route_options(route, debug=debug)
+    # Parse command arguments and get config
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '-c',
+        '--config-dir',
+        action='store',
+        help='Path to protocols failover configuration dir',
+        required=True,
+        type=Path,
+    )
 
-            for next_hop, nexthop_config in route_config.get('next_hop').items():
-                conf_iface = nexthop_config.get('interface')
-                conf_metric = int(nexthop_config.get('metric'))
-                port = nexthop_config.get('check').get('port')
-                port_opt = f'port {port}' if port else ''
-                policy = nexthop_config.get('check').get('policy')
-                proto = nexthop_config.get('check').get('type')
-                target = nexthop_config.get('check').get('target')
-                timeout = nexthop_config.get('check').get('timeout')
-                onlink = 'onlink' if 'onlink' in nexthop_config else ''
+    args = parser.parse_args()
+    config_dir = Path(args.config_dir)
+
+    last_modification_times = {}
+    all_routes = []
+
+
+    # Clean all `failover` routes now and at exit
+    flush_all_routes()
+    atexit.register(flush_all_routes)
+    signal.signal(signal.SIGINT, kill_handler)
+    signal.signal(signal.SIGTERM, kill_handler)
+
+    had_sleeps = True
+    while not kill_called:
+        # Check in case daemon was launched without routes
+        if not had_sleeps:
+            time.sleep(int(config_timeout))
+
+        update_configuration(last_modification_times, all_routes, config_dir)
+        had_sleeps = False
+
+        for route_config in all_routes:
+            if kill_called:
+                break
+            route = route_config.destination
+            vrf = route_config.vrf
+            vrf_opt = route_config.vrf_opt
+
+            for nhc in route_config.nexthops:
+                next_hop = nhc.next_hop
+                ip_args = get_ip_command_args(nhc)
+
+                is_alive = is_target_alive(
+                    nhc.targets,
+                    nhc.conf_iface,
+                    nhc.proto,
+                    nhc.port,
+                    nexthop_vrf=vrf,
+                    policy=nhc.policy,
+                )
 
                 # Route not found in the current routing table
-                if not is_route_exists(route, next_hop, conf_iface, conf_metric):
-                    if debug: print(f"    [NEW_ROUTE_DETECTED] route: [{route}]")
+                if not is_route_exists(ip_args):
+                    print_debug(f"    [NEW_ROUTE_DETECTED] route: [{route} {vrf_opt}]")
                     # Add route if check-target alive
-                    if is_target_alive(target, conf_iface, proto, port, debug=debug, policy=policy):
-                        if debug: print(f'    [ ADD ] -- ip route add {route} via {next_hop} dev {conf_iface} '
-                                        f'metric {conf_metric} proto failover\n###')
-                        rc, command = rc_cmd(f'ip route add {route} via {next_hop} dev {conf_iface} '
-                                             f'{onlink} metric {conf_metric} proto failover')
+                    if is_alive:
+                        print_debug(f'    [ ADD ] -- ip route add {ip_args}\n###')
+                        rc, command = rc_cmd(f'ip route add {ip_args}')
                         # If something is wrong and gateway not added
                         # Example: Error: Next-hop has invalid gateway.
-                        if rc !=0:
-                            if debug: print(f'{command} -- return-code [RC: {rc}] {next_hop} dev {conf_iface}')
+                        if rc != 0:
+                            print_debug(
+                                f'{command} -- return-code [RC: {rc}] {next_hop} dev {nhc.conf_iface}'
+                            )
                         else:
-                            journal.send(f'ip route add {route} via {next_hop} dev {conf_iface} '
-                                         f'{onlink} metric {conf_metric} proto failover', SYSLOG_IDENTIFIER=my_name)
+                            journal.send(
+                                f'ip route add {ip_args}',
+                                SYSLOG_IDENTIFIER=my_name,
+                            )
                     else:
-                        if debug: print(f'    [ TARGET_FAIL ] target checks fails for [{target}], do nothing')
-                        journal.send(f'Check fail for route {route} target {target} proto {proto} '
-                                     f'{port_opt}', SYSLOG_IDENTIFIER=my_name)
-
-                # Route was added, check if the target is alive
-                # We should delete route if check fails only if route exists in the routing table
-                if not is_target_alive(target, conf_iface, proto, port, debug=debug, policy=policy) and \
-                        is_route_exists(route, next_hop, conf_iface, conf_metric):
-                    if debug:
-                        print(f'Nexh_hop {next_hop} fail, target not response')
-                        print(f'    [ DEL ] -- ip route del {route} via {next_hop} dev {conf_iface} '
-                              f'metric {conf_metric} proto failover [DELETE]')
-                    rc_cmd(f'ip route del {route} via {next_hop} dev {conf_iface} metric {conf_metric} proto failover')
-                    journal.send(f'ip route del {route} via {next_hop} dev {conf_iface} '
-                                 f'metric {conf_metric} proto failover', SYSLOG_IDENTIFIER=my_name)
-
-                time.sleep(int(timeout))
+                        print_debug(
+                            f'    [ TARGET_FAIL ] target checks fails for [{nhc.pretty_targets}], do nothing'
+                        )
+                        journal.send(
+                            f'Check fail for route {route} target {nhc.pretty_targets} proto {nhc.proto} '
+                            f'{nhc.port_opt}',
+                            SYSLOG_IDENTIFIER=my_name,
+                        )
+                else:
+                    # Route was added, check if the target is alive
+                    # We should delete route if check fails only if route exists in the routing table
+                    if not is_alive:
+                        print_debug(
+                            f"Next_hop {next_hop} fail, target check didn't pass"
+                        )
+                        delete_route(ip_args)
+
+                had_sleeps = True
+                time.sleep(int(nhc.timeout))
+                if kill_called:
+                    break
+
+    print_debug(f"Out of main loop, {kill_called=}")