Merge pull request #4749 from hedrok/T7803-failover-vrf

T7803: Make failover route vrf-aware

2 files changed, 466 insertions, 141 deletions

diff --git a/src/conf_mode/protocols_failover.py b/src/conf_mode/protocols_failover.py
index 9b50cc1ed..956c060f4 100755
--- a/src/conf_mode/protocols_failover.py
+++ b/src/conf_mode/protocols_failover.py
@@ -15,12 +15,15 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import json
+import os
 
 from pathlib import Path
+from sys import argv
 
 from vyos.config import Config
 from vyos.template import render
 from vyos.utils.process import call
+from vyos.utils.process import is_systemd_service_running
 from vyos import ConfigError
 from vyos import airbag
 
@@ -28,9 +31,24 @@ airbag.enable()
 
 
 service_name = 'vyos-failover'
-service_conf = Path(f'/run/{service_name}.conf')
+service_conf_dir = Path(f'/run/{service_name}.conf.d/')
 systemd_service = '/run/systemd/system/vyos-failover.service'
-rt_proto_failover = '/etc/iproute2/rt_protos.d/failover.conf'
+rt_proto_failover = Path('/etc/iproute2/rt_protos.d/failover.conf')
+
+
+def get_vrf_name():
+    if argv and len(argv) > 1:
+        return argv[1]
+    return None
+
+
+def get_service_conf_path():
+    vrf_name = get_vrf_name()
+    if vrf_name:
+        filename = f'vrf-{vrf_name}.conf'
+    else:
+        filename = 'default.conf'
+    return service_conf_dir / filename
 
 
 def get_config(config=None):
@@ -39,7 +57,14 @@ def get_config(config=None):
     else:
         conf = Config()
 
-    base = ['protocols', 'failover']
+    vrf_name = get_vrf_name()
+    if vrf_name:
+        base = ['vrf', 'name', vrf_name]
+    else:
+        base = []
+
+    base += ['protocols', 'failover']
+
     failover = conf.get_config_dict(base, key_mangling=('-', '_'),
                                     get_first_key=True)
 
@@ -47,6 +72,9 @@ def get_config(config=None):
     if failover.get('route') is not None:
         failover = conf.merge_defaults(failover, recursive=True)
 
+    if failover:
+        failover['vrf_context'] = vrf_name
+
     return failover
 
 def verify(failover):
@@ -75,33 +103,66 @@ def verify(failover):
             if check_type == 'tcp' and 'port' not in next_hop_config['check']:
                 raise ConfigError(f'Check port for next-hop "{next_hop}" and type TCP is mandatory!')
 
+            errors = {
+                'icmp': {},
+                'tcp': {
+                    'interface': 'Check target "interface" option does nothing for type TCP. Use "vrf" if needed',
+                },
+                'arp': {
+                    'vrf': 'Check target "vrf" option is incompatible with type ARP, use "interface" option if needed',
+                },
+            }
+
+            for target, target_config in next_hop_config['check']['target'].items():
+                for key, msg in errors[check_type].items():
+                    if key in target_config:
+                        raise ConfigError(msg)
+
     return None
 
+
 def generate(failover):
+    service_conf = get_service_conf_path()
     if not failover:
         service_conf.unlink(missing_ok=True)
+        try:
+            os.rmdir(service_conf_dir)
+        # Ignore if directory doesn't exist
+        # or not empty (probably configs for other VRFs are there)
+        except (FileNotFoundError, OSError):
+            pass
         return None
 
     # Add own rt_proto 'failover'
     # Helps to detect all own routes 'proto failover'
-    with open(rt_proto_failover, 'w') as f:
-        f.write('111  failover\n')
+    rt_proto_failover.write_text('111  failover\n')
+
+    service_conf_dir.mkdir(exist_ok=True)
 
     # Write configuration file
     conf_json = json.dumps(failover, indent=4)
     service_conf.write_text(conf_json)
-    render(systemd_service, 'protocols/systemd_vyos_failover_service.j2', failover)
+    render(
+        systemd_service,
+        'protocols/systemd_vyos_failover_service.j2',
+        {'config_dir': str(service_conf_dir)},
+    )
 
     return None
 
 def apply(failover):
-    if not failover:
+    # If directory is removed - we can stop the service
+    if not service_conf_dir.is_dir():
         call(f'systemctl stop {service_name}.service')
-        call('ip route flush protocol failover')
-    else:
         call('systemctl daemon-reload')
-        call(f'systemctl restart {service_name}.service')
-        call(f'ip route flush protocol failover')
+    # Otherwise even if `failover` is False, service is
+    # still needed for other VRFs.
+    else:
+        # Daemon watches for configuration updates, so we need only
+        # to start it if it is not started yet
+        if not is_systemd_service_running(service_name):
+            call('systemctl daemon-reload')
+            call(f'systemctl start {service_name}.service')
 
     return None
 
diff --git a/src/helpers/vyos-failover.py b/src/helpers/vyos-failover.py
index 96db947e1..d255f2303 100755
--- a/src/helpers/vyos-failover.py
+++ b/src/helpers/vyos-failover.py
@@ -15,22 +15,49 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import argparse
+import atexit
 import json
-import socket
+import signal
 import time
 
+from collections import namedtuple
 from vyos.utils.process import rc_cmd
+from vyos.utils.process import run
 from pathlib import Path
 from systemd import journal
 
 
 my_name = Path(__file__).stem
 
+# Timeout between configuration reading
+# When no checks timeouts worked (e.g. no config files)
+config_timeout = 1
 
-def is_route_exists(route, gateway, interface, metric):
+# Useful debug info to console, use debug = True
+# sudo systemctl stop vyos-failover.service
+# sudo /usr/libexec/vyos/vyos-failover.py --config /run/vyos-failover.conf
+debug = False
+debug_output_journal = False
+debug_output_print = True
+
+
+def print_debug(*args, **kwargs):
+    if debug:
+        if debug_output_print:
+            print(*args, **kwargs)
+        if debug_output_journal:
+            journal.send(*args, **kwargs, SYSLOG_IDENTIFIER=my_name)
+
+
+def wrap_vrf(command, vrf):
+    if not vrf:
+        return command
+    return f"sudo ip vrf exec {vrf} {command}"
+
+
+def is_route_exists(ip_args):
     """Check if route with expected gateway, dev and metric exists"""
-    rc, data = rc_cmd(f'ip --json route show protocol failover {route} '
-                      f'via {gateway} dev {interface} metric {metric}')
+    rc, data = rc_cmd(f'ip --json route show {ip_args}')
     if rc == 0:
         data = json.loads(data)
         if len(data) > 0:
@@ -38,41 +65,7 @@ def is_route_exists(route, gateway, interface, metric):
     return False
 
 
-def get_best_route_options(route, debug=False):
-    """
-    Return current best route ('gateway, interface, metric)
-
-    % get_best_route_options('203.0.113.1')
-    ('192.168.0.1', 'eth1', 1)
-
-    % get_best_route_options('203.0.113.254')
-    (None, None, None)
-    """
-    rc, data = rc_cmd(f'ip --detail --json route show protocol failover {route}')
-    if rc == 0:
-        data = json.loads(data)
-        if len(data) == 0:
-            print(f'\nRoute {route} for protocol failover was not found')
-            return None, None, None
-        # Fake metric 999 by default
-        # Search route with the lowest metric
-        best_metric = 999
-        for entry in data:
-            if debug: print('\n', entry)
-            metric = entry.get('metric')
-            gateway = entry.get('gateway')
-            iface = entry.get('dev')
-            if metric < best_metric:
-                best_metric = metric
-                best_gateway = gateway
-                best_interface = iface
-        if debug:
-            print(f'### Best_route exists: {route}, best_gateway: {best_gateway}, '
-                  f'best_metric: {best_metric}, best_iface: {best_interface}')
-        return best_gateway, best_interface, best_metric
-
-
-def is_port_open(ip, port):
+def is_port_open(ip, port, vrf=''):
     """
     Check connection to remote host and port
     Return True if host alive
@@ -80,33 +73,29 @@ def is_port_open(ip, port):
     % is_port_open('example.com', 8080)
     True
     """
-    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP)
-    s.settimeout(2)
-    try:
-        s.connect((ip, int(port)))
-        s.shutdown(socket.SHUT_RDWR)
-        return True
-    except:
-        return False
-    finally:
-        s.close()
-
-
-def is_target_alive(target_list=None,
-                    iface='',
-                    proto='icmp',
-                    port=None,
-                    debug=False,
-                    policy='any-available') -> bool:
-    """Check the availability of each target in the target_list using
+    cmd = wrap_vrf(f"nc -w2 -z {ip} {port}", vrf)
+    rc, data = rc_cmd(cmd)
+    return rc == 0
+
+
+def is_target_alive(
+    targets=None,
+    iface='',
+    proto='icmp',
+    port=None,
+    nexthop_vrf='',
+    policy='any-available',
+) -> bool:
+    """Check the availability of each target in the target_dict using
     the specified protocol ICMP, ARP, TCP
 
     Args:
-        target_list (list): A list of IP addresses or hostnames to check.
+        targets (tuple of TargetNamedTuple): A dict: keys are IP addresses to check, values - dicts with options.
+            Possible keys (all optional): 'vrf' and 'interface'.
         iface (str): The name of the network interface to use for the check.
         proto (str): The protocol to use for the check. Options are 'icmp', 'arp', or 'tcp'.
         port (int): The port number to use for the TCP check. Only applicable if proto is 'tcp'.
-        debug (bool): If True, print debug information during the check.
+        nexthop_vrf (str): Nexthop VRF name - if specific vrf is not given for target, use this one
         policy (str): The policy to use for the check. Options are 'any-available' or 'all-available'.
 
     Returns:
@@ -120,30 +109,41 @@ def is_target_alive(target_list=None,
         iface = f'-I {iface}'
 
     num_reachable_targets = 0
-    for target in target_list:
+    for options in targets:
+        target = options.target
+        vrf = options.vrf if options.vrf else nexthop_vrf
+        # don't use nexthop interface if 'vrf' is given
+        iface_opt = iface if vrf == nexthop_vrf else ''
+        # in any case if 'interface' is given, use it
+        if options.interface:
+            iface_opt = f'-I {options.interface}'
         match proto:
             case 'icmp':
-                command = f'/usr/bin/ping -q {target} {iface} -n -c 2 -W 1'
+                command = f'/usr/bin/ping -q {target} {iface_opt} -n -c 2 -W 1'
+                command = wrap_vrf(command, vrf)
                 rc, response = rc_cmd(command)
-                if debug:
-                    print(f'    [ CHECK-TARGET ]: [{command}] -- return-code [RC: {rc}]')
+                print_debug(
+                    f'    [ CHECK-TARGET ]: [{command}] -- return-code [RC: {rc}]'
+                )
                 if rc == 0:
                     num_reachable_targets += 1
                     if policy == 'any-available':
                         return True
 
             case 'arp':
-                command = f'/usr/bin/arping -b -c 2 -f -w 1 -i 1 {iface} {target}'
+                command = f'/usr/bin/arping -b -c 2 -f -w 1 -i 1 {iface_opt} {target}'
+                command = wrap_vrf(command, vrf)
                 rc, response = rc_cmd(command)
-                if debug:
-                    print(f'    [ CHECK-TARGET ]: [{command}] -- return-code [RC: {rc}]')
+                print_debug(
+                    f'    [ CHECK-TARGET ]: [{command}] -- return-code [RC: {rc}]'
+                )
                 if rc == 0:
                     num_reachable_targets += 1
                     if policy == 'any-available':
                         return True
 
             case _ if proto == 'tcp' and port is not None:
-                if is_port_open(target, port):
+                if is_port_open(target, port, vrf=vrf):
                     num_reachable_targets += 1
                     if policy == 'any-available':
                         return True
@@ -151,85 +151,349 @@ def is_target_alive(target_list=None,
             case _:
                 return False
 
-        if policy == 'all-available' and num_reachable_targets == len(target_list):
+        if policy == 'all-available' and num_reachable_targets == len(targets):
             return True
 
     return False
 
 
-if __name__ == '__main__':
-    # Parse command arguments and get config
-    parser = argparse.ArgumentParser()
-    parser.add_argument('-c',
-                        '--config',
-                        action='store',
-                        help='Path to protocols failover configuration',
-                        required=True,
-                        type=Path)
+TargetNamedTuple = namedtuple(
+    'TargetConfig',
+    [
+        'target',
+        'vrf',
+        'interface',
+    ],
+)
+
+NextHopNamedTuple = namedtuple(
+    'NextHopConfig',
+    [
+        'route',
+        'next_hop',
+        'vrf',
+        'vrf_opt',
+        'conf_iface',
+        'conf_metric',
+        'port',
+        'port_opt',
+        'policy',
+        'proto',
+        'targets',
+        'pretty_targets',
+        'timeout',
+        'onlink',
+    ],
+)
+
+
+def get_nexthop_config_vars(destination, vrf, vrf_opt, nexthop_config, next_hop):
+    port = nexthop_config.get('check').get('port')
+
+    targets = tuple(
+        TargetNamedTuple(
+            target=target,
+            vrf=target_config.get('vrf', None),
+            interface=target_config.get('interface', None),
+        )
+        for target, target_config in nexthop_config.get('check').get('target').items()
+    )
+
+    # for print to journal and debug
+    pretty_targets = []
+    for target in targets:
+        p = target.target
+        options = []
+        if target.vrf:
+            options.append(f"vrf: {target.vrf}")
+        if target.interface:
+            options.append(f"interface: {target.interface}")
+        if options:
+            p += ' (' + ', '.join(options) + ')'
+        pretty_targets.append(p)
+    pretty_targets = ', '.join(pretty_targets)
+
+    return NextHopNamedTuple(
+        route=destination,
+        next_hop=next_hop,
+        vrf=vrf,
+        vrf_opt=vrf_opt,
+        conf_iface=nexthop_config.get('interface'),
+        conf_metric=int(nexthop_config.get('metric')),
+        port=port,
+        port_opt=f'port {port}' if port else '',
+        policy=nexthop_config.get('check').get('policy'),
+        proto=nexthop_config.get('check').get('type'),
+        targets=targets,
+        pretty_targets=pretty_targets,
+        timeout=nexthop_config.get('check').get('timeout'),
+        onlink='onlink' if 'onlink' in nexthop_config else '',
+    )
+
+
+RouteNamedTuple = namedtuple(
+    'RouteConfig',
+    [
+        'destination',
+        'vrf',
+        'vrf_opt',
+        'config_path',
+        'nexthops',
+    ],
+)
+
+
+def get_route_config(route, route_config, config_path, vrf):
+    vrf_opt = f'vrf {vrf}' if vrf else ''
+    nexthops = tuple(
+        get_nexthop_config_vars(route, vrf, vrf_opt, nexthop_config, next_hop)
+        for next_hop, nexthop_config in route_config.get('next_hop').items()
+    )
+    return RouteNamedTuple(
+        destination=route,
+        vrf=vrf,
+        vrf_opt=vrf_opt,
+        config_path=config_path,
+        nexthops=nexthops,
+    )
+
+
+def parse_config(config, path):
+    parsed = []
+    vrf = config.get('vrf_context', '')
+    for route, route_config in config.get('route').items():
+        parsed.append(get_route_config(route, route_config, path, vrf))
+    return parsed
+
+
+def flush_all_routes():
+    print_debug("flush_all_routes called")
+    flush_cmd = 'ip route flush protocol failover table all'
+    run(flush_cmd)
+    journal.send(
+        flush_cmd,
+        SYSLOG_IDENTIFIER=my_name,
+    )
+
+
+kill_called = False
+
+
+def kill_handler(*args):
+    global kill_called
+    if kill_called:
+        return
+    kill_called = True
+    print_debug(f"kill_handler called for signal {args[0]}")
+
+
+def get_ip_command_args(nhc):
+    return (
+        f'{nhc.route} via {nhc.next_hop} dev {nhc.conf_iface} '
+        f'{nhc.onlink} metric {nhc.conf_metric} {nhc.vrf_opt} proto failover'
+    )
+
+
+def delete_route(ip_args):
+    print_debug(f'    [ DEL ] -- ip route del {ip_args} [DELETE]')
+    rc_cmd(f'ip route del {ip_args}')
+    journal.send(
+        f'ip route del {ip_args}',
+        SYSLOG_IDENTIFIER=my_name,
+    )
+
+
+def update_configuration(last_modification_times, all_routes, config_dir):
+    """
+    Updates configuration:
+        rechecks config_dir for new/updated files,
+        deletes routes that were deleted from configuration
+
+    Args:
+        last_modification_times(dict): keys: relative path to file, value: last modification time.
+            Is updated.
+        all_routes(list): list of routes that were configured in previous call.
+            Is updated.
+        config_dir(Path): path to configuration directory
+    """
 
-    args = parser.parse_args()
     try:
-        config_path = Path(args.config)
-        config = json.loads(config_path.read_text())
-    except Exception as err:
-        print(
-            f'Configuration file "{config_path}" does not exist or malformed: {err}'
-        )
+        # First check if there are any changes at all
+        have_changes = False
+        for child in config_dir.iterdir():
+            file_key = str(child)
+            if file_key not in last_modification_times:
+                have_changes = True
+                print_debug(f"New file '{child}', have changes, rereading all")
+                break
+            modtime = child.stat().st_mtime_ns
+            if modtime != last_modification_times[file_key]:
+                have_changes = True
+                print_debug(f"File '{child} modified, have changes, rereading all...")
+                break
+
+        if not have_changes:
+            print_debug("No changes in configuration detected.")
+            return
+
+        last_modification_times.clear()
+        new_routes = []
+
+        # It is important that in configuration directory there MUST be
+        # only files generated by conf_mode/protocols_failover.py - otherwise
+        # the script won't be able to detect when all VRFs are disabled and
+        # won't be able to stop the service gracefully
+        for child in config_dir.iterdir():
+            if not child.is_file():
+                print(
+                    f"Path {child} under configuration dir is not a file! Please clean configuration directory {config_dir}."
+                )
+                exit(1)
+
+            modtime = child.stat().st_mtime_ns
+            file_key = str(child)
+            last_modification_times[file_key] = modtime
+
+            try:
+                config = json.loads(child.read_text())
+                print_debug(f"Config from '{child}': {config}")
+            except OSError as err:
+                print(f'Configuration file "{child}" could not be read: {err}')
+                exit(1)
+            except json.JSONDecodeError as err:
+                print(
+                    f'Configuration file "{child}" could not be parsed as JSON: {err}'
+                )
+                exit(1)
+            except UnicodeDecodeError as err:
+                print(f'Configuration file "{child}" has Unicode errors: {err}')
+                exit(1)
+
+            parsed_config = parse_config(config, file_key)
+            new_routes.extend(parsed_config)
+    except OSError as err:
+        print(f'Configuration dir "{config_dir}" does not exist or not readable: {err}')
         exit(1)
 
-    # Useful debug info to console, use debug = True
-    # sudo systemctl stop vyos-failover.service
-    # sudo /usr/libexec/vyos/vyos-failover.py --config /run/vyos-failover.conf
-    debug = False
+    old_routes_set = set(all_routes)
+    new_routes_set = set(new_routes)
+
+    delete_routes = old_routes_set - new_routes_set
+    add_routes = new_routes_set - old_routes_set
+
+    # Delete not needed routes
+    for route_config in delete_routes:
+        print_debug(
+            f"Deleting route {route_config}, not present in updated configuration"
+        )
+        for nhc in route_config.nexthops:
+            ip_args = get_ip_command_args(nhc)
+            if is_route_exists(ip_args):
+                delete_route(ip_args)
+        all_routes.remove(route_config)
+
+    # Add new routes
+    print_debug(f"Adding routes {add_routes}, new in updated configuration")
+    all_routes.extend(add_routes)
+
+    print_debug(f"All routes: {all_routes}")
 
-    while(True):
 
-        for route, route_config in config.get('route').items():
+if __name__ == '__main__':
+    print_debug(f"{my_name} started")
 
-            exists_gateway, exists_iface, exists_metric =  get_best_route_options(route, debug=debug)
+    # Parse command arguments and get config
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '-c',
+        '--config-dir',
+        action='store',
+        help='Path to protocols failover configuration dir',
+        required=True,
+        type=Path,
+    )
 
-            for next_hop, nexthop_config in route_config.get('next_hop').items():
-                conf_iface = nexthop_config.get('interface')
-                conf_metric = int(nexthop_config.get('metric'))
-                port = nexthop_config.get('check').get('port')
-                port_opt = f'port {port}' if port else ''
-                policy = nexthop_config.get('check').get('policy')
-                proto = nexthop_config.get('check').get('type')
-                target = nexthop_config.get('check').get('target')
-                timeout = nexthop_config.get('check').get('timeout')
-                onlink = 'onlink' if 'onlink' in nexthop_config else ''
+    args = parser.parse_args()
+    config_dir = Path(args.config_dir)
+
+    last_modification_times = {}
+    all_routes = []
+
+
+    # Clean all `failover` routes now and at exit
+    flush_all_routes()
+    atexit.register(flush_all_routes)
+    signal.signal(signal.SIGINT, kill_handler)
+    signal.signal(signal.SIGTERM, kill_handler)
+
+    had_sleeps = True
+    while not kill_called:
+        # Check in case daemon was launched without routes
+        if not had_sleeps:
+            time.sleep(int(config_timeout))
+
+        update_configuration(last_modification_times, all_routes, config_dir)
+        had_sleeps = False
+
+        for route_config in all_routes:
+            if kill_called:
+                break
+            route = route_config.destination
+            vrf = route_config.vrf
+            vrf_opt = route_config.vrf_opt
+
+            for nhc in route_config.nexthops:
+                next_hop = nhc.next_hop
+                ip_args = get_ip_command_args(nhc)
+
+                is_alive = is_target_alive(
+                    nhc.targets,
+                    nhc.conf_iface,
+                    nhc.proto,
+                    nhc.port,
+                    nexthop_vrf=vrf,
+                    policy=nhc.policy,
+                )
 
                 # Route not found in the current routing table
-                if not is_route_exists(route, next_hop, conf_iface, conf_metric):
-                    if debug: print(f"    [NEW_ROUTE_DETECTED] route: [{route}]")
+                if not is_route_exists(ip_args):
+                    print_debug(f"    [NEW_ROUTE_DETECTED] route: [{route} {vrf_opt}]")
                     # Add route if check-target alive
-                    if is_target_alive(target, conf_iface, proto, port, debug=debug, policy=policy):
-                        if debug: print(f'    [ ADD ] -- ip route add {route} via {next_hop} dev {conf_iface} '
-                                        f'metric {conf_metric} proto failover\n###')
-                        rc, command = rc_cmd(f'ip route add {route} via {next_hop} dev {conf_iface} '
-                                             f'{onlink} metric {conf_metric} proto failover')
+                    if is_alive:
+                        print_debug(f'    [ ADD ] -- ip route add {ip_args}\n###')
+                        rc, command = rc_cmd(f'ip route add {ip_args}')
                         # If something is wrong and gateway not added
                         # Example: Error: Next-hop has invalid gateway.
-                        if rc !=0:
-                            if debug: print(f'{command} -- return-code [RC: {rc}] {next_hop} dev {conf_iface}')
+                        if rc != 0:
+                            print_debug(
+                                f'{command} -- return-code [RC: {rc}] {next_hop} dev {nhc.conf_iface}'
+                            )
                         else:
-                            journal.send(f'ip route add {route} via {next_hop} dev {conf_iface} '
-                                         f'{onlink} metric {conf_metric} proto failover', SYSLOG_IDENTIFIER=my_name)
+                            journal.send(
+                                f'ip route add {ip_args}',
+                                SYSLOG_IDENTIFIER=my_name,
+                            )
                     else:
-                        if debug: print(f'    [ TARGET_FAIL ] target checks fails for [{target}], do nothing')
-                        journal.send(f'Check fail for route {route} target {target} proto {proto} '
-                                     f'{port_opt}', SYSLOG_IDENTIFIER=my_name)
-
-                # Route was added, check if the target is alive
-                # We should delete route if check fails only if route exists in the routing table
-                if not is_target_alive(target, conf_iface, proto, port, debug=debug, policy=policy) and \
-                        is_route_exists(route, next_hop, conf_iface, conf_metric):
-                    if debug:
-                        print(f'Nexh_hop {next_hop} fail, target not response')
-                        print(f'    [ DEL ] -- ip route del {route} via {next_hop} dev {conf_iface} '
-                              f'metric {conf_metric} proto failover [DELETE]')
-                    rc_cmd(f'ip route del {route} via {next_hop} dev {conf_iface} metric {conf_metric} proto failover')
-                    journal.send(f'ip route del {route} via {next_hop} dev {conf_iface} '
-                                 f'metric {conf_metric} proto failover', SYSLOG_IDENTIFIER=my_name)
-
-                time.sleep(int(timeout))
+                        print_debug(
+                            f'    [ TARGET_FAIL ] target checks fails for [{nhc.pretty_targets}], do nothing'
+                        )
+                        journal.send(
+                            f'Check fail for route {route} target {nhc.pretty_targets} proto {nhc.proto} '
+                            f'{nhc.port_opt}',
+                            SYSLOG_IDENTIFIER=my_name,
+                        )
+                else:
+                    # Route was added, check if the target is alive
+                    # We should delete route if check fails only if route exists in the routing table
+                    if not is_alive:
+                        print_debug(
+                            f"Next_hop {next_hop} fail, target check didn't pass"
+                        )
+                        delete_route(ip_args)
+
+                had_sleeps = True
+                time.sleep(int(nhc.timeout))
+                if kill_called:
+                    break
+
+    print_debug(f"Out of main loop, {kill_called=}")