diff options
Diffstat (limited to 'src')
| -rwxr-xr-x | src/conf_mode/protocols_failover.py | 83 | ||||
| -rwxr-xr-x | src/helpers/vyos-failover.py | 524 |
2 files changed, 466 insertions, 141 deletions
diff --git a/src/conf_mode/protocols_failover.py b/src/conf_mode/protocols_failover.py index 9b50cc1ed..956c060f4 100755 --- a/src/conf_mode/protocols_failover.py +++ b/src/conf_mode/protocols_failover.py @@ -15,12 +15,15 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. import json +import os from pathlib import Path +from sys import argv from vyos.config import Config from vyos.template import render from vyos.utils.process import call +from vyos.utils.process import is_systemd_service_running from vyos import ConfigError from vyos import airbag @@ -28,9 +31,24 @@ airbag.enable() service_name = 'vyos-failover' -service_conf = Path(f'/run/{service_name}.conf') +service_conf_dir = Path(f'/run/{service_name}.conf.d/') systemd_service = '/run/systemd/system/vyos-failover.service' -rt_proto_failover = '/etc/iproute2/rt_protos.d/failover.conf' +rt_proto_failover = Path('/etc/iproute2/rt_protos.d/failover.conf') + + +def get_vrf_name(): + if argv and len(argv) > 1: + return argv[1] + return None + + +def get_service_conf_path(): + vrf_name = get_vrf_name() + if vrf_name: + filename = f'vrf-{vrf_name}.conf' + else: + filename = 'default.conf' + return service_conf_dir / filename def get_config(config=None): @@ -39,7 +57,14 @@ def get_config(config=None): else: conf = Config() - base = ['protocols', 'failover'] + vrf_name = get_vrf_name() + if vrf_name: + base = ['vrf', 'name', vrf_name] + else: + base = [] + + base += ['protocols', 'failover'] + failover = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True) @@ -47,6 +72,9 @@ def get_config(config=None): if failover.get('route') is not None: failover = conf.merge_defaults(failover, recursive=True) + if failover: + failover['vrf_context'] = vrf_name + return failover def verify(failover): @@ -75,33 +103,66 @@ def verify(failover): if check_type == 'tcp' and 'port' not in next_hop_config['check']: raise ConfigError(f'Check port for next-hop "{next_hop}" and type TCP is mandatory!') + errors = { + 'icmp': {}, + 'tcp': { + 'interface': 'Check target "interface" option does nothing for type TCP. Use "vrf" if needed', + }, + 'arp': { + 'vrf': 'Check target "vrf" option is incompatible with type ARP, use "interface" option if needed', + }, + } + + for target, target_config in next_hop_config['check']['target'].items(): + for key, msg in errors[check_type].items(): + if key in target_config: + raise ConfigError(msg) + return None + def generate(failover): + service_conf = get_service_conf_path() if not failover: service_conf.unlink(missing_ok=True) + try: + os.rmdir(service_conf_dir) + # Ignore if directory doesn't exist + # or not empty (probably configs for other VRFs are there) + except (FileNotFoundError, OSError): + pass return None # Add own rt_proto 'failover' # Helps to detect all own routes 'proto failover' - with open(rt_proto_failover, 'w') as f: - f.write('111 failover\n') + rt_proto_failover.write_text('111 failover\n') + + service_conf_dir.mkdir(exist_ok=True) # Write configuration file conf_json = json.dumps(failover, indent=4) service_conf.write_text(conf_json) - render(systemd_service, 'protocols/systemd_vyos_failover_service.j2', failover) + render( + systemd_service, + 'protocols/systemd_vyos_failover_service.j2', + {'config_dir': str(service_conf_dir)}, + ) return None def apply(failover): - if not failover: + # If directory is removed - we can stop the service + if not service_conf_dir.is_dir(): call(f'systemctl stop {service_name}.service') - call('ip route flush protocol failover') - else: call('systemctl daemon-reload') - call(f'systemctl restart {service_name}.service') - call(f'ip route flush protocol failover') + # Otherwise even if `failover` is False, service is + # still needed for other VRFs. + else: + # Daemon watches for configuration updates, so we need only + # to start it if it is not started yet + if not is_systemd_service_running(service_name): + call('systemctl daemon-reload') + call(f'systemctl start {service_name}.service') return None diff --git a/src/helpers/vyos-failover.py b/src/helpers/vyos-failover.py index 96db947e1..d255f2303 100755 --- a/src/helpers/vyos-failover.py +++ b/src/helpers/vyos-failover.py @@ -15,22 +15,49 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. import argparse +import atexit import json -import socket +import signal import time +from collections import namedtuple from vyos.utils.process import rc_cmd +from vyos.utils.process import run from pathlib import Path from systemd import journal my_name = Path(__file__).stem +# Timeout between configuration reading +# When no checks timeouts worked (e.g. no config files) +config_timeout = 1 -def is_route_exists(route, gateway, interface, metric): +# Useful debug info to console, use debug = True +# sudo systemctl stop vyos-failover.service +# sudo /usr/libexec/vyos/vyos-failover.py --config /run/vyos-failover.conf +debug = False +debug_output_journal = False +debug_output_print = True + + +def print_debug(*args, **kwargs): + if debug: + if debug_output_print: + print(*args, **kwargs) + if debug_output_journal: + journal.send(*args, **kwargs, SYSLOG_IDENTIFIER=my_name) + + +def wrap_vrf(command, vrf): + if not vrf: + return command + return f"sudo ip vrf exec {vrf} {command}" + + +def is_route_exists(ip_args): """Check if route with expected gateway, dev and metric exists""" - rc, data = rc_cmd(f'ip --json route show protocol failover {route} ' - f'via {gateway} dev {interface} metric {metric}') + rc, data = rc_cmd(f'ip --json route show {ip_args}') if rc == 0: data = json.loads(data) if len(data) > 0: @@ -38,41 +65,7 @@ def is_route_exists(route, gateway, interface, metric): return False -def get_best_route_options(route, debug=False): - """ - Return current best route ('gateway, interface, metric) - - % get_best_route_options('203.0.113.1') - ('192.168.0.1', 'eth1', 1) - - % get_best_route_options('203.0.113.254') - (None, None, None) - """ - rc, data = rc_cmd(f'ip --detail --json route show protocol failover {route}') - if rc == 0: - data = json.loads(data) - if len(data) == 0: - print(f'\nRoute {route} for protocol failover was not found') - return None, None, None - # Fake metric 999 by default - # Search route with the lowest metric - best_metric = 999 - for entry in data: - if debug: print('\n', entry) - metric = entry.get('metric') - gateway = entry.get('gateway') - iface = entry.get('dev') - if metric < best_metric: - best_metric = metric - best_gateway = gateway - best_interface = iface - if debug: - print(f'### Best_route exists: {route}, best_gateway: {best_gateway}, ' - f'best_metric: {best_metric}, best_iface: {best_interface}') - return best_gateway, best_interface, best_metric - - -def is_port_open(ip, port): +def is_port_open(ip, port, vrf=''): """ Check connection to remote host and port Return True if host alive @@ -80,33 +73,29 @@ def is_port_open(ip, port): % is_port_open('example.com', 8080) True """ - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP) - s.settimeout(2) - try: - s.connect((ip, int(port))) - s.shutdown(socket.SHUT_RDWR) - return True - except: - return False - finally: - s.close() - - -def is_target_alive(target_list=None, - iface='', - proto='icmp', - port=None, - debug=False, - policy='any-available') -> bool: - """Check the availability of each target in the target_list using + cmd = wrap_vrf(f"nc -w2 -z {ip} {port}", vrf) + rc, data = rc_cmd(cmd) + return rc == 0 + + +def is_target_alive( + targets=None, + iface='', + proto='icmp', + port=None, + nexthop_vrf='', + policy='any-available', +) -> bool: + """Check the availability of each target in the target_dict using the specified protocol ICMP, ARP, TCP Args: - target_list (list): A list of IP addresses or hostnames to check. + targets (tuple of TargetNamedTuple): A dict: keys are IP addresses to check, values - dicts with options. + Possible keys (all optional): 'vrf' and 'interface'. iface (str): The name of the network interface to use for the check. proto (str): The protocol to use for the check. Options are 'icmp', 'arp', or 'tcp'. port (int): The port number to use for the TCP check. Only applicable if proto is 'tcp'. - debug (bool): If True, print debug information during the check. + nexthop_vrf (str): Nexthop VRF name - if specific vrf is not given for target, use this one policy (str): The policy to use for the check. Options are 'any-available' or 'all-available'. Returns: @@ -120,30 +109,41 @@ def is_target_alive(target_list=None, iface = f'-I {iface}' num_reachable_targets = 0 - for target in target_list: + for options in targets: + target = options.target + vrf = options.vrf if options.vrf else nexthop_vrf + # don't use nexthop interface if 'vrf' is given + iface_opt = iface if vrf == nexthop_vrf else '' + # in any case if 'interface' is given, use it + if options.interface: + iface_opt = f'-I {options.interface}' match proto: case 'icmp': - command = f'/usr/bin/ping -q {target} {iface} -n -c 2 -W 1' + command = f'/usr/bin/ping -q {target} {iface_opt} -n -c 2 -W 1' + command = wrap_vrf(command, vrf) rc, response = rc_cmd(command) - if debug: - print(f' [ CHECK-TARGET ]: [{command}] -- return-code [RC: {rc}]') + print_debug( + f' [ CHECK-TARGET ]: [{command}] -- return-code [RC: {rc}]' + ) if rc == 0: num_reachable_targets += 1 if policy == 'any-available': return True case 'arp': - command = f'/usr/bin/arping -b -c 2 -f -w 1 -i 1 {iface} {target}' + command = f'/usr/bin/arping -b -c 2 -f -w 1 -i 1 {iface_opt} {target}' + command = wrap_vrf(command, vrf) rc, response = rc_cmd(command) - if debug: - print(f' [ CHECK-TARGET ]: [{command}] -- return-code [RC: {rc}]') + print_debug( + f' [ CHECK-TARGET ]: [{command}] -- return-code [RC: {rc}]' + ) if rc == 0: num_reachable_targets += 1 if policy == 'any-available': return True case _ if proto == 'tcp' and port is not None: - if is_port_open(target, port): + if is_port_open(target, port, vrf=vrf): num_reachable_targets += 1 if policy == 'any-available': return True @@ -151,85 +151,349 @@ def is_target_alive(target_list=None, case _: return False - if policy == 'all-available' and num_reachable_targets == len(target_list): + if policy == 'all-available' and num_reachable_targets == len(targets): return True return False -if __name__ == '__main__': - # Parse command arguments and get config - parser = argparse.ArgumentParser() - parser.add_argument('-c', - '--config', - action='store', - help='Path to protocols failover configuration', - required=True, - type=Path) +TargetNamedTuple = namedtuple( + 'TargetConfig', + [ + 'target', + 'vrf', + 'interface', + ], +) + +NextHopNamedTuple = namedtuple( + 'NextHopConfig', + [ + 'route', + 'next_hop', + 'vrf', + 'vrf_opt', + 'conf_iface', + 'conf_metric', + 'port', + 'port_opt', + 'policy', + 'proto', + 'targets', + 'pretty_targets', + 'timeout', + 'onlink', + ], +) + + +def get_nexthop_config_vars(destination, vrf, vrf_opt, nexthop_config, next_hop): + port = nexthop_config.get('check').get('port') + + targets = tuple( + TargetNamedTuple( + target=target, + vrf=target_config.get('vrf', None), + interface=target_config.get('interface', None), + ) + for target, target_config in nexthop_config.get('check').get('target').items() + ) + + # for print to journal and debug + pretty_targets = [] + for target in targets: + p = target.target + options = [] + if target.vrf: + options.append(f"vrf: {target.vrf}") + if target.interface: + options.append(f"interface: {target.interface}") + if options: + p += ' (' + ', '.join(options) + ')' + pretty_targets.append(p) + pretty_targets = ', '.join(pretty_targets) + + return NextHopNamedTuple( + route=destination, + next_hop=next_hop, + vrf=vrf, + vrf_opt=vrf_opt, + conf_iface=nexthop_config.get('interface'), + conf_metric=int(nexthop_config.get('metric')), + port=port, + port_opt=f'port {port}' if port else '', + policy=nexthop_config.get('check').get('policy'), + proto=nexthop_config.get('check').get('type'), + targets=targets, + pretty_targets=pretty_targets, + timeout=nexthop_config.get('check').get('timeout'), + onlink='onlink' if 'onlink' in nexthop_config else '', + ) + + +RouteNamedTuple = namedtuple( + 'RouteConfig', + [ + 'destination', + 'vrf', + 'vrf_opt', + 'config_path', + 'nexthops', + ], +) + + +def get_route_config(route, route_config, config_path, vrf): + vrf_opt = f'vrf {vrf}' if vrf else '' + nexthops = tuple( + get_nexthop_config_vars(route, vrf, vrf_opt, nexthop_config, next_hop) + for next_hop, nexthop_config in route_config.get('next_hop').items() + ) + return RouteNamedTuple( + destination=route, + vrf=vrf, + vrf_opt=vrf_opt, + config_path=config_path, + nexthops=nexthops, + ) + + +def parse_config(config, path): + parsed = [] + vrf = config.get('vrf_context', '') + for route, route_config in config.get('route').items(): + parsed.append(get_route_config(route, route_config, path, vrf)) + return parsed + + +def flush_all_routes(): + print_debug("flush_all_routes called") + flush_cmd = 'ip route flush protocol failover table all' + run(flush_cmd) + journal.send( + flush_cmd, + SYSLOG_IDENTIFIER=my_name, + ) + + +kill_called = False + + +def kill_handler(*args): + global kill_called + if kill_called: + return + kill_called = True + print_debug(f"kill_handler called for signal {args[0]}") + + +def get_ip_command_args(nhc): + return ( + f'{nhc.route} via {nhc.next_hop} dev {nhc.conf_iface} ' + f'{nhc.onlink} metric {nhc.conf_metric} {nhc.vrf_opt} proto failover' + ) + + +def delete_route(ip_args): + print_debug(f' [ DEL ] -- ip route del {ip_args} [DELETE]') + rc_cmd(f'ip route del {ip_args}') + journal.send( + f'ip route del {ip_args}', + SYSLOG_IDENTIFIER=my_name, + ) + + +def update_configuration(last_modification_times, all_routes, config_dir): + """ + Updates configuration: + rechecks config_dir for new/updated files, + deletes routes that were deleted from configuration + + Args: + last_modification_times(dict): keys: relative path to file, value: last modification time. + Is updated. + all_routes(list): list of routes that were configured in previous call. + Is updated. + config_dir(Path): path to configuration directory + """ - args = parser.parse_args() try: - config_path = Path(args.config) - config = json.loads(config_path.read_text()) - except Exception as err: - print( - f'Configuration file "{config_path}" does not exist or malformed: {err}' - ) + # First check if there are any changes at all + have_changes = False + for child in config_dir.iterdir(): + file_key = str(child) + if file_key not in last_modification_times: + have_changes = True + print_debug(f"New file '{child}', have changes, rereading all") + break + modtime = child.stat().st_mtime_ns + if modtime != last_modification_times[file_key]: + have_changes = True + print_debug(f"File '{child} modified, have changes, rereading all...") + break + + if not have_changes: + print_debug("No changes in configuration detected.") + return + + last_modification_times.clear() + new_routes = [] + + # It is important that in configuration directory there MUST be + # only files generated by conf_mode/protocols_failover.py - otherwise + # the script won't be able to detect when all VRFs are disabled and + # won't be able to stop the service gracefully + for child in config_dir.iterdir(): + if not child.is_file(): + print( + f"Path {child} under configuration dir is not a file! Please clean configuration directory {config_dir}." + ) + exit(1) + + modtime = child.stat().st_mtime_ns + file_key = str(child) + last_modification_times[file_key] = modtime + + try: + config = json.loads(child.read_text()) + print_debug(f"Config from '{child}': {config}") + except OSError as err: + print(f'Configuration file "{child}" could not be read: {err}') + exit(1) + except json.JSONDecodeError as err: + print( + f'Configuration file "{child}" could not be parsed as JSON: {err}' + ) + exit(1) + except UnicodeDecodeError as err: + print(f'Configuration file "{child}" has Unicode errors: {err}') + exit(1) + + parsed_config = parse_config(config, file_key) + new_routes.extend(parsed_config) + except OSError as err: + print(f'Configuration dir "{config_dir}" does not exist or not readable: {err}') exit(1) - # Useful debug info to console, use debug = True - # sudo systemctl stop vyos-failover.service - # sudo /usr/libexec/vyos/vyos-failover.py --config /run/vyos-failover.conf - debug = False + old_routes_set = set(all_routes) + new_routes_set = set(new_routes) + + delete_routes = old_routes_set - new_routes_set + add_routes = new_routes_set - old_routes_set + + # Delete not needed routes + for route_config in delete_routes: + print_debug( + f"Deleting route {route_config}, not present in updated configuration" + ) + for nhc in route_config.nexthops: + ip_args = get_ip_command_args(nhc) + if is_route_exists(ip_args): + delete_route(ip_args) + all_routes.remove(route_config) + + # Add new routes + print_debug(f"Adding routes {add_routes}, new in updated configuration") + all_routes.extend(add_routes) + + print_debug(f"All routes: {all_routes}") - while(True): - for route, route_config in config.get('route').items(): +if __name__ == '__main__': + print_debug(f"{my_name} started") - exists_gateway, exists_iface, exists_metric = get_best_route_options(route, debug=debug) + # Parse command arguments and get config + parser = argparse.ArgumentParser() + parser.add_argument( + '-c', + '--config-dir', + action='store', + help='Path to protocols failover configuration dir', + required=True, + type=Path, + ) - for next_hop, nexthop_config in route_config.get('next_hop').items(): - conf_iface = nexthop_config.get('interface') - conf_metric = int(nexthop_config.get('metric')) - port = nexthop_config.get('check').get('port') - port_opt = f'port {port}' if port else '' - policy = nexthop_config.get('check').get('policy') - proto = nexthop_config.get('check').get('type') - target = nexthop_config.get('check').get('target') - timeout = nexthop_config.get('check').get('timeout') - onlink = 'onlink' if 'onlink' in nexthop_config else '' + args = parser.parse_args() + config_dir = Path(args.config_dir) + + last_modification_times = {} + all_routes = [] + + + # Clean all `failover` routes now and at exit + flush_all_routes() + atexit.register(flush_all_routes) + signal.signal(signal.SIGINT, kill_handler) + signal.signal(signal.SIGTERM, kill_handler) + + had_sleeps = True + while not kill_called: + # Check in case daemon was launched without routes + if not had_sleeps: + time.sleep(int(config_timeout)) + + update_configuration(last_modification_times, all_routes, config_dir) + had_sleeps = False + + for route_config in all_routes: + if kill_called: + break + route = route_config.destination + vrf = route_config.vrf + vrf_opt = route_config.vrf_opt + + for nhc in route_config.nexthops: + next_hop = nhc.next_hop + ip_args = get_ip_command_args(nhc) + + is_alive = is_target_alive( + nhc.targets, + nhc.conf_iface, + nhc.proto, + nhc.port, + nexthop_vrf=vrf, + policy=nhc.policy, + ) # Route not found in the current routing table - if not is_route_exists(route, next_hop, conf_iface, conf_metric): - if debug: print(f" [NEW_ROUTE_DETECTED] route: [{route}]") + if not is_route_exists(ip_args): + print_debug(f" [NEW_ROUTE_DETECTED] route: [{route} {vrf_opt}]") # Add route if check-target alive - if is_target_alive(target, conf_iface, proto, port, debug=debug, policy=policy): - if debug: print(f' [ ADD ] -- ip route add {route} via {next_hop} dev {conf_iface} ' - f'metric {conf_metric} proto failover\n###') - rc, command = rc_cmd(f'ip route add {route} via {next_hop} dev {conf_iface} ' - f'{onlink} metric {conf_metric} proto failover') + if is_alive: + print_debug(f' [ ADD ] -- ip route add {ip_args}\n###') + rc, command = rc_cmd(f'ip route add {ip_args}') # If something is wrong and gateway not added # Example: Error: Next-hop has invalid gateway. - if rc !=0: - if debug: print(f'{command} -- return-code [RC: {rc}] {next_hop} dev {conf_iface}') + if rc != 0: + print_debug( + f'{command} -- return-code [RC: {rc}] {next_hop} dev {nhc.conf_iface}' + ) else: - journal.send(f'ip route add {route} via {next_hop} dev {conf_iface} ' - f'{onlink} metric {conf_metric} proto failover', SYSLOG_IDENTIFIER=my_name) + journal.send( + f'ip route add {ip_args}', + SYSLOG_IDENTIFIER=my_name, + ) else: - if debug: print(f' [ TARGET_FAIL ] target checks fails for [{target}], do nothing') - journal.send(f'Check fail for route {route} target {target} proto {proto} ' - f'{port_opt}', SYSLOG_IDENTIFIER=my_name) - - # Route was added, check if the target is alive - # We should delete route if check fails only if route exists in the routing table - if not is_target_alive(target, conf_iface, proto, port, debug=debug, policy=policy) and \ - is_route_exists(route, next_hop, conf_iface, conf_metric): - if debug: - print(f'Nexh_hop {next_hop} fail, target not response') - print(f' [ DEL ] -- ip route del {route} via {next_hop} dev {conf_iface} ' - f'metric {conf_metric} proto failover [DELETE]') - rc_cmd(f'ip route del {route} via {next_hop} dev {conf_iface} metric {conf_metric} proto failover') - journal.send(f'ip route del {route} via {next_hop} dev {conf_iface} ' - f'metric {conf_metric} proto failover', SYSLOG_IDENTIFIER=my_name) - - time.sleep(int(timeout)) + print_debug( + f' [ TARGET_FAIL ] target checks fails for [{nhc.pretty_targets}], do nothing' + ) + journal.send( + f'Check fail for route {route} target {nhc.pretty_targets} proto {nhc.proto} ' + f'{nhc.port_opt}', + SYSLOG_IDENTIFIER=my_name, + ) + else: + # Route was added, check if the target is alive + # We should delete route if check fails only if route exists in the routing table + if not is_alive: + print_debug( + f"Next_hop {next_hop} fail, target check didn't pass" + ) + delete_route(ip_args) + + had_sleeps = True + time.sleep(int(nhc.timeout)) + if kill_called: + break + + print_debug(f"Out of main loop, {kill_called=}") |