Merge branch 'current' into goodnetnick-shloginotp-T4754

4 files changed, 372 insertions, 76 deletions

diff --git a/src/helpers/system-versions-foot.py b/src/helpers/system-versions-foot.py
index 2aa687221..9614f0d28 100755
--- a/src/helpers/system-versions-foot.py
+++ b/src/helpers/system-versions-foot.py
@@ -1,6 +1,6 @@
 #!/usr/bin/python3
 
-# Copyright 2019 VyOS maintainers and contributors <maintainers@vyos.io>
+# Copyright 2019, 2022 VyOS maintainers and contributors <maintainers@vyos.io>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -16,24 +16,13 @@
 # along with this library.  If not, see <http://www.gnu.org/licenses/>.
 
 import sys
-import vyos.formatversions as formatversions
-import vyos.systemversions as systemversions
 import vyos.defaults
-import vyos.version
-
-sys_versions = systemversions.get_system_component_version()
-
-component_string = formatversions.format_versions_string(sys_versions)
-
-os_version_string = vyos.version.get_version()
+from vyos.component_version import write_system_footer
 
 sys.stdout.write("\n\n")
 if vyos.defaults.cfg_vintage == 'vyos':
-    formatversions.write_vyos_versions_foot(None, component_string,
-                                            os_version_string)
+    write_system_footer(None, vintage='vyos')
 elif vyos.defaults.cfg_vintage == 'vyatta':
-    formatversions.write_vyatta_versions_foot(None, component_string,
-                                              os_version_string)
+    write_system_footer(None, vintage='vyatta')
 else:
-    formatversions.write_vyatta_versions_foot(None, component_string,
-                                              os_version_string)
+    write_system_footer(None, vintage='vyos')
diff --git a/src/helpers/vyos-domain-group-resolve.py b/src/helpers/vyos-domain-group-resolve.py
deleted file mode 100755
index 6b677670b..000000000
--- a/src/helpers/vyos-domain-group-resolve.py
+++ /dev/null
@@ -1,60 +0,0 @@
-#!/usr/bin/env python3
-#
-# Copyright (C) 2022 VyOS maintainers and contributors
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 or later as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-
-import time
-
-from vyos.configquery import ConfigTreeQuery
-from vyos.firewall import get_ips_domains_dict
-from vyos.firewall import nft_add_set_elements
-from vyos.firewall import nft_flush_set
-from vyos.firewall import nft_init_set
-from vyos.firewall import nft_update_set_elements
-from vyos.util import call
-
-
-base = ['firewall', 'group', 'domain-group']
-check_required = True
-# count_failed = 0
-# Timeout in sec between checks
-timeout = 300
-
-domain_state = {}
-
-if __name__ == '__main__':
-
-    while check_required:
-        config = ConfigTreeQuery()
-        if config.exists(base):
-            domain_groups = config.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True)
-            for set_name, domain_config in domain_groups.items():
-                list_domains = domain_config['address']
-                elements = []
-                ip_dict = get_ips_domains_dict(list_domains)
-
-                for domain in list_domains:
-                    # Resolution succeeded, update domain state
-                    if domain in ip_dict:
-                        domain_state[domain] = ip_dict[domain]
-                        elements += ip_dict[domain]
-                    # Resolution failed, use previous domain state
-                    elif domain in domain_state:
-                        elements += domain_state[domain]
-
-                # Resolve successful
-                if elements:
-                    nft_update_set_elements(f'D_{set_name}', elements)
-        time.sleep(timeout)
diff --git a/src/helpers/vyos-domain-resolver.py b/src/helpers/vyos-domain-resolver.py
new file mode 100755
index 000000000..e31d9238e
--- /dev/null
+++ b/src/helpers/vyos-domain-resolver.py
@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2022 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import json
+import os
+import time
+
+from vyos.configdict import dict_merge
+from vyos.configquery import ConfigTreeQuery
+from vyos.firewall import fqdn_config_parse
+from vyos.firewall import fqdn_resolve
+from vyos.util import cmd
+from vyos.util import commit_in_progress
+from vyos.util import dict_search_args
+from vyos.util import run
+from vyos.xml import defaults
+
+base = ['firewall']
+timeout = 300
+cache = False
+
+domain_state = {}
+
+ipv4_tables = {
+    'ip vyos_mangle',
+    'ip vyos_filter',
+    'ip vyos_nat'
+}
+
+ipv6_tables = {
+    'ip6 vyos_mangle',
+    'ip6 vyos_filter'
+}
+
+def get_config(conf):
+    firewall = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True,
+                                    no_tag_node_value_mangle=True)
+
+    default_values = defaults(base)
+    for tmp in ['name', 'ipv6_name']:
+        if tmp in default_values:
+            del default_values[tmp]
+
+    if 'zone' in default_values:
+        del default_values['zone']
+
+    firewall = dict_merge(default_values, firewall)
+
+    global timeout, cache
+
+    if 'resolver_interval' in firewall:
+        timeout = int(firewall['resolver_interval'])
+
+    if 'resolver_cache' in firewall:
+        cache = True
+
+    fqdn_config_parse(firewall)
+
+    return firewall
+
+def resolve(domains, ipv6=False):
+    global domain_state
+
+    ip_list = set()
+
+    for domain in domains:
+        resolved = fqdn_resolve(domain, ipv6=ipv6)
+
+        if resolved and cache:
+            domain_state[domain] = resolved
+        elif not resolved:
+            if domain not in domain_state:
+                continue
+            resolved = domain_state[domain]
+
+        ip_list = ip_list | resolved
+    return ip_list
+
+def nft_output(table, set_name, ip_list):
+    output = [f'flush set {table} {set_name}']
+    if ip_list:
+        ip_str = ','.join(ip_list)
+        output.append(f'add element {table} {set_name} {{ {ip_str} }}')
+    return output
+
+def nft_valid_sets():
+    try:
+        valid_sets = []
+        sets_json = cmd('nft -j list sets')
+        sets_obj = json.loads(sets_json)
+
+        for obj in sets_obj['nftables']:
+            if 'set' in obj:
+                family = obj['set']['family']
+                table = obj['set']['table']
+                name = obj['set']['name']
+                valid_sets.append((f'{family} {table}', name))
+
+        return valid_sets
+    except:
+        return []
+
+def update(firewall):
+    conf_lines = []
+    count = 0
+
+    valid_sets = nft_valid_sets()
+
+    domain_groups = dict_search_args(firewall, 'group', 'domain_group')
+    if domain_groups:
+        for set_name, domain_config in domain_groups.items():
+            if 'address' not in domain_config:
+                continue
+
+            nft_set_name = f'D_{set_name}'
+            domains = domain_config['address']
+
+            ip_list = resolve(domains, ipv6=False)
+            for table in ipv4_tables:
+                if (table, nft_set_name) in valid_sets:
+                    conf_lines += nft_output(table, nft_set_name, ip_list)
+
+            ip6_list = resolve(domains, ipv6=True)
+            for table in ipv6_tables:
+                if (table, nft_set_name) in valid_sets:
+                    conf_lines += nft_output(table, nft_set_name, ip6_list)
+            count += 1
+
+    for set_name, domain in firewall['ip_fqdn'].items():
+        table = 'ip vyos_filter'
+        nft_set_name = f'FQDN_{set_name}'
+
+        ip_list = resolve([domain], ipv6=False)
+
+        if (table, nft_set_name) in valid_sets:
+            conf_lines += nft_output(table, nft_set_name, ip_list)
+        count += 1
+
+    for set_name, domain in firewall['ip6_fqdn'].items():
+        table = 'ip6 vyos_filter'
+        nft_set_name = f'FQDN_{set_name}'
+
+        ip_list = resolve([domain], ipv6=True)
+        if (table, nft_set_name) in valid_sets:
+            conf_lines += nft_output(table, nft_set_name, ip_list)
+        count += 1
+
+    nft_conf_str = "\n".join(conf_lines) + "\n"
+    code = run(f'nft -f -', input=nft_conf_str)
+
+    print(f'Updated {count} sets - result: {code}')
+
+if __name__ == '__main__':
+    print(f'VyOS domain resolver')
+
+    count = 1
+    while commit_in_progress():
+        if ( count % 60 == 0 ):
+            print(f'Commit still in progress after {count}s - waiting')
+        count += 1
+        time.sleep(1)
+
+    conf = ConfigTreeQuery()
+    firewall = get_config(conf)
+
+    print(f'interval: {timeout}s - cache: {cache}')
+
+    while True:
+        update(firewall)
+        time.sleep(timeout)
diff --git a/src/helpers/vyos-failover.py b/src/helpers/vyos-failover.py
new file mode 100755
index 000000000..1ac193423
--- /dev/null
+++ b/src/helpers/vyos-failover.py
@@ -0,0 +1,184 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2022 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import argparse
+import json
+import subprocess
+import socket
+import time
+
+from vyos.util import rc_cmd
+from pathlib import Path
+from systemd import journal
+
+
+my_name = Path(__file__).stem
+
+
+def get_best_route_options(route, debug=False):
+    """
+    Return current best route ('gateway, interface, metric)
+
+    % get_best_route_options('203.0.113.1')
+    ('192.168.0.1', 'eth1', 1)
+
+    % get_best_route_options('203.0.113.254')
+    (None, None, None)
+    """
+    rc, data = rc_cmd(f'ip --detail --json route show protocol failover {route}')
+    if rc == 0:
+        data = json.loads(data)
+        if len(data) == 0:
+            print(f'\nRoute {route} for protocol failover was not found')
+            return None, None, None
+        # Fake metric 999 by default
+        # Search route with the lowest metric
+        best_metric = 999
+        for entry in data:
+            if debug: print('\n', entry)
+            metric = entry.get('metric')
+            gateway = entry.get('gateway')
+            iface = entry.get('dev')
+            if metric < best_metric:
+                best_metric = metric
+                best_gateway = gateway
+                best_interface = iface
+        if debug:
+            print(f'### Best_route exists: {route}, best_gateway: {best_gateway}, '
+                  f'best_metric: {best_metric}, best_iface: {best_interface}')
+        return best_gateway, best_interface, best_metric
+
+def is_port_open(ip, port):
+    """
+    Check connection to remote host and port
+    Return True if host alive
+
+    % is_port_open('example.com', 8080)
+    True
+    """
+    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP)
+    s.settimeout(2)
+    try:
+        s.connect((ip, int(port)))
+        s.shutdown(socket.SHUT_RDWR)
+        return True
+    except:
+        return False
+    finally:
+        s.close()
+
+def is_target_alive(target=None, iface='', proto='icmp', port=None, debug=False):
+    """
+    Host availability check by ICMP, ARP, TCP
+    Return True if target checks is successful
+
+    % is_target_alive('192.0.2.1', 'eth1', proto='arp')
+    True
+    """
+    if iface != '':
+        iface = f'-I {iface}'
+    if proto == 'icmp':
+        command = f'/usr/bin/ping -q {target} {iface} -n -c 2 -W 1'
+        rc, response = rc_cmd(command)
+        if debug: print(f'    [ CHECK-TARGET ]: [{command}] -- return-code [RC: {rc}]')
+        if rc == 0:
+            return True
+    elif proto == 'arp':
+        command = f'/usr/bin/arping -b -c 2 -f -w 1 -i 1 {iface} {target}'
+        rc, response = rc_cmd(command)
+        if debug: print(f'    [ CHECK-TARGET ]: [{command}] -- return-code [RC: {rc}]')
+        if rc == 0:
+            return True
+    elif proto == 'tcp' and port is not None:
+        return True if is_port_open(target, port) else False
+    else:
+        return False
+
+
+if __name__ == '__main__':
+    # Parse command arguments and get config
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-c',
+                        '--config',
+                        action='store',
+                        help='Path to protocols failover configuration',
+                        required=True,
+                        type=Path)
+
+    args = parser.parse_args()
+    try:
+        config_path = Path(args.config)
+        config = json.loads(config_path.read_text())
+    except Exception as err:
+        print(
+            f'Configuration file "{config_path}" does not exist or malformed: {err}'
+        )
+        exit(1)
+
+    # Useful debug info to console, use debug = True
+    # sudo systemctl stop vyos-failover.service
+    # sudo /usr/libexec/vyos/vyos-failover.py --config /run/vyos-failover.conf
+    debug = False
+
+    while(True):
+
+        for route, route_config in config.get('route').items():
+
+            exists_route = exists_gateway, exists_iface, exists_metric =  get_best_route_options(route, debug=debug)
+
+            for next_hop, nexthop_config in route_config.get('next_hop').items():
+                conf_iface = nexthop_config.get('interface')
+                conf_metric = int(nexthop_config.get('metric'))
+                port = nexthop_config.get('check').get('port')
+                port_opt = f'port {port}' if port else ''
+                proto = nexthop_config.get('check').get('type')
+                target = nexthop_config.get('check').get('target')
+                timeout = nexthop_config.get('check').get('timeout')
+
+                # Best route not fonund in the current routing table
+                if exists_route == (None, None, None):
+                    if debug: print(f"    [NEW_ROUTE_DETECTED] route: [{route}]")
+                    # Add route if check-target alive
+                    if is_target_alive(target, conf_iface, proto, port, debug=debug):
+                        if debug: print(f'    [ ADD ] -- ip route add {route} via {next_hop} dev {conf_iface} '
+                                        f'metric {conf_metric} proto failover\n###')
+                        rc, command = rc_cmd(f'ip route add {route} via {next_hop} dev {conf_iface} '
+                                             f'metric {conf_metric} proto failover')
+                        # If something is wrong and gateway not added
+                        # Example: Error: Next-hop has invalid gateway.
+                        if rc !=0:
+                            if debug: print(f'{command} -- return-code [RC: {rc}] {next_hop} dev {conf_iface}')
+                        else:
+                            journal.send(f'ip route add {route} via {next_hop} dev {conf_iface} '
+                                         f'metric {conf_metric} proto failover', SYSLOG_IDENTIFIER=my_name)
+                    else:
+                        if debug: print(f'    [ TARGET_FAIL ] target checks fails for [{target}], do nothing')
+                        journal.send(f'Check fail for route {route} target {target} proto {proto} '
+                                     f'{port_opt}', SYSLOG_IDENTIFIER=my_name)
+
+                # Route was added, check if the target is alive
+                # We should delete route if check fails only if route exists in the routing table
+                if not is_target_alive(target, conf_iface, proto, port, debug=debug) and \
+                        exists_route != (None, None, None):
+                    if debug:
+                        print(f'Nexh_hop {next_hop} fail, target not response')
+                        print(f'    [ DEL ] -- ip route del {route} via {next_hop} dev {conf_iface} '
+                              f'metric {conf_metric} proto failover [DELETE]')
+                    rc_cmd(f'ip route del {route} via {next_hop} dev {conf_iface} metric {conf_metric} proto failover')
+                    journal.send(f'ip route del {route} via {next_hop} dev {conf_iface} '
+                                 f'metric {conf_metric} proto failover', SYSLOG_IDENTIFIER=my_name)
+
+                time.sleep(int(timeout))