firewall: T970: Refactor domain resolver, add firewall source/destination `fqdn` node

5 files changed, 217 insertions, 109 deletions

diff --git a/src/conf_mode/firewall.py b/src/conf_mode/firewall.py
index cbd9cbe90..2bb765e65 100755
--- a/src/conf_mode/firewall.py
+++ b/src/conf_mode/firewall.py
@@ -27,12 +27,8 @@ from vyos.configdict import dict_merge
 from vyos.configdict import node_changed
 from vyos.configdiff import get_config_diff, Diff
 # from vyos.configverify import verify_interface_exists
+from vyos.firewall import fqdn_config_parse
 from vyos.firewall import geoip_update
-from vyos.firewall import get_ips_domains_dict
-from vyos.firewall import nft_add_set_elements
-from vyos.firewall import nft_flush_set
-from vyos.firewall import nft_init_set
-from vyos.firewall import nft_update_set_elements
 from vyos.template import render
 from vyos.util import call
 from vyos.util import cmd
@@ -173,6 +169,8 @@ def get_config(config=None):
 
     firewall['geoip_updated'] = geoip_updated(conf, firewall)
 
+    fqdn_config_parse(firewall)
+
     return firewall
 
 def verify_rule(firewall, rule_conf, ipv6):
@@ -232,29 +230,28 @@ def verify_rule(firewall, rule_conf, ipv6):
         if side in rule_conf:
             side_conf = rule_conf[side]
 
-            if dict_search_args(side_conf, 'geoip', 'country_code'):
-                if 'address' in side_conf:
-                    raise ConfigError('Address and GeoIP cannot both be defined')
-
-                if dict_search_args(side_conf, 'group', 'address_group'):
-                    raise ConfigError('Address-group and GeoIP cannot both be defined')
-
-                if dict_search_args(side_conf, 'group', 'network_group'):
-                    raise ConfigError('Network-group and GeoIP cannot both be defined')
+            if len({'address', 'fqdn', 'geoip'} & set(side_conf)) > 1:
+                raise ConfigError('Only one of address, fqdn or geoip can be specified')
 
             if 'group' in side_conf:
-                if {'address_group', 'network_group'} <= set(side_conf['group']):
-                    raise ConfigError('Only one address-group or network-group can be specified')
+                if len({'address_group', 'network_group', 'domain_group'} & set(side_conf['group'])) > 1:
+                    raise ConfigError('Only one address-group, network-group or domain-group can be specified')
 
                 for group in valid_groups:
                     if group in side_conf['group']:
                         group_name = side_conf['group'][group]
 
+                        fw_group = f'ipv6_{group}' if ipv6 and group in ['address_group', 'network_group'] else group
+                        error_group = fw_group.replace("_", "-")
+
+                        if group in ['address_group', 'network_group', 'domain_group']:
+                            types = [t for t in ['address', 'fqdn', 'geoip'] if t in side_conf]
+                            if types:
+                                raise ConfigError(f'{error_group} and {types[0]} cannot both be defined')
+
                         if group_name and group_name[0] == '!':
                             group_name = group_name[1:]
 
-                        fw_group = f'ipv6_{group}' if ipv6 and group in ['address_group', 'network_group'] else group
-                        error_group = fw_group.replace("_", "-")
                         group_obj = dict_search_args(firewall, 'group', fw_group, group_name)
 
                         if group_obj is None:
@@ -477,26 +474,13 @@ def apply(firewall):
     if install_result == 1:
         raise ConfigError(f'Failed to apply firewall: {output}')
 
-    # set firewall group domain-group xxx
-    if 'group' in firewall:
-        if 'domain_group' in firewall['group']:
-            # T970 Enable a resolver (systemd daemon) that checks
-            # domain-group addresses and update entries for domains by timeout
-            # If router loaded without internet connection or for synchronization
-            call('systemctl restart vyos-domain-group-resolve.service')
-            for group, group_config in firewall['group']['domain_group'].items():
-                domains = []
-                if group_config.get('address') is not None:
-                    for address in group_config.get('address'):
-                        domains.append(address)
-                # Add elements to domain-group, try to resolve domain => ip
-                # and add elements to nft set
-                ip_dict = get_ips_domains_dict(domains)
-                elements = sum(ip_dict.values(), [])
-                nft_init_set(f'D_{group}')
-                nft_add_set_elements(f'D_{group}', elements)
-        else:
-            call('systemctl stop vyos-domain-group-resolve.service')
+    # T970 Enable a resolver (systemd daemon) that checks
+    # domain-group addresses and update entries for domains by timeout
+    # If router loaded without internet connection or for synchronization
+    domain_action = 'stop'
+    if dict_search_args(firewall, 'group', 'domain_group') or firewall['ip_fqdn'] or firewall['ip6_fqdn']:
+        domain_action = 'restart'
+    call(f'systemctl {domain_action} vyos-domain-resolver.service')
 
     apply_sysfs(firewall)
 
diff --git a/src/helpers/vyos-domain-group-resolve.py b/src/helpers/vyos-domain-group-resolve.py
deleted file mode 100755
index 6b677670b..000000000
--- a/src/helpers/vyos-domain-group-resolve.py
+++ /dev/null
@@ -1,60 +0,0 @@
-#!/usr/bin/env python3
-#
-# Copyright (C) 2022 VyOS maintainers and contributors
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 or later as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-
-import time
-
-from vyos.configquery import ConfigTreeQuery
-from vyos.firewall import get_ips_domains_dict
-from vyos.firewall import nft_add_set_elements
-from vyos.firewall import nft_flush_set
-from vyos.firewall import nft_init_set
-from vyos.firewall import nft_update_set_elements
-from vyos.util import call
-
-
-base = ['firewall', 'group', 'domain-group']
-check_required = True
-# count_failed = 0
-# Timeout in sec between checks
-timeout = 300
-
-domain_state = {}
-
-if __name__ == '__main__':
-
-    while check_required:
-        config = ConfigTreeQuery()
-        if config.exists(base):
-            domain_groups = config.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True)
-            for set_name, domain_config in domain_groups.items():
-                list_domains = domain_config['address']
-                elements = []
-                ip_dict = get_ips_domains_dict(list_domains)
-
-                for domain in list_domains:
-                    # Resolution succeeded, update domain state
-                    if domain in ip_dict:
-                        domain_state[domain] = ip_dict[domain]
-                        elements += ip_dict[domain]
-                    # Resolution failed, use previous domain state
-                    elif domain in domain_state:
-                        elements += domain_state[domain]
-
-                # Resolve successful
-                if elements:
-                    nft_update_set_elements(f'D_{set_name}', elements)
-        time.sleep(timeout)
diff --git a/src/helpers/vyos-domain-resolver.py b/src/helpers/vyos-domain-resolver.py
new file mode 100755
index 000000000..2f71f15db
--- /dev/null
+++ b/src/helpers/vyos-domain-resolver.py
@@ -0,0 +1,182 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2022 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import json
+import os
+import time
+
+from vyos.configdict import dict_merge
+from vyos.configquery import ConfigTreeQuery
+from vyos.firewall import fqdn_config_parse
+from vyos.firewall import fqdn_resolve
+from vyos.util import cmd
+from vyos.util import commit_in_progress
+from vyos.util import dict_search_args
+from vyos.util import run
+from vyos.xml import defaults
+
+base = ['firewall']
+timeout = 300
+cache = False
+
+domain_state = {}
+
+ipv4_tables = {
+    'ip mangle',
+    'ip vyos_filter',
+}
+
+ipv6_tables = {
+    'ip6 mangle',
+    'ip6 vyos_filter'
+}
+
+def get_config(conf):
+    firewall = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True,
+                                    no_tag_node_value_mangle=True)
+
+    default_values = defaults(base)
+    for tmp in ['name', 'ipv6_name']:
+        if tmp in default_values:
+            del default_values[tmp]
+
+    if 'zone' in default_values:
+        del default_values['zone']
+
+    firewall = dict_merge(default_values, firewall)
+
+    global timeout, cache
+
+    if 'resolver_interval' in firewall:
+        timeout = int(firewall['resolver_interval'])
+
+    if 'resolver_cache' in firewall:
+        cache = True
+
+    fqdn_config_parse(firewall)
+
+    return firewall
+
+def resolve(domains, ipv6=False):
+    global domain_state
+
+    ip_list = set()
+
+    for domain in domains:
+        resolved = fqdn_resolve(domain, ipv6=ipv6)
+
+        if resolved and cache:
+            domain_state[domain] = resolved
+        elif not resolved:
+            if domain not in domain_state:
+                continue
+            resolved = domain_state[domain]
+
+        ip_list = ip_list | resolved
+    return ip_list
+
+def nft_output(table, set_name, ip_list):
+    output = [f'flush set {table} {set_name}']
+    if ip_list:
+        ip_str = ','.join(ip_list)
+        output.append(f'add element {table} {set_name} {{ {ip_str} }}')
+    return output
+
+def nft_valid_sets():
+    try:
+        valid_sets = []
+        sets_json = cmd('nft -j list sets')
+        sets_obj = json.loads(sets_json)
+
+        for obj in sets_obj['nftables']:
+            if 'set' in obj:
+                family = obj['set']['family']
+                table = obj['set']['table']
+                name = obj['set']['name']
+                valid_sets.append((f'{family} {table}', name))
+
+        return valid_sets
+    except:
+        return []
+
+def update(firewall):
+    conf_lines = []
+    count = 0
+
+    valid_sets = nft_valid_sets()
+
+    domain_groups = dict_search_args(firewall, 'group', 'domain_group')
+    if domain_groups:
+        for set_name, domain_config in domain_groups.items():
+            if 'address' not in domain_config:
+                continue
+
+            nft_set_name = f'D_{set_name}'
+            domains = domain_config['address']
+
+            ip_list = resolve(domains, ipv6=False)
+            for table in ipv4_tables:
+                if (table, nft_set_name) in valid_sets:
+                    conf_lines += nft_output(table, nft_set_name, ip_list)
+
+            ip6_list = resolve(domains, ipv6=True)
+            for table in ipv6_tables:
+                if (table, nft_set_name) in valid_sets:
+                    conf_lines += nft_output(table, nft_set_name, ip6_list)
+            count += 1
+
+    for set_name, domain in firewall['ip_fqdn'].items():
+        table = 'ip vyos_filter'
+        nft_set_name = f'FQDN_{set_name}'
+
+        ip_list = resolve([domain], ipv6=False)
+
+        if (table, nft_set_name) in valid_sets:
+            conf_lines += nft_output(table, nft_set_name, ip_list)
+        count += 1
+
+    for set_name, domain in firewall['ip6_fqdn'].items():
+        table = 'ip6 vyos_filter'
+        nft_set_name = f'FQDN_{set_name}'
+
+        ip_list = resolve([domain], ipv6=True)
+        if (table, nft_set_name) in valid_sets:
+            conf_lines += nft_output(table, nft_set_name, ip_list)
+        count += 1
+
+    nft_conf_str = "\n".join(conf_lines) + "\n"
+    code = run(f'nft -f -', input=nft_conf_str)
+
+    print(f'Updated {count} sets - result: {code}')
+
+if __name__ == '__main__':
+    print(f'VyOS domain resolver')
+
+    count = 1
+    while commit_in_progress():
+        if ( count % 60 == 0 ):
+            print(f'Commit still in progress after {count}s - waiting')
+        count += 1
+        time.sleep(1)
+
+    conf = ConfigTreeQuery()
+    firewall = get_config(conf)
+
+    print(f'interval: {timeout}s - cache: {cache}')
+
+    while True:
+        update(firewall)
+        time.sleep(timeout)
diff --git a/src/systemd/vyos-domain-group-resolve.service b/src/systemd/vyos-domain-group-resolve.service
deleted file mode 100644
index 29628fddb..000000000
--- a/src/systemd/vyos-domain-group-resolve.service
+++ /dev/null
@@ -1,11 +0,0 @@
-[Unit]
-Description=VyOS firewall domain-group resolver
-After=vyos-router.service
-
-[Service]
-Type=simple
-Restart=always
-ExecStart=/usr/bin/python3 /usr/libexec/vyos/vyos-domain-group-resolve.py
-
-[Install]
-WantedBy=multi-user.target
diff --git a/src/systemd/vyos-domain-resolver.service b/src/systemd/vyos-domain-resolver.service
new file mode 100644
index 000000000..c56b51f0c
--- /dev/null
+++ b/src/systemd/vyos-domain-resolver.service
@@ -0,0 +1,13 @@
+[Unit]
+Description=VyOS firewall domain resolver
+After=vyos-router.service
+
+[Service]
+Type=simple
+Restart=always
+ExecStart=/usr/bin/python3 -u /usr/libexec/vyos/vyos-domain-resolver.py
+StandardError=journal
+StandardOutput=journal
+
+[Install]
+WantedBy=multi-user.target