Merge pull request #4326 from Embezzle/T5493

firewall: T5493: Implement remote-group

10 files changed, 196 insertions, 20 deletions

diff --git a/data/templates/firewall/nftables-defines.j2 b/data/templates/firewall/nftables-defines.j2
index fa6cd74c0..3147b4c37 100644
--- a/data/templates/firewall/nftables-defines.j2
+++ b/data/templates/firewall/nftables-defines.j2
@@ -35,6 +35,15 @@
     }
 {%         endfor %}
 {%     endif %}
+{%     if group.remote_group is vyos_defined and is_l3 and not is_ipv6 %}
+{%         for name, name_config in group.remote_group.items() %}
+    set R_{{ name }} {
+        type {{ ip_type }}
+        flags interval
+        auto-merge
+    }
+{%         endfor %}
+{%     endif %}
 {%     if group.mac_group is vyos_defined %}
 {%         for group_name, group_conf in group.mac_group.items() %}
 {%             set includes = group_conf.include if group_conf.include is vyos_defined else [] %}
diff --git a/interface-definitions/firewall.xml.in b/interface-definitions/firewall.xml.in
index e4fe9a508..7538c3cc5 100644
--- a/interface-definitions/firewall.xml.in
+++ b/interface-definitions/firewall.xml.in
@@ -138,6 +138,19 @@
               </tagNode>
             </children>
           </node>
+          <tagNode name="remote-group">
+            <properties>
+              <help>Firewall remote-group</help>
+              <constraint>
+                #include <include/constraint/alpha-numeric-hyphen-underscore-dot.xml.i>
+              </constraint>
+              <constraintErrorMessage>Name of firewall group can only contain alphanumeric letters, hyphen, underscores and dot</constraintErrorMessage>
+            </properties>
+            <children>
+              #include <include/url-http-https.xml.i>
+              #include <include/generic-description.xml.i>
+            </children>
+          </tagNode>
           <tagNode name="interface-group">
             <properties>
               <help>Firewall interface-group</help>
diff --git a/interface-definitions/include/firewall/common-rule-ipv4.xml.i b/interface-definitions/include/firewall/common-rule-ipv4.xml.i
index 803b94b06..b67ef25dc 100644
--- a/interface-definitions/include/firewall/common-rule-ipv4.xml.i
+++ b/interface-definitions/include/firewall/common-rule-ipv4.xml.i
@@ -16,6 +16,7 @@
     #include <include/firewall/port.xml.i>
     #include <include/firewall/source-destination-group.xml.i>
     #include <include/firewall/source-destination-dynamic-group.xml.i>
+    #include <include/firewall/source-destination-remote-group.xml.i>
   </children>
 </node>
 <leafNode name="jump-target">
@@ -39,6 +40,7 @@
     #include <include/firewall/port.xml.i>
     #include <include/firewall/source-destination-group.xml.i>
     #include <include/firewall/source-destination-dynamic-group.xml.i>
+    #include <include/firewall/source-destination-remote-group.xml.i>
   </children>
 </node>
 <!-- include end -->
\ No newline at end of file
diff --git a/interface-definitions/include/firewall/source-destination-remote-group.xml.i b/interface-definitions/include/firewall/source-destination-remote-group.xml.i
new file mode 100644
index 000000000..16463c8eb
--- /dev/null
+++ b/interface-definitions/include/firewall/source-destination-remote-group.xml.i
@@ -0,0 +1,17 @@
+<!-- include start from firewall/source-destination-remote-group.xml.i -->
+<node name="group">
+  <properties>
+    <help>Group</help>
+  </properties>
+  <children>
+    <leafNode name="remote-group">
+      <properties>
+        <help>Group of remote addresses</help>
+        <completionHelp>
+          <path>firewall group remote-group</path>
+        </completionHelp>
+      </properties>
+    </leafNode>
+  </children>
+</node>
+<!-- include end -->
diff --git a/python/vyos/firewall.py b/python/vyos/firewall.py
index 314e8dfe3..9f01f8be1 100755
--- a/python/vyos/firewall.py
+++ b/python/vyos/firewall.py
@@ -310,6 +310,13 @@ def parse_rule(rule_conf, hook, fw_name, rule_id, ip_name):
                         operator = '!='
                         group_name = group_name[1:]
                     output.append(f'{ip_name} {prefix}addr {operator} @D_{group_name}')
+                elif 'remote_group' in group:
+                    group_name = group['remote_group']
+                    operator = ''
+                    if group_name[0] == '!':
+                        operator = '!='
+                        group_name = group_name[1:]
+                    output.append(f'{ip_name} {prefix}addr {operator} @R_{group_name}')
                 if 'mac_group' in group:
                     group_name = group['mac_group']
                     operator = ''
diff --git a/python/vyos/utils/network.py b/python/vyos/utils/network.py
index dc0c0a6d6..2f666f0ee 100644
--- a/python/vyos/utils/network.py
+++ b/python/vyos/utils/network.py
@@ -599,3 +599,19 @@ def get_nft_vrf_zone_mapping() -> dict:
     for (vrf_name, vrf_id) in vrf_list:
         output.append({'interface' : vrf_name, 'vrf_tableid' : vrf_id})
     return output
+
+def is_valid_ipv4_address_or_range(addr: str) -> bool:
+    """
+    Validates if the provided address is a valid IPv4, CIDR or IPv4 range
+    :param addr: address to test
+    :return: bool: True if provided address is valid
+    """
+    from ipaddress import ip_network
+    try:
+        if '-' in addr: # If we are checking a range, validate both address's individually
+            split = addr.split('-')
+            return is_valid_ipv4_address_or_range(split[0]) and is_valid_ipv4_address_or_range(split[1])
+        else:
+            return ip_network(addr).version == 4
+    except:
+        return False
diff --git a/smoketest/scripts/cli/test_firewall.py b/smoketest/scripts/cli/test_firewall.py
index 33144c7fa..2829edbfb 100755
--- a/smoketest/scripts/cli/test_firewall.py
+++ b/smoketest/scripts/cli/test_firewall.py
@@ -1273,5 +1273,39 @@ class TestFirewall(VyOSUnitTestSHIM.TestCase):
         with self.assertRaises(ConfigSessionError):
             self.cli_commit()
 
+    def test_ipv4_remote_group(self):
+        # Setup base config for test
+        self.cli_set(['firewall', 'group', 'remote-group', 'group01', 'url', 'http://127.0.0.1:80/list.txt'])
+        self.cli_set(['firewall', 'group', 'remote-group', 'group01', 'description', 'Example Group 01'])
+        self.cli_set(['firewall', 'ipv4', 'input', 'filter', 'rule', '10', 'action', 'drop'])
+        self.cli_set(['firewall', 'ipv4', 'input', 'filter', 'rule', '10', 'protocol', 'tcp'])
+        self.cli_set(['firewall', 'ipv4', 'input', 'filter', 'rule', '10', 'destination', 'group', 'remote-group', 'group01'])
+
+        self.cli_commit()
+
+        # Test remote-group had been loaded correctly in nft
+        nftables_search = [
+            ['R_group01'],
+            ['type ipv4_addr'],
+            ['flags interval'],
+            ['meta l4proto', 'daddr @R_group01', "ipv4-INP-filter-10"]
+        ]
+        self.verify_nftables(nftables_search, 'ip vyos_filter')
+
+        # Test remote-group cannot be configured without a URL
+        self.cli_delete(['firewall', 'group', 'remote-group', 'group01', 'url'])
+
+        with self.assertRaises(ConfigSessionError):
+            self.cli_commit()
+        self.cli_discard()
+
+        # Test remote-group cannot be set alongside address in rules
+        self.cli_set(['firewall', 'ipv4', 'input', 'filter', 'rule', '10', 'destination', 'address', '127.0.0.1'])
+
+        with self.assertRaises(ConfigSessionError):
+            self.cli_commit()
+        self.cli_discard()
+
+
 if __name__ == '__main__':
     unittest.main(verbosity=2)
diff --git a/src/conf_mode/firewall.py b/src/conf_mode/firewall.py
index 768bb127d..cebe57092 100755
--- a/src/conf_mode/firewall.py
+++ b/src/conf_mode/firewall.py
@@ -44,6 +44,7 @@ airbag.enable()
 
 nftables_conf = '/run/nftables.conf'
 domain_resolver_usage = '/run/use-vyos-domain-resolver-firewall'
+firewall_config_dir = "/config/firewall"
 
 sysctl_file = r'/run/sysctl/10-vyos-firewall.conf'
 
@@ -53,7 +54,8 @@ valid_groups = [
     'network_group',
     'port_group',
     'interface_group',
-    ## Added for group ussage in bridge firewall
+    'remote_group',
+    ## Added for group usage in bridge firewall
     'ipv4_address_group',
     'ipv6_address_group',
     'ipv4_network_group',
@@ -311,8 +313,8 @@ def verify_rule(firewall, family, hook, priority, rule_id, rule_conf):
                 raise ConfigError('Only one of address, fqdn or geoip can be specified')
 
             if 'group' in side_conf:
-                if len({'address_group', 'network_group', 'domain_group'} & set(side_conf['group'])) > 1:
-                    raise ConfigError('Only one address-group, network-group or domain-group can be specified')
+                if len({'address_group', 'network_group', 'domain_group', 'remote_group'} & set(side_conf['group'])) > 1:
+                    raise ConfigError('Only one address-group, network-group, remote-group or domain-group can be specified')
 
                 for group in valid_groups:
                     if group in side_conf['group']:
@@ -332,7 +334,7 @@ def verify_rule(firewall, family, hook, priority, rule_id, rule_conf):
 
                         error_group = fw_group.replace("_", "-")
 
-                        if group in ['address_group', 'network_group', 'domain_group']:
+                        if group in ['address_group', 'network_group', 'domain_group', 'remote_group']:
                             types = [t for t in ['address', 'fqdn', 'geoip'] if t in side_conf]
                             if types:
                                 raise ConfigError(f'{error_group} and {types[0]} cannot both be defined')
@@ -442,6 +444,11 @@ def verify(firewall):
                 for group_name, group in groups.items():
                     verify_nested_group(group_name, group, groups, [])
 
+        if 'remote_group' in firewall['group']:
+            for group_name, group in firewall['group']['remote_group'].items():
+                if 'url' not in group:
+                    raise ConfigError(f'remote-group {group_name} must have a url configured')
+
     for family in ['ipv4', 'ipv6', 'bridge']:
         if family in firewall:
             for chain in ['name','forward','input','output', 'prerouting']:
@@ -539,6 +546,15 @@ def verify(firewall):
 def generate(firewall):
     render(nftables_conf, 'firewall/nftables.j2', firewall)
     render(sysctl_file, 'firewall/sysctl-firewall.conf.j2', firewall)
+
+    # Cleanup remote-group cache files
+    if os.path.exists(firewall_config_dir):
+        for fw_file in os.listdir(firewall_config_dir):
+            # Delete matching files in 'config/firewall' that no longer exist as a remote-group in config
+            if fw_file.startswith("R_") and fw_file.endswith(".txt"):
+                if 'group' not in firewall or 'remote_group' not in firewall['group'] or fw_file[2:-4] not in firewall['group']['remote_group'].keys():
+                    os.unlink(os.path.join(firewall_config_dir, fw_file))
+
     return None
 
 def parse_firewall_error(output):
@@ -598,7 +614,7 @@ def apply(firewall):
 
     ## DOMAIN RESOLVER
     domain_action = 'restart'
-    if dict_search_args(firewall, 'group', 'domain_group') or firewall['ip_fqdn'].items() or firewall['ip6_fqdn'].items():
+    if dict_search_args(firewall, 'group', 'remote_group') or dict_search_args(firewall, 'group', 'domain_group') or firewall['ip_fqdn'].items() or firewall['ip6_fqdn'].items():
         text = f'# Automatically generated by firewall.py\nThis file indicates that vyos-domain-resolver service is used by the firewall.\n'
         Path(domain_resolver_usage).write_text(text)
     else:
diff --git a/src/op_mode/firewall.py b/src/op_mode/firewall.py
index c197ca434..7a3ab921d 100755
--- a/src/op_mode/firewall.py
+++ b/src/op_mode/firewall.py
@@ -253,15 +253,17 @@ def output_firewall_name_statistics(family, hook, prior, prior_conf, single_rule
                     if not source_addr:
                         source_addr = dict_search_args(rule_conf, 'source', 'group', 'domain_group')
                         if not source_addr:
-                            source_addr = dict_search_args(rule_conf, 'source', 'fqdn')
+                            source_addr = dict_search_args(rule_conf, 'source', 'group', 'remote_group')
                             if not source_addr:
-                                source_addr = dict_search_args(rule_conf, 'source', 'geoip', 'country_code')
-                                if source_addr:
-                                    source_addr = str(source_addr)[1:-1].replace('\'','')
-                                    if 'inverse_match' in dict_search_args(rule_conf, 'source', 'geoip'):
-                                        source_addr = 'NOT ' + str(source_addr)
+                                source_addr = dict_search_args(rule_conf, 'source', 'fqdn')
                                 if not source_addr:
-                                    source_addr = 'any'
+                                    source_addr = dict_search_args(rule_conf, 'source', 'geoip', 'country_code')
+                                    if source_addr:
+                                        source_addr = str(source_addr)[1:-1].replace('\'','')
+                                        if 'inverse_match' in dict_search_args(rule_conf, 'source', 'geoip'):
+                                            source_addr = 'NOT ' + str(source_addr)
+                                    if not source_addr:
+                                        source_addr = 'any'
 
             # Get destination
             dest_addr = dict_search_args(rule_conf, 'destination', 'address')
@@ -272,15 +274,17 @@ def output_firewall_name_statistics(family, hook, prior, prior_conf, single_rule
                     if not dest_addr:
                         dest_addr = dict_search_args(rule_conf, 'destination', 'group', 'domain_group')
                         if not dest_addr:
-                            dest_addr = dict_search_args(rule_conf, 'destination', 'fqdn')
+                            dest_addr = dict_search_args(rule_conf, 'destination', 'group', 'remote_group')
                             if not dest_addr:
-                                dest_addr = dict_search_args(rule_conf, 'destination', 'geoip', 'country_code')
-                                if dest_addr:
-                                    dest_addr = str(dest_addr)[1:-1].replace('\'','')
-                                    if 'inverse_match' in dict_search_args(rule_conf, 'destination', 'geoip'):
-                                        dest_addr = 'NOT ' + str(dest_addr)
+                                dest_addr = dict_search_args(rule_conf, 'destination', 'fqdn')
                                 if not dest_addr:
-                                    dest_addr = 'any'
+                                    dest_addr = dict_search_args(rule_conf, 'destination', 'geoip', 'country_code')
+                                    if dest_addr:
+                                        dest_addr = str(dest_addr)[1:-1].replace('\'','')
+                                        if 'inverse_match' in dict_search_args(rule_conf, 'destination', 'geoip'):
+                                            dest_addr = 'NOT ' + str(dest_addr)
+                                    if not dest_addr:
+                                        dest_addr = 'any'
 
             # Get inbound interface
             iiface = dict_search_args(rule_conf, 'inbound_interface', 'name')
@@ -571,6 +575,8 @@ def show_firewall_group(name=None):
                     row.append("\n".join(sorted(group_conf['port'])))
                 elif 'interface' in group_conf:
                     row.append("\n".join(sorted(group_conf['interface'])))
+                elif 'url' in group_conf:
+                    row.append(group_conf['url'])
                 else:
                     row.append('N/D')
                 rows.append(row)
diff --git a/src/services/vyos-domain-resolver b/src/services/vyos-domain-resolver
index 48c6b86d8..aba5ba9db 100755
--- a/src/services/vyos-domain-resolver
+++ b/src/services/vyos-domain-resolver
@@ -13,19 +13,22 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
 import json
 import time
 import logging
+import os
 
 from vyos.configdict import dict_merge
 from vyos.configquery import ConfigTreeQuery
 from vyos.firewall import fqdn_config_parse
 from vyos.firewall import fqdn_resolve
 from vyos.ifconfig import WireGuardIf
+from vyos.remote import download
 from vyos.utils.commit import commit_in_progress
 from vyos.utils.dict import dict_search_args
 from vyos.utils.kernel import WIREGUARD_REKEY_AFTER_TIME
+from vyos.utils.file import makedir, chmod_775, write_file, read_file
+from vyos.utils.network import is_valid_ipv4_address_or_range
 from vyos.utils.process import cmd
 from vyos.utils.process import run
 from vyos.xml_ref import get_defaults
@@ -37,6 +40,8 @@ base_firewall = ['firewall']
 base_nat = ['nat']
 base_interfaces = ['interfaces']
 
+firewall_config_dir = "/config/firewall"
+
 domain_state = {}
 
 ipv4_tables = {
@@ -121,6 +126,56 @@ def nft_valid_sets():
     except:
         return []
 
+def update_remote_group(config):
+    conf_lines = []
+    count = 0
+    valid_sets = nft_valid_sets()
+
+    remote_groups = dict_search_args(config, 'group', 'remote_group')
+    if remote_groups:
+        # Create directory for list files if necessary
+        if not os.path.isdir(firewall_config_dir):
+            makedir(firewall_config_dir, group='vyattacfg')
+            chmod_775(firewall_config_dir)
+
+        for set_name, remote_config in remote_groups.items():
+            if 'url' not in remote_config:
+                continue
+            nft_set_name = f'R_{set_name}'
+
+            # Create list file if necessary
+            list_file = os.path.join(firewall_config_dir, f"{nft_set_name}.txt")
+            if not os.path.exists(list_file):
+                write_file(list_file, '', user="root", group="vyattacfg", mode=0o644)
+
+            # Attempt to download file, use cached version if download fails
+            try:
+                download(list_file, remote_config['url'], raise_error=True)
+            except:
+                logger.error(f'Failed to download list-file for {set_name} remote group')
+                logger.info(f'Using cached list-file for {set_name} remote group')
+
+            # Read list file
+            ip_list = []
+            for line in read_file(list_file).splitlines():
+                line_first_word = line.strip().partition(' ')[0]
+
+                if is_valid_ipv4_address_or_range(line_first_word):
+                    ip_list.append(line_first_word)
+
+            # Load tables
+            for table in ipv4_tables:
+                if (table, nft_set_name) in valid_sets:
+                    conf_lines += nft_output(table, nft_set_name, ip_list)
+
+            count += 1
+
+    nft_conf_str = "\n".join(conf_lines) + "\n"
+    code = run(f'nft --file -', input=nft_conf_str)
+
+    logger.info(f'Updated {count} remote-groups in firewall - result: {code}')
+
+
 def update_fqdn(config, node):
     conf_lines = []
     count = 0
@@ -234,5 +289,6 @@ if __name__ == '__main__':
     while True:
         update_fqdn(firewall, 'firewall')
         update_fqdn(nat, 'nat')
+        update_remote_group(firewall)
         update_interfaces(interfaces, 'interfaces')
         time.sleep(timeout)