From 91850ce83561cdd24f21a0dfa67d0754b0ff5e80 Mon Sep 17 00:00:00 2001 From: Brad Kollmyer Date: Fri, 10 Jul 2026 14:54:38 -0700 Subject: firewall: T9076: add per-remote-group update interval Add 'set firewall group remote-group interval ' to control how often each remote group list is re-downloaded, independent of the global resolver-interval that also drives domain-group/FQDN resolution. The value accepts plain seconds or time-unit suffixes s/m/h/d/w (e.g. 4h), range 60 seconds to 4 weeks, enforced at commit time after conversion. When unset, the group keeps following 'firewall global-options resolver-interval', so existing configurations are unaffected. vyos-domain-resolver now tracks a last-update timestamp per remote group and sleeps until the next due update instead of a fixed resolver-interval tick, honoring per-group intervals both shorter and longer than the global one. A group is only stamped as updated after a successful download; failed downloads fall back to the cached list and are retried at the resolver cadence rather than after the full group interval. human_to_seconds() now treats a plain number as seconds instead of returning 0. Co-Authored-By: Claude Fable 5 --- src/conf_mode/firewall.py | 8 +++++++ src/services/vyos-domain-resolver | 49 +++++++++++++++++++++++++++++++++------ 2 files changed, 50 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/conf_mode/firewall.py b/src/conf_mode/firewall.py index 29181b4ef..893fc8970 100755 --- a/src/conf_mode/firewall.py +++ b/src/conf_mode/firewall.py @@ -30,6 +30,7 @@ from vyos.ethtool import Ethtool from vyos.firewall import fqdn_config_parse from vyos.geoip import geoip_refresh, geoip_update from vyos.template import render +from vyos.utils.convert import human_to_seconds from vyos.utils.dict import dict_search from vyos.utils.dict import dict_search_args from vyos.utils.dict import dict_search_recursive @@ -543,6 +544,13 @@ def verify(firewall): for group_name, group in firewall['group']['remote_group'].items(): if 'url' not in group: raise ConfigError(f'remote-group {group_name} must have a url configured') + if 'interval' in group: + interval = human_to_seconds(group['interval']) + if not 60 <= interval <= 2419200: + raise ConfigError( + f'remote-group {group_name} interval must be ' + 'between 60 seconds and 4 weeks' + ) offload_chains_v4 = set() offload_chains_v6 = set() diff --git a/src/services/vyos-domain-resolver b/src/services/vyos-domain-resolver index 60fcb7df4..99bca9afc 100755 --- a/src/services/vyos-domain-resolver +++ b/src/services/vyos-domain-resolver @@ -27,6 +27,7 @@ from vyos.firewall import fqdn_resolve from vyos.ifconfig import WireGuardIf from vyos.remote import download from vyos.utils.commit import commit_in_progress +from vyos.utils.convert import human_to_seconds from vyos.utils.dict import dict_search_args from vyos.utils.kernel import WIREGUARD_REKEY_AFTER_TIME from vyos.utils.file import makedir, chmod_775, write_file, read_file @@ -45,6 +46,7 @@ base_interfaces = ['interfaces'] firewall_config_dir = "/config/firewall" domain_state = {} +remote_group_last_update = {} ipv4_tables = { 'ip vyos_mangle', @@ -131,9 +133,16 @@ def nft_valid_sets(): except: return [] +def remote_group_interval(remote_config): + # Per-group update interval, falling back to the global resolver interval + if 'interval' in remote_config: + return human_to_seconds(remote_config['interval']) + return timeout + def update_remote_group(config): conf_lines = [] count = 0 + now = time.time() valid_sets = nft_valid_sets() remote_groups = dict_search_args(config, 'group', 'remote_group') @@ -146,6 +155,10 @@ def update_remote_group(config): for set_name, remote_config in remote_groups.items(): if 'url' not in remote_config: continue + interval = remote_group_interval(remote_config) + last_update = remote_group_last_update.get(set_name, 0) + if now - last_update < interval: + continue nft_ip_set_name = f'R_{set_name}' nft_ip6_set_name = f'R6_{set_name}' @@ -157,7 +170,12 @@ def update_remote_group(config): # Attempt to download file, use cached version if download fails try: download(list_file, remote_config['url'], raise_error=True) + remote_group_last_update[set_name] = now except Exception as err: + # Retry failed downloads at the resolver cadence instead of + # waiting the full group interval + retry = min(interval, timeout) + remote_group_last_update[set_name] = now - interval + retry url = urlsplit(remote_config['url']) _, _, netloc = url.netloc.rpartition('@') redacted_url = urlunsplit(url._replace(netloc=netloc, query='', fragment='')) @@ -204,10 +222,11 @@ def update_remote_group(config): count += 1 - nft_conf_str = "\n".join(conf_lines) + "\n" - code = run(f'nft --file -', input=nft_conf_str) + if count: + nft_conf_str = "\n".join(conf_lines) + "\n" + code = run(f'nft --file -', input=nft_conf_str) - logger.info(f'Updated {count} remote-groups in firewall - result: {code}') + logger.info(f'Updated {count} remote-groups in firewall - result: {code}') def update_fqdn(config, node): @@ -320,9 +339,25 @@ if __name__ == '__main__': logger.info(f'interval: {timeout}s - cache: {cache}') + remote_groups = dict_search_args(firewall, 'group', 'remote_group') or {} + + last_fqdn_update = 0 while True: - update_fqdn(firewall, 'firewall') - update_fqdn(nat, 'nat') + now = time.time() + + if now - last_fqdn_update >= timeout: + last_fqdn_update = now + update_fqdn(firewall, 'firewall') + update_fqdn(nat, 'nat') + update_interfaces(interfaces, 'interfaces') + update_remote_group(firewall) - update_interfaces(interfaces, 'interfaces') - time.sleep(timeout) + + # Sleep until the next scheduled update + next_due = [last_fqdn_update + timeout] + for set_name, remote_config in remote_groups.items(): + if 'url' not in remote_config: + continue + next_due.append(remote_group_last_update.get(set_name, 0) + + remote_group_interval(remote_config)) + time.sleep(max(min(next_due) - time.time(), 1)) -- cgit v1.2.3