From 272b6e646d64651af0b5e8e726021e4fb09cd872 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Sat, 17 Oct 2026 20:54:54 +0200 Subject: wlb: T9472: make inbound-interface "any" match traffic "inbound-interface any", offered by completion, was rendered as an interface literally named "any" and matched no packet, so the rule never balanced anything. It now matches traffic entering on any interface except the WAN interfaces themselves. Matching those as well would also restore the routing mark on replies arriving from an ISP and send them straight back to that ISP. --- python/vyos/wanloadbalance.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'python') diff --git a/python/vyos/wanloadbalance.py b/python/vyos/wanloadbalance.py index 05ae6b536..1dc9a8081 100644 --- a/python/vyos/wanloadbalance.py +++ b/python/vyos/wanloadbalance.py @@ -24,7 +24,13 @@ dhclient_lease = 'dhclient_{0}.lease' def nft_rule(rule_conf, rule_id, local=False, exclude=False, limit=False, weight=None, health_state=None, action=None, restore_mark=False): output = [] - if 'inbound_interface' in rule_conf: + if rule_conf.get('inbound_interface') == 'any': + # any interface but the WAN interfaces - matching replies arriving + # from an ISP would route them back to that ISP + wan_interfaces = ', '.join(f'"{ifname}"' for ifname in rule_conf.get('wan_interfaces', [])) + if wan_interfaces and not local: + output.append(f'iifname != {{ {wan_interfaces} }}') + elif 'inbound_interface' in rule_conf: ifname = rule_conf['inbound_interface'] if local and not exclude: output.append(f'oifname != "{ifname}"') -- cgit v1.2.3 From e71bb87517cec1818714f4dd7199a524494c1949 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Sat, 17 Oct 2026 20:55:01 +0200 Subject: wlb: T9472: honour per-packet-balancing on rules The renderer checked a key no CLI node produces, so "per-packet-balancing" had no effect and every rule balanced per flow only. With the option set, the rule no longer matches new connections only and every packet is balanced on its own. --- python/vyos/wanloadbalance.py | 3 +- smoketest/scripts/cli/test_load-balancing_wan.py | 41 ++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) (limited to 'python') diff --git a/python/vyos/wanloadbalance.py b/python/vyos/wanloadbalance.py index 1dc9a8081..fc9fb1ade 100644 --- a/python/vyos/wanloadbalance.py +++ b/python/vyos/wanloadbalance.py @@ -112,7 +112,8 @@ def nft_rule(rule_conf, rule_id, local=False, exclude=False, limit=False, weight output.append(f'{proto} {prefix}port {operator} @P_{group_name}') - if 'source_based_routing' not in rule_conf and not restore_mark: + # per flow by default - only the first packet of a connection is balanced + if 'per_packet_balancing' not in rule_conf and not restore_mark: output.append('ct state new') if limit and 'limit' in rule_conf and 'rate' in rule_conf['limit']: diff --git a/smoketest/scripts/cli/test_load-balancing_wan.py b/smoketest/scripts/cli/test_load-balancing_wan.py index 686c9e04e..c46c445bb 100755 --- a/smoketest/scripts/cli/test_load-balancing_wan.py +++ b/smoketest/scripts/cli/test_load-balancing_wan.py @@ -117,6 +117,24 @@ class TestLoadBalancingWan(VyOSUnitTestSHIM.TestCase): f'TCP:{lab_target}:{port},connect-timeout=2{bind} Date: Sat, 17 Oct 2026 20:55:12 +0200 Subject: wlb: T9472: honour limit threshold "above" on rules "limit threshold above" was ignored. The rule always matched the traffic below the rate, the same as the default "below". With "above" the rule now matches the traffic exceeding the rate. --- python/vyos/wanloadbalance.py | 4 +++- smoketest/scripts/cli/test_load-balancing_wan.py | 24 ++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) (limited to 'python') diff --git a/python/vyos/wanloadbalance.py b/python/vyos/wanloadbalance.py index fc9fb1ade..a8cd87350 100644 --- a/python/vyos/wanloadbalance.py +++ b/python/vyos/wanloadbalance.py @@ -117,7 +117,9 @@ def nft_rule(rule_conf, rule_id, local=False, exclude=False, limit=False, weight output.append('ct state new') if limit and 'limit' in rule_conf and 'rate' in rule_conf['limit']: - output.append(f'limit rate {rule_conf["limit"]["rate"]}/{rule_conf["limit"]["period"]}') + # "above" matches the traffic exceeding the rate, "below" the rest + over = 'over ' if rule_conf['limit'].get('threshold') == 'above' else '' + output.append(f'limit rate {over}{rule_conf["limit"]["rate"]}/{rule_conf["limit"]["period"]}') if 'burst' in rule_conf['limit']: output.append(f'burst {rule_conf["limit"]["burst"]} packets') diff --git a/smoketest/scripts/cli/test_load-balancing_wan.py b/smoketest/scripts/cli/test_load-balancing_wan.py index c46c445bb..c48a30a4c 100755 --- a/smoketest/scripts/cli/test_load-balancing_wan.py +++ b/smoketest/scripts/cli/test_load-balancing_wan.py @@ -787,6 +787,30 @@ printf 'WLB_INTERFACE_NAME=%s WLB_SCRIPT_IFACE=%s\\n' \\ cmd_in_netns('wlb-lan', f'ping -c 20 -i 0.05 -W 1 {lab_target}') counts = self.lab_count_echo_requests(len(isps)) self.assertTrue(all(counts), msg=f'echo requests per ISP: {counts}') + def test_limit_threshold_above(self): + # the threshold was ignored, "above" behaved like "below" + isps = ['eth231', 'eth232'] + self.lab_setup('eth230', isps) + for index, ifname in enumerate(isps, start=1): + self.lab_health(ifname, index) + # connections beyond one per second go to isp2, all others to isp1 + rule = base_path + ['wan', 'rule', '10'] + self.cli_set(rule + ['inbound-interface', 'eth230']) + self.cli_set(rule + ['interface', 'eth232']) + self.cli_set(rule + ['limit', 'rate', '1']) + self.cli_set(rule + ['limit', 'burst', '1']) + self.cli_set(rule + ['limit', 'threshold', 'above']) + self.cli_set(base_path + ['wan', 'rule', '20', 'inbound-interface', 'eth230']) + self.cli_set(base_path + ['wan', 'rule', '20', 'interface', 'eth231']) + self.cli_commit() + + self.lab_wait_status({'eth231': 'ACTIVE', 'eth232': 'ACTIVE'}) + found, out = self.wait_for_result(lambda: wlb_chain('wlb_mangle_prerouting'), + lambda out: 'limit rate over 1/second burst 1 packets' in out, timeout=15) + self.assertTrue(found, msg=out) + + answers = [self.lab_connect() for _ in range(10)] + self.assertGreaterEqual(answers.count('isp2 peer=10.250.2.1'), 5, msg=answers) if __name__ == '__main__': unittest.main(verbosity=2, failfast=VyOSUnitTestSHIM.TestCase.debug_on()) -- cgit v1.2.3 From bd8be7c142269bbe3849ab70e8e26bc2077e3f9b Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Sat, 17 Oct 2026 20:55:19 +0200 Subject: wlb: T9472: fix the inverted ttl health test The ttl test considered the interface healthy whenever ping failed. A dead uplink, where nothing answers, therefore stayed ACTIVE, while a target answering the ping marked the interface FAILED. The test now passes if the hop where the TTL runs out reports it, or if the target is closer and answers itself. If nothing answers within resp-time, the test fails. --- python/vyos/wanloadbalance.py | 11 ++++--- smoketest/scripts/cli/test_load-balancing_wan.py | 37 ++++++++++++++++++++++++ src/helpers/vyos-load-balancer.py | 3 +- 3 files changed, 46 insertions(+), 5 deletions(-) (limited to 'python') diff --git a/python/vyos/wanloadbalance.py b/python/vyos/wanloadbalance.py index a8cd87350..faea469c2 100644 --- a/python/vyos/wanloadbalance.py +++ b/python/vyos/wanloadbalance.py @@ -17,6 +17,7 @@ import os from vyos.defaults import directories +from vyos.utils.process import rc_cmd from vyos.utils.process import run dhclient_lease = 'dhclient_{0}.lease' @@ -172,10 +173,12 @@ def health_ping_host(host, ifname, count=1, wait_time=0): rc = run(cmd_str) return rc == 0 -def health_ping_host_ttl(host, ifname, count=1, ttl_limit=0): - cmd_str = f'ping -c {count} -t {ttl_limit} -I {ifname} {host}' - rc = run(cmd_str) - return rc != 0 +def health_ping_host_ttl(host, ifname, count=1, ttl_limit=0, wait_time=0): + # The path is up to ttl_limit hops if the hop where the TTL runs out + # reports it, or if the target itself is closer and answers + cmd_str = f'ping -c {count} -W {wait_time} -t {ttl_limit} -I {ifname} {host}' + rc, out = rc_cmd(cmd_str) + return rc == 0 or 'Time to live exceeded' in out def parse_dhcp_nexthop(ifname): lease_file = os.path.join(directories['isc_dhclient_dir'], dhclient_lease.format(ifname)) diff --git a/smoketest/scripts/cli/test_load-balancing_wan.py b/smoketest/scripts/cli/test_load-balancing_wan.py index c48a30a4c..ed3970abc 100755 --- a/smoketest/scripts/cli/test_load-balancing_wan.py +++ b/smoketest/scripts/cli/test_load-balancing_wan.py @@ -811,6 +811,43 @@ printf 'WLB_INTERFACE_NAME=%s WLB_SCRIPT_IFACE=%s\\n' \\ answers = [self.lab_connect() for _ in range(10)] self.assertGreaterEqual(answers.count('isp2 peer=10.250.2.1'), 5, msg=answers) + def test_health_test_ttl(self): + # the ttl test treated every ping failure as healthy and an answer + # as failure - an outage was never detected + isps = ['eth231', 'eth232'] + self.lab_setup('eth230', isps) + # 203.0.113.0/24 is behind isp1, which forwards it into a black hole + call('sudo ip route add 203.0.113.0/24 via 10.250.1.2') + self.addCleanup(call, 'sudo ip route del 203.0.113.0/24 via 10.250.1.2') + cmd_in_netns('wlb-isp1', 'sysctl -w net.ipv4.ip_forward=1') + cmd_in_netns('wlb-isp1', 'ip link add d0 type dummy') + cmd_in_netns('wlb-isp1', 'ip link set d0 up') + cmd_in_netns('wlb-isp1', 'ip addr add 192.0.2.1/30 dev d0') + cmd_in_netns('wlb-isp1', 'ip route add 203.0.113.0/24 via 192.0.2.2') + + self.lab_health('eth232', 2) + test = base_path + ['wan', 'interface-health', 'eth231', 'test', '10'] + self.cli_set(base_path + ['wan', 'interface-health', 'eth231', 'nexthop', '10.250.1.2']) + self.cli_set(test + ['type', 'ttl']) + self.cli_set(test + ['resp-time', '1']) + for ifname in isps: + self.cli_set(base_path + ['wan', 'rule', '10', 'interface', ifname]) + self.cli_set(base_path + ['wan', 'rule', '10', 'inbound-interface', 'eth230']) + + failed = {'eth231': 'FAILED', 'eth232': 'ACTIVE'} + # the target answers itself, or the hop where the TTL runs out does + for target, ttl in [('10.250.1.2', '1'), ('203.0.113.10', '1')]: + self.cli_set(test + ['target', target]) + self.cli_set(test + ['ttl-limit', ttl]) + self.cli_commit() + # the daemon starts with ACTIVE, give it two rounds of checks + found, _ = self.wait_for_result(wlb_status, failed, timeout=12) + self.assertFalse(found, msg=f'ttl test to {target} with ttl {ttl} failed') + + # nothing answers - the path is broken + self.cli_set(test + ['ttl-limit', '3']) + self.cli_commit() + self.lab_wait_status(failed) if __name__ == '__main__': unittest.main(verbosity=2, failfast=VyOSUnitTestSHIM.TestCase.debug_on()) diff --git a/src/helpers/vyos-load-balancer.py b/src/helpers/vyos-load-balancer.py index c0e03dd38..d289f1c7f 100755 --- a/src/helpers/vyos-load-balancer.py +++ b/src/helpers/vyos-load-balancer.py @@ -68,7 +68,8 @@ def health_check(ifname, conf, state, test_defaults): elif check_type == 'ttl': target = test_conf['target'] ttl_limit = test_conf['ttl_limit'] - if not health_ping_host_ttl(target, ifname, ttl_limit=ttl_limit): + resp_time = test_conf['resp_time'] + if not health_ping_host_ttl(target, ifname, ttl_limit=ttl_limit, wait_time=resp_time): return False elif check_type == 'user-defined': script = test_conf['test_script'] -- cgit v1.2.3 From 2b97c4feca89a6f780d7d7aa58262debdca00e05 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Sat, 17 Oct 2026 20:55:39 +0200 Subject: wlb: T9472: keep marks of interfaces when others change The routing table and mark of a WAN interface followed its position in the sorted interface list. Adding an interface which sorts before the others renumbered all of them. Existing connections still carried their old mark in conntrack and were routed into the wrong table - they broke. Every interface now keeps its table as long as it stays configured, new interfaces get the lowest free table. The assignment is dropped once WAN load-balancing is removed. --- python/vyos/wanloadbalance.py | 2 + smoketest/scripts/cli/test_load-balancing_wan.py | 56 ++++++++++++++++++++++++ src/conf_mode/load-balancing_wan.py | 6 +++ src/helpers/vyos-load-balancer.py | 43 +++++++++++++----- 4 files changed, 96 insertions(+), 11 deletions(-) (limited to 'python') diff --git a/python/vyos/wanloadbalance.py b/python/vyos/wanloadbalance.py index faea469c2..8acaa361e 100644 --- a/python/vyos/wanloadbalance.py +++ b/python/vyos/wanloadbalance.py @@ -21,6 +21,8 @@ from vyos.utils.process import rc_cmd from vyos.utils.process import run dhclient_lease = 'dhclient_{0}.lease' +# routing table - and mark - of every WAN interface, see vyos-load-balancer.py +wlb_tables_file = '/run/wlb_tables.json' def nft_rule(rule_conf, rule_id, local=False, exclude=False, limit=False, weight=None, health_state=None, action=None, restore_mark=False): output = [] diff --git a/smoketest/scripts/cli/test_load-balancing_wan.py b/smoketest/scripts/cli/test_load-balancing_wan.py index 5c67778e3..c2162396a 100755 --- a/smoketest/scripts/cli/test_load-balancing_wan.py +++ b/smoketest/scripts/cli/test_load-balancing_wan.py @@ -16,6 +16,7 @@ import json import os +import select import subprocess import tempfile import unittest @@ -881,6 +882,61 @@ printf 'WLB_INTERFACE_NAME=%s WLB_SCRIPT_IFACE=%s\\n' \\ found, last = self.wait_for_result(reported, {'eth231 ACTIVE', 'eth232 ACTIVE'}, timeout=20) self.assertTrue(found, msg=f'hook reported {last}') + def test_mark_stable_when_adding_interface(self): + # marks and routing tables followed the sorted interface list, adding + # an interface renumbered the others and broke their connections + isps = ['eth231', 'eth232', 'eth229'] + self.lab_setup('eth230', isps) + self.lab_health('eth231', 1) + self.lab_health('eth232', 2) + for ifname in ['eth231', 'eth232']: + self.cli_set(base_path + ['wan', 'rule', '10', 'interface', ifname]) + self.cli_set(base_path + ['wan', 'rule', '10', 'inbound-interface', 'eth230']) + self.cli_commit() + self.lab_wait_status({'eth231': 'ACTIVE', 'eth232': 'ACTIVE'}) + + def isp_marks(): + marks = {} + chain = None + for line in rc_cmd('sudo nft list table ip vyos_wanloadbalance')[1].splitlines(): + if 'chain wlb_mangle_isp_' in line: + chain = line.split('wlb_mangle_isp_')[1].split()[0] + elif chain and 'meta mark set' in line: + marks[chain] = line.split('meta mark set ')[1].split()[0] + return marks + + marks = isp_marks() + + # a long-lived connection through one of the ISPs + client = subprocess.Popen(['sudo', 'ip', 'netns', 'exec', 'wlb-lan', 'socat', '-', + f'TCP:{lab_target}:90'], stdin=subprocess.PIPE, + stdout=subprocess.PIPE, text=True) + self.addCleanup(client.kill) + + def echo(text, timeout=5): + client.stdin.write(f'{text}\n') + client.stdin.flush() + if select.select([client.stdout], [], [], timeout)[0]: + return client.stdout.readline().strip() + return None + + self.assertEqual(echo('first'), 'first') + + # eth229 sorts before the interfaces already in use + self.cli_set(base_path + ['wan', 'interface-health', 'eth229', 'nexthop', '10.250.3.2']) + self.cli_set(base_path + ['wan', 'interface-health', 'eth229', 'test', '10', 'target', '10.250.3.2']) + self.cli_commit() + self.lab_wait_status({'eth229': 'ACTIVE', 'eth231': 'ACTIVE', 'eth232': 'ACTIVE'}) + + new_marks = isp_marks() + for ifname in ['eth231', 'eth232']: + self.assertEqual(new_marks.get(ifname), marks[ifname], msg=f'mark of {ifname} changed') + self.assertEqual(echo('second'), 'second') + + # a removed WAN load-balancing starts over with the first table + self.cli_delete(base_path) + self.cli_commit() + self.assertFalse(os.path.exists('/run/wlb_tables.json')) if __name__ == '__main__': unittest.main(verbosity=2, failfast=VyOSUnitTestSHIM.TestCase.debug_on()) diff --git a/src/conf_mode/load-balancing_wan.py b/src/conf_mode/load-balancing_wan.py index 63943510f..fa95bf794 100755 --- a/src/conf_mode/load-balancing_wan.py +++ b/src/conf_mode/load-balancing_wan.py @@ -14,12 +14,15 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import os + from sys import exit from vyos.config import Config from vyos.configdep import set_dependents, call_dependents from vyos.utils.dict import dict_search_args from vyos.utils.process import cmdl +from vyos.wanloadbalance import wlb_tables_file from vyos import ConfigError from vyos import airbag airbag.enable() @@ -162,6 +165,9 @@ def generate(lb): def apply(lb): if not lb: cmdl(['systemctl', 'stop', service]) + # without WAN load-balancing the marks of existing connections are moot + if os.path.exists(wlb_tables_file): + os.unlink(wlb_tables_file) else: cmdl(['systemctl', 'restart', service]) diff --git a/src/helpers/vyos-load-balancer.py b/src/helpers/vyos-load-balancer.py index 38ba8d87b..d4db328c7 100755 --- a/src/helpers/vyos-load-balancer.py +++ b/src/helpers/vyos-load-balancer.py @@ -33,6 +33,7 @@ from vyos.wanloadbalance import health_ping_host from vyos.wanloadbalance import health_ping_host_ttl from vyos.wanloadbalance import parse_dhcp_nexthop from vyos.wanloadbalance import parse_ppp_nexthop +from vyos.wanloadbalance import wlb_tables_file nftables_wlb_conf = '/run/nftables_wlb.conf' wlb_status_file = '/run/wlb_status.json' @@ -207,12 +208,35 @@ def nftables_update(lb): return True +def assign_tables(interfaces, mark_offset): + """ Return the routing table - and mark - of every interface. Connections + keep their mark in conntrack, an interface keeps its table as long as it + stays configured. New interfaces get the lowest free table. """ + try: + with open(wlb_tables_file) as f: + previous = json.load(f) + except (OSError, ValueError): + previous = {} + + tables = {ifname: previous[ifname] for ifname in interfaces if ifname in previous} + table_num = mark_offset + for ifname in interfaces: + if ifname in tables: + continue + table_num += 1 + while table_num in tables.values(): + table_num += 1 + tables[ifname] = table_num + + with open(wlb_tables_file, 'w') as f: + json.dump(tables, f) + return tables + def cleanup(lb): - if 'interface_health' in lb: - index = 1 - for ifname, health_conf in lb['interface_health'].items(): - table_num = lb['mark_offset'] + index - suppress_prio = lb['mark_offset'] + index + if 'health_state' in lb: + for ifname, state in lb['health_state'].items(): + table_num = state['table_number'] + suppress_prio = table_num table_prio = suppress_prio + 100 run(f'ip route del table {table_num} default') run( @@ -224,7 +248,6 @@ def cleanup(lb): f'priority {table_prio}' ) run(f'ip rule del fwmark {hex(table_num)} table {table_num}') - index += 1 run(f'nft delete table ip vyos_wanloadbalance') @@ -261,9 +284,9 @@ if __name__ == '__main__': # Create state dicts, interface address and nexthop, install routes and ip rules if 'interface_health' in lb: - index = 1 + tables = assign_tables(lb['interface_health'], lb['mark_offset']) for ifname, health_conf in lb['interface_health'].items(): - table_num = lb['mark_offset'] + index + table_num = tables[ifname] addr = get_ipv4_address(ifname) lb['health_state'][ifname] = { 'if_addr': addr, @@ -284,7 +307,7 @@ if __name__ == '__main__': else: run(f'ip route replace table {table_num} default dev {ifname} via {health_conf["nexthop"]}') - suppress_prio = lb['mark_offset'] + index + suppress_prio = table_num table_prio = suppress_prio + 100 if 'only_default_route' in lb: run( @@ -298,8 +321,6 @@ if __name__ == '__main__': else: run(f'ip rule add fwmark {hex(table_num)} table {table_num}') - index += 1 - nftables_update(lb) run('ip route flush cache') -- cgit v1.2.3