wlb: T9472: keep marks of interfaces when others change

The routing table and mark of a WAN interface followed its position in the
sorted interface list. Adding an interface which sorts before the others
renumbered all of them. Existing connections still carried their old mark
in conntrack and were routed into the wrong table - they broke.

Every interface now keeps its table as long as it stays configured, new
interfaces get the lowest free table. The assignment is dropped once WAN
load-balancing is removed.

2 files changed, 38 insertions, 11 deletions

diff --git a/src/conf_mode/load-balancing_wan.py b/src/conf_mode/load-balancing_wan.py
index 63943510f..fa95bf794 100755
--- a/src/conf_mode/load-balancing_wan.py
+++ b/src/conf_mode/load-balancing_wan.py
@@ -14,12 +14,15 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import os
+
 from sys import exit
 
 from vyos.config import Config
 from vyos.configdep import set_dependents, call_dependents
 from vyos.utils.dict import dict_search_args
 from vyos.utils.process import cmdl
+from vyos.wanloadbalance import wlb_tables_file
 from vyos import ConfigError
 from vyos import airbag
 airbag.enable()
@@ -162,6 +165,9 @@ def generate(lb):
 def apply(lb):
     if not lb:
         cmdl(['systemctl', 'stop', service])
+        # without WAN load-balancing the marks of existing connections are moot
+        if os.path.exists(wlb_tables_file):
+            os.unlink(wlb_tables_file)
     else:
         cmdl(['systemctl', 'restart', service])
 
diff --git a/src/helpers/vyos-load-balancer.py b/src/helpers/vyos-load-balancer.py
index 38ba8d87b..d4db328c7 100755
--- a/src/helpers/vyos-load-balancer.py
+++ b/src/helpers/vyos-load-balancer.py
@@ -33,6 +33,7 @@ from vyos.wanloadbalance import health_ping_host
 from vyos.wanloadbalance import health_ping_host_ttl
 from vyos.wanloadbalance import parse_dhcp_nexthop
 from vyos.wanloadbalance import parse_ppp_nexthop
+from vyos.wanloadbalance import wlb_tables_file
 
 nftables_wlb_conf = '/run/nftables_wlb.conf'
 wlb_status_file = '/run/wlb_status.json'
@@ -207,12 +208,35 @@ def nftables_update(lb):
 
     return True
 
+def assign_tables(interfaces, mark_offset):
+    """ Return the routing table - and mark - of every interface. Connections
+    keep their mark in conntrack, an interface keeps its table as long as it
+    stays configured. New interfaces get the lowest free table. """
+    try:
+        with open(wlb_tables_file) as f:
+            previous = json.load(f)
+    except (OSError, ValueError):
+        previous = {}
+
+    tables = {ifname: previous[ifname] for ifname in interfaces if ifname in previous}
+    table_num = mark_offset
+    for ifname in interfaces:
+        if ifname in tables:
+            continue
+        table_num += 1
+        while table_num in tables.values():
+            table_num += 1
+        tables[ifname] = table_num
+
+    with open(wlb_tables_file, 'w') as f:
+        json.dump(tables, f)
+    return tables
+
 def cleanup(lb):
-    if 'interface_health' in lb:
-        index = 1
-        for ifname, health_conf in lb['interface_health'].items():
-            table_num = lb['mark_offset'] + index
-            suppress_prio = lb['mark_offset'] + index
+    if 'health_state' in lb:
+        for ifname, state in lb['health_state'].items():
+            table_num = state['table_number']
+            suppress_prio = table_num
             table_prio = suppress_prio + 100
             run(f'ip route del table {table_num} default')
             run(
@@ -224,7 +248,6 @@ def cleanup(lb):
                 f'priority {table_prio}'
             )
             run(f'ip rule del fwmark {hex(table_num)} table {table_num}')
-            index += 1
 
     run(f'nft delete table ip vyos_wanloadbalance')
 
@@ -261,9 +284,9 @@ if __name__ == '__main__':
 
     # Create state dicts, interface address and nexthop, install routes and ip rules
     if 'interface_health' in lb:
-        index = 1
+        tables = assign_tables(lb['interface_health'], lb['mark_offset'])
         for ifname, health_conf in lb['interface_health'].items():
-            table_num = lb['mark_offset'] + index
+            table_num = tables[ifname]
             addr = get_ipv4_address(ifname)
             lb['health_state'][ifname] = {
                 'if_addr': addr,
@@ -284,7 +307,7 @@ if __name__ == '__main__':
             else:
                 run(f'ip route replace table {table_num} default dev {ifname} via {health_conf["nexthop"]}')
 
-            suppress_prio = lb['mark_offset'] + index
+            suppress_prio = table_num
             table_prio = suppress_prio + 100
             if 'only_default_route' in lb:
                 run(
@@ -298,8 +321,6 @@ if __name__ == '__main__':
             else:
                 run(f'ip rule add fwmark {hex(table_num)} table {table_num}')
 
-            index += 1
-
         nftables_update(lb)
 
         run('ip route flush cache')