Merge pull request #4050 from jestabro/revise-migration-circinus

T6007: revise migration system

3 files changed, 218 insertions, 275 deletions

diff --git a/src/migration-scripts/vrrp/1-to-2 b/src/migration-scripts/vrrp/1-to-2
index dba5af81c..8639a7553 100755..100644
--- a/src/migration-scripts/vrrp/1-to-2
+++ b/src/migration-scripts/vrrp/1-to-2
@@ -1,37 +1,23 @@
-#!/usr/bin/env python3
+# Copyright 2018-2024 VyOS maintainers and contributors <maintainers@vyos.io>
 #
-# Copyright (C) 2018 VyOS maintainers and contributors
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 or later as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
+# This library is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
 #
+# You should have received a copy of the GNU Lesser General Public License
+# along with this library.  If not, see <http://www.gnu.org/licenses/>.
 
 import re
-import sys
 
 from vyos.configtree import ConfigTree
 
 
-if len(sys.argv) < 2:
-    print("Must specify file name!")
-    sys.exit(1)
-
-file_name = sys.argv[1]
-
-with open(file_name, 'r') as f:
-    config_file = f.read()
-
-config = ConfigTree(config_file)
-
 # Convert the old VRRP syntax to the new syntax
 
 # The old approach was to put VRRP groups inside interfaces,
@@ -109,162 +95,156 @@ def get_vrrp_group(path):
 # Only if no data is collected from any interface we can conclude that VRRP is not configured
 # and exit.
 
-groups = []
-base_paths = []
-
-if config.exists(["interfaces", "ethernet"]):
-    base_paths.append("ethernet")
-if config.exists(["interfaces", "bonding"]):
-    base_paths.append("bonding")
-
-for bp in base_paths:
-    parent_path = ["interfaces", bp]
-
-    parent_intfs = config.list_nodes(parent_path)
-
-    for pi in parent_intfs:
-        # Extract VRRP groups from the parent interface
-        vg_path =[pi, "vrrp", "vrrp-group"]
-        if config.exists(parent_path + vg_path):
-            pgroups = config.list_nodes(parent_path + vg_path)
-            for pg in pgroups:
-                g = get_vrrp_group(parent_path + vg_path + [pg])
-                g["interface"] = pi
-                g["vrid"] = pg
-                groups.append(g)
-
-            # Delete the VRRP subtree
-            # If left in place, configs will not load correctly
-            config.delete(parent_path + [pi, "vrrp"])
-
-        # Extract VRRP groups from 802.1q VLAN interfaces
-        if config.exists(parent_path + [pi, "vif"]):
-            vifs = config.list_nodes(parent_path + [pi, "vif"])
-            for vif in vifs:
-                vif_vg_path = [pi, "vif", vif, "vrrp", "vrrp-group"]
-                if config.exists(parent_path + vif_vg_path):
-                    vifgroups = config.list_nodes(parent_path + vif_vg_path)
-                    for vif_group in vifgroups:
-                        g = get_vrrp_group(parent_path + vif_vg_path + [vif_group])
-                        g["interface"] = "{0}.{1}".format(pi, vif)
-                        g["vrid"] = vif_group
-                        groups.append(g)
-
-                    config.delete(parent_path + [pi, "vif", vif, "vrrp"])
-
-        # Extract VRRP groups from 802.3ad QinQ service VLAN interfaces
-        if config.exists(parent_path + [pi, "vif-s"]):
-            vif_ss = config.list_nodes(parent_path + [pi, "vif-s"])
-            for vif_s in vif_ss:
-                vifs_vg_path = [pi, "vif-s", vif_s, "vrrp", "vrrp-group"]
-                if config.exists(parent_path + vifs_vg_path):
-                    vifsgroups = config.list_nodes(parent_path + vifs_vg_path)
-                    for vifs_group in vifsgroups:
-                        g = get_vrrp_group(parent_path + vifs_vg_path + [vifs_group])
-                        g["interface"] = "{0}.{1}".format(pi, vif_s)
-                        g["vrid"] = vifs_group
-                        groups.append(g)
-
-                    config.delete(parent_path + [pi, "vif-s", vif_s, "vrrp"])
-
-                # Extract VRRP groups from QinQ client VLAN interfaces nested in the vif-s
-                if config.exists(parent_path + [pi, "vif-s", vif_s, "vif-c"]):
-                    vif_cs = config.list_nodes(parent_path + [pi, "vif-s", vif_s, "vif-c"])
-                    for vif_c in vif_cs:
-                         vifc_vg_path = [pi, "vif-s", vif_s, "vif-c", vif_c, "vrrp", "vrrp-group"]
-                         vifcgroups = config.list_nodes(parent_path + vifc_vg_path)
-                         for vifc_group in vifcgroups:
-                              g = get_vrrp_group(parent_path + vifc_vg_path + [vifc_group])
-                              g["interface"] = "{0}.{1}.{2}".format(pi, vif_s, vif_c)
-                              g["vrid"] = vifc_group
-                              groups.append(g)
-
-                         config.delete(parent_path + [pi, "vif-s", vif_s, "vif-c", vif_c, "vrrp"])
-
-# If nothing was collected before this point, it means the config has no VRRP setup
-if not groups:
-    sys.exit(0)
-
-# Otherwise, there is VRRP to convert
- 
-# Now convert the collected groups to the new syntax
-base_group_path = ["high-availability", "vrrp", "group"]
-sync_path = ["high-availability", "vrrp", "sync-group"]
-
-for g in groups:
-    group_name = "{0}-{1}".format(g["interface"], g["vrid"])
-    group_path = base_group_path + [group_name]
-
-    config.set(group_path + ["interface"], value=g["interface"])
-    config.set(group_path + ["vrid"], value=g["vrid"])
-
-    if "advertise_interval" in g:
-        config.set(group_path + ["advertise-interval"], value=g["advertise_interval"])
-
-    if "priority" in g:
-        config.set(group_path + ["priority"], value=g["priority"])
-
-    if not g["preempt"]:
-        config.set(group_path + ["no-preempt"], value=None)
-
-    if "preempt_delay" in g:
-        config.set(group_path + ["preempt-delay"], value=g["preempt_delay"])
-
-    if g["rfc_compatibility"]:
-        config.set(group_path + ["rfc3768-compatibility"], value=None)
-
-    if g["disable"]:
-        config.set(group_path + ["disable"], value=None)
-
-    if "hello_source" in g:
-        config.set(group_path + ["hello-source-address"], value=g["hello_source"])
-
-    if "peer_address" in g:
-        config.set(group_path + ["peer-address"], value=g["peer_address"])
-
-    if "auth_password" in g:
-        config.set(group_path + ["authentication", "password"], value=g["auth_password"])
-    if "auth_type" in g:
-        config.set(group_path + ["authentication", "type"], value=g["auth_type"])
-
-    if "master_script" in g:
-        config.set(group_path + ["transition-script", "master"], value=g["master_script"])
-    if "backup_script" in g:
-        config.set(group_path + ["transition-script", "backup"], value=g["backup_script"])
-    if "fault_script" in g:
-        config.set(group_path + ["transition-script", "fault"], value=g["fault_script"])
-
-    if "health_check_interval" in g:
-        config.set(group_path + ["health-check", "interval"], value=g["health_check_interval"])
-    if "health_check_count" in g:
-        config.set(group_path + ["health-check", "failure-count"], value=g["health_check_count"])
-    if "health_check_script" in g:
-        config.set(group_path + ["health-check", "script"], value=g["health_check_script"])
-
-    # Not that it should ever be absent...
-    if "virtual_addresses" in g:
-        # The new CLI disallows addresses without prefix length
-        # Pre-rewrite configs didn't support IPv6 VRRP, but handle it anyway
-        for va in g["virtual_addresses"]:
-            if not re.search(r'/', va):
-                if re.search(r':', va):
-                    va = "{0}/128".format(va)
-                else:
-                    va = "{0}/32".format(va)
-            config.set(group_path + ["virtual-address"], value=va, replace=False)
-
-    # Sync group
-    if "sync_group" in g:
-        config.set(sync_path + [g["sync_group"], "member"], value=group_name, replace=False)
-
-# Set the tag flag
-config.set_tag(base_group_path)
-if config.exists(sync_path):
-    config.set_tag(sync_path)
-
-try:
-    with open(file_name, 'w') as f:
-        f.write(config.to_string())
-except OSError as e:
-    print("Failed to save the modified config: {}".format(e))
-    sys.exit(1)
+def migrate(config: ConfigTree) -> None:
+    groups = []
+    base_paths = []
+
+    if config.exists(["interfaces", "ethernet"]):
+        base_paths.append("ethernet")
+    if config.exists(["interfaces", "bonding"]):
+        base_paths.append("bonding")
+
+    for bp in base_paths:
+        parent_path = ["interfaces", bp]
+
+        parent_intfs = config.list_nodes(parent_path)
+
+        for pi in parent_intfs:
+            # Extract VRRP groups from the parent interface
+            vg_path =[pi, "vrrp", "vrrp-group"]
+            if config.exists(parent_path + vg_path):
+                pgroups = config.list_nodes(parent_path + vg_path)
+                for pg in pgroups:
+                    g = get_vrrp_group(parent_path + vg_path + [pg])
+                    g["interface"] = pi
+                    g["vrid"] = pg
+                    groups.append(g)
+
+                # Delete the VRRP subtree
+                # If left in place, configs will not load correctly
+                config.delete(parent_path + [pi, "vrrp"])
+
+            # Extract VRRP groups from 802.1q VLAN interfaces
+            if config.exists(parent_path + [pi, "vif"]):
+                vifs = config.list_nodes(parent_path + [pi, "vif"])
+                for vif in vifs:
+                    vif_vg_path = [pi, "vif", vif, "vrrp", "vrrp-group"]
+                    if config.exists(parent_path + vif_vg_path):
+                        vifgroups = config.list_nodes(parent_path + vif_vg_path)
+                        for vif_group in vifgroups:
+                            g = get_vrrp_group(parent_path + vif_vg_path + [vif_group])
+                            g["interface"] = "{0}.{1}".format(pi, vif)
+                            g["vrid"] = vif_group
+                            groups.append(g)
+
+                        config.delete(parent_path + [pi, "vif", vif, "vrrp"])
+
+            # Extract VRRP groups from 802.3ad QinQ service VLAN interfaces
+            if config.exists(parent_path + [pi, "vif-s"]):
+                vif_ss = config.list_nodes(parent_path + [pi, "vif-s"])
+                for vif_s in vif_ss:
+                    vifs_vg_path = [pi, "vif-s", vif_s, "vrrp", "vrrp-group"]
+                    if config.exists(parent_path + vifs_vg_path):
+                        vifsgroups = config.list_nodes(parent_path + vifs_vg_path)
+                        for vifs_group in vifsgroups:
+                            g = get_vrrp_group(parent_path + vifs_vg_path + [vifs_group])
+                            g["interface"] = "{0}.{1}".format(pi, vif_s)
+                            g["vrid"] = vifs_group
+                            groups.append(g)
+
+                        config.delete(parent_path + [pi, "vif-s", vif_s, "vrrp"])
+
+                    # Extract VRRP groups from QinQ client VLAN interfaces nested in the vif-s
+                    if config.exists(parent_path + [pi, "vif-s", vif_s, "vif-c"]):
+                        vif_cs = config.list_nodes(parent_path + [pi, "vif-s", vif_s, "vif-c"])
+                        for vif_c in vif_cs:
+                             vifc_vg_path = [pi, "vif-s", vif_s, "vif-c", vif_c, "vrrp", "vrrp-group"]
+                             vifcgroups = config.list_nodes(parent_path + vifc_vg_path)
+                             for vifc_group in vifcgroups:
+                                  g = get_vrrp_group(parent_path + vifc_vg_path + [vifc_group])
+                                  g["interface"] = "{0}.{1}.{2}".format(pi, vif_s, vif_c)
+                                  g["vrid"] = vifc_group
+                                  groups.append(g)
+
+                             config.delete(parent_path + [pi, "vif-s", vif_s, "vif-c", vif_c, "vrrp"])
+
+    # If nothing was collected before this point, it means the config has no VRRP setup
+    if not groups:
+        return
+
+    # Otherwise, there is VRRP to convert
+     
+    # Now convert the collected groups to the new syntax
+    base_group_path = ["high-availability", "vrrp", "group"]
+    sync_path = ["high-availability", "vrrp", "sync-group"]
+
+    for g in groups:
+        group_name = "{0}-{1}".format(g["interface"], g["vrid"])
+        group_path = base_group_path + [group_name]
+
+        config.set(group_path + ["interface"], value=g["interface"])
+        config.set(group_path + ["vrid"], value=g["vrid"])
+
+        if "advertise_interval" in g:
+            config.set(group_path + ["advertise-interval"], value=g["advertise_interval"])
+
+        if "priority" in g:
+            config.set(group_path + ["priority"], value=g["priority"])
+
+        if not g["preempt"]:
+            config.set(group_path + ["no-preempt"], value=None)
+
+        if "preempt_delay" in g:
+            config.set(group_path + ["preempt-delay"], value=g["preempt_delay"])
+
+        if g["rfc_compatibility"]:
+            config.set(group_path + ["rfc3768-compatibility"], value=None)
+
+        if g["disable"]:
+            config.set(group_path + ["disable"], value=None)
+
+        if "hello_source" in g:
+            config.set(group_path + ["hello-source-address"], value=g["hello_source"])
+
+        if "peer_address" in g:
+            config.set(group_path + ["peer-address"], value=g["peer_address"])
+
+        if "auth_password" in g:
+            config.set(group_path + ["authentication", "password"], value=g["auth_password"])
+        if "auth_type" in g:
+            config.set(group_path + ["authentication", "type"], value=g["auth_type"])
+
+        if "master_script" in g:
+            config.set(group_path + ["transition-script", "master"], value=g["master_script"])
+        if "backup_script" in g:
+            config.set(group_path + ["transition-script", "backup"], value=g["backup_script"])
+        if "fault_script" in g:
+            config.set(group_path + ["transition-script", "fault"], value=g["fault_script"])
+
+        if "health_check_interval" in g:
+            config.set(group_path + ["health-check", "interval"], value=g["health_check_interval"])
+        if "health_check_count" in g:
+            config.set(group_path + ["health-check", "failure-count"], value=g["health_check_count"])
+        if "health_check_script" in g:
+            config.set(group_path + ["health-check", "script"], value=g["health_check_script"])
+
+        # Not that it should ever be absent...
+        if "virtual_addresses" in g:
+            # The new CLI disallows addresses without prefix length
+            # Pre-rewrite configs didn't support IPv6 VRRP, but handle it anyway
+            for va in g["virtual_addresses"]:
+                if not re.search(r'/', va):
+                    if re.search(r':', va):
+                        va = "{0}/128".format(va)
+                    else:
+                        va = "{0}/32".format(va)
+                config.set(group_path + ["virtual-address"], value=va, replace=False)
+
+        # Sync group
+        if "sync_group" in g:
+            config.set(sync_path + [g["sync_group"], "member"], value=group_name, replace=False)
+
+    # Set the tag flag
+    config.set_tag(base_group_path)
+    if config.exists(sync_path):
+        config.set_tag(sync_path)
diff --git a/src/migration-scripts/vrrp/2-to-3 b/src/migration-scripts/vrrp/2-to-3
index ed583b489..468918f91 100755..100644
--- a/src/migration-scripts/vrrp/2-to-3
+++ b/src/migration-scripts/vrrp/2-to-3
@@ -1,62 +1,44 @@
-#!/usr/bin/env python3
+# Copyright 2021-2024 VyOS maintainers and contributors <maintainers@vyos.io>
 #
-# Copyright (C) 2021 VyOS maintainers and contributors
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 or later as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
+# This library is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# You should have received a copy of the GNU Lesser General Public License
+# along with this library.  If not, see <http://www.gnu.org/licenses/>.
 
 # T3847: vrrp config cleanup
 
-from sys import argv
 from vyos.configtree import ConfigTree
 
-if len(argv) < 2:
-    print('Must specify file name!')
-    exit(1)
-
-file_name = argv[1]
-
-with open(file_name, 'r') as f:
-    config_file = f.read()
-
 base = ['high-availability', 'vrrp']
-config = ConfigTree(config_file)
-
-if not config.exists(base):
-    # Nothing to do
-    exit(0)
-
-if config.exists(base + ['group']):
-    for group in config.list_nodes(base + ['group']):
-        group_base = base + ['group', group]
-
-        # Deprecated option
-        tmp = group_base + ['transition-script', 'mode-force']
-        if config.exists(tmp):
-            config.delete(tmp)
-
-        # Rename virtual-address -> address
-        tmp = group_base + ['virtual-address']
-        if config.exists(tmp):
-            config.rename(tmp, 'address')
-
-        # Rename virtual-address-excluded -> excluded-address
-        tmp = group_base + ['virtual-address-excluded']
-        if config.exists(tmp):
-            config.rename(tmp, 'excluded-address')
-
-try:
-    with open(file_name, 'w') as f:
-        f.write(config.to_string())
-except OSError as e:
-    print(f'Failed to save the modified config: {e}')
-    exit(1)
+
+def migrate(config: ConfigTree) -> None:
+    if not config.exists(base):
+        # Nothing to do
+        return
+
+    if config.exists(base + ['group']):
+        for group in config.list_nodes(base + ['group']):
+            group_base = base + ['group', group]
+
+            # Deprecated option
+            tmp = group_base + ['transition-script', 'mode-force']
+            if config.exists(tmp):
+                config.delete(tmp)
+
+            # Rename virtual-address -> address
+            tmp = group_base + ['virtual-address']
+            if config.exists(tmp):
+                config.rename(tmp, 'address')
+
+            # Rename virtual-address-excluded -> excluded-address
+            tmp = group_base + ['virtual-address-excluded']
+            if config.exists(tmp):
+                config.rename(tmp, 'excluded-address')
diff --git a/src/migration-scripts/vrrp/3-to-4 b/src/migration-scripts/vrrp/3-to-4
index e5d93578c..9f05cf7a1 100755..100644
--- a/src/migration-scripts/vrrp/3-to-4
+++ b/src/migration-scripts/vrrp/3-to-4
@@ -1,51 +1,32 @@
-#!/usr/bin/env python3
+# Copyright 2023-2024 VyOS maintainers and contributors <maintainers@vyos.io>
 #
-# Copyright (C) 2023 VyOS maintainers and contributors
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 or later as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
+# This library is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# You should have received a copy of the GNU Lesser General Public License
+# along with this library.  If not, see <http://www.gnu.org/licenses/>.
 
-from sys import argv
 from vyos.configtree import ConfigTree
 
-if len(argv) < 2:
-    print('Must specify file name!')
-    exit(1)
-
-file_name = argv[1]
-
-with open(file_name, 'r') as f:
-    config_file = f.read()
-
 base = ['high-availability', 'virtual-server']
-config = ConfigTree(config_file)
-
-if not config.exists(base):
-    # Nothing to do
-    exit(0)
-
-if config.exists(base):
-    for vs in config.list_nodes(base):
-        vs_base = base + [vs]
 
-        # If the fwmark is used, the address is not required
-        if not config.exists(vs_base + ['fwmark']):
-            # add option: 'virtual-server <tag> address x.x.x.x'
-            config.set(vs_base + ['address'], value=vs)
+def migrate(config: ConfigTree) -> None:
+    if not config.exists(base):
+        # Nothing to do
+        return
 
+    if config.exists(base):
+        for vs in config.list_nodes(base):
+            vs_base = base + [vs]
 
-try:
-    with open(file_name, 'w') as f:
-        f.write(config.to_string())
-except OSError as e:
-    print(f'Failed to save the modified config: {e}')
-    exit(1)
+            # If the fwmark is used, the address is not required
+            if not config.exists(vs_base + ['fwmark']):
+                # add option: 'virtual-server <tag> address x.x.x.x'
+                config.set(vs_base + ['address'], value=vs)