Merge pull request #4050 from jestabro/revise-migration-circinus

T6007: revise migration system

6 files changed, 91 insertions, 114 deletions

diff --git a/src/helpers/system-versions-foot.py b/src/helpers/add-system-version.py
index 9614f0d28..5270ee7d3 100755
--- a/src/helpers/system-versions-foot.py
+++ b/src/helpers/add-system-version.py
@@ -1,6 +1,6 @@
 #!/usr/bin/python3
 
-# Copyright 2019, 2022 VyOS maintainers and contributors <maintainers@vyos.io>
+# Copyright 2019-2024 VyOS maintainers and contributors <maintainers@vyos.io>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -15,14 +15,6 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with this library.  If not, see <http://www.gnu.org/licenses/>.
 
-import sys
-import vyos.defaults
-from vyos.component_version import write_system_footer
+from vyos.component_version import add_system_version
 
-sys.stdout.write("\n\n")
-if vyos.defaults.cfg_vintage == 'vyos':
-    write_system_footer(None, vintage='vyos')
-elif vyos.defaults.cfg_vintage == 'vyatta':
-    write_system_footer(None, vintage='vyatta')
-else:
-    write_system_footer(None, vintage='vyos')
+add_system_version()
diff --git a/src/helpers/run-config-migration.py b/src/helpers/run-config-migration.py
index ce647ad0a..e6ce97363 100755
--- a/src/helpers/run-config-migration.py
+++ b/src/helpers/run-config-migration.py
@@ -1,86 +1,78 @@
-#!/usr/bin/python3
-
-# Copyright 2019 VyOS maintainers and contributors <maintainers@vyos.io>
+#!/usr/bin/env python3
+#
+# Copyright (C) 2019-2024 VyOS maintainers and contributors
 #
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
 #
-# This library is distributed in the hope that it will be useful,
+# This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
 #
-# You should have received a copy of the GNU Lesser General Public License
-# along with this library.  If not, see <http://www.gnu.org/licenses/>.
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
 import sys
-import argparse
-import datetime
-
-from vyos.utils.process import cmd
-from vyos.migrator import Migrator, VirtualMigrator
-
-def main():
-    argparser = argparse.ArgumentParser(
-            formatter_class=argparse.RawTextHelpFormatter)
-    argparser.add_argument('config_file', type=str,
-            help="configuration file to migrate")
-    argparser.add_argument('--force', action='store_true',
-            help="Force calling of all migration scripts.")
-    argparser.add_argument('--set-vintage', type=str,
-            choices=['vyatta', 'vyos'],
-            help="Set the format for the config version footer in config"
-            " file:\n"
-            "set to 'vyatta':\n"
-            "(for '/* === vyatta-config-version ... */' format)\n"
-            "or 'vyos':\n"
-            "(for '// vyos-config-version ...' format).")
-    argparser.add_argument('--virtual', action='store_true',
-            help="Update the format of the trailing comments in"
-                 " config file,\nfrom 'vyatta' to 'vyos'; no migration"
-                 " scripts are run.")
-    args = argparser.parse_args()
+import time
+from argparse import ArgumentParser
+from shutil import copyfile
 
-    config_file_name = args.config_file
-    force_on = args.force
-    vintage = args.set_vintage
-    virtual = args.virtual
+from vyos.migrate import ConfigMigrate
+from vyos.migrate import ConfigMigrateError
 
-    if not os.access(config_file_name, os.R_OK):
-        print("Read error: {}.".format(config_file_name))
-        sys.exit(1)
+parser = ArgumentParser()
+parser.add_argument('config_file', type=str,
+                    help="configuration file to migrate")
+parser.add_argument('--test-script', type=str,
+                    help="test named script")
+parser.add_argument('--output-file', type=str,
+                    help="write to named output file instead of config file")
+parser.add_argument('--force', action='store_true',
+                    help="force run of all migration scripts")
 
-    if not os.access(config_file_name, os.W_OK):
-        print("Write error: {}.".format(config_file_name))
-        sys.exit(1)
+args = parser.parse_args()
 
-    separator = "."
-    backup_file_name = separator.join([config_file_name,
-            '{0:%Y-%m-%d-%H%M%S}'.format(datetime.datetime.now()),
-            'pre-migration'])
+config_file = args.config_file
+out_file = args.output_file
+test_script = args.test_script
+force = args.force
 
-    cmd(f'cp -p {config_file_name} {backup_file_name}')
+if not os.access(config_file, os.R_OK):
+    print(f"Config file '{config_file}' not readable")
+    sys.exit(1)
 
-    if not virtual:
-        virtual_migration = VirtualMigrator(config_file_name)
-        virtual_migration.run()
+if out_file is None:
+    if not os.access(config_file, os.W_OK):
+        print(f"Config file '{config_file}' not writeable")
+        sys.exit(1)
+else:
+    try:
+        open(out_file, 'w').close()
+    except OSError:
+        print(f"Output file '{out_file}' not writeable")
+        sys.exit(1)
 
-        migration = Migrator(config_file_name, force=force_on)
-        migration.run()
+config_migrate = ConfigMigrate(config_file, force=force, output_file=out_file)
 
-        if not migration.config_changed():
-            os.remove(backup_file_name)
-    else:
-        virtual_migration = VirtualMigrator(config_file_name,
-                                            set_vintage=vintage)
+if test_script:
+    # run_script and exit
+    config_migrate.run_script(test_script)
+    sys.exit(0)
 
-        virtual_migration.run()
+backup = None
+if out_file is None:
+    timestr = time.strftime("%Y%m%d-%H%M%S")
+    backup = f'{config_file}.{timestr}.pre-migration'
+    copyfile(config_file, backup)
 
-        if not virtual_migration.config_changed():
-            os.remove(backup_file_name)
+try:
+    config_migrate.run()
+except ConfigMigrateError as e:
+    print(f'Error: {e}')
+    sys.exit(1)
 
-if __name__ == '__main__':
-    main()
+if backup is not None and not config_migrate.config_modified:
+    os.unlink(backup)
diff --git a/src/helpers/vyos-load-config.py b/src/helpers/vyos-load-config.py
index 4ec865454..16083fd41 100755
--- a/src/helpers/vyos-load-config.py
+++ b/src/helpers/vyos-load-config.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2019 VyOS maintainers and contributors
+# Copyright (C) 2019-2024 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -30,7 +30,8 @@ import tempfile
 import vyos.defaults
 import vyos.remote
 from vyos.configsource import ConfigSourceSession, VyOSError
-from vyos.migrator import Migrator, VirtualMigrator, MigratorError
+from vyos.migrate import ConfigMigrate
+from vyos.migrate import ConfigMigrateError
 
 class LoadConfig(ConfigSourceSession):
     """A subclass for calling 'loadFile'.
@@ -81,22 +82,16 @@ with tempfile.NamedTemporaryFile() as fp:
     with open(fp.name, 'w') as fd:
         fd.write(config_string)
 
-    virtual_migration = VirtualMigrator(fp.name)
+    config_migrate = ConfigMigrate(fp.name)
     try:
-        virtual_migration.run()
-    except MigratorError as err:
-        sys.exit('{}'.format(err))
-
-    migration = Migrator(fp.name)
-    try:
-        migration.run()
-    except MigratorError as err:
-        sys.exit('{}'.format(err))
+        config_migrate.run()
+    except ConfigMigrateError as err:
+        sys.exit(err)
 
     try:
         config.load_config(fp.name)
     except VyOSError as err:
-        sys.exit('{}'.format(err))
+        sys.exit(err)
 
 if config.session_changed():
     print("Load complete. Use 'commit' to make changes effective.")
diff --git a/src/helpers/vyos-merge-config.py b/src/helpers/vyos-merge-config.py
index 35424626e..5ef845ac2 100755
--- a/src/helpers/vyos-merge-config.py
+++ b/src/helpers/vyos-merge-config.py
@@ -22,7 +22,8 @@ import vyos.remote
 
 from vyos.config import Config
 from vyos.configtree import ConfigTree
-from vyos.migrator import Migrator, VirtualMigrator
+from vyos.migrate import ConfigMigrate
+from vyos.migrate import ConfigMigrateError
 from vyos.utils.process import cmd
 from vyos.utils.process import DEVNULL
 
@@ -61,15 +62,11 @@ with tempfile.NamedTemporaryFile() as file_to_migrate:
     with open(file_to_migrate.name, 'w') as fd:
         fd.write(config_file)
 
-    virtual_migration = VirtualMigrator(file_to_migrate.name)
-    virtual_migration.run()
-
-    migration = Migrator(file_to_migrate.name)
-    migration.run()
-
-    if virtual_migration.config_changed() or migration.config_changed():
-        with open(file_to_migrate.name, 'r') as fd:
-            config_file = fd.read()
+    config_migrate = ConfigMigrate(file_to_migrate.name)
+    try:
+        config_migrate.run()
+    except ConfigMigrateError as e:
+        sys.exit(e)
 
 merge_config_tree = ConfigTree(config_file)
 
diff --git a/src/helpers/vyos-save-config.py b/src/helpers/vyos-save-config.py
index 518bd9864..fa2ea0ce4 100755
--- a/src/helpers/vyos-save-config.py
+++ b/src/helpers/vyos-save-config.py
@@ -23,7 +23,7 @@ from argparse import ArgumentParser
 
 from vyos.config import Config
 from vyos.remote import urlc
-from vyos.component_version import system_footer
+from vyos.component_version import add_system_version
 from vyos.defaults import directories
 
 DEFAULT_CONFIG_PATH = os.path.join(directories['config'], 'config.boot')
@@ -50,14 +50,13 @@ if re.match(r'\w+:/', save_file):
 config = Config()
 ct = config.get_config_tree(effective=True)
 
+# pylint: disable=consider-using-with
 write_file = save_file if remote_save is None else NamedTemporaryFile(delete=False).name
-with open(write_file, 'w') as f:
-    # config_tree is None before boot configuration is complete;
-    # automated saves should check boot_configuration_complete
-    if ct is not None:
-        f.write(ct.to_string())
-    f.write("\n")
-    f.write(system_footer())
+
+# config_tree is None before boot configuration is complete;
+# automated saves should check boot_configuration_complete
+config_str = None if ct is None else ct.to_string()
+add_system_version(config_str, write_file)
 
 if json_file is not None and ct is not None:
     try:
diff --git a/src/helpers/vyos_net_name b/src/helpers/vyos_net_name
index 8c0992414..518e204f9 100755
--- a/src/helpers/vyos_net_name
+++ b/src/helpers/vyos_net_name
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2021-2023 VyOS maintainers and contributors
+# Copyright (C) 2021-2024 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -26,7 +26,7 @@ from vyos.configtree import ConfigTree
 from vyos.defaults import directories
 from vyos.utils.process import cmd
 from vyos.utils.boot import boot_configuration_complete
-from vyos.migrator import VirtualMigrator
+from vyos.migrate import ConfigMigrate
 
 vyos_udev_dir = directories['vyos_udev_dir']
 vyos_log_dir = '/run/udev/log'
@@ -147,8 +147,10 @@ def get_configfile_interfaces() -> dict:
             with tempfile.NamedTemporaryFile() as fp:
                 with open(fp.name, 'w') as fd:
                     fd.write(config_file)
-                virtual_migration = VirtualMigrator(fp.name)
-                virtual_migration.run()
+                config_migrate = ConfigMigrate(fp.name)
+                if config_migrate.syntax_update_needed():
+                    config_migrate.update_syntax()
+                    config_migrate.write_config()
                 with open(fp.name) as fd:
                     config_file = fd.read()