T1334: Migration script runner rewrite

Python script and support code to replace the
vyatta_config_migrate.pl script.

4 files changed, 343 insertions, 1 deletions

diff --git a/python/vyos/defaults.py b/python/vyos/defaults.py
index 0603efc42..da363b8e1 100644
--- a/python/vyos/defaults.py
+++ b/python/vyos/defaults.py
@@ -16,7 +16,11 @@
 
 directories = {
   "data": "/usr/share/vyos/",
-  "config": "/opt/vyatta/etc/config"
+  "config": "/opt/vyatta/etc/config",
+  "current": "/opt/vyatta/etc/config-migrate/current",
+  "migrate": "/opt/vyatta/etc/config-migrate/migrate",
 }
 
 cfg_group = 'vyattacfg'
+
+cfg_vintage = 'vyatta'
diff --git a/python/vyos/formatversions.py b/python/vyos/formatversions.py
new file mode 100644
index 000000000..29117a5d3
--- /dev/null
+++ b/python/vyos/formatversions.py
@@ -0,0 +1,109 @@
+# Copyright 2019 VyOS maintainers and contributors <maintainers@vyos.io>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this library.  If not, see <http://www.gnu.org/licenses/>.
+
+import sys
+import os
+import re
+import fileinput
+
+def read_vyatta_versions(config_file):
+    config_file_versions = {}
+
+    with open(config_file, 'r') as config_file_handle:
+        for config_line in config_file_handle:
+            if re.match(r'/\* === vyatta-config-version:.+=== \*/$', config_line):
+                if not re.match(r'/\* === vyatta-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s+=== \*/$', config_line):
+                    raise ValueError("malformed configuration string: "
+                            "{}".format(config_line))
+
+                for pair in re.findall(r'([\w,-]+)@(\d+)', config_line):
+                    config_file_versions[pair[0]] = int(pair[1])
+
+
+    return config_file_versions
+
+def read_vyos_versions(config_file):
+    config_file_versions = {}
+
+    with open(config_file, 'r') as config_file_handle:
+        for config_line in config_file_handle:
+            if re.match(r'// vyos-config-version:.+', config_line):
+                if not re.match(r'// vyos-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s*', config_line):
+                    raise ValueError("malformed configuration string: "
+                            "{}".format(config_line))
+
+                for pair in re.findall(r'([\w,-]+)@(\d+)', config_line):
+                    config_file_versions[pair[0]] = int(pair[1])
+
+    return config_file_versions
+
+def remove_versions(config_file):
+    """
+    Remove old version string.
+    """
+    for line in fileinput.input(config_file, inplace=True):
+        if re.match(r'/\* Warning:.+ \*/$', line):
+            continue
+        if re.match(r'/\* === vyatta-config-version:.+=== \*/$', line):
+            continue
+        if re.match(r'/\* Release version:.+ \*/$', line):
+            continue
+        if re.match('// vyos-config-version:.+', line):
+            continue
+        if re.match('// Warning:.+', line):
+            continue
+        if re.match('// Release version:.+', line):
+            continue
+        sys.stdout.write(line)
+
+def format_versions_string(config_versions):
+    cfg_keys = list(config_versions.keys())
+    cfg_keys.sort()
+
+    component_version_strings = []
+
+    for key in cfg_keys:
+        cfg_vers = config_versions[key]
+        component_version_strings.append('{}@{}'.format(key, cfg_vers))
+
+    separator = ":"
+    component_version_string = separator.join(component_version_strings)
+
+    return component_version_string
+
+def write_vyatta_versions_foot(config_file, component_version_string,
+                                 os_version_string):
+    if config_file:
+        with open(config_file, 'a') as config_file_handle:
+            config_file_handle.write('/* Warning: Do not remove the following line. */\n')
+            config_file_handle.write('/* === vyatta-config-version: "{}" === */\n'.format(component_version_string))
+            config_file_handle.write('/* Release version: {} */\n'.format(os_version_string))
+    else:
+        sys.stdout.write('/* Warning: Do not remove the following line. */\n')
+        sys.stdout.write('/* === vyatta-config-version: "{}" === */\n'.format(component_version_string))
+        sys.stdout.write('/* Release version: {} */\n'.format(os_version_string))
+
+def write_vyos_versions_foot(config_file, component_version_string,
+                               os_version_string):
+    if config_file:
+        with open(config_file, 'a') as config_file_handle:
+            config_file_handle.write('// Warning: Do not remove the following line.\n')
+            config_file_handle.write('// vyos-config-version: "{}"\n'.format(component_version_string))
+            config_file_handle.write('// Release version: {}\n'.format(os_version_string))
+    else:
+        sys.stdout.write('// Warning: Do not remove the following line.\n')
+        sys.stdout.write('// vyos-config-version: "{}"\n'.format(component_version_string))
+        sys.stdout.write('// Release version: {}\n'.format(os_version_string))
+
diff --git a/python/vyos/migrator.py b/python/vyos/migrator.py
new file mode 100644
index 000000000..2d4bc7ffc
--- /dev/null
+++ b/python/vyos/migrator.py
@@ -0,0 +1,190 @@
+# Copyright 2019 VyOS maintainers and contributors <maintainers@vyos.io>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this library.  If not, see <http://www.gnu.org/licenses/>.
+
+import sys
+import os
+import subprocess
+import vyos.version
+import vyos.defaults
+import vyos.systemversions as systemversions
+import vyos.formatversions as formatversions
+
+class MigratorError(Exception):
+    pass
+
+class Migrator(object):
+    def __init__(self, config_file, force=False, set_vintage=None):
+        self._config_file = config_file
+        self._force = force
+        self._set_vintage = set_vintage
+        self._config_file_vintage = None
+        self._changed = False
+
+    def read_config_file_versions(self):
+        """
+        Get component versions from config file footer and set vintage;
+        return empty dictionary if config string is missing.
+        """
+        cfg_file = self._config_file
+        component_versions = {}
+
+        cfg_versions = formatversions.read_vyatta_versions(cfg_file)
+
+        if cfg_versions:
+            self._config_file_vintage = 'vyatta'
+            component_versions = cfg_versions
+
+        cfg_versions = formatversions.read_vyos_versions(cfg_file)
+
+        if cfg_versions:
+            self._config_file_vintage = 'vyos'
+            component_versions = cfg_versions
+
+        return component_versions
+
+    def update_vintage(self):
+        old_vintage = self._config_file_vintage
+
+        if self._set_vintage:
+            self._config_file_vintage = self._set_vintage
+
+        if not self._config_file_vintage:
+            self._config_file_vintage = vyos.defaults.cfg_vintage
+
+        if self._config_file_vintage not in ['vyatta', 'vyos']:
+            raise MigratorError("Unknown vintage.")
+
+        if self._config_file_vintage == old_vintage:
+            return False
+        else:
+            return True
+
+    def run_migration_scripts(self, config_file_versions, system_versions):
+        """
+        Run migration scripts iteratively, until config file version equals
+        system component version.
+        """
+        cfg_versions = config_file_versions
+        sys_versions = system_versions
+
+        sys_keys = list(sys_versions.keys())
+        sys_keys.sort()
+
+        rev_versions = {}
+
+        for key in sys_keys:
+            sys_ver = sys_versions[key]
+            if key in cfg_versions:
+                cfg_ver = cfg_versions[key]
+            else:
+                cfg_ver = 0
+
+            migrate_script_dir = os.path.join(
+                    vyos.defaults.directories['migrate'], key)
+
+            while cfg_ver < sys_ver:
+                next_ver = cfg_ver + 1
+
+                migrate_script = os.path.join(migrate_script_dir,
+                        '{}-to-{}'.format(cfg_ver, next_ver))
+
+                try:
+                    subprocess.check_output([migrate_script,
+                        self._config_file])
+                except FileNotFoundError:
+                    pass
+                except subprocess.CalledProcessError as err:
+                    print("Called process error: {}.".format(err))
+                    sys.exit(1)
+
+                cfg_ver = next_ver
+
+            rev_versions[key] = cfg_ver
+
+        return rev_versions
+
+    def write_config_file_versions(self, cfg_versions):
+        """
+        Write new versions string.
+        """
+        versions_string = formatversions.format_versions_string(cfg_versions)
+
+        os_version_string = vyos.version.get_version()
+
+        if self._config_file_vintage == 'vyatta':
+            formatversions.write_vyatta_versions_foot(self._config_file,
+                                                      versions_string,
+                                                      os_version_string)
+
+        if self._config_file_vintage == 'vyos':
+            formatversions.write_vyos_versions_foot(self._config_file,
+                                                    versions_string,
+                                                    os_version_string)
+
+    def run(self):
+        """
+        Gather component versions from config file and system.
+        Run migration scripts.
+        Update vintage ('vyatta' or 'vyos'), if needed.
+        If changed, remove old versions string from config file, and
+            write new versions string.
+        """
+        cfg_file = self._config_file
+
+        cfg_versions = self.read_config_file_versions()
+        if self._force:
+            # This will force calling all migration scripts:
+            cfg_versions = {}
+
+        sys_versions = systemversions.get_system_versions()
+
+        rev_versions = self.run_migration_scripts(cfg_versions, sys_versions)
+
+        if rev_versions != cfg_versions:
+            self._changed = True
+
+        if self.update_vintage():
+            self._changed = True
+
+        if not self._changed:
+            return
+
+        formatversions.remove_versions(cfg_file)
+
+        self.write_config_file_versions(rev_versions)
+
+
+class VirtualMigrator(Migrator):
+    def __init__(self, config_file, vintage='vyos'):
+        super().__init__(config_file, set_vintage = vintage)
+
+    def run(self):
+        cfg_file = self._config_file
+
+        cfg_versions = self.read_config_file_versions()
+        if not cfg_versions:
+            raise MigratorError("Config file has no version information;"
+                                " virtual migration not possible.")
+
+        if self.update_vintage():
+            self._changed = True
+
+        if not self._changed:
+            return
+
+        formatversions.remove_versions(cfg_file)
+
+        self.write_config_file_versions(cfg_versions)
+
diff --git a/python/vyos/systemversions.py b/python/vyos/systemversions.py
new file mode 100644
index 000000000..9b3f4f413
--- /dev/null
+++ b/python/vyos/systemversions.py
@@ -0,0 +1,39 @@
+# Copyright 2019 VyOS maintainers and contributors <maintainers@vyos.io>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this library.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import re
+import sys
+import vyos.defaults
+
+def get_system_versions():
+    """
+    Get component versions from running system; critical failure if
+    unable to read migration directory.
+    """
+    system_versions = {}
+
+    try:
+        version_info = os.listdir(vyos.defaults.directories['current'])
+    except OSError as err:
+        print("OS error: {}".format(err))
+        sys.exit(1)
+
+    for info in version_info:
+        if re.match(r'[\w,-]+@\d+', info):
+            pair = info.split('@')
+            system_versions[pair[0]] = int(pair[1])
+
+    return system_versions