summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJohn Estabrook <jestabro@sentrium.io>2019-06-06 09:15:03 -0500
committerGitHub <noreply@github.com>2019-06-06 09:15:03 -0500
commit04400fbf57c2e223f5f3480253d4ccd38d7965e3 (patch)
treed64f009676f44f8d49a09b91655d81ec1f22ca08
parent1483288278060c62904ba1c9984aae841600e2f5 (diff)
parent6763170830010c8cea2f17daee5f46b9203dab56 (diff)
downloadvyos-1x-04400fbf57c2e223f5f3480253d4ccd38d7965e3.tar.gz
vyos-1x-04400fbf57c2e223f5f3480253d4ccd38d7965e3.zip
Merge pull request #69 from jestabro/config-migrate
T1334: Migration script runner rewrite
-rw-r--r--python/vyos/defaults.py6
-rw-r--r--python/vyos/formatversions.py109
-rw-r--r--python/vyos/migrator.py190
-rw-r--r--python/vyos/systemversions.py39
-rwxr-xr-xsrc/helpers/run-config-migration.py83
-rwxr-xr-xsrc/helpers/system-versions-foot.py39
6 files changed, 465 insertions, 1 deletions
diff --git a/python/vyos/defaults.py b/python/vyos/defaults.py
index 0603efc42..da363b8e1 100644
--- a/python/vyos/defaults.py
+++ b/python/vyos/defaults.py
@@ -16,7 +16,11 @@
directories = {
"data": "/usr/share/vyos/",
- "config": "/opt/vyatta/etc/config"
+ "config": "/opt/vyatta/etc/config",
+ "current": "/opt/vyatta/etc/config-migrate/current",
+ "migrate": "/opt/vyatta/etc/config-migrate/migrate",
}
cfg_group = 'vyattacfg'
+
+cfg_vintage = 'vyatta'
diff --git a/python/vyos/formatversions.py b/python/vyos/formatversions.py
new file mode 100644
index 000000000..29117a5d3
--- /dev/null
+++ b/python/vyos/formatversions.py
@@ -0,0 +1,109 @@
+# Copyright 2019 VyOS maintainers and contributors <maintainers@vyos.io>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this library. If not, see <http://www.gnu.org/licenses/>.
+
+import sys
+import os
+import re
+import fileinput
+
+def read_vyatta_versions(config_file):
+ config_file_versions = {}
+
+ with open(config_file, 'r') as config_file_handle:
+ for config_line in config_file_handle:
+ if re.match(r'/\* === vyatta-config-version:.+=== \*/$', config_line):
+ if not re.match(r'/\* === vyatta-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s+=== \*/$', config_line):
+ raise ValueError("malformed configuration string: "
+ "{}".format(config_line))
+
+ for pair in re.findall(r'([\w,-]+)@(\d+)', config_line):
+ config_file_versions[pair[0]] = int(pair[1])
+
+
+ return config_file_versions
+
+def read_vyos_versions(config_file):
+ config_file_versions = {}
+
+ with open(config_file, 'r') as config_file_handle:
+ for config_line in config_file_handle:
+ if re.match(r'// vyos-config-version:.+', config_line):
+ if not re.match(r'// vyos-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s*', config_line):
+ raise ValueError("malformed configuration string: "
+ "{}".format(config_line))
+
+ for pair in re.findall(r'([\w,-]+)@(\d+)', config_line):
+ config_file_versions[pair[0]] = int(pair[1])
+
+ return config_file_versions
+
+def remove_versions(config_file):
+ """
+ Remove old version string.
+ """
+ for line in fileinput.input(config_file, inplace=True):
+ if re.match(r'/\* Warning:.+ \*/$', line):
+ continue
+ if re.match(r'/\* === vyatta-config-version:.+=== \*/$', line):
+ continue
+ if re.match(r'/\* Release version:.+ \*/$', line):
+ continue
+ if re.match('// vyos-config-version:.+', line):
+ continue
+ if re.match('// Warning:.+', line):
+ continue
+ if re.match('// Release version:.+', line):
+ continue
+ sys.stdout.write(line)
+
+def format_versions_string(config_versions):
+ cfg_keys = list(config_versions.keys())
+ cfg_keys.sort()
+
+ component_version_strings = []
+
+ for key in cfg_keys:
+ cfg_vers = config_versions[key]
+ component_version_strings.append('{}@{}'.format(key, cfg_vers))
+
+ separator = ":"
+ component_version_string = separator.join(component_version_strings)
+
+ return component_version_string
+
+def write_vyatta_versions_foot(config_file, component_version_string,
+ os_version_string):
+ if config_file:
+ with open(config_file, 'a') as config_file_handle:
+ config_file_handle.write('/* Warning: Do not remove the following line. */\n')
+ config_file_handle.write('/* === vyatta-config-version: "{}" === */\n'.format(component_version_string))
+ config_file_handle.write('/* Release version: {} */\n'.format(os_version_string))
+ else:
+ sys.stdout.write('/* Warning: Do not remove the following line. */\n')
+ sys.stdout.write('/* === vyatta-config-version: "{}" === */\n'.format(component_version_string))
+ sys.stdout.write('/* Release version: {} */\n'.format(os_version_string))
+
+def write_vyos_versions_foot(config_file, component_version_string,
+ os_version_string):
+ if config_file:
+ with open(config_file, 'a') as config_file_handle:
+ config_file_handle.write('// Warning: Do not remove the following line.\n')
+ config_file_handle.write('// vyos-config-version: "{}"\n'.format(component_version_string))
+ config_file_handle.write('// Release version: {}\n'.format(os_version_string))
+ else:
+ sys.stdout.write('// Warning: Do not remove the following line.\n')
+ sys.stdout.write('// vyos-config-version: "{}"\n'.format(component_version_string))
+ sys.stdout.write('// Release version: {}\n'.format(os_version_string))
+
diff --git a/python/vyos/migrator.py b/python/vyos/migrator.py
new file mode 100644
index 000000000..2d4bc7ffc
--- /dev/null
+++ b/python/vyos/migrator.py
@@ -0,0 +1,190 @@
+# Copyright 2019 VyOS maintainers and contributors <maintainers@vyos.io>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this library. If not, see <http://www.gnu.org/licenses/>.
+
+import sys
+import os
+import subprocess
+import vyos.version
+import vyos.defaults
+import vyos.systemversions as systemversions
+import vyos.formatversions as formatversions
+
+class MigratorError(Exception):
+ pass
+
+class Migrator(object):
+ def __init__(self, config_file, force=False, set_vintage=None):
+ self._config_file = config_file
+ self._force = force
+ self._set_vintage = set_vintage
+ self._config_file_vintage = None
+ self._changed = False
+
+ def read_config_file_versions(self):
+ """
+ Get component versions from config file footer and set vintage;
+ return empty dictionary if config string is missing.
+ """
+ cfg_file = self._config_file
+ component_versions = {}
+
+ cfg_versions = formatversions.read_vyatta_versions(cfg_file)
+
+ if cfg_versions:
+ self._config_file_vintage = 'vyatta'
+ component_versions = cfg_versions
+
+ cfg_versions = formatversions.read_vyos_versions(cfg_file)
+
+ if cfg_versions:
+ self._config_file_vintage = 'vyos'
+ component_versions = cfg_versions
+
+ return component_versions
+
+ def update_vintage(self):
+ old_vintage = self._config_file_vintage
+
+ if self._set_vintage:
+ self._config_file_vintage = self._set_vintage
+
+ if not self._config_file_vintage:
+ self._config_file_vintage = vyos.defaults.cfg_vintage
+
+ if self._config_file_vintage not in ['vyatta', 'vyos']:
+ raise MigratorError("Unknown vintage.")
+
+ if self._config_file_vintage == old_vintage:
+ return False
+ else:
+ return True
+
+ def run_migration_scripts(self, config_file_versions, system_versions):
+ """
+ Run migration scripts iteratively, until config file version equals
+ system component version.
+ """
+ cfg_versions = config_file_versions
+ sys_versions = system_versions
+
+ sys_keys = list(sys_versions.keys())
+ sys_keys.sort()
+
+ rev_versions = {}
+
+ for key in sys_keys:
+ sys_ver = sys_versions[key]
+ if key in cfg_versions:
+ cfg_ver = cfg_versions[key]
+ else:
+ cfg_ver = 0
+
+ migrate_script_dir = os.path.join(
+ vyos.defaults.directories['migrate'], key)
+
+ while cfg_ver < sys_ver:
+ next_ver = cfg_ver + 1
+
+ migrate_script = os.path.join(migrate_script_dir,
+ '{}-to-{}'.format(cfg_ver, next_ver))
+
+ try:
+ subprocess.check_output([migrate_script,
+ self._config_file])
+ except FileNotFoundError:
+ pass
+ except subprocess.CalledProcessError as err:
+ print("Called process error: {}.".format(err))
+ sys.exit(1)
+
+ cfg_ver = next_ver
+
+ rev_versions[key] = cfg_ver
+
+ return rev_versions
+
+ def write_config_file_versions(self, cfg_versions):
+ """
+ Write new versions string.
+ """
+ versions_string = formatversions.format_versions_string(cfg_versions)
+
+ os_version_string = vyos.version.get_version()
+
+ if self._config_file_vintage == 'vyatta':
+ formatversions.write_vyatta_versions_foot(self._config_file,
+ versions_string,
+ os_version_string)
+
+ if self._config_file_vintage == 'vyos':
+ formatversions.write_vyos_versions_foot(self._config_file,
+ versions_string,
+ os_version_string)
+
+ def run(self):
+ """
+ Gather component versions from config file and system.
+ Run migration scripts.
+ Update vintage ('vyatta' or 'vyos'), if needed.
+ If changed, remove old versions string from config file, and
+ write new versions string.
+ """
+ cfg_file = self._config_file
+
+ cfg_versions = self.read_config_file_versions()
+ if self._force:
+ # This will force calling all migration scripts:
+ cfg_versions = {}
+
+ sys_versions = systemversions.get_system_versions()
+
+ rev_versions = self.run_migration_scripts(cfg_versions, sys_versions)
+
+ if rev_versions != cfg_versions:
+ self._changed = True
+
+ if self.update_vintage():
+ self._changed = True
+
+ if not self._changed:
+ return
+
+ formatversions.remove_versions(cfg_file)
+
+ self.write_config_file_versions(rev_versions)
+
+
+class VirtualMigrator(Migrator):
+ def __init__(self, config_file, vintage='vyos'):
+ super().__init__(config_file, set_vintage = vintage)
+
+ def run(self):
+ cfg_file = self._config_file
+
+ cfg_versions = self.read_config_file_versions()
+ if not cfg_versions:
+ raise MigratorError("Config file has no version information;"
+ " virtual migration not possible.")
+
+ if self.update_vintage():
+ self._changed = True
+
+ if not self._changed:
+ return
+
+ formatversions.remove_versions(cfg_file)
+
+ self.write_config_file_versions(cfg_versions)
+
diff --git a/python/vyos/systemversions.py b/python/vyos/systemversions.py
new file mode 100644
index 000000000..9b3f4f413
--- /dev/null
+++ b/python/vyos/systemversions.py
@@ -0,0 +1,39 @@
+# Copyright 2019 VyOS maintainers and contributors <maintainers@vyos.io>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this library. If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import re
+import sys
+import vyos.defaults
+
+def get_system_versions():
+ """
+ Get component versions from running system; critical failure if
+ unable to read migration directory.
+ """
+ system_versions = {}
+
+ try:
+ version_info = os.listdir(vyos.defaults.directories['current'])
+ except OSError as err:
+ print("OS error: {}".format(err))
+ sys.exit(1)
+
+ for info in version_info:
+ if re.match(r'[\w,-]+@\d+', info):
+ pair = info.split('@')
+ system_versions[pair[0]] = int(pair[1])
+
+ return system_versions
diff --git a/src/helpers/run-config-migration.py b/src/helpers/run-config-migration.py
new file mode 100755
index 000000000..a57a19cdf
--- /dev/null
+++ b/src/helpers/run-config-migration.py
@@ -0,0 +1,83 @@
+#!/usr/bin/python3
+
+# Copyright 2019 VyOS maintainers and contributors <maintainers@vyos.io>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this library. If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import sys
+import argparse
+import datetime
+import subprocess
+from vyos.migrator import Migrator, VirtualMigrator
+
+def main():
+ argparser = argparse.ArgumentParser(
+ formatter_class=argparse.RawTextHelpFormatter)
+ argparser.add_argument('config_file', type=str,
+ help="configuration file to migrate")
+ argparser.add_argument('--force', action='store_true',
+ help="Force calling of all migration scripts.")
+ argparser.add_argument('--set-vintage', type=str,
+ choices=['vyatta', 'vyos'],
+ help="Set the format for the config version footer in config"
+ " file:\n"
+ "set to 'vyatta':\n"
+ "(for '/* === vyatta-config-version ... */' format)\n"
+ "or 'vyos':\n"
+ "(for '// vyos-config-version ...' format).")
+ argparser.add_argument('--virtual', action='store_true',
+ help="Update the format of the trailing comments in"
+ " config file,\nfrom 'vyatta' to 'vyos'; no migration"
+ " scripts are run.")
+ args = argparser.parse_args()
+
+ config_file_name = args.config_file
+ force_on = args.force
+ vintage = args.set_vintage
+ virtual = args.virtual
+
+ if not os.access(config_file_name, os.R_OK):
+ print("Read error: {}.".format(config_file_name))
+ sys.exit(1)
+
+ if not os.access(config_file_name, os.W_OK):
+ print("Write error: {}.".format(config_file_name))
+ sys.exit(1)
+
+ separator = "."
+ backup_file_name = separator.join([config_file_name,
+ '{0:%Y-%m-%d-%H%M%S}'.format(datetime.datetime.now()),
+ 'pre-migration'])
+
+ try:
+ subprocess.check_call(['cp', '-p', config_file_name,
+ backup_file_name])
+ except subprocess.CalledProcessError as err:
+ print("Called process error: {}.".format(err))
+ sys.exit(1)
+
+ if not virtual:
+ migration = Migrator(config_file_name, force=force_on,
+ set_vintage=vintage)
+ else:
+ migration = VirtualMigrator(config_file_name)
+
+ migration.run()
+
+ if not migration._changed:
+ os.remove(backup_file_name)
+
+if __name__ == '__main__':
+ main()
diff --git a/src/helpers/system-versions-foot.py b/src/helpers/system-versions-foot.py
new file mode 100755
index 000000000..c33e41d79
--- /dev/null
+++ b/src/helpers/system-versions-foot.py
@@ -0,0 +1,39 @@
+#!/usr/bin/python3
+
+# Copyright 2019 VyOS maintainers and contributors <maintainers@vyos.io>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this library. If not, see <http://www.gnu.org/licenses/>.
+
+import sys
+import vyos.formatversions as formatversions
+import vyos.systemversions as systemversions
+import vyos.defaults
+import vyos.version
+
+sys_versions = systemversions.get_system_versions()
+
+component_string = formatversions.format_versions_string(sys_versions)
+
+os_version_string = vyos.version.get_version()
+
+sys.stdout.write("\n\n")
+if vyos.defaults.cfg_vintage == 'vyos':
+ formatversions.write_vyos_versions_foot(None, component_string,
+ os_version_string)
+elif vyos.defaults.cfg_vintage == 'vyatta':
+ formatversions.write_vyatta_versions_foot(None, component_string,
+ os_version_string)
+else:
+ formatversions.write_vyatta_versions_foot(None, component_string,
+ os_version_string)