From 791598f2929a5b8b6bb380f7f16ec568db96aba6 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 7 Nov 2012 21:34:41 -0800 Subject: Start adding a 'migrator' module that can be used to aid in the moving of older versions of cloud-inits data to newer versions of cloud-inits data. 1. Move the semaphores for the current instance to there canonicalized names and use the canonicalized in the file 'locking' code --- cloudinit/config/cc_migrator.py | 53 +++++++++++++++++++++++++++++++++++++++++ cloudinit/helpers.py | 9 ++++++- 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 cloudinit/config/cc_migrator.py (limited to 'cloudinit') diff --git a/cloudinit/config/cc_migrator.py b/cloudinit/config/cc_migrator.py new file mode 100644 index 00000000..b71d1d17 --- /dev/null +++ b/cloudinit/config/cc_migrator.py @@ -0,0 +1,53 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import shutil + +from cloudinit import helpers +from cloudinit import util + +from cloudinit.settings import PER_ALWAYS + +frequency = PER_ALWAYS + + +def _migrate_canon_sems(cloud): + sem_path = cloud.paths.get_ipath('sem') + if not sem_path or not os.path.exists(sem_path): + return 0 + am_adjusted = 0 + for p in os.listdir(sem_path): + full_path = os.path.join(sem_path, p) + if os.path.isfile(full_path): + canon_p = helpers.canon_sem_name(p) + if canon_p != p: + new_path = os.path.join(sem_path, p) + shutil.move(full_path, new_path) + am_adjusted += 1 + return am_adjusted + + +def handle(name, cfg, cloud, log, _args): + do_migrate = util.get_cfg_option_str(cfg, "migrate", True) + if not util.translate_bool(do_migrate): + log.debug("Skipping module named %s, migration disabled", name) + return + sems_moved = _migrate_canon_sems(cloud) + log.debug("Migrated %s semaphore files to there canonicalized names", + sems_moved) diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index 985ce3e5..d26625a0 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -71,12 +71,17 @@ class FileLock(object): return "<%s using file %r>" % (util.obj_name(self), self.fn) +def canon_sem_name(name): + return name.replace("-", "_") + + class FileSemaphores(object): - def __init__(self, sem_path): + def __init__(self, sem_path): self.sem_path = sem_path @contextlib.contextmanager def lock(self, name, freq, clear_on_fail=False): + name = canon_sem_name(name) try: yield self._acquire(name, freq) except: @@ -85,6 +90,7 @@ class FileSemaphores(object): raise def clear(self, name, freq): + name = canon_sem_name(name) sem_file = self._get_path(name, freq) try: util.del_file(sem_file) @@ -119,6 +125,7 @@ class FileSemaphores(object): def has_run(self, name, freq): if not freq or freq == PER_ALWAYS: return False + name = canon_sem_name(name) sem_file = self._get_path(name, freq) # This isn't really a good atomic check # but it suffices for where and when cloudinit runs -- cgit v1.2.3 From 3de3c535f37e40a79b36997a93fa218534117397 Mon Sep 17 00:00:00 2001 From: harlowja Date: Wed, 7 Nov 2012 23:16:21 -0800 Subject: 1. Check the name and not the full path when applying the canon routine. 2. Add in a function to migrate legacy semaphores to new semaphores. --- cloudinit/config/cc_migrator.py | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_migrator.py b/cloudinit/config/cc_migrator.py index b71d1d17..56f33d42 100644 --- a/cloudinit/config/cc_migrator.py +++ b/cloudinit/config/cc_migrator.py @@ -35,14 +35,41 @@ def _migrate_canon_sems(cloud): for p in os.listdir(sem_path): full_path = os.path.join(sem_path, p) if os.path.isfile(full_path): - canon_p = helpers.canon_sem_name(p) - if canon_p != p: - new_path = os.path.join(sem_path, p) + (name, ext) = os.path.splitext(p) + canon_name = helpers.canon_sem_name(name) + if canon_name != name: + new_path = os.path.join(sem_path, canon_name + ext) shutil.move(full_path, new_path) am_adjusted += 1 return am_adjusted +def _migrate_legacy_sems(cloud, log): + sem_path = cloud.paths.get_ipath('sem') + touch_there = { + 'apt-update-upgrade': [ + 'apt-configure', + 'package-update-upgrade-install', + ], + } + sem_helper = helpers.FileSemaphores(sem_path) + for (mod_name, migrate_to) in touch_there.items(): + possibles = [mod_name, helpers.canon_sem_name(mod_name)] + old_exists = [] + for p in os.listdir(sem_path): + (name, _ext) = os.path.splitext(p) + if name in possibles and os.path.isfile(p): + old_exists.append(p) + for p in old_exists: + util.del_file(os.path.join(sem_path, p)) + (_name, freq) = os.path.splitext(p) + for m in migrate_to: + log.debug("Migrating %s => %s with the same frequency", + p, m) + with sem_helper.lock(m, freq): + pass + + def handle(name, cfg, cloud, log, _args): do_migrate = util.get_cfg_option_str(cfg, "migrate", True) if not util.translate_bool(do_migrate): @@ -51,3 +78,4 @@ def handle(name, cfg, cloud, log, _args): sems_moved = _migrate_canon_sems(cloud) log.debug("Migrated %s semaphore files to there canonicalized names", sems_moved) + _migrate_legacy_sems(cloud, log) -- cgit v1.2.3 From 196badd4cfa5f9f76be1138fb2d073649af3e031 Mon Sep 17 00:00:00 2001 From: harlowja Date: Wed, 7 Nov 2012 23:20:40 -0800 Subject: 1. Ensure that the sem_path exists and is actually a valid value returned. 2. Adjust variable naming --- cloudinit/config/cc_migrator.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_migrator.py b/cloudinit/config/cc_migrator.py index 56f33d42..58232fc9 100644 --- a/cloudinit/config/cc_migrator.py +++ b/cloudinit/config/cc_migrator.py @@ -46,14 +46,16 @@ def _migrate_canon_sems(cloud): def _migrate_legacy_sems(cloud, log): sem_path = cloud.paths.get_ipath('sem') - touch_there = { + if not sem_path or not os.path.exists(sem_path): + return + legacy_adjust = { 'apt-update-upgrade': [ 'apt-configure', 'package-update-upgrade-install', ], } sem_helper = helpers.FileSemaphores(sem_path) - for (mod_name, migrate_to) in touch_there.items(): + for (mod_name, migrate_to) in legacy_adjust.items(): possibles = [mod_name, helpers.canon_sem_name(mod_name)] old_exists = [] for p in os.listdir(sem_path): -- cgit v1.2.3