- Detect a Python version change and clear the cache (#857)

summary: Clear cache when a Python version change is detected

When a distribution gets updated it is possible that the Python version
changes. Python makes no guarantee that pickle is consistent across
versions as such we need to purge the cache and start over.

Co-authored-by: James Falcon <therealfalcon@gmail.com>

6 files changed, 93 insertions, 1 deletions

diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py
index baf1381f..21213a4a 100644
--- a/cloudinit/cmd/main.py
+++ b/cloudinit/cmd/main.py
@@ -210,6 +210,35 @@ def attempt_cmdline_url(path, network=True, cmdline=None):
             (cmdline_name, url, path))
 
 
+def purge_cache_on_python_version_change(init):
+    """Purge the cache if python version changed on us.
+
+    There could be changes not represented in our cache (obj.pkl) after we
+    upgrade to a new version of python, so at that point clear the cache
+    """
+    current_python_version = '%d.%d' % (
+        sys.version_info.major, sys.version_info.minor
+    )
+    python_version_path = os.path.join(
+        init.paths.get_cpath('data'), 'python-version'
+    )
+    if os.path.exists(python_version_path):
+        cached_python_version = open(python_version_path).read()
+        # The Python version has changed out from under us, anything that was
+        # pickled previously is likely useless due to API changes.
+        if cached_python_version != current_python_version:
+            LOG.debug('Python version change detected. Purging cache')
+            init.purge_cache(True)
+            util.write_file(python_version_path, current_python_version)
+    else:
+        if os.path.exists(init.paths.get_ipath_cur('obj_pkl')):
+            LOG.info(
+                'Writing python-version file. '
+                'Cache compatibility status is currently unknown.'
+            )
+        util.write_file(python_version_path, current_python_version)
+
+
 def main_init(name, args):
     deps = [sources.DEP_FILESYSTEM, sources.DEP_NETWORK]
     if args.local:
@@ -276,6 +305,7 @@ def main_init(name, args):
         util.logexc(LOG, "Failed to initialize, likely bad things to come!")
     # Stage 4
     path_helper = init.paths
+    purge_cache_on_python_version_change(init)
     mode = sources.DSMODE_LOCAL if args.local else sources.DSMODE_NETWORK
 
     if mode == sources.DSMODE_NETWORK:
diff --git a/cloudinit/cmd/tests/test_main.py b/cloudinit/cmd/tests/test_main.py
index 78b27441..1f5975b0 100644
--- a/cloudinit/cmd/tests/test_main.py
+++ b/cloudinit/cmd/tests/test_main.py
@@ -17,6 +17,8 @@ myargs = namedtuple('MyArgs', 'debug files force local reporter subcommand')
 
 
 class TestMain(FilesystemMockingTestCase):
+    with_logs = True
+    allowed_subp = False
 
     def setUp(self):
         super(TestMain, self).setUp()
diff --git a/tests/integration_tests/assets/test_version_change.pkl b/tests/integration_tests/assets/test_version_change.pkl
new file mode 100644
index 00000000..65ae93e5
--- /dev/null
+++ b/tests/integration_tests/assets/test_version_change.pkl
diff --git a/tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py b/tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py
index b9b0d85e..e1946cb1 100644
--- a/tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py
+++ b/tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py
@@ -18,7 +18,7 @@ USER_DATA_SSH_AUTHKEY_DISABLE = """\
 no_ssh_fingerprints: true
 """
 
-USER_DATA_SSH_AUTHKEY_ENABLE="""\
+USER_DATA_SSH_AUTHKEY_ENABLE = """\
 #cloud-config
 ssh_genkeytypes:
   - ecdsa
diff --git a/tests/integration_tests/modules/test_version_change.py b/tests/integration_tests/modules/test_version_change.py
new file mode 100644
index 00000000..4e9ab63f
--- /dev/null
+++ b/tests/integration_tests/modules/test_version_change.py
@@ -0,0 +1,56 @@
+from pathlib import Path
+
+from tests.integration_tests.instances import IntegrationInstance
+from tests.integration_tests.util import ASSETS_DIR
+
+
+PICKLE_PATH = Path('/var/lib/cloud/instance/obj.pkl')
+TEST_PICKLE = ASSETS_DIR / 'test_version_change.pkl'
+
+
+def _assert_no_pickle_problems(log):
+    assert 'Failed loading pickled blob' not in log
+    assert 'Traceback' not in log
+    assert 'WARN' not in log
+
+
+def test_reboot_without_version_change(client: IntegrationInstance):
+    log = client.read_from_file('/var/log/cloud-init.log')
+    assert 'Python version change detected' not in log
+    assert 'Cache compatibility status is currently unknown.' not in log
+    _assert_no_pickle_problems(log)
+
+    client.restart()
+    log = client.read_from_file('/var/log/cloud-init.log')
+    assert 'Python version change detected' not in log
+    assert 'Could not determine Python version used to write cache' not in log
+    _assert_no_pickle_problems(log)
+
+    # Now ensure that loading a bad pickle gives us problems
+    client.push_file(TEST_PICKLE, PICKLE_PATH)
+    client.restart()
+    log = client.read_from_file('/var/log/cloud-init.log')
+    assert 'Failed loading pickled blob from {}'.format(PICKLE_PATH) in log
+
+
+def test_cache_purged_on_version_change(client: IntegrationInstance):
+    # Start by pushing the invalid pickle so we'll hit an error if the
+    # cache didn't actually get purged
+    client.push_file(TEST_PICKLE, PICKLE_PATH)
+    client.execute("echo '1.0' > /var/lib/cloud/data/python-version")
+    client.restart()
+    log = client.read_from_file('/var/log/cloud-init.log')
+    assert 'Python version change detected. Purging cache' in log
+    _assert_no_pickle_problems(log)
+
+
+def test_log_message_on_missing_version_file(client: IntegrationInstance):
+    # Start by pushing a pickle so we can see the log message
+    client.push_file(TEST_PICKLE, PICKLE_PATH)
+    client.execute("rm /var/lib/cloud/data/python-version")
+    client.restart()
+    log = client.read_from_file('/var/log/cloud-init.log')
+    assert (
+        'Writing python-version file. '
+        'Cache compatibility status is currently unknown.'
+    ) in log
diff --git a/tests/integration_tests/util.py b/tests/integration_tests/util.py
index 3ef12358..8d726bb2 100644
--- a/tests/integration_tests/util.py
+++ b/tests/integration_tests/util.py
@@ -3,10 +3,14 @@ import multiprocessing
 import os
 import time
 from contextlib import contextmanager
+from pathlib import Path
 
 log = logging.getLogger('integration_testing')
 
 
+ASSETS_DIR = Path('tests/integration_tests/assets')
+
+
 def verify_ordered_items_in_text(to_verify: list, text: str):
     """Assert all items in list appear in order in text.