diff options
| -rw-r--r-- | cloudinit/sources/__init__.py | 18 | ||||
| -rw-r--r-- | cloudinit/stages.py | 2 | ||||
| -rw-r--r-- | tests/integration_tests/assets/trusty_with_mime.pkl | 572 | ||||
| -rw-r--r-- | tests/integration_tests/modules/test_persistence.py | 30 |
4 files changed, 622 insertions, 0 deletions
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index a07c4b4f..9d25b0ee 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -75,6 +75,10 @@ NetworkConfigSource = namedtuple('NetworkConfigSource', _NETCFG_SOURCE_NAMES)(*_NETCFG_SOURCE_NAMES) +class DatasourceUnpickleUserDataError(Exception): + """Raised when userdata is unable to be unpickled due to python upgrades""" + + class DataSourceNotFoundException(Exception): pass @@ -239,6 +243,20 @@ class DataSource(CloudInitPickleMixin, metaclass=abc.ABCMeta): self.vendordata2 = None if not hasattr(self, 'vendordata2_raw'): self.vendordata2_raw = None + if hasattr(self, 'userdata') and self.userdata is not None: + # If userdata stores MIME data, on < python3.6 it will be + # missing the 'policy' attribute that exists on >=python3.6. + # Calling str() on the userdata will attempt to access this + # policy attribute. This will raise an exception, causing + # the pickle load to fail, so cloud-init will discard the cache + try: + str(self.userdata) + except AttributeError as e: + LOG.debug( + "Unable to unpickle datasource: %s." + " Ignoring current cache.", e + ) + raise DatasourceUnpickleUserDataError() from e def __str__(self): return type_utils.obj_name(self) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 3688be2e..06e0d9b1 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -1070,6 +1070,8 @@ def _pkl_load(fname): return None try: return pickle.loads(pickle_contents) + except sources.DatasourceUnpickleUserDataError: + return None except Exception: util.logexc(LOG, "Failed loading pickled blob from %s", fname) return None diff --git a/tests/integration_tests/assets/trusty_with_mime.pkl b/tests/integration_tests/assets/trusty_with_mime.pkl new file mode 100644 index 00000000..a4089ecf --- /dev/null +++ b/tests/integration_tests/assets/trusty_with_mime.pkl @@ -0,0 +1,572 @@ +ccopy_reg +_reconstructor +p1 +(ccloudinit.sources.DataSourceNoCloud +DataSourceNoCloudNet +p2 +c__builtin__ +object +p3 +NtRp4 +(dp5 +S'paths' +p6 +g1 +(ccloudinit.helpers +Paths +p7 +g3 +NtRp8 +(dp9 +S'lookups' +p10 +(dp11 +S'cloud_config' +p12 +S'cloud-config.txt' +p13 +sS'userdata' +p14 +S'user-data.txt.i' +p15 +sS'vendordata' +p16 +S'vendor-data.txt.i' +p17 +sS'userdata_raw' +p18 +S'user-data.txt' +p19 +sS'boothooks' +p20 +g20 +sS'scripts' +p21 +g21 +sS'sem' +p22 +g22 +sS'data' +p23 +g23 +sS'vendor_scripts' +p24 +S'scripts/vendor' +p25 +sS'handlers' +p26 +g26 +sS'obj_pkl' +p27 +S'obj.pkl' +p28 +sS'vendordata_raw' +p29 +S'vendor-data.txt' +p30 +sS'vendor_cloud_config' +p31 +S'vendor-cloud-config.txt' +p32 +ssS'template_tpl' +p33 +S'/etc/cloud/templates/%s.tmpl' +p34 +sS'cfgs' +p35 +(dp36 +S'cloud_dir' +p37 +S'/var/lib/cloud/' +p38 +sS'templates_dir' +p39 +S'/etc/cloud/templates/' +p40 +sS'upstart_dir' +p41 +S'/etc/init/' +p42 +ssS'cloud_dir' +p43 +g38 +sS'datasource' +p44 +NsS'upstart_conf_d' +p45 +g42 +sS'boot_finished' +p46 +S'/var/lib/cloud/instance/boot-finished' +p47 +sS'instance_link' +p48 +S'/var/lib/cloud/instance' +p49 +sS'seed_dir' +p50 +S'/var/lib/cloud/seed' +p51 +sbsS'supported_seed_starts' +p52 +(S'http://' +p53 +S'https://' +p54 +S'ftp://' +p55 +tp56 +sS'sys_cfg' +p57 +(dp58 +S'output' +p59 +(dp60 +S'all' +p61 +S'| tee -a /var/log/cloud-init-output.log' +p62 +ssS'users' +p63 +(lp64 +S'default' +p65 +asS'def_log_file' +p66 +S'/var/log/cloud-init.log' +p67 +sS'cloud_final_modules' +p68 +(lp69 +S'rightscale_userdata' +p70 +aS'scripts-vendor' +p71 +aS'scripts-per-once' +p72 +aS'scripts-per-boot' +p73 +aS'scripts-per-instance' +p74 +aS'scripts-user' +p75 +aS'ssh-authkey-fingerprints' +p76 +aS'keys-to-console' +p77 +aS'phone-home' +p78 +aS'final-message' +p79 +aS'power-state-change' +p80 +asS'disable_root' +p81 +I01 +sS'syslog_fix_perms' +p82 +S'syslog:adm' +p83 +sS'log_cfgs' +p84 +(lp85 +(lp86 +S'[loggers]\nkeys=root,cloudinit\n\n[handlers]\nkeys=consoleHandler,cloudLogHandler\n\n[formatters]\nkeys=simpleFormatter,arg0Formatter\n\n[logger_root]\nlevel=DEBUG\nhandlers=consoleHandler,cloudLogHandler\n\n[logger_cloudinit]\nlevel=DEBUG\nqualname=cloudinit\nhandlers=\npropagate=1\n\n[handler_consoleHandler]\nclass=StreamHandler\nlevel=WARNING\nformatter=arg0Formatter\nargs=(sys.stderr,)\n\n[formatter_arg0Formatter]\nformat=%(asctime)s - %(filename)s[%(levelname)s]: %(message)s\n\n[formatter_simpleFormatter]\nformat=[CLOUDINIT] %(filename)s[%(levelname)s]: %(message)s\n' +p87 +aS'[handler_cloudLogHandler]\nclass=handlers.SysLogHandler\nlevel=DEBUG\nformatter=simpleFormatter\nargs=("/dev/log", handlers.SysLogHandler.LOG_USER)\n' +p88 +aa(lp89 +g87 +aS"[handler_cloudLogHandler]\nclass=FileHandler\nlevel=DEBUG\nformatter=arg0Formatter\nargs=('/var/log/cloud-init.log',)\n" +p90 +aasS'cloud_init_modules' +p91 +(lp92 +S'migrator' +p93 +aS'seed_random' +p94 +aS'bootcmd' +p95 +aS'write-files' +p96 +aS'growpart' +p97 +aS'resizefs' +p98 +aS'set_hostname' +p99 +aS'update_hostname' +p100 +aS'update_etc_hosts' +p101 +aS'ca-certs' +p102 +aS'rsyslog' +p103 +aS'users-groups' +p104 +aS'ssh' +p105 +asS'preserve_hostname' +p106 +I00 +sS'_log' +p107 +(lp108 +g87 +ag90 +ag88 +asS'datasource_list' +p109 +(lp110 +S'NoCloud' +p111 +aS'ConfigDrive' +p112 +aS'OpenNebula' +p113 +aS'Azure' +p114 +aS'AltCloud' +p115 +aS'OVF' +p116 +aS'MAAS' +p117 +aS'GCE' +p118 +aS'OpenStack' +p119 +aS'CloudSigma' +p120 +aS'Ec2' +p121 +aS'CloudStack' +p122 +aS'SmartOS' +p123 +aS'None' +p124 +asS'vendor_data' +p125 +(dp126 +S'prefix' +p127 +(lp128 +sS'enabled' +p129 +I01 +ssS'cloud_config_modules' +p130 +(lp131 +S'emit_upstart' +p132 +aS'disk_setup' +p133 +aS'mounts' +p134 +aS'ssh-import-id' +p135 +aS'locale' +p136 +aS'set-passwords' +p137 +aS'grub-dpkg' +p138 +aS'apt-pipelining' +p139 +aS'apt-configure' +p140 +aS'package-update-upgrade-install' +p141 +aS'landscape' +p142 +aS'timezone' +p143 +aS'puppet' +p144 +aS'chef' +p145 +aS'salt-minion' +p146 +aS'mcollective' +p147 +aS'disable-ec2-metadata' +p148 +aS'runcmd' +p149 +aS'byobu' +p150 +assg14 +(iemail.mime.multipart +MIMEMultipart +p151 +(dp152 +S'_headers' +p153 +(lp154 +(S'Content-Type' +p155 +S'multipart/mixed; boundary="===============4291038100093149247=="' +tp156 +a(S'MIME-Version' +p157 +S'1.0' +p158 +tp159 +a(S'Number-Attachments' +p160 +S'1' +tp161 +asS'_payload' +p162 +(lp163 +(iemail.mime.base +MIMEBase +p164 +(dp165 +g153 +(lp166 +(g157 +g158 +tp167 +a(S'Content-Type' +p168 +S'text/x-not-multipart' +tp169 +a(S'Content-Disposition' +p170 +S'attachment; filename="part-001"' +tp171 +asg162 +S'' +sS'_charset' +p172 +NsS'_default_type' +p173 +S'text/plain' +p174 +sS'preamble' +p175 +NsS'defects' +p176 +(lp177 +sS'_unixfrom' +p178 +NsS'epilogue' +p179 +Nsbasg172 +Nsg173 +g174 +sg175 +Nsg176 +(lp180 +sg178 +Nsg179 +Nsbsg16 +S'#cloud-config\n{}\n\n' +p181 +sg18 +S'Content-Type: multipart/mixed; boundary="===============1378281702283945349=="\nMIME-Version: 1.0\n\n--===============1378281702283945349==\nContent-Type: text/x-shellscript; charset="utf-8"\nMIME-Version: 1.0\nContent-Transfer-Encoding: base64\nContent-Disposition: attachment; filename="script1.sh"\n\nIyEvYmluL3NoCgplY2hvICdoaScgPiAvdmFyL3RtcC9oaQo=\n\n--===============1378281702283945349==\nContent-Type: text/x-shellscript; charset="utf-8"\nMIME-Version: 1.0\nContent-Transfer-Encoding: base64\nContent-Disposition: attachment; filename="script2.sh"\n\nIyEvYmluL2Jhc2gKCmVjaG8gJ2hpMicgPiAvdmFyL3RtcC9oaTIK\n\n--===============1378281702283945349==--\n\n#cloud-config\n# final_message: |\n# This is my final message!\n# $version\n# $timestamp\n# $datasource\n# $uptime\n# updates:\n# network:\n# when: [\'hotplug\']\n' +p182 +sg29 +NsS'dsmode' +p183 +S'net' +p184 +sS'seed' +p185 +S'/var/lib/cloud/seed/nocloud-net' +p186 +sS'cmdline_id' +p187 +S'ds=nocloud-net' +p188 +sS'ud_proc' +p189 +g1 +(ccloudinit.user_data +UserDataProcessor +p190 +g3 +NtRp191 +(dp192 +g6 +g8 +sS'ssl_details' +p193 +(dp194 +sbsg50 +g186 +sS'ds_cfg' +p195 +(dp196 +sS'distro' +p197 +g1 +(ccloudinit.distros.ubuntu +Distro +p198 +g3 +NtRp199 +(dp200 +S'osfamily' +p201 +S'debian' +p202 +sS'_paths' +p203 +g8 +sS'name' +p204 +S'ubuntu' +p205 +sS'_runner' +p206 +g1 +(ccloudinit.helpers +Runners +p207 +g3 +NtRp208 +(dp209 +g6 +g8 +sS'sems' +p210 +(dp211 +sbsS'_cfg' +p212 +(dp213 +S'paths' +p214 +(dp215 +g37 +g38 +sg39 +g40 +sg41 +g42 +ssS'default_user' +p216 +(dp217 +S'shell' +p218 +S'/bin/bash' +p219 +sS'name' +p220 +S'ubuntu' +p221 +sS'sudo' +p222 +(lp223 +S'ALL=(ALL) NOPASSWD:ALL' +p224 +asS'lock_passwd' +p225 +I01 +sS'gecos' +p226 +S'Ubuntu' +p227 +sS'groups' +p228 +(lp229 +S'adm' +p230 +aS'audio' +p231 +aS'cdrom' +p232 +aS'dialout' +p233 +aS'dip' +p234 +aS'floppy' +p235 +aS'netdev' +p236 +aS'plugdev' +p237 +aS'sudo' +p238 +aS'video' +p239 +assS'package_mirrors' +p240 +(lp241 +(dp242 +S'arches' +p243 +(lp244 +S'i386' +p245 +aS'amd64' +p246 +asS'failsafe' +p247 +(dp248 +S'security' +p249 +S'http://security.ubuntu.com/ubuntu' +p250 +sS'primary' +p251 +S'http://archive.ubuntu.com/ubuntu' +p252 +ssS'search' +p253 +(dp254 +S'security' +p255 +(lp256 +sS'primary' +p257 +(lp258 +S'http://%(ec2_region)s.ec2.archive.ubuntu.com/ubuntu/' +p259 +aS'http://%(availability_zone)s.clouds.archive.ubuntu.com/ubuntu/' +p260 +aS'http://%(region)s.clouds.archive.ubuntu.com/ubuntu/' +p261 +assa(dp262 +S'arches' +p263 +(lp264 +S'armhf' +p265 +aS'armel' +p266 +aS'default' +p267 +asS'failsafe' +p268 +(dp269 +S'security' +p270 +S'http://ports.ubuntu.com/ubuntu-ports' +p271 +sS'primary' +p272 +S'http://ports.ubuntu.com/ubuntu-ports' +p273 +ssasS'ssh_svcname' +p274 +S'ssh' +p275 +ssbsS'metadata' +p276 +(dp277 +g183 +g184 +sS'local-hostname' +p278 +S'me' +p279 +sS'instance-id' +p280 +S'me' +p281 +ssb.
\ No newline at end of file diff --git a/tests/integration_tests/modules/test_persistence.py b/tests/integration_tests/modules/test_persistence.py new file mode 100644 index 00000000..00fdeaea --- /dev/null +++ b/tests/integration_tests/modules/test_persistence.py @@ -0,0 +1,30 @@ +# This file is part of cloud-init. See LICENSE file for license information. +"""Test the behavior of loading/discarding pickle data""" +from pathlib import Path + +import pytest + +from tests.integration_tests.instances import IntegrationInstance +from tests.integration_tests.util import ( + ASSETS_DIR, + verify_ordered_items_in_text, +) + + +PICKLE_PATH = Path('/var/lib/cloud/instance/obj.pkl') +TEST_PICKLE = ASSETS_DIR / 'trusty_with_mime.pkl' + + +@pytest.mark.lxd_container +def test_log_message_on_missing_version_file(client: IntegrationInstance): + client.push_file(TEST_PICKLE, PICKLE_PATH) + client.restart() + assert client.execute('cloud-init status --wait').ok + log = client.read_from_file('/var/log/cloud-init.log') + verify_ordered_items_in_text([ + "Unable to unpickle datasource: 'MIMEMultipart' object has no " + "attribute 'policy'. Ignoring current cache.", + 'no cache found', + 'Searching for local data source', + 'SUCCESS: found local data from DataSourceNoCloud' + ], log) |