From 6ce134c1868478345471ba9166f1523f7d9bf19d Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 22 Mar 2016 03:02:31 -0400 Subject: move some of the pickle loading out of Init, into private methods I plan to re-use these methods later. They stand alone even if they dont end up getting used, though. --- cloudinit/stages.py | 65 ++++++++++++++++++++++++++++------------------------- 1 file changed, 35 insertions(+), 30 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index edad6450..c230ec0d 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -193,40 +193,12 @@ class Init(object): # We try to restore from a current link and static path # by using the instance link, if purge_cache was called # the file wont exist. - pickled_fn = self.paths.get_ipath_cur('obj_pkl') - pickle_contents = None - try: - pickle_contents = util.load_file(pickled_fn, decode=False) - except Exception as e: - if os.path.isfile(pickled_fn): - LOG.warn("failed loading pickle in %s: %s" % (pickled_fn, e)) - pass - - # This is expected so just return nothing - # successfully loaded... - if not pickle_contents: - return None - try: - return pickle.loads(pickle_contents) - except Exception: - util.logexc(LOG, "Failed loading pickled blob from %s", pickled_fn) - return None + return _pkl_load(self.paths.get_ipath_cur('obj_pkl')) def _write_to_cache(self): if self.datasource is NULL_DATA_SOURCE: return False - pickled_fn = self.paths.get_ipath_cur("obj_pkl") - try: - pk_contents = pickle.dumps(self.datasource) - except Exception: - util.logexc(LOG, "Failed pickling datasource %s", self.datasource) - return False - try: - util.write_file(pickled_fn, pk_contents, omode="wb", mode=0o400) - except Exception: - util.logexc(LOG, "Failed pickling datasource to %s", pickled_fn) - return False - return True + return _pkl_store(self.datasource, self.paths.get_ipath_cur("obj_pkl")) def _get_datasources(self): # Any config provided??? @@ -796,3 +768,36 @@ def fetch_base_config(): base_cfgs.append(default_cfg) return util.mergemanydict(base_cfgs) + + +def _pkl_store(obj, fname): + try: + pk_contents = pickle.dumps(obj) + except Exception: + util.logexc(LOG, "Failed pickling datasource %s", obj) + return False + try: + util.write_file(fname, pk_contents, omode="wb", mode=0o400) + except Exception: + util.logexc(LOG, "Failed pickling datasource to %s", fname) + return False + return True + + +def _pkl_load(fname): + pickle_contents = None + try: + pickle_contents = util.load_file(fname, decode=False) + except Exception as e: + if os.path.isfile(fname): + LOG.warn("failed loading pickle in %s: %s" % (fname, e)) + pass + + # This is allowed so just return nothing successfully loaded... + if not pickle_contents: + return None + try: + return pickle.loads(pickle_contents) + except Exception: + util.logexc(LOG, "Failed loading pickled blob from %s", fname) + return None -- cgit v1.2.3 From 9c0a2abc8d2c0e390745ddb163f5eae07b20d61d Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 22 Mar 2016 03:50:28 -0400 Subject: add code to invoke networking config there is no data source that has a populated network_config() so at this point this doesn't do anything. --- bin/cloud-init | 4 ++++ cloudinit/distros/__init__.py | 2 +- cloudinit/net/__init__.py | 17 +++++++++++++++++ cloudinit/sources/__init__.py | 4 ++++ cloudinit/stages.py | 24 ++++++++++++++++++++++++ 5 files changed, 50 insertions(+), 1 deletion(-) (limited to 'cloudinit/stages.py') diff --git a/bin/cloud-init b/bin/cloud-init index 63aa765b..8875d2f6 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -263,6 +263,10 @@ def main_init(name, args): return (None, []) else: return (None, ["No instance datasource found."]) + + if args.local: + init.apply_network_config() + # Stage 6 iid = init.instancify() LOG.debug("%s will now be targeting instance id: %s", name, iid) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 74b484a7..418421b9 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -135,7 +135,7 @@ class Distro(object): return self._bring_up_interfaces(dev_names) return False - def apply_network_config(self, netconfig, bring_up=True): + def apply_network_config(self, netconfig, bring_up=False): # Write it out dev_names = self._write_network_config(netconfig) # Now try to bring them up diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 3cf99604..799cb97e 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -434,4 +434,21 @@ def render_network_state(target, network_state): with open(netrules, 'w+') as f: f.write(render_persistent_net(network_state)) + +def is_disabled_cfg(cfg): + if not cfg or not isinstance(cfg, dict): + return False + return cfg.get('config') == "disabled" + + +def generate_fallback_config(): + # FIXME: add implementation here + return None + + +def read_kernel_cmdline_config(): + # FIXME: add implementation here + return None + + # vi: ts=4 expandtab syntax=python diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 28540a7b..c63464b2 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -221,6 +221,10 @@ class DataSource(object): # quickly (local check only) if self.instance_id is still return False + @property + def network_config(self): + return None + def normalize_pubkey_data(pubkey_data): keys = [] diff --git a/cloudinit/stages.py b/cloudinit/stages.py index c230ec0d..8e681e29 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -43,6 +43,7 @@ from cloudinit import distros from cloudinit import helpers from cloudinit import importer from cloudinit import log as logging +from cloudinit import net from cloudinit import sources from cloudinit import type_utils from cloudinit import util @@ -567,6 +568,29 @@ class Init(object): # Run the handlers self._do_handlers(user_data_msg, c_handlers_list, frequency) + def _find_networking_config(self): + cmdline_cfg = ('cmdline', net.read_kernel_cmdline_config()) + dscfg = ('ds', None) + if self.datasource and hasattr(self.datasource, 'network_config'): + dscfg = ('ds', self.datasource.network_config) + sys_cfg = ('system_cfg', self.cfg.get('network')) + + for loc, ncfg in (cmdline_cfg, dscfg, sys_cfg): + if net.is_disabled_cfg(ncfg): + LOG.debug("network config disabled by %s", loc) + return None + if ncfg: + return ncfg + return net.generate_fallback_config() + + def apply_network_config(self): + netcfg = self._find_networking_config() + if netcfg is None: + LOG.info("network config is disabled") + return + + return self.distro.apply_network_config(netcfg) + class Modules(object): def __init__(self, init, cfg_files=None, reporter=None): -- cgit v1.2.3 From ca00b0f1f8c8a40409328c595d44234bb61c24c4 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 22 Mar 2016 04:49:34 -0400 Subject: make NoCloud work for seeding network. Tested now with the generated fallback config in an lxc container. Had to change to return a config rather than a network state. Also this makes nocloud look in nocloud-net's seed dir. This way it will read the seed and clame the datasource but not do anything other than apply networking and the init_modules early. It is a change in behavior of the time that boothooks woudl run to do this. May need to change that back. --- cloudinit/net/__init__.py | 20 +++++--------------- cloudinit/sources/DataSourceNoCloud.py | 34 +++++++++++++++++++--------------- cloudinit/stages.py | 1 + 3 files changed, 25 insertions(+), 30 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index b45153f4..63fad2fa 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -448,11 +448,7 @@ def generate_fallback_config(): """Determine which attached net dev is most likely to have a connection and generate network state to run dhcp on that interface""" # by default use eth0 as primary interface - nconf = {'config': {'interfaces': {}, - 'dns': {'search': [], 'nameservers': []}, 'routes': [] - }, - 'version': 1 - } + nconf = {'config': [], 'version': 1} # get list of interfaces that could have connections invalid_interfaces = set(['lo']) @@ -506,21 +502,15 @@ def generate_fallback_config(): if DEFAULT_PRIMARY_INTERFACE in potential_interfaces: name = DEFAULT_PRIMARY_INTERFACE else: - potential_interfaces.sort( - key=lambda x: int(''.join(i for i in x if i in string.digits))) - name = potential_interfaces[0] + name = sorted(potential_interfaces)[0] sysfs_mac = os.path.join(SYS_CLASS_NET, name, 'address') mac = util.load_file(sysfs_mac).strip() target_name = name - # generate net config for interface - nconf['config']['interfaces'][target_name] = { - 'mac_address': mac, 'name': target_name, 'type': 'physical', - 'mode': 'manual', 'inet': 'inet', - 'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}] - } - + nconf['config'].append( + {'type': 'physical', 'name': target_name, + 'mac_address': mac, 'subnets': [{'type': 'dhcp4'}]}) return nconf diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 538df7d9..bd04a6fe 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -36,7 +36,9 @@ class DataSourceNoCloud(sources.DataSource): self.dsmode = 'local' self.seed = None self.cmdline_id = "ds=nocloud" - self.seed_dir = os.path.join(paths.seed_dir, 'nocloud') + self.seed_dirs = [os.path.join(paths.seed_dir, 'nocloud'), + os.path.join(paths.seed_dir, 'nocloud-net')] + self.seed_dir = None self.supported_seed_starts = ("/", "file://") def __str__(self): @@ -67,15 +69,15 @@ class DataSourceNoCloud(sources.DataSource): pp2d_kwargs = {'required': ['user-data', 'meta-data'], 'optional': ['vendor-data', 'network-config']} - try: - seeded = util.pathprefix2dict(self.seed_dir, **pp2d_kwargs) - found.append(self.seed_dir) - LOG.debug("Using seeded data from %s", self.seed_dir) - except ValueError as e: - pass - - if self.seed_dir in found: - mydata = _merge_new_seed(mydata, seeded) + for path in self.seed_dirs: + try: + seeded = util.pathprefix2dict(path, **pp2d_kwargs) + found.append(path) + LOG.debug("Using seeded data from %s", path) + mydata = _merge_new_seed(mydata, seeded) + break + except ValueError as e: + pass # If the datasource config had a 'seedfrom' entry, then that takes # precedence over a 'seedfrom' that was found in a filesystem @@ -188,21 +190,19 @@ class DataSourceNoCloud(sources.DataSource): # if this is the local datasource or 'seedfrom' was used # and the source of the seed was self.dsmode. # Then see if there is network config to apply. + # note this is obsolete network-interfaces style seeding. if self.dsmode in ("local", seeded_network): if mydata['meta-data'].get('network-interfaces'): LOG.debug("Updating network interfaces from %s", self) self.distro.apply_network( mydata['meta-data']['network-interfaces']) - elif mydata.get('network-config'): - LOG.debug("Updating network config from %s", self) - self.distro.apply_network_config(mydata['network-config'], - bring_up=False) if mydata['meta-data']['dsmode'] == self.dsmode: self.seed = ",".join(found) self.metadata = mydata['meta-data'] self.userdata_raw = mydata['user-data'] self.vendordata_raw = mydata['vendor-data'] + self._network_config = mydata['network-config'] return True LOG.debug("%s: not claiming datasource, dsmode=%s", self, @@ -222,6 +222,10 @@ class DataSourceNoCloud(sources.DataSource): return None return quick_id == current + @property + def network_config(self): + return self._network_config + def _quick_read_instance_id(cmdline_id, dirs=None): if dirs is None: @@ -312,7 +316,7 @@ class DataSourceNoCloudNet(DataSourceNoCloud): DataSourceNoCloud.__init__(self, sys_cfg, distro, paths) self.cmdline_id = "ds=nocloud-net" self.supported_seed_starts = ("http://", "https://", "ftp://") - self.seed_dir = os.path.join(paths.seed_dir, 'nocloud-net') + self.seed_dirs = [os.path.join(paths.seed_dir, 'nocloud-net')] self.dsmode = "net" diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 8e681e29..73090025 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -589,6 +589,7 @@ class Init(object): LOG.info("network config is disabled") return + LOG.info("Applying configuration: %s", netcfg) return self.distro.apply_network_config(netcfg) -- cgit v1.2.3 From ed55d6bac52c53b9473b9644ce50f61404bfd438 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 22 Mar 2016 20:02:29 -0400 Subject: better log message about applying networking. --- cloudinit/stages.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 73090025..8ebbe6a9 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -578,18 +578,18 @@ class Init(object): for loc, ncfg in (cmdline_cfg, dscfg, sys_cfg): if net.is_disabled_cfg(ncfg): LOG.debug("network config disabled by %s", loc) - return None + return (None, loc) if ncfg: - return ncfg - return net.generate_fallback_config() + return (ncfg, loc) + return (net.generate_fallback_config(), "fallback") def apply_network_config(self): - netcfg = self._find_networking_config() + netcfg, src = self._find_networking_config() if netcfg is None: - LOG.info("network config is disabled") + LOG.info("network config is disabled by %s", src) return - LOG.info("Applying configuration: %s", netcfg) + LOG.info("Applying network configuration from %s: %s", src, netcfg) return self.distro.apply_network_config(netcfg) -- cgit v1.2.3 From eb8b2f0e7b777b756a4965ea784ce1354b5c6396 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 24 Mar 2016 12:51:31 -0400 Subject: provide datasource.check_instance_id with access to system config Changing this interface to allow for easy change later. The thing that this will enable is: a.) maas datasource to look at the system config and see if it is configured with the same consumer_key b.) datasource config could allow setting a variable that it would look at. --- cloudinit/sources/DataSourceAzure.py | 2 +- cloudinit/sources/DataSourceNoCloud.py | 3 ++- cloudinit/sources/DataSourceOpenStack.py | 2 +- cloudinit/sources/__init__.py | 2 +- cloudinit/stages.py | 3 ++- 5 files changed, 7 insertions(+), 5 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 832b3063..698f4cac 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -254,7 +254,7 @@ class DataSourceAzureNet(sources.DataSource): def get_config_obj(self): return self.cfg - def check_instance_id(self): + def check_instance_id(self, sys_cfg): # quickly (local check only) if self.instance_id is still valid return sources.instance_id_matches_system_uuid(self.get_instance_id()) diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index afd08935..f786516b 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -209,13 +209,14 @@ class DataSourceNoCloud(sources.DataSource): mydata['meta-data']['dsmode']) return False - def check_instance_id(self): + def check_instance_id(self, sys_cfg): # quickly (local check only) if self.instance_id is still valid # we check kernel command line or files. current = self.get_instance_id() if not current: return None + LOG.info("Hi, I got some system config: %s", sys_cfg) quick_id = _quick_read_instance_id(cmdline_id=self.cmdline_id, dirs=self.seed_dirs) if not quick_id: diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index 79bb9d63..f7f4590b 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -150,7 +150,7 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): return True - def check_instance_id(self): + def check_instance_id(self, sys_cfg): # quickly (local check only) if self.instance_id is still valid return sources.instance_id_matches_system_uuid(self.get_instance_id()) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index c63464b2..82cd3553 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -217,7 +217,7 @@ class DataSource(object): def get_package_mirror_info(self): return self.distro.get_package_mirror_info(data_source=self) - def check_instance_id(self): + def check_instance_id(self, sys_cfg): # quickly (local check only) if self.instance_id is still return False diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 8ebbe6a9..5d6b0447 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -223,7 +223,8 @@ class Init(object): if ds and existing == "trust": myrep.description = "restored from cache: %s" % ds elif ds and existing == "check": - if hasattr(ds, 'check_instance_id') and ds.check_instance_id(): + if (hasattr(ds, 'check_instance_id') and + ds.check_instance_id(self.cfg)): myrep.description = "restored from checked cache: %s" % ds else: myrep.description = "cache invalid in datasource: %s" % ds -- cgit v1.2.3 From 6c49afad6134c5094c5e21784e76735faf510a29 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 24 Mar 2016 17:11:26 -0400 Subject: some final changes a.) do not write systemd link files if we do not have a mac address. the check is updated to check for value rather than just presense (ie, 'mac_address': None) b.) DataSourceNoCloudNet: search in the nocloud seed dir this is important because NoCloud if dsmode is Net will look only would pass by, expecting NoCloudNet to pick it up but NoCloudNet would not look in /var/lib/cloud/seed/nocloud and thus skip it. c.) support the disabling of network configuration via /var/lib/cloud/data/upgraded-network This is what the package upgrader is writing. --- cloudinit/net/__init__.py | 4 ++-- cloudinit/sources/DataSourceNoCloud.py | 1 - cloudinit/stages.py | 5 +++++ 3 files changed, 7 insertions(+), 3 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 57beb837..40929c6e 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -407,7 +407,7 @@ def render_persistent_net(network_state): for iface in interfaces.values(): # for physical interfaces write out a persist net udev rule if iface['type'] == 'physical' and \ - 'name' in iface and 'mac_address' in iface: + 'name' in iface and iface.get('mac_address'): content += generate_udev_rule(iface['name'], iface['mac_address']) @@ -598,7 +598,7 @@ def render_systemd_links(target, network_state, interfaces = network_state.get('interfaces') for iface in interfaces.values(): if (iface['type'] == 'physical' and 'name' in iface and - 'mac_address' in iface): + iface.get('mac_address')): fname = fp_prefix + iface['name'] + ".link" with open(fname, "w") as fp: fp.write("\n".join([ diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 802d515b..c2fba4d2 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -316,7 +316,6 @@ class DataSourceNoCloudNet(DataSourceNoCloud): DataSourceNoCloud.__init__(self, sys_cfg, distro, paths) self.cmdline_id = "ds=nocloud-net" self.supported_seed_starts = ("http://", "https://", "ftp://") - self.seed_dirs = [os.path.join(paths.seed_dir, 'nocloud-net')] self.dsmode = "net" diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 5d6b0447..143a4fc9 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -570,6 +570,11 @@ class Init(object): self._do_handlers(user_data_msg, c_handlers_list, frequency) def _find_networking_config(self): + disable_file = os.path.join( + self.paths.get_cpath('data'), 'upgraded-network') + if os.path.exists(disable_file): + return (None, disable_file) + cmdline_cfg = ('cmdline', net.read_kernel_cmdline_config()) dscfg = ('ds', None) if self.datasource and hasattr(self.datasource, 'network_config'): -- cgit v1.2.3