From 0b3e3f898f70cc98c6e694c7b7a11e654fff9967 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Sun, 1 May 2016 13:03:23 -0400 Subject: initial commit of rework --- cloudinit/sources/DataSourceSmartOS.py | 450 +++++++++++++++++++++------------ 1 file changed, 286 insertions(+), 164 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 6cbd8dfa..46cf117a 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -32,8 +32,10 @@ # http://us-east.manta.joyent.com/jmc/public/mdata/datadict.html # Comments with "@datadictionary" are snippets of the definition +import base64 import binascii import contextlib +import json import os import random import re @@ -66,12 +68,26 @@ SMARTOS_ATTRIB_MAP = { DS_NAME = 'SmartOS' DS_CFG_PATH = ['datasource', DS_NAME] +NO_BASE64_DECODE = [ + 'iptables_disable', + 'motd_sys_info', + 'root_authorized_keys', + 'sdc:datacenter_name', + 'sdc:uuid' + 'user-data', + 'user-script', +] + +METADATA_SOCKFILE = '/native/.zonecontrol/metadata.sock' +SERIAL_DEVICE = '/dev/ttyS1' +SERIAL_TIMEOUT = 60 + # BUILT-IN DATASOURCE CONFIGURATION # The following is the built-in configuration. If the values # are not set via the system configuration, then these default # will be used: # serial_device: which serial device to use for the meta-data -# seed_timeout: how long to wait on the device +# serial_timeout: how long to wait on the device # no_base64_decode: values which are not base64 encoded and # are fetched directly from SmartOS, not meta-data values # base64_keys: meta-data keys that are delivered in base64 @@ -81,16 +97,10 @@ DS_CFG_PATH = ['datasource', DS_NAME] # fs_setup: describes how to format the ephemeral drive # BUILTIN_DS_CONFIG = { - 'serial_device': '/dev/ttyS1', - 'metadata_sockfile': '/native/.zonecontrol/metadata.sock', - 'seed_timeout': 60, - 'no_base64_decode': ['root_authorized_keys', - 'motd_sys_info', - 'iptables_disable', - 'user-data', - 'user-script', - 'sdc:datacenter_name', - 'sdc:uuid'], + 'serial_device': SERIAL_DEVICE, + 'serial_timeout': SERIAL_TIMEOUT, + 'metadata_sockfile': METADATA_SOCKFILE, + 'no_base64_decode': NO_BASE64_DECODE, 'base64_keys': [], 'base64_all': False, 'disk_aliases': {'ephemeral0': '/dev/vdb'}, @@ -154,9 +164,12 @@ LEGACY_USER_D = "/var/db" class DataSourceSmartOS(sources.DataSource): + _unset = "_unset" + smartos_environ = _unset + md_client = _unset + def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) - self.is_smartdc = None self.ds_cfg = util.mergemanydict([ self.ds_cfg, util.get_cfg_by_path(sys_cfg, DS_CFG_PATH, {}), @@ -164,49 +177,24 @@ class DataSourceSmartOS(sources.DataSource): self.metadata = {} - # SDC LX-Brand Zones lack dmidecode (no /dev/mem) but - # report 'BrandZ virtual linux' as the kernel version - if os.uname()[3].lower() == 'brandz virtual linux': - LOG.debug("Host is SmartOS, guest in Zone") - self.is_smartdc = True - self.smartos_type = 'lx-brand' - self.cfg = {} - self.seed = self.ds_cfg.get("metadata_sockfile") - else: - self.is_smartdc = True - self.smartos_type = 'kvm' - self.seed = self.ds_cfg.get("serial_device") - self.cfg = BUILTIN_CLOUD_CONFIG - self.seed_timeout = self.ds_cfg.get("serial_timeout") - self.smartos_no_base64 = self.ds_cfg.get('no_base64_decode') - self.b64_keys = self.ds_cfg.get('base64_keys') - self.b64_all = self.ds_cfg.get('base64_all') self.script_base_d = os.path.join(self.paths.get_cpath("scripts")) + self._init() + def __str__(self): root = sources.DataSource.__str__(self) - return "%s [seed=%s]" % (root, self.seed) - - def _get_seed_file_object(self): - if not self.seed: - raise AttributeError("seed device is not set") - - if self.smartos_type == 'lx-brand': - if not stat.S_ISSOCK(os.stat(self.seed).st_mode): - LOG.debug("Seed %s is not a socket", self.seed) - return None - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - sock.connect(self.seed) - return sock.makefile('rwb') - else: - if not stat.S_ISCHR(os.stat(self.seed).st_mode): - LOG.debug("Seed %s is not a character device") - return None - ser = serial.Serial(self.seed, timeout=self.seed_timeout) - if not ser.isOpen(): - raise SystemError("Unable to open %s" % self.seed) - return ser - return None + return "%s [client=%s]" % (root, self.md_client) + + def _init(self): + if self.smartos_environ == self._unset: + self.smartos_env = get_smartos_environ() + + if self.md_client == self._unset: + self.md_client = jmc_client_factory( + metadata_sockfile=self.ds_cfg['metadata_sockfile'], + serial_device=self.ds_cfg['serial_device'], + serial_timeout=self.ds_cfg['serial_timeout']) + def _set_provisioned(self): '''Mark the instance provisioning state as successful. @@ -225,50 +213,23 @@ class DataSourceSmartOS(sources.DataSource): '/'.join([svc_path, 'provision_success'])) def get_data(self): + self._init() + md = {} ud = "" - if not device_exists(self.seed): - LOG.debug("No metadata device '%s' found for SmartOS datasource", - self.seed) - return False - - uname_arch = os.uname()[4] - if uname_arch.startswith("arm") or uname_arch == "aarch64": - # Disabling because dmidcode in dmi_data() crashes kvm process - LOG.debug("Disabling SmartOS datasource on arm (LP: #1243287)") + if not self.smartos_env: + LOG.debug("Not running on smartos") return False - - # SDC KVM instances will provide dmi data, LX-brand does not - if self.smartos_type == 'kvm': - dmi_info = dmi_data() - if dmi_info is None: - LOG.debug("No dmidata utility found") - return False - - system_type = dmi_info - if 'smartdc' not in system_type.lower(): - LOG.debug("Host is not on SmartOS. system_type=%s", - system_type) - return False - LOG.debug("Host is SmartOS, guest in KVM") - - seed_obj = self._get_seed_file_object() - if seed_obj is None: - LOG.debug('Seed file object not found.') + + if not self.md_client.exists(): + LOG.debug("No metadata device '%r' found for SmartOS datasource", + self.md_client) return False - with contextlib.closing(seed_obj) as seed: - b64_keys = self.query('base64_keys', seed, strip=True, b64=False) - if b64_keys is not None: - self.b64_keys = [k.strip() for k in str(b64_keys).split(',')] - b64_all = self.query('base64_all', seed, strip=True, b64=False) - if b64_all is not None: - self.b64_all = util.is_true(b64_all) - - for ci_noun, attribute in SMARTOS_ATTRIB_MAP.items(): - smartos_noun, strip = attribute - md[ci_noun] = self.query(smartos_noun, seed, strip=strip) + for ci_noun, attribute in SMARTOS_ATTRIB_MAP.items(): + smartos_noun, strip = attribute + md[ci_noun] = self.md_client.get(smartos_noun, strip=strip) # @datadictionary: This key may contain a program that is written # to a file in the filesystem of the guest on each boot and then @@ -331,65 +292,6 @@ class DataSourceSmartOS(sources.DataSource): def get_instance_id(self): return self.metadata['instance-id'] - def query(self, noun, seed_file, strip=False, default=None, b64=None): - if b64 is None: - if noun in self.smartos_no_base64: - b64 = False - elif self.b64_all or noun in self.b64_keys: - b64 = True - - return self._query_data(noun, seed_file, strip=strip, - default=default, b64=b64) - - def _query_data(self, noun, seed_file, strip=False, - default=None, b64=None): - """Makes a request via "GET " - - In the response, the first line is the status, while subsequent - lines are is the value. A blank line with a "." is used to - indicate end of response. - - If the response is expected to be base64 encoded, then set - b64encoded to true. Unfortantely, there is no way to know if - something is 100% encoded, so this method relies on being told - if the data is base64 or not. - """ - - if not noun: - return False - - response = JoyentMetadataClient(seed_file).get_metadata(noun) - - if response is None: - return default - - if b64 is None: - b64 = self._query_data('b64-%s' % noun, seed_file, b64=False, - default=False, strip=True) - b64 = util.is_true(b64) - - resp = None - if b64 or strip: - resp = "".join(response).rstrip() - else: - resp = "".join(response) - - if b64: - try: - return util.b64d(resp) - # Bogus input produces different errors in Python 2 and 3; - # catch both. - except (TypeError, binascii.Error): - LOG.warn("Failed base64 decoding key '%s'", noun) - return resp - - return resp - - -def device_exists(device): - """Symplistic method to determine if the device exists or not""" - return os.path.exists(device) - class JoyentMetadataFetchException(Exception): pass @@ -407,8 +309,11 @@ class JoyentMetadataClient(object): r' (?P(?P[0-9a-f]+) (?PSUCCESS|NOTFOUND)' r'( (?P.+))?)') - def __init__(self, metasource): - self.metasource = metasource + def __init__(self, smartos_type=None): + if smartos_type is None: + smartos_type = get_smartos_environ() + self.smartos_type = smartos_type + self.fp = None def _checksum(self, body): return '{0:08x}'.format( @@ -436,37 +341,227 @@ class JoyentMetadataClient(object): LOG.debug('Value "%s" found.', value) return value - def get_metadata(self, metadata_key): - LOG.debug('Fetching metadata key "%s"...', metadata_key) + def request(self, rtype, param=None): request_id = '{0:08x}'.format(random.randint(0, 0xffffffff)) - message_body = '{0} GET {1}'.format(request_id, - util.b64e(metadata_key)) + message_body = ' '.join((request_id, rtype,)) + if param: + message_body += ' ' + base64.b64encode(param.encode()).decode() msg = 'V2 {0} {1} {2}\n'.format( len(message_body), self._checksum(message_body), message_body) LOG.debug('Writing "%s" to metadata transport.', msg) - self.metasource.write(msg.encode('ascii')) - self.metasource.flush() + + need_close = False + if not self.fp: + self.open_transport() + need_close = True + + self.fp.write(msg.encode('ascii')) + self.fp.flush() response = bytearray() - response.extend(self.metasource.read(1)) + response.extend(self.fp.read(1)) while response[-1:] != b'\n': - response.extend(self.metasource.read(1)) + response.extend(self.fp.read(1)) + + if need_close: + self.close_transport() + response = response.rstrip().decode('ascii') LOG.debug('Read "%s" from metadata transport.', response) if 'SUCCESS' not in response: return None - return self._get_value_from_frame(request_id, response) + value = self._get_value_from_frame(request_id, response) + if value is None: + return default + return value -def dmi_data(): - sys_type = util.read_dmi_data("system-product-name") + def get(self, key, default=None, strip=False): + result = self.request(rtype='GET', param=key) + if result is None: + return default + if result and strip: + result = result.strip() + return result + + def get_json(self, key, default=None): + result = self.get(key) + if result is None: + return default + return json.loads(result) + + def list(self): + result = self.request(rtype='KEYS') + if result: + result = result.split('\n') + return result + + def put(self, key, val): + param = b' '.join([base64.b64encode(i.encode()) + for i in (key, val)]).decode() + return self.request(rtype='PUT', param=param) + + def delete(self, key): + return self.request(rtype='DELETE', param=key) + + def close_transport(self): + if self.fp: + self.fp.close() + self.fp = None + + def __enter__(self): + if self.fp: + return self + self.open_transport() + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.close_transport() + return - if not sys_type: - return None - return sys_type +class JoyentMetadataSocketClient(JoyentMetadataClient): + def __init__(self, socketpath): + self.socketpath = metadata_socketfile + + def open_transport(self): + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.connect(path) + self.fp = sock.makefile('rwb') + + def exists(self): + return os.path.exists(self.socketpath) + + def __repr__(self): + return "%s(socketpath=%s)" % (self.__class__.__name__, self.socketpath) + + +class JoyentMetadataSerialClient(JoyentMetadataClient): + def __init__(self, device, timeout=10, smartos_type=None): + super(JoyentMetadataSerialClient, self).__init__(smartos_type) + self.device = device + self.timeout = timeout + + def exists(self): + return os.path.exists(self.device) + + def open_transport(self): + ser = serial.Serial(self.device, timeout=self.timeout) + if not ser.isOpen(): + raise SystemError("Unable to open %s" % self.device) + self.fp = ser + + def __repr__(self): + return "%s(device=%s, timeout=%s)" % ( + self.__class__.__name__, self.device, self.timeout) + + +class JoyentMetadataLegacySerialClient(JoyentMetadataSerialClient): + """V1 of the protocol was not safe for all values. + Thus, we allowed the user to pass values in as base64 encoded. + Users may still reasonably expect to be able to send base64 data + and have it transparently decoded. So even though the V2 format is + now used, and is safe (using base64 itself), we keep legacy support. + + The way for a user to do this was: + a.) specify 'base64_keys' key whose value is a comma delimited + list of keys that were base64 encoded. + b.) base64_all: string interpreted as a boolean that indicates + if all keys are base64 encoded. + c.) set a key named b64- with a boolean indicating that + is base64 encoded.""" + + def __init__(self, device, timeout=10, smartos_type=None): + s = super(JoyentMetadataLegacySerialClient, self) + s.__init__(device, timeout, smartos_type) + self.base64_keys = None + self.base64_all = None + + def _init_base64_keys(self, reset=False): + if reset: + self.base64_keys = None + self.base64_all = None + + keys = None + if self.base64_all is None: + keys = self.list() + if 'base64_all' in keys: + self.base64_all = util.is_true(self._get("base64_all")) + else: + self.base64_all = False + + if self.base64_all: + # short circuit if base64_all is true + return + + if self.base64_keys is None: + if keys is None: + keys = self.list() + b64_keys = set() + if 'base64_keys' in keys: + b64_keys = set(self._get("base64_keys").split(",")) + + # now add any b64- that has a true value + for key in [k[3:] for k in keys if k.startswith("b64-")]: + if util.is_true(self._get(key)): + b64_keys.append(key) + else: + if key in b64_keys: + b64_keys.remove(key) + + self.base64_keys = b64_keys + + def _get(self, key, default=None, strip=False): + return (super(JoyentMetadataLegacySerialClient, self). + get(key, default=default, strip=strip)) + + def is_b64_encoded(self, key, reset=False): + if key in NO_BASE64_DECODE: + return False + + self._init_base64_keys(reset=reset) + if self.base64_all: + return True + + return key in self.base64_keys + + def get(self, key, default=None, strip=False): + mdefault = object() + val = self._get(key, strip=False, default=mdefault) + if val is mdefault: + return default + + if self.is_b64_encoded(key): + try: + val = base64.b64decode(val.encode()).decode() + # Bogus input produces different errors in Python 2 and 3 + except (TypeError, binascii.Error): + LOG.warn("Failed base64 decoding key '%s': %s", key, val) + + if strip: + val = val.strip() + + return val + + +def jmc_client_factory( + smartos_type=None, metadata_sockfile=METADATA_SOCKFILE, + serial_device=SERIAL_DEVICE, serial_timeout=SERIAL_TIMEOUT, + uname_version=None): + + if smartos_type is None: + smartos_type = get_smartos_environ(uname_version) + + if smartos_type == 'kvm': + return JoyentMetadataLegacySerialClient( + device=serial_device, timeout=serial_timeout, + smartos_type=smartos_type) + elif smartos_type == 'lx-brand': + return JoyentMetadataSerialClient(socketpath=metadata_socketfile) + + raise ValueError("Unknown value for smartos_type: %s" % smartos_type) def write_boot_content(content, content_f, link=None, shebang=False, @@ -525,6 +620,29 @@ def write_boot_content(content, content_f, link=None, shebang=False, util.logexc(LOG, "failed establishing content link", e) +def get_smartos_environ(uname_version=None, product_name=None, + uname_arch=None): + uname = os.uname() + if uname_arch is None: + uname_arch = uname[4] + + if uname_arch.startswith("arm") or uname_arch == "aarch64": + return None + + # SDC LX-Brand Zones lack dmidecode (no /dev/mem) but + # report 'BrandZ virtual linux' as the kernel version + if uname_version is None: + uname_version = uname[3] + if uname_version.lower() == 'brandz virtual linux': + return 'lx-brand' + + system_type = util.read_dmi_data("system-product-name") + if system_type and 'smartdc' in system_type.lower(): + return 'kvm' + + return None + + # Used to match classes to dependencies datasources = [ (DataSourceSmartOS, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), @@ -534,3 +652,7 @@ datasources = [ # Return a list of data sources that match this set of dependencies def get_datasource_list(depends): return sources.list_from_depends(depends, datasources) + + +if __name__ == "__main__": + jmc = JoyentMetadataClient(seed_file).get_metadata(noun) -- cgit v1.2.3 From 7f2e99f5345c227d07849da68acdf8562b44c3e1 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 25 May 2016 17:05:09 -0400 Subject: commit to push for fear of loss. == background == DataSource Mode (dsmode) is present in many datasources in cloud-init. dsmode was originally added to cloud-init to specify when this datasource should be 'realized'. cloud-init has 4 stages of boot. a.) cloud-init --local . network is guaranteed not present. b.) cloud-init (--network). network is guaranteed present. c.) cloud-config d.) cloud-init final 'init_modules' [1] are run "as early as possible". And as such, are executed in either 'a' or 'b' based on the datasource. However, executing them means that user-data has been fully consumed. User-data and vendor-data may have '#include http://...' which then rely on the network being present. boothooks are an example of the things run in init_modules. The 'dsmode' was a way for a user to indicate that init_modules should run at 'a' (dsmode=local) or 'b' (dsmode=net) directly. Things were further confused when a datasource could provide networking configuration. Then, we needed to apply the networking config at 'a' but if the user had provided boothooks that expected networking, then the init_modules would need to be executed at 'b'. The config drive datasource hacked its way through this and applies networking if *it* detects it is a new instance. == Suggested Change == The plan is to 1. incorporate 'dsmode' into DataSource superclass 2. make all existing datasources default to network 3. apply any networking configuration from a datasource on first boot only apply_networking will always rename network devices when it runs. for bug 1579130. 4. run init_modules at cloud-init (network) time frame unless datasource is 'local'. 5. Datasources can provide a 'first_boot' method that will be called when a new instance_id is found. This will allow the config drive's write_files to be applied once. Over all, this will very much simplify things. We'll no longer have 2 sources like DataSourceNoCloud and DataSourceNoCloudNet, but would just have one source with a dsmode. == Concerns == Some things have odd reliance on dsmode. For example, OpenNebula's get_hostname uses it to determine if it should do a lookup of an ip address. == Bugs to fix here == http://pad.lv/1577982 ConfigDrive: cloud-init fails to configure network from network_data.json http://pad.lv/1579130 need to support systemd.link renaming of devices in container http://pad.lv/1577844 Drop unnecessary blocking of all net udev rules --- bin/cloud-init | 20 ++++--- cloudinit/distros/__init__.py | 2 + cloudinit/helpers.py | 23 ++++---- cloudinit/net/__init__.py | 45 +++++++++++++++ cloudinit/sources/DataSourceCloudSigma.py | 18 +----- cloudinit/sources/DataSourceConfigDrive.py | 90 ++++++++---------------------- cloudinit/sources/DataSourceNoCloud.py | 78 +++++++++++--------------- cloudinit/sources/DataSourceOpenNebula.py | 39 ++----------- cloudinit/sources/DataSourceOpenStack.py | 9 +-- cloudinit/sources/__init__.py | 33 +++++++++++ cloudinit/stages.py | 66 ++++++++++++++++------ tox.ini | 3 +- 12 files changed, 223 insertions(+), 203 deletions(-) (limited to 'cloudinit/sources') diff --git a/bin/cloud-init b/bin/cloud-init index 5857af32..482b8402 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -236,6 +236,7 @@ def main_init(name, args): else: LOG.debug("Execution continuing, no previous run detected that" " would allow us to stop early.") + else: existing = "check" if util.get_cfg_option_bool(init.cfg, 'manual_cache_clean', False): @@ -265,17 +266,20 @@ def main_init(name, args): else: return (None, ["No instance datasource found."]) - if args.local: - if not init.ds_restored: - # if local mode and the datasource was not restored from cache - # (this is not first boot) then apply networking. - init.apply_network_config() - else: - LOG.debug("skipping networking config from restored datasource.") - # Stage 6 iid = init.instancify() LOG.debug("%s will now be targeting instance id: %s", name, iid) + + if init.is_new_instance(): + # on new instance, apply network config. if not in local mode, + # then we just bring up new networking as the OS has already + # brought up the configured networking. + init.apply_network_config(bringup=not args.local) + + if args.local and init.datasource.dsmode != sources.DSMODE_LOCAL: + return (init.datasource, []) + + # update fully realizes user-data (pulling in #include if necessary) init.update() # Stage 7 try: diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 0f222c8c..3bfbc484 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -128,6 +128,8 @@ class Distro(object): mirror_info=arch_info) def apply_network(self, settings, bring_up=True): + # this applies network where 'settings' is interfaces(5) style + # it is obsolete compared to apply_network_config # Write it out dev_names = self._write_network(settings) # Now try to bring them up diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index 09d75e65..abfb0cbb 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -328,6 +328,7 @@ class Paths(object): self.cfgs = path_cfgs # Populate all the initial paths self.cloud_dir = path_cfgs.get('cloud_dir', '/var/lib/cloud') + self.run_dir = path_cfgs.get('run_dir', '/run/cloud-init') self.instance_link = os.path.join(self.cloud_dir, 'instance') self.boot_finished = os.path.join(self.instance_link, "boot-finished") self.upstart_conf_d = path_cfgs.get('upstart_dir') @@ -349,26 +350,19 @@ class Paths(object): "data": "data", "vendordata_raw": "vendor-data.txt", "vendordata": "vendor-data.txt.i", + "instance_id": "instance-id", } # Set when a datasource becomes active self.datasource = ds # get_ipath_cur: get the current instance path for an item def get_ipath_cur(self, name=None): - ipath = self.instance_link - add_on = self.lookups.get(name) - if add_on: - ipath = os.path.join(ipath, add_on) - return ipath + return self._get_path(self.instance_link, name) # get_cpath : get the "clouddir" (/var/lib/cloud/) # for a name in dirmap def get_cpath(self, name=None): - cpath = self.cloud_dir - add_on = self.lookups.get(name) - if add_on: - cpath = os.path.join(cpath, add_on) - return cpath + return self._get_path(self.cloud_dir, name) # _get_ipath : get the instance path for a name in pathmap # (/var/lib/cloud/instances//) @@ -397,6 +391,15 @@ class Paths(object): else: return ipath + def _get_path(self, base, name=None): + add_on = self.lookups.get(name) + if not add_on: + return base + return os.path.join(base, add_on) + + def get_runpath(self, name=None): + return self._get_path(self.run_dir, name) + # This config parser will not throw when sections don't exist # and you are setting values on those sections which is useful diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 91e36aca..40d330b5 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -768,4 +768,49 @@ def read_kernel_cmdline_config(files=None, mac_addrs=None, cmdline=None): return config_from_klibc_net_cfg(files=files, mac_addrs=mac_addrs) +def convert_eni_data(eni_data): + # return a network config representation of what is in eni_data + ifaces = {} + parse_deb_config_data(ifaces, eni_data, src_dir=None, src_path=None) + return _ifaces_to_net_config_data(ifaces) + + +def _ifaces_to_net_config_data(ifaces): + """Return network config that represents the ifaces data provided. + ifaces = parse_deb_config("/etc/network/interfaces") + config = ifaces_to_net_config_data(ifaces) + state = parse_net_config_data(config).""" + devs = {} + for name, data in ifaces.items(): + # devname is 'eth0' for name='eth0:1' + devname = name.partition(":")[0] + if devname not in devs: + devs[devname] = {'type': 'physical', 'name': devname, + 'subnets': []} + # this isnt strictly correct, but some might specify + # hwaddress on a nic for matching / declaring name. + if 'hwaddress' in data: + devs[devname]['mac_address'] = data['hwaddress'] + subnet = {'_orig_eni_name': name, 'type': data['method']} + if data.get('auto'): + subnet['control'] = 'auto' + else: + subnet['control'] = 'manual' + + if data.get('method') == 'static': + subnet['address'] = data['address'] + + if 'gateway' in data: + subnet['gateway'] = data['gateway'] + + if 'dns' in data: + for n in ('nameservers', 'search'): + if n in data['dns'] and data['dns'][n]: + subnet['dns_' + n] = data['dns'][n] + devs[devname]['subnets'].append(subnet) + + return {'version': 1, + 'config': [devs[d] for d in sorted(devs)]} + + # vi: ts=4 expandtab syntax=python diff --git a/cloudinit/sources/DataSourceCloudSigma.py b/cloudinit/sources/DataSourceCloudSigma.py index 33fe78b9..07e8ae11 100644 --- a/cloudinit/sources/DataSourceCloudSigma.py +++ b/cloudinit/sources/DataSourceCloudSigma.py @@ -27,8 +27,6 @@ from cloudinit import util LOG = logging.getLogger(__name__) -VALID_DSMODES = ("local", "net", "disabled") - class DataSourceCloudSigma(sources.DataSource): """ @@ -38,7 +36,6 @@ class DataSourceCloudSigma(sources.DataSource): http://cloudsigma-docs.readthedocs.org/en/latest/server_context.html """ def __init__(self, sys_cfg, distro, paths): - self.dsmode = 'local' self.cepko = Cepko() self.ssh_public_key = '' sources.DataSource.__init__(self, sys_cfg, distro, paths) @@ -84,11 +81,9 @@ class DataSourceCloudSigma(sources.DataSource): LOG.debug("CloudSigma: Unable to read from serial port") return False - dsmode = server_meta.get('cloudinit-dsmode', self.dsmode) - if dsmode not in VALID_DSMODES: - LOG.warn("Invalid dsmode %s, assuming default of 'net'", dsmode) - dsmode = 'net' - if dsmode == "disabled" or dsmode != self.dsmode: + self.dsmode = self._determine_dsmode( + [server_meta.get('cloudinit-dsmode')]) + if dsmode == sources.DSMODE_DISABLED: return False base64_fields = server_meta.get('base64_fields', '').split(',') @@ -120,17 +115,10 @@ class DataSourceCloudSigma(sources.DataSource): return self.metadata['uuid'] -class DataSourceCloudSigmaNet(DataSourceCloudSigma): - def __init__(self, sys_cfg, distro, paths): - DataSourceCloudSigma.__init__(self, sys_cfg, distro, paths) - self.dsmode = 'net' - - # Used to match classes to dependencies. Since this datasource uses the serial # port network is not really required, so it's okay to load without it, too. datasources = [ (DataSourceCloudSigma, (sources.DEP_FILESYSTEM)), - (DataSourceCloudSigmaNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), ] diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 52a9f543..20df5fcd 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -22,6 +22,7 @@ import copy import os from cloudinit import log as logging +from cloudinit import net from cloudinit import sources from cloudinit import util @@ -35,7 +36,6 @@ DEFAULT_MODE = 'pass' DEFAULT_METADATA = { "instance-id": DEFAULT_IID, } -VALID_DSMODES = ("local", "net", "pass", "disabled") FS_TYPES = ('vfat', 'iso9660') LABEL_TYPES = ('config-2',) POSSIBLE_MOUNTS = ('sr', 'cd') @@ -47,12 +47,12 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): def __init__(self, sys_cfg, distro, paths): super(DataSourceConfigDrive, self).__init__(sys_cfg, distro, paths) self.source = None - self.dsmode = 'local' self.seed_dir = os.path.join(paths.seed_dir, 'config_drive') self.version = None self.ec2_metadata = None self._network_config = None self.network_json = None + self.network_eni = None self.files = {} def __str__(self): @@ -98,38 +98,22 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): md = results.get('metadata', {}) md = util.mergemanydict([md, DEFAULT_METADATA]) - user_dsmode = results.get('dsmode', None) - if user_dsmode not in VALID_DSMODES + (None,): - LOG.warn("User specified invalid mode: %s", user_dsmode) - user_dsmode = None - dsmode = get_ds_mode(cfgdrv_ver=results['version'], - ds_cfg=self.ds_cfg.get('dsmode'), - user=user_dsmode) + self.dsmode = self._determine_dsmode( + [results.get('dsmode'), self.ds_cfg.get('dsmode'), + sources.DSMODE_PASS if results['version'] == 1 else None]) - if dsmode == "disabled": - # most likely user specified + if self.dsmode == sources.DSMODE_DISABLED: return False - # TODO(smoser): fix this, its dirty. - # we want to do some things (writing files and network config) - # only on first boot, and even then, we want to do so in the - # local datasource (so they happen earlier) even if the configured - # dsmode is 'net' or 'pass'. To do this, we check the previous - # instance-id + # This is legacy and sneaky. If dsmode is 'pass' then write + # 'injected files' and apply legacy ENI network format. prev_iid = get_previous_iid(self.paths) cur_iid = md['instance-id'] - if prev_iid != cur_iid and self.dsmode == "local": + if prev_iid != cur_iid and self.dsmode == sources.DSMODE_PASS: on_first_boot(results, distro=self.distro) - - # dsmode != self.dsmode here if: - # * dsmode = "pass", pass means it should only copy files and then - # pass to another datasource - # * dsmode = "net" and self.dsmode = "local" - # so that user boothooks would be applied with network, the - # local datasource just gets out of the way, and lets the net claim - if dsmode != self.dsmode: - LOG.debug("%s: not claiming datasource, dsmode=%s", self, dsmode) + LOG.debug("%s: not claiming datasource, dsmode=%s", self, + self.dsmode) return False self.source = found @@ -147,12 +131,11 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): LOG.warn("Invalid content in vendor-data: %s", e) self.vendordata_raw = None - try: - self.network_json = results.get('networkdata') - except ValueError as e: - LOG.warn("Invalid content in network-data: %s", e) - self.network_json = None - + # network_config is an /etc/network/interfaces formated file and is + # obsolete compared to networkdata (from network_data.json) but both + # might be present. + self.network_eni = results.get("network_config") + self.network_json = results.get('networkdata') return True def check_instance_id(self, sys_cfg): @@ -164,40 +147,11 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): if self._network_config is None: if self.network_json is not None: self._network_config = convert_network_data(self.network_json) + elif self.network_eni is not None: + self._network_config = net.convert_eni_data(self.network_eni) return self._network_config -class DataSourceConfigDriveNet(DataSourceConfigDrive): - def __init__(self, sys_cfg, distro, paths): - DataSourceConfigDrive.__init__(self, sys_cfg, distro, paths) - self.dsmode = 'net' - - -def get_ds_mode(cfgdrv_ver, ds_cfg=None, user=None): - """Determine what mode should be used. - valid values are 'pass', 'disabled', 'local', 'net' - """ - # user passed data trumps everything - if user is not None: - return user - - if ds_cfg is not None: - return ds_cfg - - # at config-drive version 1, the default behavior was pass. That - # meant to not use use it as primary data source, but expect a ec2 metadata - # source. for version 2, we default to 'net', which means - # the DataSourceConfigDriveNet, would be used. - # - # this could change in the future. If there was definitive metadata - # that indicated presense of an openstack metadata service, then - # we could change to 'pass' by default also. The motivation for that - # would be 'cloud-init query' as the web service could be more dynamic - if cfgdrv_ver == 1: - return "pass" - return "net" - - def read_config_drive(source_dir): reader = openstack.ConfigDriveReader(source_dir) finders = [ @@ -231,9 +185,12 @@ def on_first_boot(data, distro=None): % (type(data))) net_conf = data.get("network_config", '') if net_conf and distro: - LOG.debug("Updating network interfaces from config drive") + LOG.warn("Updating network interfaces from config drive") distro.apply_network(net_conf) - files = data.get('files', {}) + write_injected_files(data.get('files')) + + +def write_injected_files(files): if files: LOG.debug("Writing %s injected files", len(files)) for (filename, content) in files.items(): @@ -296,7 +253,6 @@ def find_candidate_devs(probe_optical=True): # Used to match classes to dependencies datasources = [ (DataSourceConfigDrive, (sources.DEP_FILESYSTEM, )), - (DataSourceConfigDriveNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), ] diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 48c61a90..7e30118c 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -24,6 +24,7 @@ import errno import os from cloudinit import log as logging +from cloudinit import net from cloudinit import sources from cloudinit import util @@ -35,7 +36,6 @@ class DataSourceNoCloud(sources.DataSource): sources.DataSource.__init__(self, sys_cfg, distro, paths) self.dsmode = 'local' self.seed = None - self.cmdline_id = "ds=nocloud" self.seed_dirs = [os.path.join(paths.seed_dir, 'nocloud'), os.path.join(paths.seed_dir, 'nocloud-net')] self.seed_dir = None @@ -58,7 +58,7 @@ class DataSourceNoCloud(sources.DataSource): try: # Parse the kernel command line, getting data passed in md = {} - if parse_cmdline_data(self.cmdline_id, md): + if load_cmdline_data(md): found.append("cmdline") mydata = _merge_new_seed(mydata, {'meta-data': md}) except Exception: @@ -123,12 +123,6 @@ class DataSourceNoCloud(sources.DataSource): mydata = _merge_new_seed(mydata, seeded) - # For seed from a device, the default mode is 'net'. - # that is more likely to be what is desired. If they want - # dsmode of local, then they must specify that. - if 'dsmode' not in mydata['meta-data']: - mydata['meta-data']['dsmode'] = "net" - LOG.debug("Using data from %s", dev) found.append(dev) break @@ -144,7 +138,6 @@ class DataSourceNoCloud(sources.DataSource): if len(found) == 0: return False - seeded_network = None # The special argument "seedfrom" indicates we should # attempt to seed the userdata / metadata from its value # its primarily value is in allowing the user to type less @@ -160,10 +153,6 @@ class DataSourceNoCloud(sources.DataSource): LOG.debug("Seed from %s not supported by %s", seedfrom, self) return False - if (mydata['meta-data'].get('network-interfaces') or - mydata.get('network-config')): - seeded_network = self.dsmode - # This could throw errors, but the user told us to do it # so if errors are raised, let them raise (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) @@ -179,35 +168,21 @@ class DataSourceNoCloud(sources.DataSource): mydata['meta-data'] = util.mergemanydict([mydata['meta-data'], defaults]) - netdata = {'format': None, 'data': None} - if mydata['meta-data'].get('network-interfaces'): - netdata['format'] = 'interfaces' - netdata['data'] = mydata['meta-data']['network-interfaces'] - elif mydata.get('network-config'): - netdata['format'] = 'network-config' - netdata['data'] = mydata['network-config'] - - # if this is the local datasource or 'seedfrom' was used - # and the source of the seed was self.dsmode. - # Then see if there is network config to apply. - # note this is obsolete network-interfaces style seeding. - if self.dsmode in ("local", seeded_network): - if mydata['meta-data'].get('network-interfaces'): - LOG.debug("Updating network interfaces from %s", self) - self.distro.apply_network( - mydata['meta-data']['network-interfaces']) - - if mydata['meta-data']['dsmode'] == self.dsmode: - self.seed = ",".join(found) - self.metadata = mydata['meta-data'] - self.userdata_raw = mydata['user-data'] - self.vendordata_raw = mydata['vendor-data'] - self._network_config = mydata['network-config'] - return True + self.dsmode = self._determine_dsmode( + [mydata['meta-data'].get('dsmode')]) - LOG.debug("%s: not claiming datasource, dsmode=%s", self, - mydata['meta-data']['dsmode']) - return False + if self.dsmode == sources.DSMODE_DISABLED: + LOG.debug("%s: not claiming datasource, dsmode=%s", self, + self.dsmode) + return False + + self.seed = ",".join(found) + self.metadata = mydata['meta-data'] + self.userdata_raw = mydata['user-data'] + self.vendordata_raw = mydata['vendor-data'] + self._network_config = mydata['network-config'] + self._network_eni = mydata['meta-data'].get('network-interfaces') + return True def check_instance_id(self, sys_cfg): # quickly (local check only) if self.instance_id is still valid @@ -227,6 +202,9 @@ class DataSourceNoCloud(sources.DataSource): @property def network_config(self): + if self._network_config is None: + if self.network_eni is not None: + self._network_config = net.convert_eni_data(self.network_eni) return self._network_config @@ -254,8 +232,22 @@ def _quick_read_instance_id(cmdline_id, dirs=None): return None +def load_cmdline_data(fill, cmdline=None): + pairs = [("ds=nocloud", sources.DSMODE_LOCAL), + ("ds=nocloud-net", sources.DSMODE_NETWORK)] + for idstr, dsmode in pairs: + if parse_cmdline_data(idstr, fill, cmdline): + # if dsmode was explicitly in the commanad line, then + # prefer it to the dsmode based on the command line id + if 'dsmode' not in fill: + fill['dsmode'] = dsmode + return True + return False + + # Returns true or false indicating if cmdline indicated -# that this module should be used +# that this module should be used. Updates dictionary 'fill' +# with data that was found. # Example cmdline: # root=LABEL=uec-rootfs ro ds=nocloud def parse_cmdline_data(ds_id, fill, cmdline=None): @@ -319,9 +311,7 @@ def _merge_new_seed(cur, seeded): class DataSourceNoCloudNet(DataSourceNoCloud): def __init__(self, sys_cfg, distro, paths): DataSourceNoCloud.__init__(self, sys_cfg, distro, paths) - self.cmdline_id = "ds=nocloud-net" self.supported_seed_starts = ("http://", "https://", "ftp://") - self.dsmode = "net" # Used to match classes to dependencies diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 681f3a96..15819a4f 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -37,16 +37,13 @@ from cloudinit import util LOG = logging.getLogger(__name__) DEFAULT_IID = "iid-dsopennebula" -DEFAULT_MODE = 'net' DEFAULT_PARSEUSER = 'nobody' CONTEXT_DISK_FILES = ["context.sh"] -VALID_DSMODES = ("local", "net", "disabled") class DataSourceOpenNebula(sources.DataSource): def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) - self.dsmode = 'local' self.seed = None self.seed_dir = os.path.join(paths.seed_dir, 'opennebula') @@ -93,52 +90,27 @@ class DataSourceOpenNebula(sources.DataSource): md = util.mergemanydict([md, defaults]) # check for valid user specified dsmode - user_dsmode = results['metadata'].get('DSMODE', None) - if user_dsmode not in VALID_DSMODES + (None,): - LOG.warn("user specified invalid mode: %s", user_dsmode) - user_dsmode = None - - # decide dsmode - if user_dsmode: - dsmode = user_dsmode - elif self.ds_cfg.get('dsmode'): - dsmode = self.ds_cfg.get('dsmode') - else: - dsmode = DEFAULT_MODE - - if dsmode == "disabled": - # most likely user specified - return False - - # apply static network configuration only in 'local' dsmode - if ('network-interfaces' in results and self.dsmode == "local"): - LOG.debug("Updating network interfaces from %s", self) - self.distro.apply_network(results['network-interfaces']) + self.dsmode = self._determine_dsmode( + [results.get('DSMODE'), self.ds_cfg.get('dsmode')]) - if dsmode != self.dsmode: - LOG.debug("%s: not claiming datasource, dsmode=%s", self, dsmode) + if self.dsmode == sources.DSMODE_DISABLED: return False self.seed = seed + self.network_eni = results.get("network_config") self.metadata = md self.userdata_raw = results.get('userdata') return True def get_hostname(self, fqdn=False, resolve_ip=None): if resolve_ip is None: - if self.dsmode == 'net': + if self.dsmode == sources.DSMODE_NET: resolve_ip = True else: resolve_ip = False return sources.DataSource.get_hostname(self, fqdn, resolve_ip) -class DataSourceOpenNebulaNet(DataSourceOpenNebula): - def __init__(self, sys_cfg, distro, paths): - DataSourceOpenNebula.__init__(self, sys_cfg, distro, paths) - self.dsmode = 'net' - - class NonContextDiskDir(Exception): pass @@ -446,7 +418,6 @@ def read_context_disk_dir(source_dir, asuser=None): # Used to match classes to dependencies datasources = [ (DataSourceOpenNebula, (sources.DEP_FILESYSTEM, )), - (DataSourceOpenNebulaNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), ] diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index dfd96035..c06d17f3 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -33,13 +33,11 @@ DEFAULT_IID = "iid-dsopenstack" DEFAULT_METADATA = { "instance-id": DEFAULT_IID, } -VALID_DSMODES = ("net", "disabled") class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): def __init__(self, sys_cfg, distro, paths): super(DataSourceOpenStack, self).__init__(sys_cfg, distro, paths) - self.dsmode = 'net' self.metadata_address = None self.ssl_details = util.fetch_ssl_details(self.paths) self.version = None @@ -125,11 +123,8 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): self.metadata_address) return False - user_dsmode = results.get('dsmode', None) - if user_dsmode not in VALID_DSMODES + (None,): - LOG.warn("User specified invalid mode: %s", user_dsmode) - user_dsmode = None - if user_dsmode == 'disabled': + self.dsmode = self._determine_dsmode([results.get('dsmode')]) + if self.dsmode == sources.DSMODE_DISABLED: return False md = results.get('metadata', {}) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 43e4fd57..e0171e8c 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -34,6 +34,13 @@ from cloudinit import util from cloudinit.filters import launch_index from cloudinit.reporting import events +DSMODE_DISABLED = "disabled" +DSMODE_LOCAL = "net" +DSMODE_NETWORK = "local" +DSMODE_PASS = "pass" + +VALID_DSMODES = [DSMODE_DISABLED, DSMODE_LOCAL, DSMODE_NETWORK] + DEP_FILESYSTEM = "FILESYSTEM" DEP_NETWORK = "NETWORK" DS_PREFIX = 'DataSource' @@ -57,6 +64,7 @@ class DataSource(object): self.userdata_raw = None self.vendordata = None self.vendordata_raw = None + self.dsmode = DSMODE_NETWORK # find the datasource config name. # remove 'DataSource' from classname on front, and remove 'Net' on end. @@ -223,10 +231,35 @@ class DataSource(object): # quickly (local check only) if self.instance_id is still return False + @staticmethod + def _determine_dsmode(candidates, default=None, valid=None): + # return the first candidate that is non None, warn if not valid + if default is None: + default = DSMODE_NETWORK + + if valid is None: + valid = VALID_DSMODES + + for candidate in candidates: + if candidate is None: + continue + if candidate in valid: + return candidate + else: + LOG.warn("invalid dsmode '%s', using default=%s", + candidate, default) + return default + + return default + @property def network_config(self): return None + @property + def first_instance_boot(self): + return + def normalize_pubkey_data(pubkey_data): keys = [] diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 62d066de..53ebcb45 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -67,6 +67,7 @@ class Init(object): # Changed only when a fetch occurs self.datasource = NULL_DATA_SOURCE self.ds_restored = False + self._previous_iid = None if reporter is None: reporter = events.ReportEventStack( @@ -213,6 +214,31 @@ class Init(object): cfg_list = self.cfg.get('datasource_list') or [] return (cfg_list, pkg_list) + def _restore_from_checked_cache(self, existing): + if existing not in ("check", "trust"): + raise ValueError("Unexpected value for existing: %s" % existing) + + ds = self._restore_from_cache() + if not ds: + return (None, "no cache found") + + run_iid_fn = self.paths.get_runpath('instance-id') + if os.path.exists(run_iid_fn): + run_iid = util.load_file(run_iid_fn).strip() + else: + run_iid = None + + if run_iid == ds.get_instance_id: + return (ds, "restored from cache with run check: %s" % ds) + elif existing == "trust": + return (ds, "restored from cache: %s" % ds) + else: + if (hasattr(ds, 'check_instance_id') and + ds.check_instance_id(self.cfg)): + return (ds, "restored from checked cache: %s" % ds) + else: + return (None, "cache invalid in datasource: %s" % ds) + def _get_data_source(self, existing): if self.datasource is not NULL_DATA_SOURCE: return self.datasource @@ -221,19 +247,9 @@ class Init(object): name="check-cache", description="attempting to read from cache [%s]" % existing, parent=self.reporter) as myrep: - ds = self._restore_from_cache() - if ds and existing == "trust": - myrep.description = "restored from cache: %s" % ds - elif ds and existing == "check": - if (hasattr(ds, 'check_instance_id') and - ds.check_instance_id(self.cfg)): - myrep.description = "restored from checked cache: %s" % ds - else: - myrep.description = "cache invalid in datasource: %s" % ds - ds = None - else: - myrep.description = "no cache found" + ds, desc = self._restore_from_checked_cache(existing) + myrep.description = desc self.ds_restored = bool(ds) LOG.debug(myrep.description) @@ -301,15 +317,15 @@ class Init(object): # What the instance id was and is... iid = self.datasource.get_instance_id() - previous_iid = None iid_fn = os.path.join(dp, 'instance-id') try: previous_iid = util.load_file(iid_fn).strip() except Exception: - pass + previous_iid = None if not previous_iid: previous_iid = iid util.write_file(iid_fn, "%s\n" % iid) + util.write_file(self.paths.get_runpath('instance-id'), "%s\n" % iid) util.write_file(os.path.join(dp, 'previous-instance-id'), "%s\n" % (previous_iid)) # Ensure needed components are regenerated @@ -318,6 +334,21 @@ class Init(object): self._reset() return iid + def previous_iid(self): + if self._previous_iid is not None: + return self._previous_iid + + dp = self.paths.get_cpath('data') + iid_fn = os.path.join(dp, 'instance-id') + try: + self._previous_iid = util.load_file(iid_fn).strip() + except Exception: + pass + return self._previous_iid + + def is_new_instance(self): + return self.datasource.get_instance_id() == self.previous_iid() + def fetch(self, existing="check"): return self._get_data_source(existing=existing) @@ -593,15 +624,16 @@ class Init(object): return (ncfg, loc) return (net.generate_fallback_config(), "fallback") - def apply_network_config(self): + def apply_network_config(self, bringup): netcfg, src = self._find_networking_config() if netcfg is None: LOG.info("network config is disabled by %s", src) return - LOG.info("Applying network configuration from %s: %s", src, netcfg) + LOG.info("Applying network configuration from %s bringup=%s: %s", + src, bringup, netcfg) try: - return self.distro.apply_network_config(netcfg) + return self.distro.apply_network_config(netcfg, bringup=bringup) except NotImplementedError: LOG.warn("distro '%s' does not implement apply_network_config. " "networking may not be configured properly." % diff --git a/tox.ini b/tox.ini index dafaaf6d..18d059df 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,7 @@ [tox] envlist = py27,py3,flake8 -recreate = True +recreate = False +skip_install = True [testenv] commands = python -m nose {posargs:tests} -- cgit v1.2.3 From 399eb29662ee67f7744d3a482f63e8af377e75db Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 May 2016 11:22:39 -0400 Subject: config drive: log where network config came from --- cloudinit/sources/DataSourceConfigDrive.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 20df5fcd..2d13a32f 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -146,9 +146,13 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): def network_config(self): if self._network_config is None: if self.network_json is not None: + LOG.debug("network config provided via network_json") self._network_config = convert_network_data(self.network_json) elif self.network_eni is not None: self._network_config = net.convert_eni_data(self.network_eni) + LOG.debug("network config provided via converted eni data") + else: + LOG.debug("no network configuration available") return self._network_config -- cgit v1.2.3 From 48b7526425055a3c636f11135305f1e77469562a Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 May 2016 15:50:59 -0400 Subject: fix typos in names --- cloudinit/sources/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index e0171e8c..2a6b8d90 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -35,8 +35,8 @@ from cloudinit.filters import launch_index from cloudinit.reporting import events DSMODE_DISABLED = "disabled" -DSMODE_LOCAL = "net" -DSMODE_NETWORK = "local" +DSMODE_LOCAL = "local" +DSMODE_NETWORK = "net" DSMODE_PASS = "pass" VALID_DSMODES = [DSMODE_DISABLED, DSMODE_LOCAL, DSMODE_NETWORK] -- cgit v1.2.3 From 22bd075cd77ccc1021e049333f498afd56de2c6a Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Thu, 26 May 2016 16:10:22 -0500 Subject: Add smartos sdc:nics converter and network_config property for smartos configdrive --- cloudinit/sources/DataSourceSmartOS.py | 95 ++++++++++++++++++++++++++++++++-- 1 file changed, 91 insertions(+), 4 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 46cf117a..6355fc2a 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -64,6 +64,7 @@ SMARTOS_ATTRIB_MAP = { 'availability_zone': ('sdc:datacenter_name', True), 'vendor-data': ('sdc:vendor-data', False), 'operator-script': ('sdc:operator-script', False), + 'network-data': ('sdc:nics', False), } DS_NAME = 'SmartOS' @@ -176,6 +177,8 @@ class DataSourceSmartOS(sources.DataSource): BUILTIN_DS_CONFIG]) self.metadata = {} + self.network_data = None + self._network_config = None self.script_base_d = os.path.join(self.paths.get_cpath("scripts")) @@ -195,7 +198,6 @@ class DataSourceSmartOS(sources.DataSource): serial_device=self.ds_cfg['serial_device'], serial_timeout=self.ds_cfg['serial_timeout']) - def _set_provisioned(self): '''Mark the instance provisioning state as successful. @@ -221,7 +223,7 @@ class DataSourceSmartOS(sources.DataSource): if not self.smartos_env: LOG.debug("Not running on smartos") return False - + if not self.md_client.exists(): LOG.debug("No metadata device '%r' found for SmartOS datasource", self.md_client) @@ -279,6 +281,11 @@ class DataSourceSmartOS(sources.DataSource): self.metadata = util.mergemanydict([md, self.metadata]) self.userdata_raw = ud self.vendordata_raw = md['vendor-data'] + if not md['network-data']: + smartos_noun, strip = SMARTOS_ATTRIB_MAP.get('network-data') + self.network_data = self.md_client.get_json(smartos_noun, + strip=strip) + md['network-data'] = self.network_data self._set_provisioned() return True @@ -292,6 +299,14 @@ class DataSourceSmartOS(sources.DataSource): def get_instance_id(self): return self.metadata['instance-id'] + @property + def network_config(self): + if self._network_config is None: + if self.network_data is not None: + self._network_config = ( + convert_smartos_network_data(self.network_data)) + return self._network_config + class JoyentMetadataFetchException(Exception): pass @@ -386,8 +401,8 @@ class JoyentMetadataClient(object): result = result.strip() return result - def get_json(self, key, default=None): - result = self.get(key) + def get_json(self, key, default=None, strip=False): + result = self.get(key, default=default, strip=strip) if result is None: return default return json.loads(result) @@ -654,5 +669,77 @@ def get_datasource_list(depends): return sources.list_from_depends(depends, datasources) +# Covert SMARTOS 'sdc:nics' data to network_config yaml +def convert_smartos_network_data(network_data=None): + """Return a dictionary of network_config by parsing provided + SMARTOS sdc:nics configuration data + + sdc:nics data is a dictionary of properties of a nic and the ip + configuration desired. Additional nic dictionaries are appended + to the list. + + Converting the format is straightforward though it does include + duplicate information as well as data which appears to be relevant + to the hostOS rather than the guest. + + For each entry in the nics list returned from query sdc:nics, we + create a type: physical entry, and extract the interface properties: + 'mac' -> 'mac_address', 'mtu', 'interface' -> 'name'. The remaining + keys are related to ip configuration. For each ip in the 'ips' list + we create a subnet entry under 'subnets' pairing the ip to a one in + the 'gateways' list. + """ + + valid_keys = { + 'physical': [ + 'mac_address', + 'mtu', + 'name', + 'params', + 'subnets', + 'type', + ], + 'subnet': [ + 'address', + 'broadcast', + 'dns_nameservers', + 'dns_search', + 'gateway', + 'metric', + 'netmask', + 'pointopoint', + 'routes', + 'scope', + 'type', + ], + } + + config = [] + for nic in network_data: + cfg = {k: v for k, v in nic.items() + if k in valid_keys['physical']} + cfg.update({ + 'type': 'physical', + 'name': nic['interface'] + }) + if 'mac' in nic: + cfg.update({'mac_address': nic['mac']}) + + subnets = [] + for ip, gw in zip(nic['ips'], nic['gateways']): + subnet = {k: v for k, v in nic.items() + if k in valid_keys['subnet']} + subnet.update({ + 'type': 'static', + 'address': ip, + 'gateway': gw, + }) + subnets.append(subnet) + cfg.update({'subnets': subnets}) + config.append(cfg) + + return {'version': 1, 'config': config} + + if __name__ == "__main__": jmc = JoyentMetadataClient(seed_file).get_metadata(noun) -- cgit v1.2.3 From c1c1550d74d067d78cf4ba1cf64f38c2dae8c9e1 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Thu, 26 May 2016 21:17:29 -0500 Subject: Move sdc:nics to a JSON map. Add unittest for sdc:nics --- cloudinit/sources/DataSourceSmartOS.py | 19 +++++---- tests/unittests/test_datasource/test_smartos.py | 51 +++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 8 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 6355fc2a..35fa8c33 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -64,7 +64,11 @@ SMARTOS_ATTRIB_MAP = { 'availability_zone': ('sdc:datacenter_name', True), 'vendor-data': ('sdc:vendor-data', False), 'operator-script': ('sdc:operator-script', False), - 'network-data': ('sdc:nics', False), +} + +SMARTOS_ATTRIB_JSON = { + # Cloud-init Key : (SmartOS Key known JSON) + 'network-data': 'sdc:nics', } DS_NAME = 'SmartOS' @@ -233,6 +237,9 @@ class DataSourceSmartOS(sources.DataSource): smartos_noun, strip = attribute md[ci_noun] = self.md_client.get(smartos_noun, strip=strip) + for ci_noun, smartos_noun in SMARTOS_ATTRIB_JSON.items(): + md[ci_noun] = self.md_client.get_json(smartos_noun) + # @datadictionary: This key may contain a program that is written # to a file in the filesystem of the guest on each boot and then # executed. It may be of any format that would be considered @@ -281,11 +288,7 @@ class DataSourceSmartOS(sources.DataSource): self.metadata = util.mergemanydict([md, self.metadata]) self.userdata_raw = ud self.vendordata_raw = md['vendor-data'] - if not md['network-data']: - smartos_noun, strip = SMARTOS_ATTRIB_MAP.get('network-data') - self.network_data = self.md_client.get_json(smartos_noun, - strip=strip) - md['network-data'] = self.network_data + self.network_data = md['network-data'] self._set_provisioned() return True @@ -401,8 +404,8 @@ class JoyentMetadataClient(object): result = result.strip() return result - def get_json(self, key, default=None, strip=False): - result = self.get(key, default=default, strip=strip) + def get_json(self, key, default=None): + result = self.get(key, default=default) if result is None: return default return json.loads(result) diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 5c49966a..1ee64d60 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -24,6 +24,7 @@ from __future__ import print_function +import json import os import os.path import re @@ -47,6 +48,48 @@ try: except ImportError: import mock +SDC_NICS = json.loads(""" +[ + { + "nic_tag": "external", + "primary": true, + "mtu": 1500, + "model": "virtio", + "gateway": "8.12.42.1", + "netmask": "255.255.255.0", + "ip": "8.12.42.102", + "network_uuid": "992fc7ce-6aac-4b74-aed6-7b9d2c6c0bfe", + "gateways": [ + "8.12.42.1" + ], + "vlan_id": 324, + "mac": "90:b8:d0:f5:e4:f5", + "interface": "net0", + "ips": [ + "8.12.42.102/24" + ] + }, + { + "nic_tag": "sdc_overlay/16187209", + "gateway": "192.168.128.1", + "model": "virtio", + "mac": "90:b8:d0:a5:ff:cd", + "netmask": "255.255.252.0", + "ip": "192.168.128.93", + "network_uuid": "4cad71da-09bc-452b-986d-03562a03a0a9", + "gateways": [ + "192.168.128.1" + ], + "vlan_id": 2, + "mtu": 8500, + "interface": "net1", + "ips": [ + "192.168.128.93/22" + ] + } +] +""") + MOCK_RETURNS = { 'hostname': 'test-host', 'root_authorized_keys': 'ssh-rsa AAAAB3Nz...aC1yc2E= keyname', @@ -60,6 +103,7 @@ MOCK_RETURNS = { 'sdc:vendor-data': '\n'.join(['VENDOR_DATA', '']), 'user-data': '\n'.join(['something', '']), 'user-script': '\n'.join(['/bin/true', '']), + 'sdc:nics': SDC_NICS, } DMI_DATA_RETURN = 'smartdc' @@ -275,6 +319,13 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): self.assertEquals(MOCK_RETURNS['cloud-init:user-data'], dsrc.userdata_raw) + def test_sdc_nics(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEquals(MOCK_RETURNS['sdc:nics'], + dsrc.metadata['network-data']) + def test_sdc_scripts(self): dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() -- cgit v1.2.3 From 5cd30a36eaa6ca1a239019a5409faa603f063f6c Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 May 2016 09:24:23 -0400 Subject: fix pyflakes, move datasources= to bottom --- cloudinit/sources/DataSourceSmartOS.py | 37 ++++++++++++++++------------------ 1 file changed, 17 insertions(+), 20 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 35fa8c33..4224f2ba 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -34,13 +34,11 @@ import base64 import binascii -import contextlib import json import os import random import re import socket -import stat import serial @@ -391,9 +389,6 @@ class JoyentMetadataClient(object): return None value = self._get_value_from_frame(request_id, response) - if value is None: - return default - return value def get(self, key, default=None, strip=False): @@ -442,11 +437,11 @@ class JoyentMetadataClient(object): class JoyentMetadataSocketClient(JoyentMetadataClient): def __init__(self, socketpath): - self.socketpath = metadata_socketfile + self.socketpath = socketpath def open_transport(self): sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - sock.connect(path) + sock.connect(self.socketpath) self.fp = sock.makefile('rwb') def exists(self): @@ -577,7 +572,7 @@ def jmc_client_factory( device=serial_device, timeout=serial_timeout, smartos_type=smartos_type) elif smartos_type == 'lx-brand': - return JoyentMetadataSerialClient(socketpath=metadata_socketfile) + return JoyentMetadataSerialClient(socketpath=metadata_sockfile) raise ValueError("Unknown value for smartos_type: %s" % smartos_type) @@ -661,17 +656,6 @@ def get_smartos_environ(uname_version=None, product_name=None, return None -# Used to match classes to dependencies -datasources = [ - (DataSourceSmartOS, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), -] - - -# Return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return sources.list_from_depends(depends, datasources) - - # Covert SMARTOS 'sdc:nics' data to network_config yaml def convert_smartos_network_data(network_data=None): """Return a dictionary of network_config by parsing provided @@ -744,5 +728,18 @@ def convert_smartos_network_data(network_data=None): return {'version': 1, 'config': config} +# Used to match classes to dependencies +datasources = [ + (DataSourceSmartOS, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), +] + + +# Return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return sources.list_from_depends(depends, datasources) + + if __name__ == "__main__": - jmc = JoyentMetadataClient(seed_file).get_metadata(noun) + import sys + jmc = jmc_client_factory() + jmc.get_metadata(sys.argv[1]) -- cgit v1.2.3 From 949cebd48c9100d4fd00b74232bcf048980e6e0d Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 May 2016 09:49:29 -0400 Subject: fix pyflakes and some pylint errors/warnings --- cloudinit/sources/DataSourceSmartOS.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 4224f2ba..cf4e00e5 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -69,6 +69,9 @@ SMARTOS_ATTRIB_JSON = { 'network-data': 'sdc:nics', } +SMARTOS_ENV_LX_BRAND = "lx-brand" +SMARTOS_ENV_KVM = "kvm" + DS_NAME = 'SmartOS' DS_CFG_PATH = ['datasource', DS_NAME] NO_BASE64_DECODE = [ @@ -183,6 +186,7 @@ class DataSourceSmartOS(sources.DataSource): self._network_config = None self.script_base_d = os.path.join(self.paths.get_cpath("scripts")) + self.smartos_env = None self._init() @@ -295,7 +299,9 @@ class DataSourceSmartOS(sources.DataSource): return self.ds_cfg['disk_aliases'].get(name) def get_config_obj(self): - return self.cfg + if self.smartos_env == SMARTOS_ENV_KVM: + return BUILTIN_CLOUD_CONFIG + return None def get_instance_id(self): return self.metadata['instance-id'] @@ -434,6 +440,9 @@ class JoyentMetadataClient(object): self.close_transport() return + def open_transport(self): + raise NotImplementedError + class JoyentMetadataSocketClient(JoyentMetadataClient): def __init__(self, socketpath): @@ -519,7 +528,7 @@ class JoyentMetadataLegacySerialClient(JoyentMetadataSerialClient): # now add any b64- that has a true value for key in [k[3:] for k in keys if k.startswith("b64-")]: if util.is_true(self._get(key)): - b64_keys.append(key) + b64_keys.add(key) else: if key in b64_keys: b64_keys.remove(key) @@ -572,7 +581,7 @@ def jmc_client_factory( device=serial_device, timeout=serial_timeout, smartos_type=smartos_type) elif smartos_type == 'lx-brand': - return JoyentMetadataSerialClient(socketpath=metadata_sockfile) + return JoyentMetadataSocketClient(socketpath=metadata_sockfile) raise ValueError("Unknown value for smartos_type: %s" % smartos_type) @@ -647,11 +656,15 @@ def get_smartos_environ(uname_version=None, product_name=None, if uname_version is None: uname_version = uname[3] if uname_version.lower() == 'brandz virtual linux': - return 'lx-brand' + return SMARTOS_ENV_LX_BRAND + + if product_name is None: + system_type = util.read_dmi_data("system-product-name") + else: + system_type = product_name - system_type = util.read_dmi_data("system-product-name") if system_type and 'smartdc' in system_type.lower(): - return 'kvm' + return SMARTOS_ENV_KVM return None @@ -742,4 +755,4 @@ def get_datasource_list(depends): if __name__ == "__main__": import sys jmc = jmc_client_factory() - jmc.get_metadata(sys.argv[1]) + jmc.get(sys.argv[1]) -- cgit v1.2.3 From e6e768769d67960cde12dee0b0c2b58deb2eb376 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 May 2016 14:06:59 -0400 Subject: fix a bunch of the tests --- cloudinit/sources/DataSourceSmartOS.py | 12 +- tests/unittests/test_datasource/test_smartos.py | 312 ++++++++++++++++++++++-- 2 files changed, 298 insertions(+), 26 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index cf4e00e5..e64dea68 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -186,7 +186,7 @@ class DataSourceSmartOS(sources.DataSource): self._network_config = None self.script_base_d = os.path.join(self.paths.get_cpath("scripts")) - self.smartos_env = None + self.smartos_type = None self._init() @@ -196,10 +196,11 @@ class DataSourceSmartOS(sources.DataSource): def _init(self): if self.smartos_environ == self._unset: - self.smartos_env = get_smartos_environ() + self.smartos_type = get_smartos_environ() if self.md_client == self._unset: self.md_client = jmc_client_factory( + smartos_type=self.smartos_type, metadata_sockfile=self.ds_cfg['metadata_sockfile'], serial_device=self.ds_cfg['serial_device'], serial_timeout=self.ds_cfg['serial_timeout']) @@ -226,7 +227,7 @@ class DataSourceSmartOS(sources.DataSource): md = {} ud = "" - if not self.smartos_env: + if not self.smartos_type: LOG.debug("Not running on smartos") return False @@ -299,7 +300,7 @@ class DataSourceSmartOS(sources.DataSource): return self.ds_cfg['disk_aliases'].get(name) def get_config_obj(self): - if self.smartos_env == SMARTOS_ENV_KVM: + if self.smartos_type == SMARTOS_ENV_KVM: return BUILTIN_CLOUD_CONFIG return None @@ -608,6 +609,7 @@ def write_boot_content(content, content_f, link=None, shebang=False, bit and to the SmartOS default of assuming that bash. """ + print("content_f=%s" % content_f) if not content and os.path.exists(content_f): os.unlink(content_f) if link and os.path.islink(link): @@ -639,7 +641,7 @@ def write_boot_content(content, content_f, link=None, shebang=False, util.ensure_dir(os.path.dirname(link)) os.symlink(content_f, link) except IOError as e: - util.logexc(LOG, "failed establishing content link", e) + util.logexc(LOG, "failed establishing content link: %s", e) def get_smartos_environ(uname_version=None, product_name=None, diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index ea20777a..946286bd 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -40,13 +40,9 @@ import six from cloudinit import helpers as c_helpers from cloudinit.sources import DataSourceSmartOS from cloudinit.util import b64e +from cloudinit import util -from .. import helpers - -try: - from unittest import mock -except ImportError: - import mock +from ..helpers import mock, TestCase, FilesystemMockingTestCase SDC_NICS = json.loads(""" [ @@ -103,7 +99,7 @@ MOCK_RETURNS = { 'sdc:vendor-data': '\n'.join(['VENDOR_DATA', '']), 'user-data': '\n'.join(['something', '']), 'user-script': '\n'.join(['/bin/true', '']), - 'sdc:nics': SDC_NICS, + 'sdc:nics': json.dumps(SDC_NICS), } DMI_DATA_RETURN = 'smartdc' @@ -115,12 +111,286 @@ def get_mock_client(mockdata): def __init__(self, serial): pass - def get_metadata(self, metadata_key): + def get(self, metadata_key): return mockdata.get(metadata_key) return MockMetadataClient -class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): +class PsuedoJoyentClient(object): + def __init__(self, data=None): + if data is None: + data = MOCK_RETURNS.copy() + self.data = data + return + + def get(self, key, default=None, strip=False): + if key in self.data: + r = self.data[key] + if strip: + r = r.strip() + else: + r = default + return r + + def get_json(self, key, default=None): + result = self.get(key, default=default) + if result is None: + return default + return json.loads(result) + + def exists(self): + return True + + +class TestSmartOSDataSource(FilesystemMockingTestCase): + def setUp(self): + super(TestSmartOSDataSource, self).setUp() + + dsmos = 'cloudinit.sources.DataSourceSmartOS' + patcher = mock.patch(dsmos + ".jmc_client_factory") + self.jmc_cfact = patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch(dsmos + ".get_smartos_environ") + self.get_smartos_environ = patcher.start() + self.addCleanup(patcher.stop) + + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp) + self.paths = c_helpers.Paths({'cloud_dir': self.tmp}) + + self.legacy_user_d = tempfile.mkdtemp() + self.orig_lud = DataSourceSmartOS.LEGACY_USER_D + DataSourceSmartOS.LEGACY_USER_D = self.legacy_user_d + + def tearDown(self): + DataSourceSmartOS.LEGACY_USER_D = self.orig_lud + super(TestSmartOSDataSource, self).tearDown() + + def _get_ds(self, mockdata=None, mode=DataSourceSmartOS.SMARTOS_ENV_KVM, + sys_cfg=None, ds_cfg=None): + self.jmc_cfact.return_value = PsuedoJoyentClient(mockdata) + self.get_smartos_environ.return_value = mode + + if sys_cfg is None: + sys_cfg = {} + + if ds_cfg is not None: + sys_cfg['datasource'] = sys_cfg.get('datasource', {}) + sys_cfg['datasource']['SmartOS'] = ds_cfg + + return DataSourceSmartOS.DataSourceSmartOS( + sys_cfg, distro=None, paths=self.paths) + + def test_it_got_here(self): + dsrc = self._get_ds() + ret = dsrc.get_data() + + def test_no_base64(self): + ds_cfg = {'no_base64_decode': ['test_var1'], 'all_base': True} + dsrc = self._get_ds(ds_cfg=ds_cfg) + ret = dsrc.get_data() + self.assertTrue(ret) + + def test_uuid(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(MOCK_RETURNS['sdc:uuid'], + dsrc.metadata['instance-id']) + + def test_root_keys(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(MOCK_RETURNS['root_authorized_keys'], + dsrc.metadata['public-keys']) + + def test_hostname_b64(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(MOCK_RETURNS['hostname'], + dsrc.metadata['local-hostname']) + + def test_hostname(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(MOCK_RETURNS['hostname'], + dsrc.metadata['local-hostname']) + + def test_userdata(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(MOCK_RETURNS['user-data'], + dsrc.metadata['legacy-user-data']) + self.assertEqual(MOCK_RETURNS['cloud-init:user-data'], + dsrc.userdata_raw) + + def test_sdc_nics(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEquals(json.loads(MOCK_RETURNS['sdc:nics']), + dsrc.metadata['network-data']) + + def test_sdc_scripts(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(MOCK_RETURNS['user-script'], + dsrc.metadata['user-script']) + + legacy_script_f = "%s/user-script" % self.legacy_user_d + self.assertTrue(os.path.exists(legacy_script_f)) + self.assertTrue(os.path.islink(legacy_script_f)) + user_script_perm = oct(os.stat(legacy_script_f)[stat.ST_MODE])[-3:] + self.assertEqual(user_script_perm, '700') + + def test_scripts_shebanged(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(MOCK_RETURNS['user-script'], + dsrc.metadata['user-script']) + + legacy_script_f = "%s/user-script" % self.legacy_user_d + self.assertTrue(os.path.exists(legacy_script_f)) + self.assertTrue(os.path.islink(legacy_script_f)) + shebang = None + with open(legacy_script_f, 'r') as f: + shebang = f.readlines()[0].strip() + self.assertEqual(shebang, "#!/bin/bash") + user_script_perm = oct(os.stat(legacy_script_f)[stat.ST_MODE])[-3:] + self.assertEqual(user_script_perm, '700') + + def test_scripts_shebang_not_added(self): + """ + Test that the SmartOS requirement that plain text scripts + are executable. This test makes sure that plain texts scripts + with out file magic have it added appropriately by cloud-init. + """ + + my_returns = MOCK_RETURNS.copy() + my_returns['user-script'] = '\n'.join(['#!/usr/bin/perl', + 'print("hi")', '']) + + dsrc = self._get_ds(mockdata=my_returns) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(my_returns['user-script'], + dsrc.metadata['user-script']) + + legacy_script_f = "%s/user-script" % self.legacy_user_d + self.assertTrue(os.path.exists(legacy_script_f)) + self.assertTrue(os.path.islink(legacy_script_f)) + shebang = None + with open(legacy_script_f, 'r') as f: + shebang = f.readlines()[0].strip() + self.assertEqual(shebang, "#!/usr/bin/perl") + + def test_userdata_removed(self): + """ + User-data in the SmartOS world is supposed to be written to a file + each and every boot. This tests to make sure that in the event the + legacy user-data is removed, the existing user-data is backed-up + and there is no /var/db/user-data left. + """ + + user_data_f = "%s/mdata-user-data" % self.legacy_user_d + with open(user_data_f, 'w') as f: + f.write("PREVIOUS") + + my_returns = MOCK_RETURNS.copy() + del my_returns['user-data'] + + dsrc = self._get_ds(mockdata=my_returns) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertFalse(dsrc.metadata.get('legacy-user-data')) + + found_new = False + for root, _dirs, files in os.walk(self.legacy_user_d): + for name in files: + name_f = os.path.join(root, name) + permissions = oct(os.stat(name_f)[stat.ST_MODE])[-3:] + if re.match(r'.*\/mdata-user-data$', name_f): + found_new = True + print(name_f) + self.assertEqual(permissions, '400') + + self.assertFalse(found_new) + + def test_vendor_data_not_default(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(MOCK_RETURNS['sdc:vendor-data'], + dsrc.metadata['vendor-data']) + + def test_default_vendor_data(self): + my_returns = MOCK_RETURNS.copy() + def_op_script = my_returns['sdc:vendor-data'] + del my_returns['sdc:vendor-data'] + dsrc = self._get_ds(mockdata=my_returns) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertNotEqual(def_op_script, dsrc.metadata['vendor-data']) + + # we expect default vendor-data is a boothook + self.assertTrue(dsrc.vendordata_raw.startswith("#cloud-boothook")) + + def test_disable_iptables_flag(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(MOCK_RETURNS['disable_iptables_flag'], + dsrc.metadata['iptables_disable']) + + def test_motd_sys_info(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(MOCK_RETURNS['enable_motd_sys_info'], + dsrc.metadata['motd_sys_info']) + + def test_default_ephemeral(self): + # Test to make sure that the builtin config has the ephemeral + # configuration. + dsrc = self._get_ds() + cfg = dsrc.get_config_obj() + + ret = dsrc.get_data() + self.assertTrue(ret) + + assert 'disk_setup' in cfg + assert 'fs_setup' in cfg + self.assertIsInstance(cfg['disk_setup'], dict) + self.assertIsInstance(cfg['fs_setup'], list) + + def test_override_disk_aliases(self): + # Test to make sure that the built-in DS is overriden + builtin = DataSourceSmartOS.BUILTIN_DS_CONFIG + + mydscfg = {'disk_aliases': {'FOO': '/dev/bar'}} + + # expect that these values are in builtin, or this is pointless + for k in mydscfg: + self.assertIn(k, builtin) + + dsrc = self._get_ds(ds_cfg=mydscfg) + ret = dsrc.get_data() + self.assertTrue(ret) + + self.assertEqual(mydscfg['disk_aliases']['FOO'], + dsrc.ds_cfg['disk_aliases']['FOO']) + + self.assertEqual(dsrc.device_name_to_device('FOO'), + mydscfg['disk_aliases']['FOO']) + + +class OldTestSmartOSDataSource(FilesystemMockingTestCase): def setUp(self): super(TestSmartOSDataSource, self).setUp() @@ -141,7 +411,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): super(TestSmartOSDataSource, self).setUp() def tearDown(self): - helpers.FilesystemMockingTestCase.tearDown(self) + FilesystemMockingTestCase.tearDown(self) if self._log_handler and self._log: self._log.removeHandler(self._log_handler) apply_patches([i for i in reversed(self.unapply)]) @@ -166,7 +436,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): if dmi_data is None: dmi_data = DMI_DATA_RETURN - def _dmi_data(): + def _dmi_data(item): return dmi_data def _os_uname(): @@ -188,12 +458,12 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): self.apply_patches([(mod, 'LEGACY_USER_D', self.legacy_user_d)]) self.apply_patches([ (mod, 'JoyentMetadataClient', get_mock_client(mockdata))]) - self.apply_patches([(mod, 'dmi_data', _dmi_data)]) + self.apply_patches([(util, 'read_dmi_data', _dmi_data)]) self.apply_patches([(os, 'uname', _os_uname)]) - self.apply_patches([(mod, 'device_exists', lambda d: True)]) + self.apply_patches([(os.path, 'exists', lambda d: True)]) dsrc = mod.DataSourceSmartOS(sys_cfg, distro=None, paths=self.paths) - self.apply_patches([(dsrc, '_get_seed_file_object', mock.MagicMock())]) + #self.apply_patches([(dsrc, '_get_seed_file_object', mock.MagicMock())]) return dsrc def test_seed(self): @@ -492,7 +762,7 @@ def apply_patches(patches): return ret -class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase): +class TestJoyentMetadataClient(FilesystemMockingTestCase): def setUp(self): super(TestJoyentMetadataClient, self).setUp() @@ -532,7 +802,7 @@ class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase): mock.Mock(return_value=self.request_id))) def _get_client(self): - return DataSourceSmartOS.JoyentMetadataClient(self.serial) + return DataSourceSmartOS.JoyentMetadataSerialClient(self.serial) def assertEndsWith(self, haystack, prefix): self.assertTrue(haystack.endswith(prefix), @@ -546,7 +816,7 @@ class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase): def test_get_metadata_writes_a_single_line(self): client = self._get_client() - client.get_metadata('some_key') + client.get('some_key') self.assertEqual(1, self.serial.write.call_count) written_line = self.serial.write.call_args[0][0] print(type(written_line)) @@ -556,7 +826,7 @@ class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase): def _get_written_line(self, key='some_key'): client = self._get_client() - client.get_metadata(key) + client.get(key) return self.serial.write.call_args[0][0] def test_get_metadata_writes_bytes(self): @@ -600,12 +870,12 @@ class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase): def test_get_metadata_reads_a_line(self): client = self._get_client() - client.get_metadata('some_key') + client.get('some_key') self.assertEqual(self.metasource_data_len, self.serial.read.call_count) def test_get_metadata_returns_valid_value(self): client = self._get_client() - value = client.get_metadata('some_key') + value = client.get('some_key') self.assertEqual(self.metadata_value, value) def test_get_metadata_throws_exception_for_incorrect_length(self): @@ -633,4 +903,4 @@ class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase): self.response_parts['length'] = 17 client = self._get_client() client._checksum = lambda _: self.response_parts['crc'] - self.assertIsNone(client.get_metadata('some_key')) + self.assertIsNone(client.get('some_key')) -- cgit v1.2.3 From 8a148ed934156c63a76a28c9d3a33278e52d71d1 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 May 2016 14:27:50 -0400 Subject: fix the remaining tests --- cloudinit/sources/DataSourceSmartOS.py | 4 +- tests/unittests/test_datasource/test_smartos.py | 391 +----------------------- 2 files changed, 7 insertions(+), 388 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index e64dea68..e9ff1235 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -332,11 +332,11 @@ class JoyentMetadataClient(object): r' (?P(?P[0-9a-f]+) (?PSUCCESS|NOTFOUND)' r'( (?P.+))?)') - def __init__(self, smartos_type=None): + def __init__(self, smartos_type=None, fp=None): if smartos_type is None: smartos_type = get_smartos_environ() self.smartos_type = smartos_type - self.fp = None + self.fp = fp def _checksum(self, body): return '{0:08x}'.format( diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 946286bd..56d85f99 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -105,17 +105,6 @@ MOCK_RETURNS = { DMI_DATA_RETURN = 'smartdc' -def get_mock_client(mockdata): - class MockMetadataClient(object): - - def __init__(self, serial): - pass - - def get(self, metadata_key): - return mockdata.get(metadata_key) - return MockMetadataClient - - class PsuedoJoyentClient(object): def __init__(self, data=None): if data is None: @@ -390,377 +379,6 @@ class TestSmartOSDataSource(FilesystemMockingTestCase): mydscfg['disk_aliases']['FOO']) -class OldTestSmartOSDataSource(FilesystemMockingTestCase): - def setUp(self): - super(TestSmartOSDataSource, self).setUp() - - self.tmp = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, self.tmp) - self.legacy_user_d = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, self.legacy_user_d) - - # If you should want to watch the logs... - self._log = None - self._log_file = None - self._log_handler = None - - # patch cloud_dir, so our 'seed_dir' is guaranteed empty - self.paths = c_helpers.Paths({'cloud_dir': self.tmp}) - - self.unapply = [] - super(TestSmartOSDataSource, self).setUp() - - def tearDown(self): - FilesystemMockingTestCase.tearDown(self) - if self._log_handler and self._log: - self._log.removeHandler(self._log_handler) - apply_patches([i for i in reversed(self.unapply)]) - super(TestSmartOSDataSource, self).tearDown() - - def _patchIn(self, root): - self.restore() - self.patchOS(root) - self.patchUtils(root) - - def apply_patches(self, patches): - ret = apply_patches(patches) - self.unapply += ret - - def _get_ds(self, sys_cfg=None, ds_cfg=None, mockdata=None, dmi_data=None, - is_lxbrand=False): - mod = DataSourceSmartOS - - if mockdata is None: - mockdata = MOCK_RETURNS - - if dmi_data is None: - dmi_data = DMI_DATA_RETURN - - def _dmi_data(item): - return dmi_data - - def _os_uname(): - if not is_lxbrand: - # LP: #1243287. tests assume this runs, but running test on - # arm would cause them all to fail. - return ('LINUX', 'NODENAME', 'RELEASE', 'VERSION', 'x86_64') - else: - return ('LINUX', 'NODENAME', 'RELEASE', 'BRANDZ VIRTUAL LINUX', - 'X86_64') - - if sys_cfg is None: - sys_cfg = {} - - if ds_cfg is not None: - sys_cfg['datasource'] = sys_cfg.get('datasource', {}) - sys_cfg['datasource']['SmartOS'] = ds_cfg - - self.apply_patches([(mod, 'LEGACY_USER_D', self.legacy_user_d)]) - self.apply_patches([ - (mod, 'JoyentMetadataClient', get_mock_client(mockdata))]) - self.apply_patches([(util, 'read_dmi_data', _dmi_data)]) - self.apply_patches([(os, 'uname', _os_uname)]) - self.apply_patches([(os.path, 'exists', lambda d: True)]) - dsrc = mod.DataSourceSmartOS(sys_cfg, distro=None, - paths=self.paths) - #self.apply_patches([(dsrc, '_get_seed_file_object', mock.MagicMock())]) - return dsrc - - def test_seed(self): - # default seed should be /dev/ttyS1 - dsrc = self._get_ds() - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual('kvm', dsrc.smartos_type) - self.assertEqual('/dev/ttyS1', dsrc.seed) - - def test_seed_lxbrand(self): - # default seed should be /dev/ttyS1 - dsrc = self._get_ds(is_lxbrand=True) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual('lx-brand', dsrc.smartos_type) - self.assertEqual('/native/.zonecontrol/metadata.sock', dsrc.seed) - - def test_issmartdc(self): - dsrc = self._get_ds() - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertTrue(dsrc.is_smartdc) - - def test_issmartdc_lxbrand(self): - dsrc = self._get_ds(is_lxbrand=True) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertTrue(dsrc.is_smartdc) - - def test_no_base64(self): - ds_cfg = {'no_base64_decode': ['test_var1'], 'all_base': True} - dsrc = self._get_ds(ds_cfg=ds_cfg) - ret = dsrc.get_data() - self.assertTrue(ret) - - def test_uuid(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(MOCK_RETURNS['sdc:uuid'], - dsrc.metadata['instance-id']) - - def test_root_keys(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(MOCK_RETURNS['root_authorized_keys'], - dsrc.metadata['public-keys']) - - def test_hostname_b64(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(MOCK_RETURNS['hostname'], - dsrc.metadata['local-hostname']) - - def test_hostname(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(MOCK_RETURNS['hostname'], - dsrc.metadata['local-hostname']) - - def test_base64_all(self): - # metadata provided base64_all of true - my_returns = MOCK_RETURNS.copy() - my_returns['base64_all'] = "true" - for k in ('hostname', 'cloud-init:user-data'): - my_returns[k] = b64e(my_returns[k]) - - dsrc = self._get_ds(mockdata=my_returns) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(MOCK_RETURNS['hostname'], - dsrc.metadata['local-hostname']) - self.assertEqual(MOCK_RETURNS['cloud-init:user-data'], - dsrc.userdata_raw) - self.assertEqual(MOCK_RETURNS['root_authorized_keys'], - dsrc.metadata['public-keys']) - self.assertEqual(MOCK_RETURNS['disable_iptables_flag'], - dsrc.metadata['iptables_disable']) - self.assertEqual(MOCK_RETURNS['enable_motd_sys_info'], - dsrc.metadata['motd_sys_info']) - - def test_b64_userdata(self): - my_returns = MOCK_RETURNS.copy() - my_returns['b64-cloud-init:user-data'] = "true" - my_returns['b64-hostname'] = "true" - for k in ('hostname', 'cloud-init:user-data'): - my_returns[k] = b64e(my_returns[k]) - - dsrc = self._get_ds(mockdata=my_returns) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(MOCK_RETURNS['hostname'], - dsrc.metadata['local-hostname']) - self.assertEqual(MOCK_RETURNS['cloud-init:user-data'], - dsrc.userdata_raw) - self.assertEqual(MOCK_RETURNS['root_authorized_keys'], - dsrc.metadata['public-keys']) - - def test_b64_keys(self): - my_returns = MOCK_RETURNS.copy() - my_returns['base64_keys'] = 'hostname,ignored' - for k in ('hostname',): - my_returns[k] = b64e(my_returns[k]) - - dsrc = self._get_ds(mockdata=my_returns) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(MOCK_RETURNS['hostname'], - dsrc.metadata['local-hostname']) - self.assertEqual(MOCK_RETURNS['cloud-init:user-data'], - dsrc.userdata_raw) - - def test_userdata(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(MOCK_RETURNS['user-data'], - dsrc.metadata['legacy-user-data']) - self.assertEqual(MOCK_RETURNS['cloud-init:user-data'], - dsrc.userdata_raw) - - def test_sdc_nics(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEquals(MOCK_RETURNS['sdc:nics'], - dsrc.metadata['network-data']) - - def test_sdc_scripts(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(MOCK_RETURNS['user-script'], - dsrc.metadata['user-script']) - - legacy_script_f = "%s/user-script" % self.legacy_user_d - self.assertTrue(os.path.exists(legacy_script_f)) - self.assertTrue(os.path.islink(legacy_script_f)) - user_script_perm = oct(os.stat(legacy_script_f)[stat.ST_MODE])[-3:] - self.assertEqual(user_script_perm, '700') - - def test_scripts_shebanged(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(MOCK_RETURNS['user-script'], - dsrc.metadata['user-script']) - - legacy_script_f = "%s/user-script" % self.legacy_user_d - self.assertTrue(os.path.exists(legacy_script_f)) - self.assertTrue(os.path.islink(legacy_script_f)) - shebang = None - with open(legacy_script_f, 'r') as f: - shebang = f.readlines()[0].strip() - self.assertEqual(shebang, "#!/bin/bash") - user_script_perm = oct(os.stat(legacy_script_f)[stat.ST_MODE])[-3:] - self.assertEqual(user_script_perm, '700') - - def test_scripts_shebang_not_added(self): - """ - Test that the SmartOS requirement that plain text scripts - are executable. This test makes sure that plain texts scripts - with out file magic have it added appropriately by cloud-init. - """ - - my_returns = MOCK_RETURNS.copy() - my_returns['user-script'] = '\n'.join(['#!/usr/bin/perl', - 'print("hi")', '']) - - dsrc = self._get_ds(mockdata=my_returns) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(my_returns['user-script'], - dsrc.metadata['user-script']) - - legacy_script_f = "%s/user-script" % self.legacy_user_d - self.assertTrue(os.path.exists(legacy_script_f)) - self.assertTrue(os.path.islink(legacy_script_f)) - shebang = None - with open(legacy_script_f, 'r') as f: - shebang = f.readlines()[0].strip() - self.assertEqual(shebang, "#!/usr/bin/perl") - - def test_userdata_removed(self): - """ - User-data in the SmartOS world is supposed to be written to a file - each and every boot. This tests to make sure that in the event the - legacy user-data is removed, the existing user-data is backed-up - and there is no /var/db/user-data left. - """ - - user_data_f = "%s/mdata-user-data" % self.legacy_user_d - with open(user_data_f, 'w') as f: - f.write("PREVIOUS") - - my_returns = MOCK_RETURNS.copy() - del my_returns['user-data'] - - dsrc = self._get_ds(mockdata=my_returns) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertFalse(dsrc.metadata.get('legacy-user-data')) - - found_new = False - for root, _dirs, files in os.walk(self.legacy_user_d): - for name in files: - name_f = os.path.join(root, name) - permissions = oct(os.stat(name_f)[stat.ST_MODE])[-3:] - if re.match(r'.*\/mdata-user-data$', name_f): - found_new = True - print(name_f) - self.assertEqual(permissions, '400') - - self.assertFalse(found_new) - - def test_vendor_data_not_default(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(MOCK_RETURNS['sdc:vendor-data'], - dsrc.metadata['vendor-data']) - - def test_default_vendor_data(self): - my_returns = MOCK_RETURNS.copy() - def_op_script = my_returns['sdc:vendor-data'] - del my_returns['sdc:vendor-data'] - dsrc = self._get_ds(mockdata=my_returns) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertNotEqual(def_op_script, dsrc.metadata['vendor-data']) - - # we expect default vendor-data is a boothook - self.assertTrue(dsrc.vendordata_raw.startswith("#cloud-boothook")) - - def test_disable_iptables_flag(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(MOCK_RETURNS['disable_iptables_flag'], - dsrc.metadata['iptables_disable']) - - def test_motd_sys_info(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(MOCK_RETURNS['enable_motd_sys_info'], - dsrc.metadata['motd_sys_info']) - - def test_default_ephemeral(self): - # Test to make sure that the builtin config has the ephemeral - # configuration. - dsrc = self._get_ds() - cfg = dsrc.get_config_obj() - - ret = dsrc.get_data() - self.assertTrue(ret) - - assert 'disk_setup' in cfg - assert 'fs_setup' in cfg - self.assertIsInstance(cfg['disk_setup'], dict) - self.assertIsInstance(cfg['fs_setup'], list) - - def test_override_disk_aliases(self): - # Test to make sure that the built-in DS is overriden - builtin = DataSourceSmartOS.BUILTIN_DS_CONFIG - - mydscfg = {'disk_aliases': {'FOO': '/dev/bar'}} - - # expect that these values are in builtin, or this is pointless - for k in mydscfg: - self.assertIn(k, builtin) - - dsrc = self._get_ds(ds_cfg=mydscfg) - ret = dsrc.get_data() - self.assertTrue(ret) - - self.assertEqual(mydscfg['disk_aliases']['FOO'], - dsrc.ds_cfg['disk_aliases']['FOO']) - - self.assertEqual(dsrc.device_name_to_device('FOO'), - mydscfg['disk_aliases']['FOO']) - - -def apply_patches(patches): - ret = [] - for (ref, name, replace) in patches: - if replace is None: - continue - orig = getattr(ref, name) - setattr(ref, name, replace) - ret.append((ref, name, orig)) - return ret - class TestJoyentMetadataClient(FilesystemMockingTestCase): @@ -802,7 +420,8 @@ class TestJoyentMetadataClient(FilesystemMockingTestCase): mock.Mock(return_value=self.request_id))) def _get_client(self): - return DataSourceSmartOS.JoyentMetadataSerialClient(self.serial) + return DataSourceSmartOS.JoyentMetadataClient( + fp=self.serial, smartos_type=DataSourceSmartOS.SMARTOS_ENV_KVM) def assertEndsWith(self, haystack, prefix): self.assertTrue(haystack.endswith(prefix), @@ -882,20 +501,20 @@ class TestJoyentMetadataClient(FilesystemMockingTestCase): self.response_parts['length'] = 0 client = self._get_client() self.assertRaises(DataSourceSmartOS.JoyentMetadataFetchException, - client.get_metadata, 'some_key') + client.get, 'some_key') def test_get_metadata_throws_exception_for_incorrect_crc(self): self.response_parts['crc'] = 'deadbeef' client = self._get_client() self.assertRaises(DataSourceSmartOS.JoyentMetadataFetchException, - client.get_metadata, 'some_key') + client.get, 'some_key') def test_get_metadata_throws_exception_for_request_id_mismatch(self): self.response_parts['request_id'] = 'deadbeef' client = self._get_client() client._checksum = lambda _: self.response_parts['crc'] self.assertRaises(DataSourceSmartOS.JoyentMetadataFetchException, - client.get_metadata, 'some_key') + client.get, 'some_key') def test_get_metadata_returns_None_if_value_not_found(self): self.response_parts['payload'] = '' -- cgit v1.2.3 From 91734552a2d338938ed0e3aa4885f77b99409ead Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 May 2016 14:32:43 -0400 Subject: fix pyflakes and flake8 --- cloudinit/sources/DataSourceSmartOS.py | 3 +-- tests/unittests/test_datasource/test_smartos.py | 10 ++-------- 2 files changed, 3 insertions(+), 10 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index e9ff1235..f5820696 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -722,8 +722,7 @@ def convert_smartos_network_data(network_data=None): if k in valid_keys['physical']} cfg.update({ 'type': 'physical', - 'name': nic['interface'] - }) + 'name': nic['interface']}) if 'mac' in nic: cfg.update({'mac_address': nic['mac']}) diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 56d85f99..ae45513d 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -40,9 +40,8 @@ import six from cloudinit import helpers as c_helpers from cloudinit.sources import DataSourceSmartOS from cloudinit.util import b64e -from cloudinit import util -from ..helpers import mock, TestCase, FilesystemMockingTestCase +from ..helpers import mock, FilesystemMockingTestCase SDC_NICS = json.loads(""" [ @@ -111,7 +110,7 @@ class PsuedoJoyentClient(object): data = MOCK_RETURNS.copy() self.data = data return - + def get(self, key, default=None, strip=False): if key in self.data: r = self.data[key] @@ -169,10 +168,6 @@ class TestSmartOSDataSource(FilesystemMockingTestCase): return DataSourceSmartOS.DataSourceSmartOS( sys_cfg, distro=None, paths=self.paths) - - def test_it_got_here(self): - dsrc = self._get_ds() - ret = dsrc.get_data() def test_no_base64(self): ds_cfg = {'no_base64_decode': ['test_var1'], 'all_base': True} @@ -379,7 +374,6 @@ class TestSmartOSDataSource(FilesystemMockingTestCase): mydscfg['disk_aliases']['FOO']) - class TestJoyentMetadataClient(FilesystemMockingTestCase): def setUp(self): -- cgit v1.2.3 From 71e4da45263e6cb3eb5d5938908656ed04c3db9f Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 May 2016 14:33:20 -0400 Subject: remove debug print --- cloudinit/sources/DataSourceSmartOS.py | 1 - 1 file changed, 1 deletion(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index f5820696..9c249ddf 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -609,7 +609,6 @@ def write_boot_content(content, content_f, link=None, shebang=False, bit and to the SmartOS default of assuming that bash. """ - print("content_f=%s" % content_f) if not content and os.path.exists(content_f): os.unlink(content_f) if link and os.path.islink(link): -- cgit v1.2.3 From a9533cd924e8eae89234a19d8359a87c23a30e12 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 May 2016 14:52:13 -0400 Subject: add nicer main --- cloudinit/sources/DataSourceSmartOS.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 9c249ddf..3d7297c9 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -755,4 +755,23 @@ def get_datasource_list(depends): if __name__ == "__main__": import sys jmc = jmc_client_factory() - jmc.get(sys.argv[1]) + if len(sys.argv) == 1: + keys = (list(SMARTOS_ATTRIB_JSON.keys()) + + list(SMARTOS_ATTRIB_MAP.keys())) + else: + keys = sys.argv[1:] + + data = {} + for key in keys: + if key in SMARTOS_ATTRIB_JSON: + keyname = SMARTOS_ATTRIB_JSON[key] + data[key] = jmc.get_json(keyname) + else: + if key in SMARTOS_ATTRIB_MAP: + keyname, strip = SMARTOS_ATTRIB_MAP[key] + else: + keyname, strip = (key, False) + val = jmc.get(keyname, strip=strip) + data[key] = jmc.get(keyname, strip=strip) + + print(json.dumps(data, indent=1)) -- cgit v1.2.3 From 9a43053c564210aa088f96ab7877a66a1c3b48fa Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 May 2016 14:55:52 -0400 Subject: Smartos datasource is local. --- cloudinit/sources/DataSourceSmartOS.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 3d7297c9..24331d37 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -743,7 +743,7 @@ def convert_smartos_network_data(network_data=None): # Used to match classes to dependencies datasources = [ - (DataSourceSmartOS, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), + (DataSourceConfigDrive, (sources.DEP_FILESYSTEM, )), ] -- cgit v1.2.3 From 7ab03cab899c5bd355c24a4b894c74d63c6dd8b6 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 May 2016 15:03:08 -0400 Subject: smartos is local, but it is named DataSourceSmartOS not DataSourceConfigDrive --- cloudinit/sources/DataSourceSmartOS.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 24331d37..2821402a 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -743,7 +743,7 @@ def convert_smartos_network_data(network_data=None): # Used to match classes to dependencies datasources = [ - (DataSourceConfigDrive, (sources.DEP_FILESYSTEM, )), + (DataSourceSmartOS, (sources.DEP_FILESYSTEM, )), ] -- cgit v1.2.3 From 75109ffd12ad4dcbbccf1b0603506efcae413433 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 May 2016 15:31:52 -0400 Subject: return dict not None on get_config_obj --- cloudinit/sources/DataSourceSmartOS.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 2821402a..7c8928b3 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -302,7 +302,7 @@ class DataSourceSmartOS(sources.DataSource): def get_config_obj(self): if self.smartos_type == SMARTOS_ENV_KVM: return BUILTIN_CLOUD_CONFIG - return None + return {} def get_instance_id(self): return self.metadata['instance-id'] -- cgit v1.2.3 From 0ce31b75247458b735b1b52dd5985519190d48fb Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 31 May 2016 13:14:17 -0400 Subject: use constants for kvm and lx-brand --- cloudinit/sources/DataSourceSmartOS.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 7c8928b3..d6a9bf28 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -577,11 +577,11 @@ def jmc_client_factory( if smartos_type is None: smartos_type = get_smartos_environ(uname_version) - if smartos_type == 'kvm': + if smartos_type == SMARTOS_ENV_KVM: return JoyentMetadataLegacySerialClient( device=serial_device, timeout=serial_timeout, smartos_type=smartos_type) - elif smartos_type == 'lx-brand': + elif smartos_type == SMARTOS_ENV_LX_BRAND: return JoyentMetadataSocketClient(socketpath=metadata_sockfile) raise ValueError("Unknown value for smartos_type: %s" % smartos_type) -- cgit v1.2.3 From b071e4bbbbe1b5a6ced02796696b05d2e1b16778 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 2 Jun 2016 13:18:23 -0400 Subject: openstack: support decoding when reading files, use that for network_config The network config file is /etc/network/interfaces formated. We will decode that here so that the user can expect that it is a string. The issue was that it was bytes but convert_eni_data was expecting a string. --- cloudinit/sources/helpers/openstack.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 156aba6c..3ccb11d3 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -190,14 +190,14 @@ class BaseReader(object): versions_available) return selected_version - def _read_content_path(self, item): + def _read_content_path(self, item, decode=False): path = item.get('content_path', '').lstrip("/") path_pieces = path.split("/") valid_pieces = [p for p in path_pieces if len(p)] if not valid_pieces: raise BrokenMetadata("Item %s has no valid content path" % (item)) path = self._path_join(self.base_path, "openstack", *path_pieces) - return self._path_read(path) + return self._path_read(path, decode=decode) def read_v2(self): """Reads a version 2 formatted location. @@ -298,7 +298,8 @@ class BaseReader(object): net_item = metadata.get("network_config", None) if net_item: try: - results['network_config'] = self._read_content_path(net_item) + content = self._read_content_path(net_item, decode=True) + results['network_config'] = content except IOError as e: raise BrokenMetadata("Failed to read network" " configuration: %s" % (e)) @@ -333,8 +334,8 @@ class ConfigDriveReader(BaseReader): components = [base] + list(add_ons) return os.path.join(*components) - def _path_read(self, path): - return util.load_file(path, decode=False) + def _path_read(self, path, decode=False): + return util.load_file(path, decode=decode) def _fetch_available_versions(self): if self._versions is None: @@ -446,7 +447,7 @@ class MetadataReader(BaseReader): self._versions = found return self._versions - def _path_read(self, path): + def _path_read(self, path, decode=False): def should_retry_cb(_request_args, cause): try: @@ -463,7 +464,10 @@ class MetadataReader(BaseReader): ssl_details=self.ssl_details, timeout=self.timeout, exception_cb=should_retry_cb) - return response.contents + if decode: + return response.contents.decode() + else: + return response.contents def _path_join(self, base, *add_ons): return url_helper.combine_url(base, *add_ons) -- cgit v1.2.3 From 1abff1453a24ed14375cb6294364295a0c2c7ee3 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 2 Jun 2016 14:08:44 -0400 Subject: smartos: do not raise error when not on smartos if get_smartos_environ() returned a None, then the datasoure would raise a ValueError when get_data was called. Fix that. --- cloudinit/sources/DataSourceSmartOS.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index d6a9bf28..55dc94bb 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -171,7 +171,7 @@ LEGACY_USER_D = "/var/db" class DataSourceSmartOS(sources.DataSource): _unset = "_unset" - smartos_environ = _unset + smartos_type = _unset md_client = _unset def __init__(self, sys_cfg, distro, paths): @@ -195,8 +195,10 @@ class DataSourceSmartOS(sources.DataSource): return "%s [client=%s]" % (root, self.md_client) def _init(self): - if self.smartos_environ == self._unset: + if self.smartos_type == self._unset: self.smartos_type = get_smartos_environ() + if self.smartos_type is None: + self.md_client = None if self.md_client == self._unset: self.md_client = jmc_client_factory( @@ -577,7 +579,9 @@ def jmc_client_factory( if smartos_type is None: smartos_type = get_smartos_environ(uname_version) - if smartos_type == SMARTOS_ENV_KVM: + if smartos_type is None: + return None + elif smartos_type == SMARTOS_ENV_KVM: return JoyentMetadataLegacySerialClient( device=serial_device, timeout=serial_timeout, smartos_type=smartos_type) @@ -755,6 +759,9 @@ def get_datasource_list(depends): if __name__ == "__main__": import sys jmc = jmc_client_factory() + if jmc is None: + print("Do not appear to be on smartos.") + sys.exit(1) if len(sys.argv) == 1: keys = (list(SMARTOS_ATTRIB_JSON.keys()) + list(SMARTOS_ATTRIB_MAP.keys())) -- cgit v1.2.3 From bd31ab1e78f59c88b4aba031ffdaca506b3b04ae Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 2 Jun 2016 14:36:51 -0400 Subject: fix untested previous change to smartos --- cloudinit/sources/DataSourceSmartOS.py | 1 - 1 file changed, 1 deletion(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 55dc94bb..0e03b04f 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -186,7 +186,6 @@ class DataSourceSmartOS(sources.DataSource): self._network_config = None self.script_base_d = os.path.join(self.paths.get_cpath("scripts")) - self.smartos_type = None self._init() -- cgit v1.2.3 From d709524941ba2b4e06940a9eb0861f0819d5560f Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 2 Jun 2016 15:18:27 -0400 Subject: re-add the 'Net' classes for datasources When the .pkl file is loaded, the module that it is loaded from must have the same symbol. Ie, if booted once and got DataSourceConfigDriveNet then upgraded and rebooted, then next boot would show Can't get attribute 'DataSourceConfigDriveNet' --- cloudinit/sources/DataSourceCloudSigma.py | 3 +++ cloudinit/sources/DataSourceConfigDrive.py | 25 ++++++++++++++----------- cloudinit/sources/DataSourceOpenNebula.py | 3 +++ 3 files changed, 20 insertions(+), 11 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceCloudSigma.py b/cloudinit/sources/DataSourceCloudSigma.py index 07e8ae11..d1f806d6 100644 --- a/cloudinit/sources/DataSourceCloudSigma.py +++ b/cloudinit/sources/DataSourceCloudSigma.py @@ -115,6 +115,9 @@ class DataSourceCloudSigma(sources.DataSource): return self.metadata['uuid'] +# Legacy: Must be present in case we load an old pkl object +DataSourceCloudSigmaNet = DataSourceCloudSigma + # Used to match classes to dependencies. Since this datasource uses the serial # port network is not really required, so it's okay to load without it, too. datasources = [ diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 2d13a32f..61d967d9 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -254,17 +254,6 @@ def find_candidate_devs(probe_optical=True): return devices -# Used to match classes to dependencies -datasources = [ - (DataSourceConfigDrive, (sources.DEP_FILESYSTEM, )), -] - - -# Return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return sources.list_from_depends(depends, datasources) - - # Convert OpenStack ConfigDrive NetworkData json to network_config yaml def convert_network_data(network_json=None): """Return a dictionary of network_config by parsing provided @@ -382,3 +371,17 @@ def convert_network_data(network_json=None): config.append(cfg) return {'version': 1, 'config': config} + + +# Legacy: Must be present in case we load an old pkl object +DataSourceConfigDriveNet = DataSourceConfigDrive + +# Used to match classes to dependencies +datasources = [ + (DataSourceConfigDrive, (sources.DEP_FILESYSTEM, )), +] + + +# Return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return sources.list_from_depends(depends, datasources) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 15819a4f..8f85b115 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -415,6 +415,9 @@ def read_context_disk_dir(source_dir, asuser=None): return results +# Legacy: Must be present in case we load an old pkl object +DataSourceOpenNebulaNet = DataSourceOpenNebula + # Used to match classes to dependencies datasources = [ (DataSourceOpenNebula, (sources.DEP_FILESYSTEM, )), -- cgit v1.2.3 From 6bd7fbc35ac8726a8a0f422cd802d290c236bf3b Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 2 Jun 2016 23:03:38 -0400 Subject: ConfigDrive: do not use 'id' on a link for the device name 'id' on a link in the openstack spec should be "Generic, generated ID". current implementation was to use the host's name for the host side nic. Which provided names like 'tap-adfasdffd'. We do not want to name devices like that as its quite unexpected and non user friendly. So here we use the system name for any nic that is present, but then require that the nics found also be present at the time of rendering. The end result is that if the system boots with net.ifnames=0 then it will get 'eth0' like names. and if it boots without net.ifnames then it will get enp0s1 like names. --- cloudinit/net/__init__.py | 15 +++++--- cloudinit/sources/DataSourceConfigDrive.py | 29 ++++++++++++--- .../unittests/test_datasource/test_configdrive.py | 41 ++++++++++++++++++++-- 3 files changed, 74 insertions(+), 11 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 05152ead..f47053b2 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -970,9 +970,16 @@ def get_interface_mac(ifname): return read_sys_net(ifname, "address", enoent=False) -def get_ifname_mac_pairs(): - """Build a list of tuples (ifname, mac)""" - return [(ifname, get_interface_mac(ifname)) for ifname in get_devicelist()] - +def get_interfaces_by_mac(devs=None): + """Build a dictionary of tuples {mac: name}""" + if devs is None: + devs = get_devicelist() + ret = {} + for name in devs: + mac = get_interface_mac(name) + # some devices may not have a mac (tun0) + if mac: + ret[mac] = name + return ret # vi: ts=4 expandtab syntax=python diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 61d967d9..3cc9155d 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -255,7 +255,7 @@ def find_candidate_devs(probe_optical=True): # Convert OpenStack ConfigDrive NetworkData json to network_config yaml -def convert_network_data(network_json=None): +def convert_network_data(network_json=None, known_macs=None): """Return a dictionary of network_config by parsing provided OpenStack ConfigDrive NetworkData json format @@ -319,9 +319,15 @@ def convert_network_data(network_json=None): subnets = [] cfg = {k: v for k, v in link.items() if k in valid_keys['physical']} - cfg.update({'name': link['id']}) - for network in [net for net in networks - if net['link'] == link['id']]: + # 'name' is not in openstack spec yet, but we will support it if it is + # present. The 'id' in the spec is currently implemented as the host + # nic's name, meaning something like 'tap-adfasdffd'. We do not want + # to name guest devices with such ugly names. + if 'name' in link: + cfg['name'] = link['name'] + + for network in [n for n in networks + if n['link'] == link['id']]: subnet = {k: v for k, v in network.items() if k in valid_keys['subnet']} if 'dhcp' in network['type']: @@ -365,6 +371,21 @@ def convert_network_data(network_json=None): config.append(cfg) + need_names = [d for d in config + if d.get('type') == 'physical' and 'name' not in d] + + if need_names: + if known_macs is None: + known_macs = net.get_interfaces_by_mac() + + for d in need_names: + mac = d.get('mac_address') + if not mac: + raise ValueError("No mac_address or name entry for %s" % d) + if mac not in known_macs: + raise ValueError("Unable to find a system nic for %s" % d) + d['name'] = known_macs[mac] + for service in services: cfg = service cfg.update({'type': 'nameserver'}) diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index 195b8207..1364b39d 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -73,7 +73,7 @@ NETWORK_DATA = { 'type': 'ovs', 'mtu': None, 'id': 'tap2f88d109-5b'}, {'vif_id': '1a5382f8-04c5-4d75-ab98-d666c1ef52cc', 'ethernet_mac_address': 'fa:16:3e:05:30:fe', - 'type': 'ovs', 'mtu': None, 'id': 'tap1a5382f8-04'} + 'type': 'ovs', 'mtu': None, 'id': 'tap1a5382f8-04', 'name': 'nic0'} ], 'networks': [ {'link': 'tap2ecc7709-b3', 'type': 'ipv4_dhcp', @@ -88,6 +88,10 @@ NETWORK_DATA = { ] } +KNOWN_MACS = { + 'fa:16:3e:69:b0:58': 'enp0s1', + 'fa:16:3e:d4:57:ad': 'enp0s2'} + CFG_DRIVE_FILES_V2 = { 'ec2/2009-04-04/meta-data.json': json.dumps(EC2_META), 'ec2/2009-04-04/user-data': USER_DATA, @@ -365,10 +369,40 @@ class TestConfigDriveDataSource(TestCase): """Verify that network_data is converted and present on ds object.""" populate_dir(self.tmp, CFG_DRIVE_FILES_V2) myds = cfg_ds_from_dir(self.tmp) - network_config = ds.convert_network_data(NETWORK_DATA) + network_config = ds.convert_network_data(NETWORK_DATA, + known_macs=KNOWN_MACS) self.assertEqual(myds.network_config, network_config) +class TestConvertNetworkData(TestCase): + def _getnames_in_config(self, ncfg): + return set([n['name'] for n in ncfg['config'] + if n['type'] == 'physical']) + + def test_conversion_fills_names(self): + ncfg = ds.convert_network_data(NETWORK_DATA, known_macs=KNOWN_MACS) + expected = set(['nic0', 'enp0s1', 'enp0s2']) + found = self._getnames_in_config(ncfg) + self.assertEqual(found, expected) + + @mock.patch('cloudinit.net.get_interfaces_by_mac') + def test_convert_reads_system_prefers_name(self, get_interfaces_by_mac): + macs = KNOWN_MACS.copy() + macs.update({'fa:16:3e:05:30:fe': 'foonic1', + 'fa:16:3e:69:b0:58': 'ens1'}) + get_interfaces_by_mac.return_value = macs + + ncfg = ds.convert_network_data(NETWORK_DATA) + expected = set(['nic0', 'ens1', 'enp0s2']) + found = self._getnames_in_config(ncfg) + self.assertEqual(found, expected) + + def test_convert_raises_value_error_on_missing_name(self): + macs = {'aa:aa:aa:aa:aa:00': 'ens1'} + self.assertRaises(ValueError, ds.convert_network_data, + NETWORK_DATA, known_macs=macs) + + def cfg_ds_from_dir(seed_d): found = ds.read_config_drive(seed_d) cfg_ds = ds.DataSourceConfigDrive(settings.CFG_BUILTIN, None, @@ -387,7 +421,8 @@ def populate_ds_from_read_config(cfg_ds, source, results): cfg_ds.userdata_raw = results.get('userdata') cfg_ds.version = results.get('version') cfg_ds.network_json = results.get('networkdata') - cfg_ds._network_config = ds.convert_network_data(cfg_ds.network_json) + cfg_ds._network_config = ds.convert_network_data( + cfg_ds.network_json, known_macs=KNOWN_MACS) def populate_dir(seed_dir, files): -- cgit v1.2.3 From 42a7d2b6d44be5fd6e41734902e08897b709015d Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 3 Jun 2016 15:06:55 -0400 Subject: config drive conversion: recognize 'bridge' as a physical type, fix mtu the network json in openstack provides a type of 'bridge' when the underlying (host) type is a bridge. Silly, but we need to consider that a physical device as it will be for us. also, the 'mtu' will appear on the link, not on the route --- cloudinit/sources/DataSourceConfigDrive.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 3cc9155d..c87f57fd 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -293,6 +293,7 @@ def convert_network_data(network_json=None, known_macs=None): 'mac_address', 'subnets', 'params', + 'mtu', ], 'subnet': [ 'type', @@ -302,7 +303,6 @@ def convert_network_data(network_json=None, known_macs=None): 'metric', 'gateway', 'pointopoint', - 'mtu', 'scope', 'dns_nameservers', 'dns_search', @@ -342,7 +342,7 @@ def convert_network_data(network_json=None, known_macs=None): }) subnets.append(subnet) cfg.update({'subnets': subnets}) - if link['type'] in ['ethernet', 'vif', 'ovs', 'phy']: + if link['type'] in ['ethernet', 'vif', 'ovs', 'phy', 'bridge']: cfg.update({ 'type': 'physical', 'mac_address': link['ethernet_mac_address']}) -- cgit v1.2.3