From 4a60af54957634920e84a928aa22b4fc9a6dfd11 Mon Sep 17 00:00:00 2001 From: Junjie Wang Date: Fri, 21 Apr 2017 20:06:09 +0800 Subject: AliYun: Enable platform identification and enable by default. AliYun cloud platform is now identifying themselves by setting the dmi product id to the well known value "Alibaba Cloud ECS". The changes here identify that properly in tools/ds-identify and in the DataSourceAliYun. Since the 'get_data' for AliYun now identifies itself correctly, we can enable AliYun by default. LP: #1638931 --- cloudinit/settings.py | 1 + cloudinit/sources/DataSourceAliYun.py | 14 +++++++++++++- cloudinit/sources/DataSourceEc2.py | 7 +++++++ 3 files changed, 21 insertions(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/settings.py b/cloudinit/settings.py index 411960d8..0abd8a4a 100644 --- a/cloudinit/settings.py +++ b/cloudinit/settings.py @@ -29,6 +29,7 @@ CFG_BUILTIN = { 'MAAS', 'GCE', 'OpenStack', + 'AliYun', 'Ec2', 'CloudSigma', 'CloudStack', diff --git a/cloudinit/sources/DataSourceAliYun.py b/cloudinit/sources/DataSourceAliYun.py index 9debe947..380e27cb 100644 --- a/cloudinit/sources/DataSourceAliYun.py +++ b/cloudinit/sources/DataSourceAliYun.py @@ -4,8 +4,10 @@ import os from cloudinit import sources from cloudinit.sources import DataSourceEc2 as EC2 +from cloudinit import util DEF_MD_VERSION = "2016-01-01" +ALIYUN_PRODUCT = "Alibaba Cloud ECS" class DataSourceAliYun(EC2.DataSourceEc2): @@ -24,7 +26,17 @@ class DataSourceAliYun(EC2.DataSourceEc2): @property def cloud_platform(self): - return EC2.Platforms.ALIYUN + if self._cloud_platform is None: + if _is_aliyun(): + self._cloud_platform = EC2.Platforms.ALIYUN + else: + self._cloud_platform = EC2.Platforms.NO_EC2_METADATA + + return self._cloud_platform + + +def _is_aliyun(): + return util.read_dmi_data('system-product-name') == ALIYUN_PRODUCT def parse_public_keys(public_keys): diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 2f9c7edf..9e2fdc0a 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -32,7 +32,12 @@ class Platforms(object): AWS = "AWS" BRIGHTBOX = "Brightbox" SEEDED = "Seeded" + # UNKNOWN indicates no positive id. If strict_id is 'warn' or 'false', + # then an attempt at the Ec2 Metadata service will be made. UNKNOWN = "Unknown" + # NO_EC2_METADATA indicates this platform does not have a Ec2 metadata + # service available. No attempt at the Ec2 Metadata service will be made. + NO_EC2_METADATA = "No-EC2-Metadata" class DataSourceEc2(sources.DataSource): @@ -65,6 +70,8 @@ class DataSourceEc2(sources.DataSource): strict_mode, self.cloud_platform) if strict_mode == "true" and self.cloud_platform == Platforms.UNKNOWN: return False + elif self.cloud_platform == Platforms.NO_EC2_METADATA: + return False try: if not self.wait_for_metadata_service(): -- cgit v1.2.3 From 003c6678e9c873b3b787a814016872b6592f5069 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Thu, 25 May 2017 15:37:15 -0500 Subject: net: remove systemd link file writing from eni renderer During the network v2 merge, we inadvertently re-enabled rendering systemd .link files. This files are not required as cloud-init already has to do interface renaming due to issues with udevd which may refuse to rename certain interfaces (such as veth devices in a LXD container). As such, removing the code altogether. --- cloudinit/net/eni.py | 25 ------------------------- tests/unittests/test_net.py | 9 +++------ 2 files changed, 3 insertions(+), 31 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index 9819d4f5..0cc4e3b2 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -304,8 +304,6 @@ class Renderer(renderer.Renderer): config = {} self.eni_path = config.get('eni_path', 'etc/network/interfaces') self.eni_header = config.get('eni_header', None) - self.links_path_prefix = config.get( - 'links_path_prefix', 'etc/systemd/network/50-cloud-init-') self.netrules_path = config.get( 'netrules_path', 'etc/udev/rules.d/70-persistent-net.rules') @@ -451,28 +449,6 @@ class Renderer(renderer.Renderer): util.write_file(netrules, self._render_persistent_net(network_state)) - if self.links_path_prefix: - self._render_systemd_links(target, network_state, - links_prefix=self.links_path_prefix) - - def _render_systemd_links(self, target, network_state, links_prefix): - fp_prefix = util.target_path(target, links_prefix) - for f in glob.glob(fp_prefix + "*"): - os.unlink(f) - for iface in network_state.iter_interfaces(): - if (iface['type'] == 'physical' and 'name' in iface and - iface.get('mac_address')): - fname = fp_prefix + iface['name'] + ".link" - content = "\n".join([ - "[Match]", - "MACAddress=" + iface['mac_address'], - "", - "[Link]", - "Name=" + iface['name'], - "" - ]) - util.write_file(fname, content) - def network_state_to_eni(network_state, header=None, render_hwaddress=False): # render the provided network state, return a string of equivalent eni @@ -480,7 +456,6 @@ def network_state_to_eni(network_state, header=None, render_hwaddress=False): renderer = Renderer(config={ 'eni_path': eni_path, 'eni_header': header, - 'links_path_prefix': None, 'netrules_path': None, }) if not header: diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 167ed01e..5a0fcbcf 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -992,9 +992,7 @@ class TestEniNetRendering(CiTestCase): os.makedirs(render_dir) renderer = eni.Renderer( - {'links_path_prefix': None, - 'eni_path': 'interfaces', 'netrules_path': None, - }) + {'eni_path': 'interfaces', 'netrules_path': None}) renderer.render_network_state(ns, render_dir) self.assertTrue(os.path.exists(os.path.join(render_dir, @@ -1376,7 +1374,7 @@ class TestNetplanRoundTrip(CiTestCase): class TestEniRoundTrip(CiTestCase): def _render_and_read(self, network_config=None, state=None, eni_path=None, - links_prefix=None, netrules_path=None, dir=None): + netrules_path=None, dir=None): if dir is None: dir = self.tmp_dir() @@ -1391,8 +1389,7 @@ class TestEniRoundTrip(CiTestCase): eni_path = 'etc/network/interfaces' renderer = eni.Renderer( - config={'eni_path': eni_path, 'links_path_prefix': links_prefix, - 'netrules_path': netrules_path}) + config={'eni_path': eni_path, 'netrules_path': netrules_path}) renderer.render_network_state(ns, dir) return dir2dict(dir) -- cgit v1.2.3 From 00b678c61a54f176625d3f937971215faf6af2cd Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Fri, 26 May 2017 14:29:24 -0500 Subject: Fix eni rendering for bridge params that require repeated key for values. There are a few bridge parameters which require repeating the key with each value in the list when rendering eni. Extend the network unittests to cover all of the known bridge parameters and check we render eni and netplan correctly. --- cloudinit/net/eni.py | 13 +++++++++++++ tests/unittests/test_net.py | 39 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 50 insertions(+), 2 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index 0cc4e3b2..20e19f5b 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -75,6 +75,15 @@ def _iface_add_attrs(iface, index): 'subnets', 'type', ] + + # The following parameters require repetitive entries of the key for + # each of the values + multiline_keys = [ + 'bridge_pathcost', + 'bridge_portprio', + 'bridge_waitport', + ] + renames = {'mac_address': 'hwaddress'} if iface['type'] not in ['bond', 'bridge', 'vlan']: ignore_map.append('mac_address') @@ -82,6 +91,10 @@ def _iface_add_attrs(iface, index): for key, value in iface.items(): if not value or key in ignore_map: continue + if key in multiline_keys: + for v in value: + content.append(" {0} {1}".format(renames.get(key, key), v)) + continue if type(value) == list: value = " ".join(value) content.append(" {0} {1}".format(renames.get(key, key), value)) diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 5a0fcbcf..5f7e902a 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -511,8 +511,20 @@ iface bond0 inet6 dhcp auto br0 iface br0 inet static address 192.168.14.2/24 + bridge_ageing 250 + bridge_bridgeprio 22 + bridge_fd 1 + bridge_gcint 2 + bridge_hello 1 + bridge_maxage 10 + bridge_pathcost eth3 50 + bridge_pathcost eth4 75 + bridge_portprio eth3 28 + bridge_portprio eth4 14 bridge_ports eth3 eth4 bridge_stp off + bridge_waitport 1 eth3 + bridge_waitport 2 eth4 # control-alias br0 iface br0 inet6 static @@ -642,6 +654,15 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true interfaces: - eth3 - eth4 + parameters: + ageing-time: 250 + forward-delay: 1 + hello-time: 1 + max-age: 10 + path-cost: + eth3: 50 + eth4: 75 + priority: 22 vlans: bond0.200: dhcp4: true @@ -752,9 +773,23 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true forwarding: 1 # basically anything in /proc/sys/net/ipv6/conf/.../ params: - bridge_stp: 'off' - bridge_fd: 0 + bridge_ageing: 250 + bridge_bridgeprio: 22 + bridge_fd: 1 + bridge_gcint: 2 + bridge_hello: 1 + bridge_maxage: 10 bridge_maxwait: 0 + bridge_pathcost: + - eth3 50 + - eth4 75 + bridge_portprio: + - eth3 28 + - eth4 14 + bridge_stp: 'off' + bridge_waitport: + - 1 eth3 + - 2 eth4 subnets: - type: static address: 192.168.14.2/24 -- cgit v1.2.3 From 0a448dd034883c07f85091dbfc9117de7227eb8d Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 25 May 2017 11:04:55 -0600 Subject: ntp: Add schema definition and passive schema validation. cloud-config files are very flexible and permissive. This adds a jsonsschema definition to the cc_ntp module and validation functions in cloudinit/config/schema which will log warnings about invalid configuration values in the ntp section. A cmdline tools/cloudconfig-schema is added which can be used in our dev environments to quickly attempt to exercise the ntp schema. It is also exposed as a main in cloudinit.config.schema. (python3 -m cloudinit.config.schema) LP: #1692916 --- cloudinit/config/cc_ntp.py | 69 ++++++- cloudinit/config/schema.py | 222 +++++++++++++++++++++++ requirements.txt | 3 + tests/unittests/helpers.py | 6 +- tests/unittests/test_handler/test_handler_ntp.py | 109 +++++++++++ tests/unittests/test_handler/test_schema.py | 220 ++++++++++++++++++++++ tools/cloudconfig-schema | 35 ++++ 7 files changed, 657 insertions(+), 7 deletions(-) create mode 100644 cloudinit/config/schema.py create mode 100644 tests/unittests/test_handler/test_schema.py create mode 100755 tools/cloudconfig-schema (limited to 'cloudinit') diff --git a/cloudinit/config/cc_ntp.py b/cloudinit/config/cc_ntp.py index 5cc54536..31ed64e3 100644 --- a/cloudinit/config/cc_ntp.py +++ b/cloudinit/config/cc_ntp.py @@ -36,6 +36,7 @@ servers or pools are provided, 4 pools will be used in the format - 192.168.23.2 """ +from cloudinit.config.schema import validate_cloudconfig_schema from cloudinit import log as logging from cloudinit.settings import PER_INSTANCE from cloudinit import templater @@ -43,6 +44,7 @@ from cloudinit import type_utils from cloudinit import util import os +from textwrap import dedent LOG = logging.getLogger(__name__) @@ -52,21 +54,84 @@ NR_POOL_SERVERS = 4 distros = ['centos', 'debian', 'fedora', 'opensuse', 'ubuntu'] +# The schema definition for each cloud-config module is a strict contract for +# describing supported configuration parameters for each cloud-config section. +# It allows cloud-config to validate and alert users to invalid or ignored +# configuration options before actually attempting to deploy with said +# configuration. + +schema = { + 'id': 'cc_ntp', + 'name': 'NTP', + 'title': 'enable and configure ntp', + 'description': dedent("""\ + Handle ntp configuration. If ntp is not installed on the system and + ntp configuration is specified, ntp will be installed. If there is a + default ntp config file in the image or one is present in the + distro's ntp package, it will be copied to ``/etc/ntp.conf.dist`` + before any changes are made. A list of ntp pools and ntp servers can + be provided under the ``ntp`` config key. If no ntp ``servers`` or + ``pools`` are provided, 4 pools will be used in the format + ``{0-3}.{distro}.pool.ntp.org``."""), + 'distros': distros, + 'examples': [ + {'ntp': {'pools': ['0.company.pool.ntp.org', '1.company.pool.ntp.org', + 'ntp.myorg.org'], + 'servers': ['my.ntp.server.local', 'ntp.ubuntu.com', + '192.168.23.2']}}], + 'frequency': PER_INSTANCE, + 'type': 'object', + 'properties': { + 'ntp': { + 'type': ['object', 'null'], + 'properties': { + 'pools': { + 'type': 'array', + 'items': { + 'type': 'string', + 'format': 'hostname' + }, + 'uniqueItems': True, + 'description': dedent("""\ + List of ntp pools. If both pools and servers are + empty, 4 default pool servers will be provided of + the format ``{0-3}.{distro}.pool.ntp.org``.""") + }, + 'servers': { + 'type': 'array', + 'items': { + 'type': 'string', + 'format': 'hostname' + }, + 'uniqueItems': True, + 'description': dedent("""\ + List of ntp servers. If both pools and servers are + empty, 4 default pool servers will be provided with + the format ``{0-3}.{distro}.pool.ntp.org``.""") + } + }, + 'required': [], + 'additionalProperties': False + } + } +} + + def handle(name, cfg, cloud, log, _args): """Enable and configure ntp.""" - if 'ntp' not in cfg: LOG.debug( "Skipping module named %s, not present or disabled by cfg", name) return - ntp_cfg = cfg.get('ntp', {}) + # TODO drop this when validate_cloudconfig_schema is strict=True if not isinstance(ntp_cfg, (dict)): raise RuntimeError(("'ntp' key existed in config," " but not a dictionary type," " is a %s %instead"), type_utils.obj_name(ntp_cfg)) + validate_cloudconfig_schema(cfg, schema) rename_ntp_conf() # ensure when ntp is installed it has a configuration file # to use instead of starting up with packaged defaults diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py new file mode 100644 index 00000000..6400f005 --- /dev/null +++ b/cloudinit/config/schema.py @@ -0,0 +1,222 @@ +# This file is part of cloud-init. See LICENSE file for license information. +"""schema.py: Set of module functions for processing cloud-config schema.""" + +from __future__ import print_function + +from cloudinit.util import read_file_or_url + +import argparse +import logging +import os +import sys +import yaml + +SCHEMA_UNDEFINED = b'UNDEFINED' +CLOUD_CONFIG_HEADER = b'#cloud-config' +SCHEMA_DOC_TMPL = """ +{name} +--- +**Summary:** {title} + +{description} + +**Internal name:** ``{id}`` + +**Module frequency:** {frequency} + +**Supported distros:** {distros} + +**Config schema**: +{property_doc} +{examples} +""" +SCHEMA_PROPERTY_TMPL = '{prefix}**{prop_name}:** ({type}) {description}' + + +class SchemaValidationError(ValueError): + """Raised when validating a cloud-config file against a schema.""" + + def __init__(self, schema_errors=()): + """Init the exception an n-tuple of schema errors. + + @param schema_errors: An n-tuple of the format: + ((flat.config.key, msg),) + """ + self.schema_errors = schema_errors + error_messages = [ + '{0}: {1}'.format(config_key, message) + for config_key, message in schema_errors] + message = "Cloud config schema errors: {0}".format( + ', '.join(error_messages)) + super(SchemaValidationError, self).__init__(message) + + +def validate_cloudconfig_schema(config, schema, strict=False): + """Validate provided config meets the schema definition. + + @param config: Dict of cloud configuration settings validated against + schema. + @param schema: jsonschema dict describing the supported schema definition + for the cloud config module (config.cc_*). + @param strict: Boolean, when True raise SchemaValidationErrors instead of + logging warnings. + + @raises: SchemaValidationError when provided config does not validate + against the provided schema. + """ + try: + from jsonschema import Draft4Validator, FormatChecker + except ImportError: + logging.warning( + 'Ignoring schema validation. python-jsonschema is not present') + return + validator = Draft4Validator(schema, format_checker=FormatChecker()) + errors = () + for error in sorted(validator.iter_errors(config), key=lambda e: e.path): + path = '.'.join([str(p) for p in error.path]) + errors += ((path, error.message),) + if errors: + if strict: + raise SchemaValidationError(errors) + else: + messages = ['{0}: {1}'.format(k, msg) for k, msg in errors] + logging.warning('Invalid config:\n%s', '\n'.join(messages)) + + +def validate_cloudconfig_file(config_path, schema): + """Validate cloudconfig file adheres to a specific jsonschema. + + @param config_path: Path to the yaml cloud-config file to parse. + @param schema: Dict describing a valid jsonschema to validate against. + + @raises SchemaValidationError containing any of schema_errors encountered. + @raises RuntimeError when config_path does not exist. + """ + if not os.path.exists(config_path): + raise RuntimeError('Configfile {0} does not exist'.format(config_path)) + content = read_file_or_url('file://{0}'.format(config_path)).contents + if not content.startswith(CLOUD_CONFIG_HEADER): + errors = ( + ('header', 'File {0} needs to begin with "{1}"'.format( + config_path, CLOUD_CONFIG_HEADER.decode())),) + raise SchemaValidationError(errors) + + try: + cloudconfig = yaml.safe_load(content) + except yaml.parser.ParserError as e: + errors = ( + ('format', 'File {0} is not valid yaml. {1}'.format( + config_path, str(e))),) + raise SchemaValidationError(errors) + validate_cloudconfig_schema( + cloudconfig, schema, strict=True) + + +def _get_property_type(property_dict): + """Return a string representing a property type from a given jsonschema.""" + property_type = property_dict.get('type', SCHEMA_UNDEFINED) + if isinstance(property_type, list): + property_type = '/'.join(property_type) + item_type = property_dict.get('items', {}).get('type') + if item_type: + property_type = '{0} of {1}'.format(property_type, item_type) + return property_type + + +def _get_property_doc(schema, prefix=' '): + """Return restructured text describing the supported schema properties.""" + new_prefix = prefix + ' ' + properties = [] + for prop_key, prop_config in schema.get('properties', {}).items(): + # Define prop_name and dscription for SCHEMA_PROPERTY_TMPL + description = prop_config.get('description', '') + properties.append(SCHEMA_PROPERTY_TMPL.format( + prefix=prefix, + prop_name=prop_key, + type=_get_property_type(prop_config), + description=description.replace('\n', ''))) + if 'properties' in prop_config: + properties.append( + _get_property_doc(prop_config, prefix=new_prefix)) + return '\n\n'.join(properties) + + +def _get_schema_examples(schema, prefix=''): + """Return restructured text describing the schema examples if present.""" + examples = schema.get('examples') + if not examples: + return '' + rst_content = '\n**Examples**::\n\n' + for example in examples: + example_yaml = yaml.dump(example, default_flow_style=False) + # Python2.6 is missing textwrapper.indent + lines = example_yaml.split('\n') + indented_lines = [' {0}'.format(line) for line in lines] + rst_content += '\n'.join(indented_lines) + return rst_content + + +def get_schema_doc(schema): + """Return reStructured text rendering the provided jsonschema. + + @param schema: Dict of jsonschema to render. + @raise KeyError: If schema lacks an expected key. + """ + schema['property_doc'] = _get_property_doc(schema) + schema['examples'] = _get_schema_examples(schema) + schema['distros'] = ', '.join(schema['distros']) + return SCHEMA_DOC_TMPL.format(**schema) + + +def get_schema(section_key=None): + """Return a dict of jsonschema defined in any cc_* module. + + @param: section_key: Optionally limit schema to a specific top-level key. + """ + # TODO use util.find_modules in subsequent branch + from cloudinit.config.cc_ntp import schema + return schema + + +def error(message): + print(message, file=sys.stderr) + return 1 + + +def get_parser(): + """Return a parser for supported cmdline arguments.""" + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--config-file', + help='Path of the cloud-config yaml file to validate') + parser.add_argument('-d', '--doc', action="store_true", default=False, + help='Print schema documentation') + parser.add_argument('-k', '--key', + help='Limit validation or docs to a section key') + return parser + + +def main(): + """Tool to validate schema of a cloud-config file or print schema docs.""" + parser = get_parser() + args = parser.parse_args() + exclusive_args = [args.config_file, args.doc] + if not any(exclusive_args) or all(exclusive_args): + return error('Expected either --config-file argument or --doc') + + schema = get_schema() + if args.config_file: + try: + validate_cloudconfig_file(args.config_file, schema) + except (SchemaValidationError, RuntimeError) as e: + return error(str(e)) + print("Valid cloud-config file {0}".format(args.config_file)) + if args.doc: + print(get_schema_doc(schema)) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) + + +# vi: ts=4 expandtab diff --git a/requirements.txt b/requirements.txt index 0c4951f5..60abab16 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,5 +36,8 @@ requests # For patching pieces of cloud-config together jsonpatch +# For validating cloud-config sections per schema definitions +jsonschema + # For Python 2/3 compatibility six diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 9ff15993..e78abce2 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -19,10 +19,6 @@ try: from contextlib import ExitStack except ImportError: from contextlib2 import ExitStack -try: - from cStringIO import StringIO -except ImportError: - from io import StringIO from cloudinit import helpers as ch from cloudinit import util @@ -102,7 +98,7 @@ class CiTestCase(TestCase): if self.with_logs: # Create a log handler so unit tests can search expected logs. logger = logging.getLogger() - self.logs = StringIO() + self.logs = six.StringIO() handler = logging.StreamHandler(self.logs) self.old_handlers = logger.handlers logger.handlers = [handler] diff --git a/tests/unittests/test_handler/test_handler_ntp.py b/tests/unittests/test_handler/test_handler_ntp.py index bc4277b7..6cafa63d 100644 --- a/tests/unittests/test_handler/test_handler_ntp.py +++ b/tests/unittests/test_handler/test_handler_ntp.py @@ -212,4 +212,113 @@ class TestNtp(FilesystemMockingTestCase): 'Skipping module named cc_ntp, not present or disabled by cfg\n', self.logs.getvalue()) + def test_ntp_handler_schema_validation_allows_empty_ntp_config(self): + """Ntp schema validation allows for an empty ntp: configuration.""" + invalid_config = {'ntp': {}} + distro = 'ubuntu' + cc = self._get_cloud(distro) + ntp_conf = os.path.join(self.new_root, 'ntp.conf') + with open('{0}.tmpl'.format(ntp_conf), 'wb') as stream: + stream.write(NTP_TEMPLATE) + with mock.patch('cloudinit.config.cc_ntp.NTP_CONF', ntp_conf): + cc_ntp.handle('cc_ntp', invalid_config, cc, None, []) + self.assertNotIn('Invalid config:', self.logs.getvalue()) + with open(ntp_conf) as stream: + content = stream.read() + default_pools = [ + "{0}.{1}.pool.ntp.org".format(x, distro) + for x in range(0, cc_ntp.NR_POOL_SERVERS)] + self.assertEqual( + "servers []\npools {0}\n".format(default_pools), + content) + + def test_ntp_handler_schema_validation_warns_non_string_item_type(self): + """Ntp schema validation warns of non-strings in pools or servers. + + Schema validation is not strict, so ntp config is still be rendered. + """ + invalid_config = {'ntp': {'pools': [123], 'servers': ['valid', None]}} + cc = self._get_cloud('ubuntu') + ntp_conf = os.path.join(self.new_root, 'ntp.conf') + with open('{0}.tmpl'.format(ntp_conf), 'wb') as stream: + stream.write(NTP_TEMPLATE) + with mock.patch('cloudinit.config.cc_ntp.NTP_CONF', ntp_conf): + cc_ntp.handle('cc_ntp', invalid_config, cc, None, []) + self.assertIn( + "Invalid config:\nntp.pools.0: 123 is not of type 'string'\n" + "ntp.servers.1: None is not of type 'string'", + self.logs.getvalue()) + with open(ntp_conf) as stream: + content = stream.read() + self.assertEqual("servers ['valid', None]\npools [123]\n", content) + + def test_ntp_handler_schema_validation_warns_of_non_array_type(self): + """Ntp schema validation warns of non-array pools or servers types. + + Schema validation is not strict, so ntp config is still be rendered. + """ + invalid_config = {'ntp': {'pools': 123, 'servers': 'non-array'}} + cc = self._get_cloud('ubuntu') + ntp_conf = os.path.join(self.new_root, 'ntp.conf') + with open('{0}.tmpl'.format(ntp_conf), 'wb') as stream: + stream.write(NTP_TEMPLATE) + with mock.patch('cloudinit.config.cc_ntp.NTP_CONF', ntp_conf): + cc_ntp.handle('cc_ntp', invalid_config, cc, None, []) + self.assertIn( + "Invalid config:\nntp.pools: 123 is not of type 'array'\n" + "ntp.servers: 'non-array' is not of type 'array'", + self.logs.getvalue()) + with open(ntp_conf) as stream: + content = stream.read() + self.assertEqual("servers non-array\npools 123\n", content) + + def test_ntp_handler_schema_validation_warns_invalid_key_present(self): + """Ntp schema validation warns of invalid keys present in ntp config. + + Schema validation is not strict, so ntp config is still be rendered. + """ + invalid_config = { + 'ntp': {'invalidkey': 1, 'pools': ['0.mycompany.pool.ntp.org']}} + cc = self._get_cloud('ubuntu') + ntp_conf = os.path.join(self.new_root, 'ntp.conf') + with open('{0}.tmpl'.format(ntp_conf), 'wb') as stream: + stream.write(NTP_TEMPLATE) + with mock.patch('cloudinit.config.cc_ntp.NTP_CONF', ntp_conf): + cc_ntp.handle('cc_ntp', invalid_config, cc, None, []) + self.assertIn( + "Invalid config:\nntp: Additional properties are not allowed " + "('invalidkey' was unexpected)", + self.logs.getvalue()) + with open(ntp_conf) as stream: + content = stream.read() + self.assertEqual( + "servers []\npools ['0.mycompany.pool.ntp.org']\n", + content) + + def test_ntp_handler_schema_validation_warns_of_duplicates(self): + """Ntp schema validation warns of duplicates in servers or pools. + + Schema validation is not strict, so ntp config is still be rendered. + """ + invalid_config = { + 'ntp': {'pools': ['0.mypool.org', '0.mypool.org'], + 'servers': ['10.0.0.1', '10.0.0.1']}} + cc = self._get_cloud('ubuntu') + ntp_conf = os.path.join(self.new_root, 'ntp.conf') + with open('{0}.tmpl'.format(ntp_conf), 'wb') as stream: + stream.write(NTP_TEMPLATE) + with mock.patch('cloudinit.config.cc_ntp.NTP_CONF', ntp_conf): + cc_ntp.handle('cc_ntp', invalid_config, cc, None, []) + self.assertIn( + "Invalid config:\nntp.pools: ['0.mypool.org', '0.mypool.org'] has " + "non-unique elements\nntp.servers: ['10.0.0.1', '10.0.0.1'] has " + "non-unique elements", + self.logs.getvalue()) + with open(ntp_conf) as stream: + content = stream.read() + self.assertEqual( + "servers ['10.0.0.1', '10.0.0.1']\n" + "pools ['0.mypool.org', '0.mypool.org']\n", + content) + # vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py new file mode 100644 index 00000000..3239e326 --- /dev/null +++ b/tests/unittests/test_handler/test_schema.py @@ -0,0 +1,220 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit.config.schema import ( + CLOUD_CONFIG_HEADER, SchemaValidationError, get_schema_doc, + validate_cloudconfig_file, validate_cloudconfig_schema, + main) +from cloudinit.util import write_file + +from ..helpers import CiTestCase, mock + +from copy import copy +from six import StringIO +from textwrap import dedent + + +class SchemaValidationErrorTest(CiTestCase): + """Test validate_cloudconfig_schema""" + + def test_schema_validation_error_expects_schema_errors(self): + """SchemaValidationError is initialized from schema_errors.""" + errors = (('key.path', 'unexpected key "junk"'), + ('key2.path', '"-123" is not a valid "hostname" format')) + exception = SchemaValidationError(schema_errors=errors) + self.assertIsInstance(exception, Exception) + self.assertEqual(exception.schema_errors, errors) + self.assertEqual( + 'Cloud config schema errors: key.path: unexpected key "junk", ' + 'key2.path: "-123" is not a valid "hostname" format', + str(exception)) + self.assertTrue(isinstance(exception, ValueError)) + + +class ValidateCloudConfigSchemaTest(CiTestCase): + """Tests for validate_cloudconfig_schema.""" + + with_logs = True + + def test_validateconfig_schema_non_strict_emits_warnings(self): + """When strict is False validate_cloudconfig_schema emits warnings.""" + schema = {'properties': {'p1': {'type': 'string'}}} + validate_cloudconfig_schema({'p1': -1}, schema, strict=False) + self.assertIn( + "Invalid config:\np1: -1 is not of type 'string'\n", + self.logs.getvalue()) + + def test_validateconfig_schema_emits_warning_on_missing_jsonschema(self): + """Warning from validate_cloudconfig_schema when missing jsonschema.""" + schema = {'properties': {'p1': {'type': 'string'}}} + with mock.patch.dict('sys.modules', **{'jsonschema': ImportError()}): + validate_cloudconfig_schema({'p1': -1}, schema, strict=True) + self.assertIn( + 'Ignoring schema validation. python-jsonschema is not present', + self.logs.getvalue()) + + def test_validateconfig_schema_strict_raises_errors(self): + """When strict is True validate_cloudconfig_schema raises errors.""" + schema = {'properties': {'p1': {'type': 'string'}}} + with self.assertRaises(SchemaValidationError) as context_mgr: + validate_cloudconfig_schema({'p1': -1}, schema, strict=True) + self.assertEqual( + "Cloud config schema errors: p1: -1 is not of type 'string'", + str(context_mgr.exception)) + + def test_validateconfig_schema_honors_formats(self): + """When strict is True validate_cloudconfig_schema raises errors.""" + schema = { + 'properties': {'p1': {'type': 'string', 'format': 'hostname'}}} + with self.assertRaises(SchemaValidationError) as context_mgr: + validate_cloudconfig_schema({'p1': '-1'}, schema, strict=True) + self.assertEqual( + "Cloud config schema errors: p1: '-1' is not a 'hostname'", + str(context_mgr.exception)) + + +class ValidateCloudConfigFileTest(CiTestCase): + """Tests for validate_cloudconfig_file.""" + + def setUp(self): + super(ValidateCloudConfigFileTest, self).setUp() + self.config_file = self.tmp_path('cloudcfg.yaml') + + def test_validateconfig_file_error_on_absent_file(self): + """On absent config_path, validate_cloudconfig_file errors.""" + with self.assertRaises(RuntimeError) as context_mgr: + validate_cloudconfig_file('/not/here', {}) + self.assertEqual( + 'Configfile /not/here does not exist', + str(context_mgr.exception)) + + def test_validateconfig_file_error_on_invalid_header(self): + """On invalid header, validate_cloudconfig_file errors. + + A SchemaValidationError is raised when the file doesn't begin with + CLOUD_CONFIG_HEADER. + """ + write_file(self.config_file, '#junk') + with self.assertRaises(SchemaValidationError) as context_mgr: + validate_cloudconfig_file(self.config_file, {}) + self.assertEqual( + 'Cloud config schema errors: header: File {0} needs to begin with ' + '"{1}"'.format(self.config_file, CLOUD_CONFIG_HEADER.decode()), + str(context_mgr.exception)) + + def test_validateconfig_file_error_on_non_yaml_format(self): + """On non-yaml format, validate_cloudconfig_file errors.""" + write_file(self.config_file, '#cloud-config\n{}}') + with self.assertRaises(SchemaValidationError) as context_mgr: + validate_cloudconfig_file(self.config_file, {}) + self.assertIn( + 'schema errors: format: File {0} is not valid yaml.'.format( + self.config_file), + str(context_mgr.exception)) + + def test_validateconfig_file_sctricty_validates_schema(self): + """validate_cloudconfig_file raises errors on invalid schema.""" + schema = { + 'properties': {'p1': {'type': 'string', 'format': 'hostname'}}} + write_file(self.config_file, '#cloud-config\np1: "-1"') + with self.assertRaises(SchemaValidationError) as context_mgr: + validate_cloudconfig_file(self.config_file, schema) + self.assertEqual( + "Cloud config schema errors: p1: '-1' is not a 'hostname'", + str(context_mgr.exception)) + + +class GetSchemaDocTest(CiTestCase): + """Tests for get_schema_doc.""" + + def setUp(self): + super(GetSchemaDocTest, self).setUp() + self.required_schema = { + 'title': 'title', 'description': 'description', 'id': 'id', + 'name': 'name', 'frequency': 'frequency', + 'distros': ['debian', 'rhel']} + + def test_get_schema_doc_returns_restructured_text(self): + """get_schema_doc returns restructured text for a cloudinit schema.""" + full_schema = copy(self.required_schema) + full_schema.update( + {'properties': { + 'prop1': {'type': 'array', 'description': 'prop-description', + 'items': {'type': 'int'}}}}) + self.assertEqual( + dedent(""" + name + --- + **Summary:** title + + description + + **Internal name:** ``id`` + + **Module frequency:** frequency + + **Supported distros:** debian, rhel + + **Config schema**: + **prop1:** (array of int) prop-description\n\n"""), + get_schema_doc(full_schema)) + + def test_get_schema_doc_returns_restructured_text_with_examples(self): + """get_schema_doc returns indented examples when present in schema.""" + full_schema = copy(self.required_schema) + full_schema.update( + {'examples': {'ex1': [1, 2, 3]}, + 'properties': { + 'prop1': {'type': 'array', 'description': 'prop-description', + 'items': {'type': 'int'}}}}) + self.assertIn( + dedent(""" + **Config schema**: + **prop1:** (array of int) prop-description + + **Examples**:: + + ex1"""), + get_schema_doc(full_schema)) + + def test_get_schema_doc_raises_key_errors(self): + """get_schema_doc raises KeyErrors on missing keys.""" + for key in self.required_schema: + invalid_schema = copy(self.required_schema) + invalid_schema.pop(key) + with self.assertRaises(KeyError) as context_mgr: + get_schema_doc(invalid_schema) + self.assertEqual("'{0}'".format(key), str(context_mgr.exception)) + + +class MainTest(CiTestCase): + + def test_main_missing_args(self): + """Main exits non-zero and reports an error on missing parameters.""" + with mock.patch('sys.argv', ['mycmd']): + with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr: + self.assertEqual(1, main(), 'Expected non-zero exit code') + self.assertEqual( + 'Expected either --config-file argument or --doc\n', + m_stderr.getvalue()) + + def test_main_prints_docs(self): + """When --doc parameter is provided, main generates documentation.""" + myargs = ['mycmd', '--doc'] + with mock.patch('sys.argv', myargs): + with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout: + self.assertEqual(0, main(), 'Expected 0 exit code') + self.assertIn('\nNTP\n---\n', m_stdout.getvalue()) + + def test_main_validates_config_file(self): + """When --config-file parameter is provided, main validates schema.""" + myyaml = self.tmp_path('my.yaml') + myargs = ['mycmd', '--config-file', myyaml] + with open(myyaml, 'wb') as stream: + stream.write(b'#cloud-config\nntp:') # shortest ntp schema + with mock.patch('sys.argv', myargs): + with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout: + self.assertEqual(0, main(), 'Expected 0 exit code') + self.assertIn( + 'Valid cloud-config file {0}'.format(myyaml), m_stdout.getvalue()) + +# vi: ts=4 expandtab syntax=python diff --git a/tools/cloudconfig-schema b/tools/cloudconfig-schema new file mode 100755 index 00000000..32f0d61e --- /dev/null +++ b/tools/cloudconfig-schema @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +# This file is part of cloud-init. See LICENSE file for license information. + +"""cloudconfig-schema + +Validate existing files against cloud-config schema or provide supported schema +documentation. +""" + +import os +import sys + + +def call_entry_point(name): + (istr, dot, ent) = name.rpartition('.') + try: + __import__(istr) + except ImportError: + # if that import failed, check dirname(__file__/..) + # to support ./bin/program with modules in . + _tdir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) + sys.path.insert(0, _tdir) + try: + __import__(istr) + except ImportError as e: + sys.stderr.write("Unable to find %s: %s\n" % (name, e)) + sys.exit(2) + + sys.exit(getattr(sys.modules[istr], ent)()) + + +if __name__ == '__main__': + call_entry_point("cloudinit.config.schema.main") + +# vi: ts=4 expandtab syntax=python -- cgit v1.2.3 From 5fb49bacf7441d8d20a7b4e0e7008ca586f5ebab Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Tue, 30 May 2017 10:28:05 -0600 Subject: azure: identify platform by well known value in chassis asset tag. Azure sets a known chassis asset tag to 7783-7084-3265-9085-8269-3286-77. We can inspect this in both ds-identify and DataSource.get_data to determine whether we are on Azure. Added unit tests to cover these changes and some minor tweaks to Exception error message content to give more context on malformed or missing ovf-env.xml files. LP: #1693939 --- cloudinit/sources/DataSourceAzure.py | 9 +++- tests/unittests/test_datasource/test_azure.py | 66 +++++++++++++++++++++++++-- tests/unittests/test_ds_identify.py | 39 ++++++++++++++++ tools/ds-identify | 35 +++++++++----- 4 files changed, 134 insertions(+), 15 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index b9458ffa..314848e4 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -36,6 +36,8 @@ RESOURCE_DISK_PATH = '/dev/disk/cloud/azure_resource' DEFAULT_PRIMARY_NIC = 'eth0' LEASE_FILE = '/var/lib/dhcp/dhclient.eth0.leases' DEFAULT_FS = 'ext4' +# DMI chassis-asset-tag is set static for all azure instances +AZURE_CHASSIS_ASSET_TAG = '7783-7084-3265-9085-8269-3286-77' def find_storvscid_from_sysctl_pnpinfo(sysctl_out, deviceid): @@ -320,6 +322,11 @@ class DataSourceAzureNet(sources.DataSource): # azure removes/ejects the cdrom containing the ovf-env.xml # file on reboot. So, in order to successfully reboot we # need to look in the datadir and consider that valid + asset_tag = util.read_dmi_data('chassis-asset-tag') + if asset_tag != AZURE_CHASSIS_ASSET_TAG: + LOG.debug("Non-Azure DMI asset tag '%s' discovered.", asset_tag) + return False + asset_tag = util.read_dmi_data('chassis-asset-tag') ddir = self.ds_cfg['data_dir'] candidates = [self.seed_dir] @@ -694,7 +701,7 @@ def read_azure_ovf(contents): try: dom = minidom.parseString(contents) except Exception as e: - raise BrokenAzureDataSource("invalid xml: %s" % e) + raise BrokenAzureDataSource("Invalid ovf-env.xml: %s" % e) results = find_child(dom.documentElement, lambda n: n.localName == "ProvisioningSection") diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 852ec703..42f49e06 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -76,7 +76,9 @@ def construct_valid_ovf_env(data=None, pubkeys=None, userdata=None): return content -class TestAzureDataSource(TestCase): +class TestAzureDataSource(CiTestCase): + + with_logs = True def setUp(self): super(TestAzureDataSource, self).setUp() @@ -160,6 +162,12 @@ scbus-1 on xpt0 bus 0 self.instance_id = 'test-instance-id' + def _dmi_mocks(key): + if key == 'system-uuid': + return self.instance_id + elif key == 'chassis-asset-tag': + return '7783-7084-3265-9085-8269-3286-77' + self.apply_patches([ (dsaz, 'list_possible_azure_ds_devs', dsdevs), (dsaz, 'invoke_agent', _invoke_agent), @@ -170,7 +178,7 @@ scbus-1 on xpt0 bus 0 (dsaz, 'set_hostname', mock.MagicMock()), (dsaz, 'get_metadata_from_fabric', self.get_metadata_from_fabric), (dsaz.util, 'read_dmi_data', mock.MagicMock( - return_value=self.instance_id)), + side_effect=_dmi_mocks)), ]) dsrc = dsaz.DataSourceAzureNet( @@ -241,6 +249,23 @@ fdescfs /dev/fd fdescfs rw 0 0 res = get_path_dev_freebsd('/etc', mnt_list) self.assertIsNotNone(res) + @mock.patch('cloudinit.sources.DataSourceAzure.util.read_dmi_data') + def test_non_azure_dmi_chassis_asset_tag(self, m_read_dmi_data): + """Report non-azure when DMI's chassis asset tag doesn't match. + + Return False when the asset tag doesn't match Azure's static + AZURE_CHASSIS_ASSET_TAG. + """ + # Return a non-matching asset tag value + nonazure_tag = dsaz.AZURE_CHASSIS_ASSET_TAG + 'X' + m_read_dmi_data.return_value = nonazure_tag + dsrc = dsaz.DataSourceAzureNet( + {}, distro=None, paths=self.paths) + self.assertFalse(dsrc.get_data()) + self.assertEqual( + "Non-Azure DMI asset tag '{0}' discovered.\n".format(nonazure_tag), + self.logs.getvalue()) + def test_basic_seed_dir(self): odata = {'HostName': "myhost", 'UserName': "myuser"} data = {'ovfcontent': construct_valid_ovf_env(data=odata), @@ -531,9 +556,17 @@ class TestAzureBounce(TestCase): self.patches.enter_context( mock.patch.object(dsaz, 'get_metadata_from_fabric', mock.MagicMock(return_value={}))) + + def _dmi_mocks(key): + if key == 'system-uuid': + return 'test-instance-id' + elif key == 'chassis-asset-tag': + return '7783-7084-3265-9085-8269-3286-77' + raise RuntimeError('should not get here') + self.patches.enter_context( mock.patch.object(dsaz.util, 'read_dmi_data', - mock.MagicMock(return_value='test-instance-id'))) + mock.MagicMock(side_effect=_dmi_mocks))) def setUp(self): super(TestAzureBounce, self).setUp() @@ -696,6 +729,33 @@ class TestAzureBounce(TestCase): self.assertEqual(0, self.set_hostname.call_count) +class TestLoadAzureDsDir(CiTestCase): + """Tests for load_azure_ds_dir.""" + + def setUp(self): + self.source_dir = self.tmp_dir() + super(TestLoadAzureDsDir, self).setUp() + + def test_missing_ovf_env_xml_raises_non_azure_datasource_error(self): + """load_azure_ds_dir raises an error When ovf-env.xml doesn't exit.""" + with self.assertRaises(dsaz.NonAzureDataSource) as context_manager: + dsaz.load_azure_ds_dir(self.source_dir) + self.assertEqual( + 'No ovf-env file found', + str(context_manager.exception)) + + def test_wb_invalid_ovf_env_xml_calls_read_azure_ovf(self): + """load_azure_ds_dir calls read_azure_ovf to parse the xml.""" + ovf_path = os.path.join(self.source_dir, 'ovf-env.xml') + with open(ovf_path, 'wb') as stream: + stream.write(b'invalid xml') + with self.assertRaises(dsaz.BrokenAzureDataSource) as context_manager: + dsaz.load_azure_ds_dir(self.source_dir) + self.assertEqual( + 'Invalid ovf-env.xml: syntax error: line 1, column 0', + str(context_manager.exception)) + + class TestReadAzureOvf(TestCase): def test_invalid_xml_raises_non_azure_ds(self): invalid_xml = "" + construct_valid_ovf_env(data={}) diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py index 5c26e65f..8ccfe55c 100644 --- a/tests/unittests/test_ds_identify.py +++ b/tests/unittests/test_ds_identify.py @@ -39,9 +39,11 @@ RC_FOUND = 0 RC_NOT_FOUND = 1 DS_NONE = 'None' +P_CHASSIS_ASSET_TAG = "sys/class/dmi/id/chassis_asset_tag" P_PRODUCT_NAME = "sys/class/dmi/id/product_name" P_PRODUCT_SERIAL = "sys/class/dmi/id/product_serial" P_PRODUCT_UUID = "sys/class/dmi/id/product_uuid" +P_SEED_DIR = "var/lib/cloud/seed" P_DSID_CFG = "etc/cloud/ds-identify.cfg" MOCK_VIRT_IS_KVM = {'name': 'detect_virt', 'RET': 'kvm', 'ret': 0} @@ -160,6 +162,30 @@ class TestDsIdentify(CiTestCase): _print_run_output(rc, out, err, cfg, files) return rc, out, err, cfg, files + def test_wb_print_variables(self): + """_print_info reports an array of discovered variables to stderr.""" + data = VALID_CFG['Azure-dmi-detection'] + _, _, err, _, _ = self._call_via_dict(data) + expected_vars = [ + 'DMI_PRODUCT_NAME', 'DMI_SYS_VENDOR', 'DMI_PRODUCT_SERIAL', + 'DMI_PRODUCT_UUID', 'PID_1_PRODUCT_NAME', 'DMI_CHASSIS_ASSET_TAG', + 'FS_LABELS', 'KERNEL_CMDLINE', 'VIRT', 'UNAME_KERNEL_NAME', + 'UNAME_KERNEL_RELEASE', 'UNAME_KERNEL_VERSION', 'UNAME_MACHINE', + 'UNAME_NODENAME', 'UNAME_OPERATING_SYSTEM', 'DSNAME', 'DSLIST', + 'MODE', 'ON_FOUND', 'ON_MAYBE', 'ON_NOTFOUND'] + for var in expected_vars: + self.assertIn('{0}='.format(var), err) + + def test_azure_dmi_detection_from_chassis_asset_tag(self): + """Azure datasource is detected from DMI chassis-asset-tag""" + self._test_ds_found('Azure-dmi-detection') + + def test_azure_seed_file_detection(self): + """Azure datasource is detected due to presence of a seed file. + + The seed file tested is /var/lib/cloud/seed/azure/ovf-env.xml.""" + self._test_ds_found('Azure-seed-detection') + def test_aws_ec2_hvm(self): """EC2: hvm instances use dmi serial and uuid starting with 'ec2'.""" self._test_ds_found('Ec2-hvm') @@ -272,6 +298,19 @@ VALID_CFG = { 'ds': 'AliYun', 'files': {P_PRODUCT_NAME: 'Alibaba Cloud ECS\n'}, }, + 'Azure-dmi-detection': { + 'ds': 'Azure', + 'files': { + P_CHASSIS_ASSET_TAG: '7783-7084-3265-9085-8269-3286-77\n', + } + }, + 'Azure-seed-detection': { + 'ds': 'Azure', + 'files': { + P_CHASSIS_ASSET_TAG: 'No-match\n', + os.path.join(P_SEED_DIR, 'azure', 'ovf-env.xml'): 'present\n', + } + }, 'Ec2-hvm': { 'ds': 'Ec2', 'mocks': [{'name': 'detect_virt', 'RET': 'kvm', 'ret': 0}], diff --git a/tools/ds-identify b/tools/ds-identify index 5fc500b9..546e0f59 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -85,6 +85,7 @@ DI_MAIN=${DI_MAIN:-main} DI_DEFAULT_POLICY="search,found=all,maybe=all,notfound=${DI_DISABLED}" DI_DEFAULT_POLICY_NO_DMI="search,found=all,maybe=all,notfound=${DI_ENABLED}" +DI_DMI_CHASSIS_ASSET_TAG="" DI_DMI_PRODUCT_NAME="" DI_DMI_SYS_VENDOR="" DI_DMI_PRODUCT_SERIAL="" @@ -259,6 +260,12 @@ read_kernel_cmdline() { DI_KERNEL_CMDLINE="$cmdline" } +read_dmi_chassis_asset_tag() { + cached "${DI_DMI_CHASSIS_ASSET_TAG}" && return + get_dmi_field chassis_asset_tag + DI_DMI_CHASSIS_ASSET_TAG="$_RET" +} + read_dmi_sys_vendor() { cached "${DI_DMI_SYS_VENDOR}" && return get_dmi_field sys_vendor @@ -386,6 +393,14 @@ read_pid1_product_name() { DI_PID_1_PRODUCT_NAME="$product_name" } +dmi_chassis_asset_tag_matches() { + is_container && return 1 + case "${DI_DMI_CHASSIS_ASSET_TAG}" in + $1) return 0;; + esac + return 1 +} + dmi_product_name_matches() { is_container && return 1 case "${DI_DMI_PRODUCT_NAME}" in @@ -402,11 +417,6 @@ dmi_product_serial_matches() { return 1 } -dmi_product_name_is() { - is_container && return 1 - [ "${DI_DMI_PRODUCT_NAME}" = "$1" ] -} - dmi_sys_vendor_is() { is_container && return 1 [ "${DI_DMI_SYS_VENDOR}" = "$1" ] @@ -478,7 +488,7 @@ dscheck_CloudStack() { dscheck_CloudSigma() { # http://paste.ubuntu.com/23624795/ - dmi_product_name_is "CloudSigma" && return $DS_FOUND + dmi_product_name_matches "CloudSigma" && return $DS_FOUND return $DS_NOT_FOUND } @@ -654,6 +664,8 @@ dscheck_Azure() { # UUID="112D211272645f72" LABEL="rd_rdfe_stable.161212-1209" # TYPE="udf">/dev/sr0 # + local azure_chassis="7783-7084-3265-9085-8269-3286-77" + dmi_chassis_asset_tag_matches "${azure_chassis}" && return $DS_FOUND check_seed_dir azure ovf-env.xml && return ${DS_FOUND} [ "${DI_VIRT}" = "microsoft" ] || return ${DS_NOT_FOUND} @@ -786,7 +798,7 @@ dscheck_Ec2() { } dscheck_GCE() { - if dmi_product_name_is "Google Compute Engine"; then + if dmi_product_name_matches "Google Compute Engine"; then return ${DS_FOUND} fi # product name is not guaranteed (LP: #1674861) @@ -807,10 +819,10 @@ dscheck_OpenStack() { return ${DS_NOT_FOUND} fi local nova="OpenStack Nova" compute="OpenStack Compute" - if dmi_product_name_is "$nova"; then + if dmi_product_name_matches "$nova"; then return ${DS_FOUND} fi - if dmi_product_name_is "$compute"; then + if dmi_product_name_matches "$compute"; then # RDO installed nova (LP: #1675349). return ${DS_FOUND} fi @@ -823,7 +835,7 @@ dscheck_OpenStack() { dscheck_AliYun() { check_seed_dir "AliYun" meta-data user-data && return ${DS_FOUND} - if dmi_product_name_is "Alibaba Cloud ECS"; then + if dmi_product_name_matches "Alibaba Cloud ECS"; then return $DS_FOUND fi return $DS_NOT_FOUND @@ -889,6 +901,7 @@ collect_info() { read_config read_datasource_list read_dmi_sys_vendor + read_dmi_chassis_asset_tag read_dmi_product_name read_dmi_product_serial read_dmi_product_uuid @@ -903,7 +916,7 @@ print_info() { _print_info() { local n="" v="" vars="" vars="DMI_PRODUCT_NAME DMI_SYS_VENDOR DMI_PRODUCT_SERIAL" - vars="$vars DMI_PRODUCT_UUID PID_1_PRODUCT_NAME" + vars="$vars DMI_PRODUCT_UUID PID_1_PRODUCT_NAME DMI_CHASSIS_ASSET_TAG" vars="$vars FS_LABELS KERNEL_CMDLINE VIRT" vars="$vars UNAME_KERNEL_NAME UNAME_KERNEL_RELEASE UNAME_KERNEL_VERSION" vars="$vars UNAME_MACHINE UNAME_NODENAME UNAME_OPERATING_SYSTEM" -- cgit v1.2.3 From 1cd4323b940408aa34dcaa01bd8a7ed43d9a966a Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 1 Jun 2017 12:40:12 -0400 Subject: azure: remove accidental duplicate line in merge. In previous commit I inadvertantly left two calls to asset_tag = util.read_dmi_data('chassis-asset-tag') The second did not do anything useful. Thus, remove it. --- cloudinit/sources/DataSourceAzure.py | 1 - 1 file changed, 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 314848e4..a0b9eaef 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -326,7 +326,6 @@ class DataSourceAzureNet(sources.DataSource): if asset_tag != AZURE_CHASSIS_ASSET_TAG: LOG.debug("Non-Azure DMI asset tag '%s' discovered.", asset_tag) return False - asset_tag = util.read_dmi_data('chassis-asset-tag') ddir = self.ds_cfg['data_dir'] candidates = [self.seed_dir] -- cgit v1.2.3 From 543e25cda6235d18adb6485e4266944d59e1979d Mon Sep 17 00:00:00 2001 From: Marc-Aurèle Brothier Date: Wed, 24 May 2017 14:45:25 +0200 Subject: net: when selecting a network device, use natural sort order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The code deciding which interface to choose as the default to request the IP address through DHCP does not sort the interfaces correctly. On Ubuntu Xenial images for example, the interfaces are named ens1, ens2, ens3..., ens11, ... depending on the pci bus address. The python sorting will list 'ens11' before 'ens3' for example despite the fact that 'ens3' should be before 'ens11'. This patch address this issue and sort the interface names according to a human sorting. Signed-off-by: Marc-Aurèle Brothier --- cloudinit/net/__init__.py | 13 ++++++++++++- tests/unittests/test_net.py | 14 ++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 8c6cd057..65accbb0 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -17,6 +17,17 @@ SYS_CLASS_NET = "/sys/class/net/" DEFAULT_PRIMARY_INTERFACE = 'eth0' +def _natural_sort_key(s, _nsre=re.compile('([0-9]+)')): + """Sorting for Humans: natural sort order. Can be use as the key to sort + functions. + This will sort ['eth0', 'ens3', 'ens10', 'ens12', 'ens8', 'ens0'] as + ['ens0', 'ens3', 'ens8', 'ens10', 'ens12', 'eth0'] instead of the simple + python way which will produce ['ens0', 'ens10', 'ens12', 'ens3', 'ens8', + 'eth0'].""" + return [int(text) if text.isdigit() else text.lower() + for text in re.split(_nsre, s)] + + def sys_dev_path(devname, path=""): return SYS_CLASS_NET + devname + "/" + path @@ -169,7 +180,7 @@ def generate_fallback_config(): # if eth0 exists use it above anything else, otherwise get the interface # that we can read 'first' (using the sorted defintion of first). - names = list(sorted(potential_interfaces)) + names = list(sorted(potential_interfaces, key=_natural_sort_key)) if DEFAULT_PRIMARY_INTERFACE in names: names.remove(DEFAULT_PRIMARY_INTERFACE) names.insert(0, DEFAULT_PRIMARY_INTERFACE) diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 5f7e902a..0000b075 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -1,6 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. from cloudinit import net +from cloudinit.net import _natural_sort_key from cloudinit.net import cmdline from cloudinit.net import eni from cloudinit.net import netplan @@ -1659,6 +1660,19 @@ class TestGetInterfacesByMac(CiTestCase): self.assertEqual('lo', ret[empty_mac]) +class TestInterfacesSorting(CiTestCase): + + def test_natural_order(self): + data = ['ens5', 'ens6', 'ens3', 'ens20', 'ens13', 'ens2'] + self.assertEqual( + sorted(data, key=_natural_sort_key), + ['ens2', 'ens3', 'ens5', 'ens6', 'ens13', 'ens20']) + data2 = ['enp2s0', 'enp2s3', 'enp0s3', 'enp0s13', 'enp0s8', 'enp1s2'] + self.assertEqual( + sorted(data2, key=_natural_sort_key), + ['enp0s3', 'enp0s8', 'enp0s13', 'enp1s2', 'enp2s0', 'enp2s3']) + + def _gzip_data(data): with io.BytesIO() as iobuf: gzfp = gzip.GzipFile(mode="wb", fileobj=iobuf) -- cgit v1.2.3 From 802e7cb2da8e2d0225525160e6edd6b58b275b8c Mon Sep 17 00:00:00 2001 From: Vladimir Pouzanov Date: Tue, 2 May 2017 16:08:34 +0100 Subject: NoCloud: support seed of nocloud from smbios information This allows the user to seed NoCloud in a trivial way from qemu/libvirt, by using a stock image and passing a single command line flag. No custom command line, no filesystem modification, no bootstrap disk image. This is particularly handy now that Ec2 backend is discouraged from use under bug 1660385. LP: #1691772 --- cloudinit/sources/DataSourceNoCloud.py | 12 ++++++++++++ doc/rtd/topics/datasources/nocloud.rst | 22 ++++++++++++++++++++++ tools/ds-identify | 3 +++ 3 files changed, 37 insertions(+) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index c68f6b8c..e641244d 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -42,6 +42,18 @@ class DataSourceNoCloud(sources.DataSource): mydata = {'meta-data': {}, 'user-data': "", 'vendor-data': "", 'network-config': None} + try: + # Parse the system serial label from dmi. If not empty, try parsing + # like the commandline + md = {} + serial = util.read_dmi_data('system-serial-number') + if serial and load_cmdline_data(md, serial): + found.append("dmi") + mydata = _merge_new_seed(mydata, {'meta-data': md}) + except Exception: + util.logexc(LOG, "Unable to parse dmi data") + return False + try: # Parse the kernel command line, getting data passed in md = {} diff --git a/doc/rtd/topics/datasources/nocloud.rst b/doc/rtd/topics/datasources/nocloud.rst index 0159e853..665057f3 100644 --- a/doc/rtd/topics/datasources/nocloud.rst +++ b/doc/rtd/topics/datasources/nocloud.rst @@ -11,6 +11,28 @@ You can provide meta-data and user-data to a local vm boot via files on a `vfat`_ or `iso9660`_ filesystem. The filesystem volume label must be ``cidata``. +Alternatively, you can provide meta-data via kernel command line or SMBIOS +"serial number" option. The data must be passed in the form of a string: + +:: + + ds=nocloud[;key=val;key=val] + +or + +:: + + ds=nocloud-net[;key=val;key=val] + +e.g. you can pass this option to QEMU: + +:: + + -smbios type=1,serial=ds=nocloud-net;s=http://10.10.0.1:8000/ + +to cause NoCloud to fetch the full meta-data from http://10.10.0.1:8000/meta-data +after the network initialization is complete. + These user-data and meta-data files are expected to be in the following format. :: diff --git a/tools/ds-identify b/tools/ds-identify index 546e0f59..7c8b144b 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -555,6 +555,9 @@ dscheck_NoCloud() { case " ${DI_KERNEL_CMDLINE} " in *\ ds=nocloud*) return ${DS_FOUND};; esac + case " ${DI_DMI_PRODUCT_SERIAL} " in + *\ ds=nocloud*) return ${DS_FOUND};; + esac for d in nocloud nocloud-net; do check_seed_dir "$d" meta-data user-data && return ${DS_FOUND} done -- cgit v1.2.3 From f99745cf916e707eaa1ded6f12e8b69837b7242d Mon Sep 17 00:00:00 2001 From: Andreas Karis Date: Tue, 6 Jun 2017 12:55:50 -0400 Subject: RHEL/CentOS: Fix default routes for IPv4/IPv6 configuration. Since f38fa413176, default routes get added to both ifcfg-* and route-* and route6-* files. Default routes should only go to ifcfg-* files, otherwise the information is redundant. LP: #1696176 --- cloudinit/net/sysconfig.py | 12 +++++++----- tests/unittests/test_net.py | 8 -------- 2 files changed, 7 insertions(+), 13 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index 58c5713f..f7d45482 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -372,11 +372,13 @@ class Renderer(renderer.Renderer): nm_key = 'NETMASK%s' % route_cfg.last_idx addr_key = 'ADDRESS%s' % route_cfg.last_idx route_cfg.last_idx += 1 - for (old_key, new_key) in [('gateway', gw_key), - ('netmask', nm_key), - ('network', addr_key)]: - if old_key in route: - route_cfg[new_key] = route[old_key] + # add default routes only to ifcfg files, not + # to route-* or route6-* + for (old_key, new_key) in [('gateway', gw_key), + ('netmask', nm_key), + ('network', addr_key)]: + if old_key in route: + route_cfg[new_key] = route[old_key] @classmethod def _render_bonding_opts(cls, iface_cfg, iface): diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 0000b075..0a88caf1 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -149,14 +149,6 @@ NM_CONTROLLED=no ONBOOT=yes TYPE=Ethernet USERCTL=no -""".lstrip()), - ('etc/sysconfig/network-scripts/route-eth0', - """ -# Created by cloud-init on instance boot automatically, do not edit. -# -ADDRESS0=0.0.0.0 -GATEWAY0=172.19.3.254 -NETMASK0=0.0.0.0 """.lstrip()), ('etc/resolv.conf', """ -- cgit v1.2.3 From 41d46bfb85929c79dabcec3cf21c8d71401fd2b8 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 28 Sep 2016 13:20:55 -0700 Subject: cloud.cfg: move to a template. setup.py changes along the way. Here we move the config/cloud.cfg to be rendered as a template. That allows us to maintain deltas between distros in one place. Currently we use 'variant' variable to make decisions. A tools/render-cloudcfg is provided to render the file. There were changes to setup.py, MANIFEST.in to allow us to put all files into a virtual env installation and to render the cloud-config file in 'install' or 'bdist' targets. We have also included some config changes that were found in the redhat distro spec. * include some config changes from the redhat distro spec. The rendered cloud.cfg has some differences. Ubuntu: white space and comment changes only. Freebsd: - whitespace changes and comment changes - datasource_list definition moved to be closer to 'datasource'. - enable modules: migrator, write_files - move package-update-upgrade-install to final. The initial work was done by Josh Harlow. --- MANIFEST.in | 11 ++- Makefile | 3 + cloudinit/util.py | 29 ++++++- config/cloud.cfg | 117 ---------------------------- config/cloud.cfg-freebsd | 88 --------------------- config/cloud.cfg.tmpl | 194 +++++++++++++++++++++++++++++++++++++++++++++++ setup.py | 168 ++++++++++++++++++++++++---------------- tools/render-cloudcfg | 43 +++++++++++ 8 files changed, 379 insertions(+), 274 deletions(-) delete mode 100644 config/cloud.cfg delete mode 100644 config/cloud.cfg-freebsd create mode 100644 config/cloud.cfg.tmpl create mode 100755 tools/render-cloudcfg (limited to 'cloudinit') diff --git a/MANIFEST.in b/MANIFEST.in index 94264640..1a4d7711 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,15 @@ -include *.py MANIFEST.in ChangeLog +include *.py MANIFEST.in LICENSE* ChangeLog global-include *.txt *.rst *.ini *.in *.conf *.cfg *.sh +graft config +graft doc +graft packages +graft systemd +graft sysvinit +graft templates +graft tests graft tools +graft udev +graft upstart prune build prune dist prune .tox diff --git a/Makefile b/Makefile index 66d1dcad..a3bfaf79 100644 --- a/Makefile +++ b/Makefile @@ -69,6 +69,9 @@ check_version: "not equal to code version '$(CODE_VERSION)'"; exit 2; \ else true; fi +config/cloud.cfg: + $(PYVER) ./tools/render-cloudcfg config/cloud.cfg.tmpl config/cloud.cfg + clean_pyc: @find . -type f -name "*.pyc" -delete diff --git a/cloudinit/util.py b/cloudinit/util.py index 135e4608..b8c3e4ee 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -592,13 +592,40 @@ def get_cfg_option_int(yobj, key, default=0): def system_info(): - return { + info = { 'platform': platform.platform(), 'release': platform.release(), 'python': platform.python_version(), 'uname': platform.uname(), 'dist': platform.linux_distribution(), # pylint: disable=W1505 } + plat = info['platform'].lower() + # Try to get more info about what it actually is, in a format + # that we can easily use across linux and variants... + if plat.startswith('darwin'): + info['variant'] = 'darwin' + elif plat.endswith("bsd"): + info['variant'] = 'bsd' + elif plat.startswith('win'): + info['variant'] = 'windows' + elif 'linux' in plat: + # Try to get a single string out of these... + linux_dist, _version, _id = info['dist'] + linux_dist = linux_dist.lower() + if linux_dist in ('ubuntu', 'linuxmint', 'mint'): + info['variant'] = 'ubuntu' + else: + for prefix, variant in [('redhat', 'rhel'), + ('centos', 'centos'), + ('fedora', 'fedora'), + ('debian', 'debian')]: + if linux_dist.startswith(prefix): + info['variant'] = variant + if 'variant' not in info: + info['variant'] = 'linux' + if 'variant' not in info: + info['variant'] = 'unknown' + return info def get_cfg_option_list(yobj, key, default=None): diff --git a/config/cloud.cfg b/config/cloud.cfg deleted file mode 100644 index 1b93e7f9..00000000 --- a/config/cloud.cfg +++ /dev/null @@ -1,117 +0,0 @@ -# The top level settings are used as module -# and system configuration. - -# A set of users which may be applied and/or used by various modules -# when a 'default' entry is found it will reference the 'default_user' -# from the distro configuration specified below -users: - - default - -# If this is set, 'root' will not be able to ssh in and they -# will get a message to login instead as the above $user (ubuntu) -disable_root: true - -# This will cause the set+update hostname module to not operate (if true) -preserve_hostname: false - -# Example datasource config -# datasource: -# Ec2: -# metadata_urls: [ 'blah.com' ] -# timeout: 5 # (defaults to 50 seconds) -# max_wait: 10 # (defaults to 120 seconds) - -# The modules that run in the 'init' stage -cloud_init_modules: - - migrator - - ubuntu-init-switch - - seed_random - - bootcmd - - write-files - - growpart - - resizefs - - disk_setup - - mounts - - set_hostname - - update_hostname - - update_etc_hosts - - ca-certs - - rsyslog - - users-groups - - ssh - -# The modules that run in the 'config' stage -cloud_config_modules: -# Emit the cloud config ready event -# this can be used by upstart jobs for 'start on cloud-config'. - - emit_upstart - - snap_config - - ssh-import-id - - locale - - set-passwords - - grub-dpkg - - apt-pipelining - - apt-configure - - ntp - - timezone - - disable-ec2-metadata - - runcmd - - byobu - -# The modules that run in the 'final' stage -cloud_final_modules: - - snappy - - package-update-upgrade-install - - fan - - landscape - - lxd - - puppet - - chef - - salt-minion - - mcollective - - rightscale_userdata - - scripts-vendor - - scripts-per-once - - scripts-per-boot - - scripts-per-instance - - scripts-user - - ssh-authkey-fingerprints - - keys-to-console - - phone-home - - final-message - - power-state-change - -# System and/or distro specific settings -# (not accessible to handlers/transforms) -system_info: - # This will affect which distro class gets used - distro: ubuntu - # Default user name + that default users groups (if added/used) - default_user: - name: ubuntu - lock_passwd: True - gecos: Ubuntu - groups: [adm, audio, cdrom, dialout, dip, floppy, lxd, netdev, plugdev, sudo, video] - sudo: ["ALL=(ALL) NOPASSWD:ALL"] - shell: /bin/bash - # Other config here will be given to the distro class and/or path classes - paths: - cloud_dir: /var/lib/cloud/ - templates_dir: /etc/cloud/templates/ - upstart_dir: /etc/init/ - package_mirrors: - - arches: [i386, amd64] - failsafe: - primary: http://archive.ubuntu.com/ubuntu - security: http://security.ubuntu.com/ubuntu - search: - primary: - - http://%(ec2_region)s.ec2.archive.ubuntu.com/ubuntu/ - - http://%(availability_zone)s.clouds.archive.ubuntu.com/ubuntu/ - - http://%(region)s.clouds.archive.ubuntu.com/ubuntu/ - security: [] - - arches: [armhf, armel, default] - failsafe: - primary: http://ports.ubuntu.com/ubuntu-ports - security: http://ports.ubuntu.com/ubuntu-ports - ssh_svcname: ssh diff --git a/config/cloud.cfg-freebsd b/config/cloud.cfg-freebsd deleted file mode 100644 index d666c397..00000000 --- a/config/cloud.cfg-freebsd +++ /dev/null @@ -1,88 +0,0 @@ -# The top level settings are used as module -# and system configuration. - -syslog_fix_perms: root:wheel - -# This should not be required, but leave it in place until the real cause of -# not beeing able to find -any- datasources is resolved. -datasource_list: ['ConfigDrive', 'Azure', 'OpenStack', 'Ec2'] - -# A set of users which may be applied and/or used by various modules -# when a 'default' entry is found it will reference the 'default_user' -# from the distro configuration specified below -users: - - default - -# If this is set, 'root' will not be able to ssh in and they -# will get a message to login instead as the above $user (ubuntu) -disable_root: false - -# This will cause the set+update hostname module to not operate (if true) -preserve_hostname: false - -# Example datasource config -# datasource: -# Ec2: -# metadata_urls: [ 'blah.com' ] -# timeout: 5 # (defaults to 50 seconds) -# max_wait: 10 # (defaults to 120 seconds) - -# The modules that run in the 'init' stage -cloud_init_modules: -# - migrator - - seed_random - - bootcmd -# - write-files - - growpart - - resizefs - - set_hostname - - update_hostname -# - update_etc_hosts -# - ca-certs -# - rsyslog - - users-groups - - ssh - -# The modules that run in the 'config' stage -cloud_config_modules: -# - disk_setup -# - mounts - - ssh-import-id - - locale - - set-passwords - - package-update-upgrade-install -# - landscape - - timezone -# - puppet -# - chef -# - salt-minion -# - mcollective - - disable-ec2-metadata - - runcmd -# - byobu - -# The modules that run in the 'final' stage -cloud_final_modules: - - rightscale_userdata - - scripts-vendor - - scripts-per-once - - scripts-per-boot - - scripts-per-instance - - scripts-user - - ssh-authkey-fingerprints - - keys-to-console - - phone-home - - final-message - - power-state-change - -# System and/or distro specific settings -# (not accessible to handlers/transforms) -system_info: - distro: freebsd - default_user: - name: freebsd - lock_passwd: True - gecos: FreeBSD - groups: [wheel] - sudo: ["ALL=(ALL) NOPASSWD:ALL"] - shell: /bin/tcsh diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl new file mode 100644 index 00000000..5af2a88f --- /dev/null +++ b/config/cloud.cfg.tmpl @@ -0,0 +1,194 @@ +## template:jinja +# The top level settings are used as module +# and system configuration. + +{% if variant in ["bsd"] %} +syslog_fix_perms: root:wheel +{% endif %} +# A set of users which may be applied and/or used by various modules +# when a 'default' entry is found it will reference the 'default_user' +# from the distro configuration specified below +users: + - default + +# If this is set, 'root' will not be able to ssh in and they +# will get a message to login instead as the default $user +{% if variant in ["bsd"] %} +disable_root: false +{% else %} +disable_root: true +{% endif %} + +{% if variant in ["centos", "fedora", "rhel"] %} +mount_default_fields: [~, ~, 'auto', 'defaults,nofail', '0', '2'] +resize_rootfs_tmp: /dev +ssh_deletekeys: 0 +ssh_genkeytypes: ~ +ssh_pwauth: 0 + +{% endif %} +# This will cause the set+update hostname module to not operate (if true) +preserve_hostname: false + +{% if variant in ["bsd"] %} +# This should not be required, but leave it in place until the real cause of +# not beeing able to find -any- datasources is resolved. +datasource_list: ['ConfigDrive', 'Azure', 'OpenStack', 'Ec2'] +{% endif %} +# Example datasource config +# datasource: +# Ec2: +# metadata_urls: [ 'blah.com' ] +# timeout: 5 # (defaults to 50 seconds) +# max_wait: 10 # (defaults to 120 seconds) + +# The modules that run in the 'init' stage +cloud_init_modules: + - migrator +{% if variant in ["ubuntu", "unknown", "debian"] %} + - ubuntu-init-switch +{% endif %} + - seed_random + - bootcmd + - write-files + - growpart + - resizefs +{% if variant not in ["bsd"] %} + - disk_setup + - mounts +{% endif %} + - set_hostname + - update_hostname +{% if variant not in ["bsd"] %} + - update_etc_hosts + - ca-certs + - rsyslog +{% endif %} + - users-groups + - ssh + +# The modules that run in the 'config' stage +cloud_config_modules: +{% if variant in ["ubuntu", "unknown", "debian"] %} +# Emit the cloud config ready event +# this can be used by upstart jobs for 'start on cloud-config'. + - emit_upstart + - snap_config +{% endif %} + - ssh-import-id + - locale + - set-passwords +{% if variant in ["rhel", "fedora"] %} + - spacewalk + - yum-add-repo +{% endif %} +{% if variant in ["ubuntu", "unknown", "debian"] %} + - grub-dpkg + - apt-pipelining + - apt-configure +{% endif %} +{% if variant not in ["bsd"] %} + - ntp +{% endif %} + - timezone + - disable-ec2-metadata + - runcmd +{% if variant in ["ubuntu", "unknown", "debian"] %} + - byobu +{% endif %} + +# The modules that run in the 'final' stage +cloud_final_modules: +{% if variant in ["ubuntu", "unknown", "debian"] %} + - snappy +{% endif %} + - package-update-upgrade-install +{% if variant in ["ubuntu", "unknown", "debian"] %} + - fan + - landscape + - lxd +{% endif %} +{% if variant not in ["bsd"] %} + - puppet + - chef + - salt-minion + - mcollective +{% endif %} + - rightscale_userdata + - scripts-vendor + - scripts-per-once + - scripts-per-boot + - scripts-per-instance + - scripts-user + - ssh-authkey-fingerprints + - keys-to-console + - phone-home + - final-message + - power-state-change + +# System and/or distro specific settings +# (not accessible to handlers/transforms) +system_info: + # This will affect which distro class gets used +{% if variant in ["centos", "debian", "fedora", "rhel", "ubuntu"] %} + distro: {{ variant }} +{% elif variant in ["bsd"] %} + distro: freebsd +{% else %} + # Unknown/fallback distro. + distro: ubuntu +{% endif %} +{% if variant in ["ubuntu", "unknown", "debian"] %} + # Default user name + that default users groups (if added/used) + default_user: + name: ubuntu + lock_passwd: True + gecos: Ubuntu + groups: [adm, audio, cdrom, dialout, dip, floppy, lxd, netdev, plugdev, sudo, video] + sudo: ["ALL=(ALL) NOPASSWD:ALL"] + shell: /bin/bash + # Other config here will be given to the distro class and/or path classes + paths: + cloud_dir: /var/lib/cloud/ + templates_dir: /etc/cloud/templates/ + upstart_dir: /etc/init/ + package_mirrors: + - arches: [i386, amd64] + failsafe: + primary: http://archive.ubuntu.com/ubuntu + security: http://security.ubuntu.com/ubuntu + search: + primary: + - http://%(ec2_region)s.ec2.archive.ubuntu.com/ubuntu/ + - http://%(availability_zone)s.clouds.archive.ubuntu.com/ubuntu/ + - http://%(region)s.clouds.archive.ubuntu.com/ubuntu/ + security: [] + - arches: [armhf, armel, default] + failsafe: + primary: http://ports.ubuntu.com/ubuntu-ports + security: http://ports.ubuntu.com/ubuntu-ports + ssh_svcname: ssh +{% elif variant in ["centos", "rhel", "fedora"] %} + # Default user name + that default users groups (if added/used) + default_user: + name: {{ variant }} + lock_passwd: True + gecos: {{ variant }} Cloud User + groups: [wheel, adm, systemd-journal] + sudo: ["ALL=(ALL) NOPASSWD:ALL"] + shell: /bin/bash + # Other config here will be given to the distro class and/or path classes + paths: + cloud_dir: /var/lib/cloud/ + templates_dir: /etc/cloud/templates/ + ssh_svcname: sshd +{% elif variant in ["bsd"] %} + # Default user name + that default users groups (if added/used) + default_user: + name: freebsd + lock_passwd: True + gecos: FreeBSD + groups: [wheel] + sudo: ["ALL=(ALL) NOPASSWD:ALL"] + shell: /bin/tcsh +{% endif %} diff --git a/setup.py b/setup.py index 4616599b..d5223285 100755 --- a/setup.py +++ b/setup.py @@ -10,8 +10,11 @@ from glob import glob +import atexit import os +import shutil import sys +import tempfile import setuptools from setuptools.command.install import install @@ -53,47 +56,15 @@ def pkg_config_read(library, var): cmd = ['pkg-config', '--variable=%s' % var, library] try: (path, err) = tiny_p(cmd) + path = path.strip() except Exception: - return fallbacks[library][var] - return str(path).strip() + path = fallbacks[library][var] + if path.startswith("/"): + path = path[1:] + return path -INITSYS_FILES = { - 'sysvinit': [f for f in glob('sysvinit/redhat/*') if is_f(f)], - 'sysvinit_freebsd': [f for f in glob('sysvinit/freebsd/*') if is_f(f)], - 'sysvinit_deb': [f for f in glob('sysvinit/debian/*') if is_f(f)], - 'sysvinit_openrc': [f for f in glob('sysvinit/gentoo/*') if is_f(f)], - 'systemd': [f for f in (glob('systemd/*.service') + - glob('systemd/*.target')) if is_f(f)], - 'systemd.generators': [f for f in glob('systemd/*-generator') if is_f(f)], - 'upstart': [f for f in glob('upstart/*') if is_f(f)], -} -INITSYS_ROOTS = { - 'sysvinit': '/etc/rc.d/init.d', - 'sysvinit_freebsd': '/usr/local/etc/rc.d', - 'sysvinit_deb': '/etc/init.d', - 'sysvinit_openrc': '/etc/init.d', - 'systemd': pkg_config_read('systemd', 'systemdsystemunitdir'), - 'systemd.generators': pkg_config_read('systemd', - 'systemdsystemgeneratordir'), - 'upstart': '/etc/init/', -} -INITSYS_TYPES = sorted([f.partition(".")[0] for f in INITSYS_ROOTS.keys()]) - -# Install everything in the right location and take care of Linux (default) and -# FreeBSD systems. -USR = "/usr" -ETC = "/etc" -USR_LIB_EXEC = "/usr/lib" -LIB = "/lib" -if os.uname()[0] == 'FreeBSD': - USR = "/usr/local" - USR_LIB_EXEC = "/usr/local/lib" -elif os.path.isfile('/etc/redhat-release'): - USR_LIB_EXEC = "/usr/libexec" - -# Avoid having datafiles installed in a virtualenv... def in_virtualenv(): try: if sys.real_prefix == sys.prefix: @@ -116,6 +87,66 @@ def read_requires(): return str(deps).splitlines() +def render_cloud_cfg(): + """render cloud.cfg into a tmpdir under same dir as setup.py + + This is rendered to a temporary directory under the top level + directory with the name 'cloud.cfg'. The reason for not just rendering + to config/cloud.cfg is for a.) don't want to write over contents + in that file if user had something there. b.) debuild will complain + that files are different outside of the debian directory.""" + + # older versions of tox use bdist (xenial), and then install from there. + # newer versions just use install. + if not (sys.argv[1] == 'install' or sys.argv[1].startswith('bdist*')): + return 'config/cloud.cfg.tmpl' + topdir = os.path.dirname(sys.argv[0]) + tmpd = tempfile.mkdtemp(dir=topdir) + atexit.register(shutil.rmtree, tmpd) + fpath = os.path.join(tmpd, 'cloud.cfg') + tiny_p([sys.executable, './tools/render-cloudcfg', + 'config/cloud.cfg.tmpl', fpath]) + # relpath is relative to setup.py + relpath = os.path.join(os.path.basename(tmpd), 'cloud.cfg') + return relpath + + +INITSYS_FILES = { + 'sysvinit': [f for f in glob('sysvinit/redhat/*') if is_f(f)], + 'sysvinit_freebsd': [f for f in glob('sysvinit/freebsd/*') if is_f(f)], + 'sysvinit_deb': [f for f in glob('sysvinit/debian/*') if is_f(f)], + 'sysvinit_openrc': [f for f in glob('sysvinit/gentoo/*') if is_f(f)], + 'systemd': [f for f in (glob('systemd/*.service') + + glob('systemd/*.target')) if is_f(f)], + 'systemd.generators': [f for f in glob('systemd/*-generator') if is_f(f)], + 'upstart': [f for f in glob('upstart/*') if is_f(f)], +} +INITSYS_ROOTS = { + 'sysvinit': 'etc/rc.d/init.d', + 'sysvinit_freebsd': 'usr/local/etc/rc.d', + 'sysvinit_deb': 'etc/init.d', + 'sysvinit_openrc': 'etc/init.d', + 'systemd': pkg_config_read('systemd', 'systemdsystemunitdir'), + 'systemd.generators': pkg_config_read('systemd', + 'systemdsystemgeneratordir'), + 'upstart': 'etc/init/', +} +INITSYS_TYPES = sorted([f.partition(".")[0] for f in INITSYS_ROOTS.keys()]) + + +# Install everything in the right location and take care of Linux (default) and +# FreeBSD systems. +USR = "usr" +ETC = "etc" +USR_LIB_EXEC = "usr/lib" +LIB = "lib" +if os.uname()[0] == 'FreeBSD': + USR = "usr/local" + USR_LIB_EXEC = "usr/local/lib" +elif os.path.isfile('/etc/redhat-release'): + USR_LIB_EXEC = "usr/libexec" + + # TODO: Is there a better way to do this?? class InitsysInstallData(install): init_system = None @@ -155,36 +186,39 @@ class InitsysInstallData(install): self.distribution.reinitialize_command('install_data', True) -if in_virtualenv(): - data_files = [] - cmdclass = {} -else: - data_files = [ - (ETC + '/cloud', glob('config/*.cfg')), - (ETC + '/cloud/cloud.cfg.d', glob('config/cloud.cfg.d/*')), - (ETC + '/cloud/templates', glob('templates/*')), - (USR_LIB_EXEC + '/cloud-init', ['tools/ds-identify', - 'tools/uncloud-init', - 'tools/write-ssh-key-fingerprints']), - (USR + '/share/doc/cloud-init', [f for f in glob('doc/*') if is_f(f)]), - (USR + '/share/doc/cloud-init/examples', - [f for f in glob('doc/examples/*') if is_f(f)]), - (USR + '/share/doc/cloud-init/examples/seed', - [f for f in glob('doc/examples/seed/*') if is_f(f)]), - ] - if os.uname()[0] != 'FreeBSD': - data_files.extend([ - (ETC + '/NetworkManager/dispatcher.d/', - ['tools/hook-network-manager']), - (ETC + '/dhcp/dhclient-exit-hooks.d/', ['tools/hook-dhclient']), - (LIB + '/udev/rules.d', [f for f in glob('udev/*.rules')]) - ]) - # Use a subclass for install that handles - # adding on the right init system configuration files - cmdclass = { - 'install': InitsysInstallData, - } - +if not in_virtualenv(): + USR = "/" + USR + ETC = "/" + ETC + USR_LIB_EXEC = "/" + USR_LIB_EXEC + LIB = "/" + LIB + for k in INITSYS_ROOTS.keys(): + INITSYS_ROOTS[k] = "/" + INITSYS_ROOTS[k] + +data_files = [ + (ETC + '/cloud', [render_cloud_cfg()]), + (ETC + '/cloud/cloud.cfg.d', glob('config/cloud.cfg.d/*')), + (ETC + '/cloud/templates', glob('templates/*')), + (USR_LIB_EXEC + '/cloud-init', ['tools/ds-identify', + 'tools/uncloud-init', + 'tools/write-ssh-key-fingerprints']), + (USR + '/share/doc/cloud-init', [f for f in glob('doc/*') if is_f(f)]), + (USR + '/share/doc/cloud-init/examples', + [f for f in glob('doc/examples/*') if is_f(f)]), + (USR + '/share/doc/cloud-init/examples/seed', + [f for f in glob('doc/examples/seed/*') if is_f(f)]), +] +if os.uname()[0] != 'FreeBSD': + data_files.extend([ + (ETC + '/NetworkManager/dispatcher.d/', + ['tools/hook-network-manager']), + (ETC + '/dhcp/dhclient-exit-hooks.d/', ['tools/hook-dhclient']), + (LIB + '/udev/rules.d', [f for f in glob('udev/*.rules')]) + ]) +# Use a subclass for install that handles +# adding on the right init system configuration files +cmdclass = { + 'install': InitsysInstallData, +} requirements = read_requires() if sys.version_info < (3,): diff --git a/tools/render-cloudcfg b/tools/render-cloudcfg new file mode 100755 index 00000000..e624541a --- /dev/null +++ b/tools/render-cloudcfg @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +import argparse +import os +import sys + +if "avoid-pep8-E402-import-not-top-of-file": + _tdir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) + sys.path.insert(0, _tdir) + from cloudinit import templater + from cloudinit import util + from cloudinit.atomic_helper import write_file + + +def main(): + parser = argparse.ArgumentParser() + variants = ["bsd", "centos", "fedora", "rhel", "ubuntu", "unknown"] + platform = util.system_info() + parser.add_argument( + "--variant", default=platform['variant'], action="store", + help="define the variant.", choices=variants) + parser.add_argument( + "template", nargs="?", action="store", + default='./config/cloud.cfg.tmpl', + help="Path to the cloud.cfg template") + parser.add_argument( + "output", nargs="?", action="store", default="-", + help="Output file. Use '-' to write to stdout") + + args = parser.parse_args() + + with open(args.template, 'r') as fh: + contents = fh.read() + tpl_params = {'variant': args.variant} + contents = (templater.render_string(contents, tpl_params)).rstrip() + "\n" + util.load_yaml(contents) + if args.output == "-": + sys.stdout.write(contents) + else: + write_file(args.output, contents, omode="w") + +if __name__ == '__main__': + main() -- cgit v1.2.3 From 914822a765007be7e17539e456b3e6ff12b19442 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 7 Jun 2017 11:32:56 -0400 Subject: rhel/centos spec cleanups. Many changes here to get us able to build rpms on CentOS 5 or 6 and RHEL. * add 'Requires' as 'BuildRequires' also. This allows us to run cloud-init tools in the build environment, and also will allow us to run tests in the build process. * build for both systemd and upstart (centos 5) init systems. * Add 'centos' as a variant Adding the variant means we can use the 'centos' user as default on centos rather than a 'fedora' or 'rhel'. * drop argparse from the requirements. On any system other than python 2.6, having a 'requirements' that mentions argparse just causes problems. Instead we add that Requires to the spec directly. * list dependency on dmidecode (as redhat distro spec had) * remove duplicate line in files section ({_unitdir}/cloud-*) * Use rpm macros for init-system chunks and drop use of init_system variable template * Add el6 only build-req on python-argparse * python-cheetah is not required in the build environment as the the spec is already rendered. (We will soon move the spec to jinja). --- cloudinit/distros/__init__.py | 2 +- cloudinit/distros/centos.py | 12 +++++ packages/redhat/cloud-init.spec.in | 92 +++++++++++++++++++++----------------- requirements.txt | 3 -- 4 files changed, 65 insertions(+), 44 deletions(-) create mode 100644 cloudinit/distros/centos.py (limited to 'cloudinit') diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index f56c0cf7..1fd48a7b 100755 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -32,7 +32,7 @@ from cloudinit.distros.parsers import hosts OSFAMILIES = { 'debian': ['debian', 'ubuntu'], - 'redhat': ['fedora', 'rhel'], + 'redhat': ['centos', 'fedora', 'rhel'], 'gentoo': ['gentoo'], 'freebsd': ['freebsd'], 'suse': ['sles'], diff --git a/cloudinit/distros/centos.py b/cloudinit/distros/centos.py new file mode 100644 index 00000000..4b803d2e --- /dev/null +++ b/cloudinit/distros/centos.py @@ -0,0 +1,12 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit.distros import rhel +from cloudinit import log as logging + +LOG = logging.getLogger(__name__) + + +class Distro(rhel.Distro): + pass + +# vi: ts=4 expandtab diff --git a/packages/redhat/cloud-init.spec.in b/packages/redhat/cloud-init.spec.in index fd3cf938..1939ca88 100644 --- a/packages/redhat/cloud-init.spec.in +++ b/packages/redhat/cloud-init.spec.in @@ -1,6 +1,12 @@ ## template: cheetah %{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")} +%if %{undefined systemd_requires} +%define init_system sysvinit +%else +%define init_system systemd +%endif + # See: http://www.zarb.org/~jasonc/macros.php # Or: http://fedoraproject.org/wiki/Packaging:ScriptletSnippets # Or: http://www.rpm.org/max-rpm/ch-rpm-inside.html @@ -20,9 +26,20 @@ BuildRoot: %{_tmppath} BuildRequires: python-devel BuildRequires: python-setuptools -BuildRequires: python-cheetah +%if "%{?el6}" == "1" +BuildRequires: python-argparse +%endif +# These are runtime dependencies, but declared as BuildRequires so that +# - tests can be run here. +# - parts of cloud-init such (setup.py) use these dependencies. +#for $r in $requires +BuildRequires: ${r} +#end for # System util packages needed +%ifarch %{?ix86} x86_64 ia64 +Requires: dmidecode +%endif Requires: shadow-utils Requires: rsyslog Requires: iproute @@ -32,6 +49,12 @@ Requires: procps Requires: shadow-utils Requires: sudo >= 1.7.2p2-3 +Requires: python-setuptools +# python2.6 needs argparse +%if "%{?el6}" == "1" +Requires: python-argparse +%endif + # Install pypi 'dynamic' requirements #for $r in $requires Requires: ${r} @@ -44,19 +67,15 @@ Patch${size}: $p #set $size += 1 #end for -#if $sysvinit +%if "%{init_system}" == "systemd" +BuildRequires: systemd-units +%{systemd_requires} +%else Requires(post): chkconfig Requires(postun): initscripts Requires(preun): chkconfig Requires(preun): initscripts -#end if - -#if $systemd -BuildRequires: systemd-units -Requires(post): systemd-units -Requires(postun): systemd-units -Requires(preun): systemd-units -#end if +%endif %description Cloud-init is a set of init scripts for cloud instances. Cloud instances @@ -80,7 +99,7 @@ ssh keys and to let the user run various scripts. %{__python} setup.py install -O1 \ --skip-build --root \$RPM_BUILD_ROOT \ - --init-system=${init_sys} + --init-system=%{init_system} # Note that /etc/rsyslog.d didn't exist by default until F15. # el6 request: https://bugzilla.redhat.com/show_bug.cgi?id=740420 @@ -95,17 +114,17 @@ rm -rf \$RPM_BUILD_ROOT%{python_sitelib}/tests mkdir -p \$RPM_BUILD_ROOT/%{_sharedstatedir}/cloud mkdir -p \$RPM_BUILD_ROOT/%{_libexecdir}/%{name} -#if $systemd +%if "%{init_system}" == "systemd" mkdir -p \$RPM_BUILD_ROOT/%{_unitdir} cp -p systemd/* \$RPM_BUILD_ROOT/%{_unitdir} -#end if +%endif %clean rm -rf \$RPM_BUILD_ROOT %post -#if $systemd +%if "%{init_system}" == "systemd" if [ \$1 -eq 1 ] then /bin/systemctl enable cloud-config.service >/dev/null 2>&1 || : @@ -113,18 +132,24 @@ then /bin/systemctl enable cloud-init.service >/dev/null 2>&1 || : /bin/systemctl enable cloud-init-local.service >/dev/null 2>&1 || : fi -#end if - -#if $sysvinit +%else /sbin/chkconfig --add %{_initrddir}/cloud-init-local /sbin/chkconfig --add %{_initrddir}/cloud-init /sbin/chkconfig --add %{_initrddir}/cloud-config /sbin/chkconfig --add %{_initrddir}/cloud-final -#end if +%endif %preun -#if $sysvinit +%if "%{init_system}" == "systemd" +if [ \$1 -eq 0 ] +then + /bin/systemctl --no-reload disable cloud-config.service >/dev/null 2>&1 || : + /bin/systemctl --no-reload disable cloud-final.service >/dev/null 2>&1 || : + /bin/systemctl --no-reload disable cloud-init.service >/dev/null 2>&1 || : + /bin/systemctl --no-reload disable cloud-init-local.service >/dev/null 2>&1 || : +fi +%else if [ \$1 -eq 0 ] then /sbin/service cloud-init stop >/dev/null 2>&1 || : @@ -136,40 +161,27 @@ then /sbin/service cloud-final stop >/dev/null 2>&1 || : /sbin/chkconfig --del cloud-final || : fi -#end if - -#if $systemd -if [ \$1 -eq 0 ] -then - /bin/systemctl --no-reload disable cloud-config.service >/dev/null 2>&1 || : - /bin/systemctl --no-reload disable cloud-final.service >/dev/null 2>&1 || : - /bin/systemctl --no-reload disable cloud-init.service >/dev/null 2>&1 || : - /bin/systemctl --no-reload disable cloud-init-local.service >/dev/null 2>&1 || : -fi -#end if +%endif %postun -#if $systemd +%if "%{init_system}" == "systemd" /bin/systemctl daemon-reload >/dev/null 2>&1 || : -#end if +%endif %files /lib/udev/rules.d/66-azure-ephemeral.rules -#if $sysvinit +%if "%{init_system}" == "systemd" +/usr/lib/systemd/system-generators/cloud-init-generator +%{_unitdir}/cloud-* +%else %attr(0755, root, root) %{_initddir}/cloud-config %attr(0755, root, root) %{_initddir}/cloud-final %attr(0755, root, root) %{_initddir}/cloud-init-local %attr(0755, root, root) %{_initddir}/cloud-init -#end if - -#if $systemd -/usr/lib/systemd/system-generators/cloud-init-generator -%{_unitdir}/cloud-* -%{_unitdir}/cloud-* -#end if +%endif %{_sysconfdir}/NetworkManager/dispatcher.d/hook-network-manager %{_sysconfdir}/dhcp/dhclient-exit-hooks.d/hook-dhclient diff --git a/requirements.txt b/requirements.txt index 60abab16..59ec06d0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,9 +27,6 @@ configobj>=5.0.2 # All new style configurations are in the yaml format pyyaml -# The new main entrypoint uses argparse instead of optparse -argparse - # Requests handles ssl correctly! requests -- cgit v1.2.3 From ad2680a689ab78847ccce7766d6591797d99e219 Mon Sep 17 00:00:00 2001 From: JJ Asghar Date: Mon, 5 Jun 2017 20:36:12 -0500 Subject: Chef: Update omnibus url to chef.io, minor doc changes. - Updated to standard chef.io url - Removed the port 4000, due to that has been deprecated - Added Note about the run_list not being required Signed-off-by: JJ Asghar --- cloudinit/config/cc_chef.py | 2 +- doc/examples/cloud-config-chef.txt | 12 ++++++------ .../configs/examples/install_run_chef_recipes.yaml | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 2be2532c..02c70b10 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -92,7 +92,7 @@ REQUIRED_CHEF_DIRS = tuple([ ]) # Used if fetching chef from a omnibus style package -OMNIBUS_URL = "https://www.getchef.com/chef/install.sh" +OMNIBUS_URL = "https://www.chef.io/chef/install.sh" OMNIBUS_URL_RETRIES = 5 CHEF_VALIDATION_PEM_PATH = '/etc/chef/validation.pem' diff --git a/doc/examples/cloud-config-chef.txt b/doc/examples/cloud-config-chef.txt index 3cb62006..9d235817 100644 --- a/doc/examples/cloud-config-chef.txt +++ b/doc/examples/cloud-config-chef.txt @@ -1,6 +1,6 @@ #cloud-config # -# This is an example file to automatically install chef-client and run a +# This is an example file to automatically install chef-client and run a # list of recipes when the instance boots for the first time. # Make sure that this file is valid yaml before starting instances. # It should be passed as user-data when starting the instance. @@ -8,7 +8,7 @@ # This example assumes the instance is 16.04 (xenial) -# The default is to install from packages. +# The default is to install from packages. # Key from https://packages.chef.io/chef.asc apt: @@ -60,7 +60,7 @@ chef: force_install: false # Chef settings - server_url: "https://chef.yourorg.com:4000" + server_url: "https://chef.yourorg.com" # Node Name # Defaults to the instance-id if not present @@ -78,8 +78,8 @@ chef: -----BEGIN RSA PRIVATE KEY----- YOUR-ORGS-VALIDATION-KEY-HERE -----END RSA PRIVATE KEY----- - - # A run list for a first boot json + + # A run list for a first boot json, an example (not required) run_list: - "recipe[apache2]" - "role[db]" @@ -92,7 +92,7 @@ chef: keepalive: "off" # if install_type is 'omnibus', change the url to download - omnibus_url: "https://www.opscode.com/chef/install.sh" + omnibus_url: "https://www.chef.io/chef/install.sh" # Capture all subprocess output into a logfile diff --git a/tests/cloud_tests/configs/examples/install_run_chef_recipes.yaml b/tests/cloud_tests/configs/examples/install_run_chef_recipes.yaml index 0bec305e..66b635a8 100644 --- a/tests/cloud_tests/configs/examples/install_run_chef_recipes.yaml +++ b/tests/cloud_tests/configs/examples/install_run_chef_recipes.yaml @@ -56,7 +56,7 @@ cloud_config: | force_install: false # Chef settings - server_url: "https://chef.yourorg.com:4000" + server_url: "https://chef.yourorg.com" # Node Name # Defaults to the instance-id if not present @@ -75,7 +75,7 @@ cloud_config: | YOUR-ORGS-VALIDATION-KEY-HERE -----END RSA PRIVATE KEY----- - # A run list for a first boot json + # A run list for a first boot json, this is an example (not required) run_list: - "recipe[apache2]" - "role[db]" @@ -88,7 +88,7 @@ cloud_config: | keepalive: "off" # if install_type is 'omnibus', change the url to download - omnibus_url: "https://www.opscode.com/chef/install.sh" + omnibus_url: "https://www.chef.io/chef/install.sh" # Capture all subprocess output into a logfile -- cgit v1.2.3 From d00da2d5b0d45db5670622a66d833d2abb907388 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 24 May 2017 21:10:50 -0400 Subject: net: normalize data in network_state object The network_state object's network and route keys would have different information depending upon how the network_state object was populated. This change cleans that up. Now: * address will always contain an IP address. * prefix will always include an integer value that is the network_prefix for the address. * netmask will be present only if the address is ipv4, and its value will always correlate to the 'prefix'. --- cloudinit/net/eni.py | 4 + cloudinit/net/netplan.py | 14 +- cloudinit/net/network_state.py | 244 ++++++++++++++++++++----- cloudinit/net/sysconfig.py | 23 +-- tests/unittests/test_distros/test_netconfig.py | 5 +- tests/unittests/test_net.py | 6 +- 6 files changed, 215 insertions(+), 81 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index 20e19f5b..98ce01e4 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -46,6 +46,10 @@ def _iface_add_subnet(iface, subnet): 'dns_nameservers', ] for key, value in subnet.items(): + if key == 'netmask': + continue + if key == 'address': + value = "%s/%s" % (subnet['address'], subnet['prefix']) if value and key in valid_map: if type(value) == list: value = " ".join(value) diff --git a/cloudinit/net/netplan.py b/cloudinit/net/netplan.py index a715f3b0..67543305 100644 --- a/cloudinit/net/netplan.py +++ b/cloudinit/net/netplan.py @@ -4,7 +4,7 @@ import copy import os from . import renderer -from .network_state import mask2cidr, subnet_is_ipv6 +from .network_state import subnet_is_ipv6 from cloudinit import log as logging from cloudinit import util @@ -118,10 +118,9 @@ def _extract_addresses(config, entry): sn_type += '4' entry.update({sn_type: True}) elif sn_type in ['static']: - addr = '%s' % subnet.get('address') - netmask = subnet.get('netmask') - if netmask and '/' not in addr: - addr += '/%s' % mask2cidr(netmask) + addr = "%s" % subnet.get('address') + if 'prefix' in subnet: + addr += "/%d" % subnet.get('prefix') if 'gateway' in subnet and subnet.get('gateway'): gateway = subnet.get('gateway') if ":" in gateway: @@ -138,9 +137,8 @@ def _extract_addresses(config, entry): mtukey += '6' entry.update({mtukey: subnet.get('mtu')}) for route in subnet.get('routes', []): - network = route.get('network') - netmask = route.get('netmask') - to_net = '%s/%s' % (network, mask2cidr(netmask)) + to_net = "%s/%s" % (route.get('network'), + route.get('prefix')) route = { 'via': route.get('gateway'), 'to': to_net, diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 9e9c05a0..87a7222d 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -289,19 +289,15 @@ class NetworkStateInterpreter(object): iface.update({param: val}) # convert subnet ipv6 netmask to cidr as needed - subnets = command.get('subnets') - if subnets: + subnets = _normalize_subnets(command.get('subnets')) + + # automatically set 'use_ipv6' if any addresses are ipv6 + if not self.use_ipv6: for subnet in subnets: - if subnet['type'] == 'static': - if ':' in subnet['address']: - self.use_ipv6 = True - if 'netmask' in subnet and ':' in subnet['address']: - subnet['netmask'] = mask2cidr(subnet['netmask']) - for route in subnet.get('routes', []): - if 'netmask' in route: - route['netmask'] = mask2cidr(route['netmask']) - elif subnet['type'].endswith('6'): + if (subnet.get('type').endswith('6') or + is_ipv6_addr(subnet.get('address'))): self.use_ipv6 = True + break iface.update({ 'name': command.get('name'), @@ -456,16 +452,7 @@ class NetworkStateInterpreter(object): @ensure_command_keys(['destination']) def handle_route(self, command): - routes = self._network_state.get('routes', []) - network, cidr = command['destination'].split("/") - netmask = cidr2mask(int(cidr)) - route = { - 'network': network, - 'netmask': netmask, - 'gateway': command.get('gateway'), - 'metric': command.get('metric'), - } - routes.append(route) + self._network_state['routes'].append(_normalize_route(command)) # V2 handlers def handle_bonds(self, command): @@ -666,18 +653,9 @@ class NetworkStateInterpreter(object): routes = [] for route in cfg.get('routes', []): - route_addr = route.get('to') - if "/" in route_addr: - route_addr, route_cidr = route_addr.split("/") - route_netmask = cidr2mask(route_cidr) - subnet_route = { - 'address': route_addr, - 'netmask': route_netmask, - 'gateway': route.get('via') - } - routes.append(subnet_route) - if len(routes) > 0: - subnet.update({'routes': routes}) + routes.append(_normalize_route( + {'address': route.get('to'), 'gateway': route.get('via')})) + subnet['routes'] = routes if ":" in address: if 'gateway6' in cfg and gateway6 is None: @@ -692,53 +670,219 @@ class NetworkStateInterpreter(object): return subnets +def _normalize_subnet(subnet): + # Prune all keys with None values. + subnet = copy.deepcopy(subnet) + normal_subnet = dict((k, v) for k, v in subnet.items() if v) + + if subnet.get('type') in ('static', 'static6'): + normal_subnet.update( + _normalize_net_keys(normal_subnet, address_keys=('address',))) + normal_subnet['routes'] = [_normalize_route(r) + for r in subnet.get('routes', [])] + return normal_subnet + + +def _normalize_net_keys(network, address_keys=()): + """Normalize dictionary network keys returning prefix and address keys. + + @param network: A dict of network-related definition containing prefix, + netmask and address_keys. + @param address_keys: A tuple of keys to search for representing the address + or cidr. The first address_key discovered will be used for + normalization. + + @returns: A dict containing normalized prefix and matching addr_key. + """ + net = dict((k, v) for k, v in network.items() if v) + addr_key = None + for key in address_keys: + if net.get(key): + addr_key = key + break + if not addr_key: + message = ( + 'No config network address keys [%s] found in %s' % + (','.join(address_keys), network)) + LOG.error(message) + raise ValueError(message) + + addr = net.get(addr_key) + ipv6 = is_ipv6_addr(addr) + netmask = net.get('netmask') + if "/" in addr: + addr_part, _, maybe_prefix = addr.partition("/") + net[addr_key] = addr_part + try: + prefix = int(maybe_prefix) + except ValueError: + # this supports input of
/255.255.255.0 + prefix = mask_to_net_prefix(maybe_prefix) + elif netmask: + prefix = mask_to_net_prefix(netmask) + elif 'prefix' in net: + prefix = int(prefix) + else: + prefix = 64 if ipv6 else 24 + + if 'prefix' in net and str(net['prefix']) != str(prefix): + LOG.warning("Overwriting existing 'prefix' with '%s' in " + "network info: %s", prefix, net) + net['prefix'] = prefix + + if ipv6: + # TODO: we could/maybe should add this back with the very uncommon + # 'netmask' for ipv6. We need a 'net_prefix_to_ipv6_mask' for that. + if 'netmask' in net: + del net['netmask'] + else: + net['netmask'] = net_prefix_to_ipv4_mask(net['prefix']) + + return net + + +def _normalize_route(route): + """normalize a route. + return a dictionary with only: + 'type': 'route' (only present if it was present in input) + 'network': the network portion of the route as a string. + 'prefix': the network prefix for address as an integer. + 'metric': integer metric (only if present in input). + 'netmask': netmask (string) equivalent to prefix iff network is ipv4. + """ + # Prune None-value keys. Specifically allow 0 (a valid metric). + normal_route = dict((k, v) for k, v in route.items() + if v not in ("", None)) + if 'destination' in normal_route: + normal_route['network'] = normal_route['destination'] + del normal_route['destination'] + + normal_route.update( + _normalize_net_keys( + normal_route, address_keys=('network', 'destination'))) + + metric = normal_route.get('metric') + if metric: + try: + normal_route['metric'] = int(metric) + except ValueError: + raise TypeError( + 'Route config metric {} is not an integer'.format(metric)) + return normal_route + + +def _normalize_subnets(subnets): + if not subnets: + subnets = [] + return [_normalize_subnet(s) for s in subnets] + + +def is_ipv6_addr(address): + if not address: + return False + return ":" in str(address) + + def subnet_is_ipv6(subnet): """Common helper for checking network_state subnets for ipv6.""" # 'static6' or 'dhcp6' if subnet['type'].endswith('6'): # This is a request for DHCPv6. return True - elif subnet['type'] == 'static' and ":" in subnet['address']: + elif subnet['type'] == 'static' and is_ipv6_addr(subnet.get('address')): return True return False -def cidr2mask(cidr): +def net_prefix_to_ipv4_mask(prefix): + """Convert a network prefix to an ipv4 netmask. + + This is the inverse of ipv4_mask_to_net_prefix. + 24 -> "255.255.255.0" + Also supports input as a string.""" + mask = [0, 0, 0, 0] - for i in list(range(0, cidr)): + for i in list(range(0, int(prefix))): idx = int(i / 8) mask[idx] = mask[idx] + (1 << (7 - i % 8)) return ".".join([str(x) for x in mask]) -def ipv4mask2cidr(mask): - if '.' not in mask: +def ipv4_mask_to_net_prefix(mask): + """Convert an ipv4 netmask into a network prefix length. + + If the input is already an integer or a string representation of + an integer, then int(mask) will be returned. + "255.255.255.0" => 24 + str(24) => 24 + "24" => 24 + """ + if isinstance(mask, int): return mask - return sum([bin(int(x)).count('1') for x in mask.split('.')]) + if isinstance(mask, six.string_types): + try: + return int(mask) + except ValueError: + pass + else: + raise TypeError("mask '%s' is not a string or int") + if '.' not in mask: + raise ValueError("netmask '%s' does not contain a '.'" % mask) -def ipv6mask2cidr(mask): - if ':' not in mask: + toks = mask.split(".") + if len(toks) != 4: + raise ValueError("netmask '%s' had only %d parts" % (mask, len(toks))) + + return sum([bin(int(x)).count('1') for x in toks]) + + +def ipv6_mask_to_net_prefix(mask): + """Convert an ipv6 netmask (very uncommon) or prefix (64) to prefix. + + If 'mask' is an integer or string representation of one then + int(mask) will be returned. + """ + + if isinstance(mask, int): return mask + if isinstance(mask, six.string_types): + try: + return int(mask) + except ValueError: + pass + else: + raise TypeError("mask '%s' is not a string or int") + + if ':' not in mask: + raise ValueError("mask '%s' does not have a ':'") bitCount = [0, 0x8000, 0xc000, 0xe000, 0xf000, 0xf800, 0xfc00, 0xfe00, 0xff00, 0xff80, 0xffc0, 0xffe0, 0xfff0, 0xfff8, 0xfffc, 0xfffe, 0xffff] - cidr = 0 + prefix = 0 for word in mask.split(':'): if not word or int(word, 16) == 0: break - cidr += bitCount.index(int(word, 16)) + prefix += bitCount.index(int(word, 16)) + + return prefix - return cidr +def mask_to_net_prefix(mask): + """Return the network prefix for the netmask provided. -def mask2cidr(mask): - if ':' in str(mask): - return ipv6mask2cidr(mask) - elif '.' in str(mask): - return ipv4mask2cidr(mask) + Supports ipv4 or ipv6 netmasks.""" + try: + # if 'mask' is a prefix that is an integer. + # then just return it. + return int(mask) + except ValueError: + pass + if is_ipv6_addr(mask): + return ipv6_mask_to_net_prefix(mask) else: - return mask + return ipv4_mask_to_net_prefix(mask) + # vi: ts=4 expandtab diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index f7d45482..5d9b3d10 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -9,7 +9,7 @@ from cloudinit.distros.parsers import resolv_conf from cloudinit import util from . import renderer -from .network_state import subnet_is_ipv6 +from .network_state import subnet_is_ipv6, net_prefix_to_ipv4_mask def _make_header(sep='#'): @@ -26,11 +26,8 @@ def _make_header(sep='#'): def _is_default_route(route): - if route['network'] == '::' and route['netmask'] == 0: - return True - if route['network'] == '0.0.0.0' and route['netmask'] == '0.0.0.0': - return True - return False + default_nets = ('::', '0.0.0.0') + return route['prefix'] == 0 and route['network'] in default_nets def _quote_value(value): @@ -323,16 +320,10 @@ class Renderer(renderer.Renderer): " " + ipv6_cidr) else: ipv4_index = ipv4_index + 1 - if ipv4_index == 0: - iface_cfg['IPADDR'] = subnet['address'] - if 'netmask' in subnet: - iface_cfg['NETMASK'] = subnet['netmask'] - else: - iface_cfg['IPADDR' + str(ipv4_index)] = \ - subnet['address'] - if 'netmask' in subnet: - iface_cfg['NETMASK' + str(ipv4_index)] = \ - subnet['netmask'] + suff = "" if ipv4_index == 0 else str(ipv4_index) + iface_cfg['IPADDR' + suff] = subnet['address'] + iface_cfg['NETMASK' + suff] = \ + net_prefix_to_ipv4_mask(subnet['prefix']) @classmethod def _render_subnet_routes(cls, iface_cfg, route_cfg, subnets): diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py index be9a8318..83580cc0 100644 --- a/tests/unittests/test_distros/test_netconfig.py +++ b/tests/unittests/test_distros/test_netconfig.py @@ -92,10 +92,9 @@ iface lo inet loopback auto eth0 iface eth0 inet static - address 192.168.1.5 + address 192.168.1.5/24 broadcast 192.168.1.0 gateway 192.168.1.254 - netmask 255.255.255.0 auto eth1 iface eth1 inet dhcp @@ -156,7 +155,7 @@ network: ethernets: eth7: addresses: - - 192.168.1.5/255.255.255.0 + - 192.168.1.5/24 gateway4: 192.168.1.254 eth9: dhcp4: true diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 0a88caf1..91e5fb59 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -334,17 +334,15 @@ iface lo inet loopback auto eth0 iface eth0 inet static - address 1.2.3.12 + address 1.2.3.12/29 broadcast 1.2.3.15 dns-nameservers 69.9.160.191 69.9.191.4 gateway 1.2.3.9 - netmask 255.255.255.248 auto eth1 iface eth1 inet static - address 10.248.2.4 + address 10.248.2.4/29 broadcast 10.248.2.7 - netmask 255.255.255.248 """.lstrip() NETWORK_CONFIGS = { -- cgit v1.2.3 From 67bab5bb804e2346673430868935f6bbcdb88f13 Mon Sep 17 00:00:00 2001 From: Ryan McCabe Date: Thu, 8 Jun 2017 13:24:23 -0400 Subject: net: Allow for NetworkManager configuration In cases where the config json specifies nameserver entries, if there are interfaces configured to use dhcp, NetworkManager, if enabled, will clobber the /etc/resolv.conf that cloud-init has produced, which can break dns. If there are no interfaces configured to use dhcp, NetworkManager could clobber /etc/resolv.conf with an empty file. This patch adds a mechanism for dropping additional configuration into /etc/NetworkManager/conf.d/ and disables management of /etc/resolv.conf by NetworkManager when nameserver information is provided in the config. LP: #1693251 Signed-off-by: Ryan McCabe --- cloudinit/distros/parsers/networkmanager_conf.py | 23 ++++++++++++++++++++++ cloudinit/net/sysconfig.py | 25 ++++++++++++++++++++++++ tests/unittests/test_net.py | 21 ++++++++++++++++++++ 3 files changed, 69 insertions(+) create mode 100644 cloudinit/distros/parsers/networkmanager_conf.py (limited to 'cloudinit') diff --git a/cloudinit/distros/parsers/networkmanager_conf.py b/cloudinit/distros/parsers/networkmanager_conf.py new file mode 100644 index 00000000..ac51f122 --- /dev/null +++ b/cloudinit/distros/parsers/networkmanager_conf.py @@ -0,0 +1,23 @@ +# Copyright (C) 2017 Red Hat, Inc. +# +# Author: Ryan McCabe +# +# This file is part of cloud-init. See LICENSE file for license information. + +import configobj + +# This module is used to set additional NetworkManager configuration +# in /etc/NetworkManager/conf.d +# + + +class NetworkManagerConf(configobj.ConfigObj): + def __init__(self, contents): + configobj.ConfigObj.__init__(self, contents, + interpolation=False, + write_empty_values=False) + + def set_section_keypair(self, section_name, key, value): + if section_name not in self.sections: + self.main[section_name] = {} + self.main[section_name] = {key: value} diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index 5d9b3d10..7ed11d1e 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -5,6 +5,7 @@ import re import six +from cloudinit.distros.parsers import networkmanager_conf from cloudinit.distros.parsers import resolv_conf from cloudinit import util @@ -249,6 +250,9 @@ class Renderer(renderer.Renderer): self.netrules_path = config.get( 'netrules_path', 'etc/udev/rules.d/70-persistent-net.rules') self.dns_path = config.get('dns_path', 'etc/resolv.conf') + nm_conf_path = 'etc/NetworkManager/conf.d/99-cloud-init.conf' + self.networkmanager_conf_path = config.get('networkmanager_conf_path', + nm_conf_path) @classmethod def _render_iface_shared(cls, iface, iface_cfg): @@ -438,6 +442,21 @@ class Renderer(renderer.Renderer): content.add_search_domain(searchdomain) return "\n".join([_make_header(';'), str(content)]) + @staticmethod + def _render_networkmanager_conf(network_state): + content = networkmanager_conf.NetworkManagerConf("") + + # If DNS server information is provided, configure + # NetworkManager to not manage dns, so that /etc/resolv.conf + # does not get clobbered. + if network_state.dns_nameservers: + content.set_section_keypair('main', 'dns', 'none') + + if len(content) == 0: + return None + out = "".join([_make_header(), "\n", "\n".join(content.write()), "\n"]) + return out + @classmethod def _render_bridge_interfaces(cls, network_state, iface_contents): bridge_filter = renderer.filter_by_type('bridge') @@ -498,6 +517,12 @@ class Renderer(renderer.Renderer): resolv_content = self._render_dns(network_state, existing_dns_path=dns_path) util.write_file(dns_path, resolv_content, file_mode) + if self.networkmanager_conf_path: + nm_conf_path = util.target_path(target, + self.networkmanager_conf_path) + nm_conf_content = self._render_networkmanager_conf(network_state) + if nm_conf_content: + util.write_file(nm_conf_path, nm_conf_content, file_mode) if self.netrules_path: netrules_content = self._render_persistent_net(network_state) netrules_path = util.target_path(target, self.netrules_path) diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 91e5fb59..8edc0b89 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -155,6 +155,13 @@ USERCTL=no ; Created by cloud-init on instance boot automatically, do not edit. ; nameserver 172.19.0.12 +""".lstrip()), + ('etc/NetworkManager/conf.d/99-cloud-init.conf', + """ +# Created by cloud-init on instance boot automatically, do not edit. +# +[main] +dns = none """.lstrip()), ('etc/udev/rules.d/70-persistent-net.rules', "".join(['SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ', @@ -216,6 +223,13 @@ USERCTL=no ; Created by cloud-init on instance boot automatically, do not edit. ; nameserver 172.19.0.12 +""".lstrip()), + ('etc/NetworkManager/conf.d/99-cloud-init.conf', + """ +# Created by cloud-init on instance boot automatically, do not edit. +# +[main] +dns = none """.lstrip()), ('etc/udev/rules.d/70-persistent-net.rules', "".join(['SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ', @@ -299,6 +313,13 @@ USERCTL=no ; Created by cloud-init on instance boot automatically, do not edit. ; nameserver 172.19.0.12 +""".lstrip()), + ('etc/NetworkManager/conf.d/99-cloud-init.conf', + """ +# Created by cloud-init on instance boot automatically, do not edit. +# +[main] +dns = none """.lstrip()), ('etc/udev/rules.d/70-persistent-net.rules', "".join(['SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ', -- cgit v1.2.3 From a01ffd65492e2882408aa18972ff80021eee624a Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Thu, 27 Apr 2017 20:01:38 +0000 Subject: net: Allow netinfo subprocesses to return 0 or 1. On systems with selinux enabled, some of the networking commands executed successfully do not return 0. Allow these commands to return 1 since the output is valid. Ultimately we need to get this information in some way so that we can display it correctly. For now, work around the stack trace when selinux does not allow us to collect it. LP: #1686751 --- cloudinit/netinfo.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/netinfo.py b/cloudinit/netinfo.py index ed374a36..39c79dee 100644 --- a/cloudinit/netinfo.py +++ b/cloudinit/netinfo.py @@ -20,7 +20,7 @@ LOG = logging.getLogger() def netdev_info(empty=""): fields = ("hwaddr", "addr", "bcast", "mask") - (ifcfg_out, _err) = util.subp(["ifconfig", "-a"]) + (ifcfg_out, _err) = util.subp(["ifconfig", "-a"], rcs=[0, 1]) devs = {} for line in str(ifcfg_out).splitlines(): if len(line) == 0: @@ -85,7 +85,7 @@ def netdev_info(empty=""): def route_info(): - (route_out, _err) = util.subp(["netstat", "-rn"]) + (route_out, _err) = util.subp(["netstat", "-rn"], rcs=[0, 1]) routes = {} routes['ipv4'] = [] @@ -125,7 +125,8 @@ def route_info(): routes['ipv4'].append(entry) try: - (route_out6, _err6) = util.subp(["netstat", "-A", "inet6", "-n"]) + (route_out6, _err6) = util.subp(["netstat", "-A", "inet6", "-n"], + rcs=[0, 1]) except util.ProcessExecutionError: pass else: -- cgit v1.2.3 From 95fd5a2b5426f3420e05ac190eb9f286df630484 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Thu, 27 Apr 2017 20:34:28 +0000 Subject: selinux: Allow restorecon to be non-fatal. On some systems with python-libselinux a bug[1] related to recursive restorecon fails but the distro release does not yet include an update. This change will accept the error and log a warning. 1. https://bugzilla.redhat.com/show_bug.cgi?id=1406520 LP: #1686751 --- cloudinit/util.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/util.py b/cloudinit/util.py index b8c3e4ee..415ca374 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -330,7 +330,11 @@ class SeLinuxGuard(object): LOG.debug("Restoring selinux mode for %s (recursive=%s)", path, self.recursive) - self.selinux.restorecon(path, recursive=self.recursive) + try: + self.selinux.restorecon(path, recursive=self.recursive) + except OSError as e: + LOG.warning('restorecon failed on %s,%s maybe badness? %s', + path, self.recursive, e) class MountFailedError(Exception): -- cgit v1.2.3 From 0fe6a0607408d387f4b0d4482b95afbc5d3f3909 Mon Sep 17 00:00:00 2001 From: Andrew Jorgensen Date: Fri, 5 Dec 2014 14:34:10 -0800 Subject: write_file(s): Print permissions as octal, not decimal Unix file modes are usually represented as octal, but they were being interpreted as decimal, for example 0o644 would be printed as '420'. Reviewed-by: Tom Kirchner --- cloudinit/config/cc_write_files.py | 10 ++++++- cloudinit/util.py | 6 ++++- tests/unittests/helpers.py | 8 +++--- tests/unittests/test_datasource/test_azure.py | 3 ++- tests/unittests/test_handler/test_handler_ntp.py | 3 ++- .../test_handler/test_handler_write_files.py | 31 ++++++++++++++++++++-- 6 files changed, 52 insertions(+), 9 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_write_files.py b/cloudinit/config/cc_write_files.py index 72e1cdd6..1835a31b 100644 --- a/cloudinit/config/cc_write_files.py +++ b/cloudinit/config/cc_write_files.py @@ -53,6 +53,7 @@ import six from cloudinit.settings import PER_INSTANCE from cloudinit import util + frequency = PER_INSTANCE DEFAULT_OWNER = "root:root" @@ -119,7 +120,14 @@ def decode_perms(perm, default, log): # Force to string and try octal conversion return int(str(perm), 8) except (TypeError, ValueError): - log.warn("Undecodable permissions %s, assuming %s", perm, default) + reps = [] + for r in (perm, default): + try: + reps.append("%o" % r) + except TypeError: + reps.append("%r" % r) + log.warning( + "Undecodable permissions {0}, returning default {1}".format(*reps)) return default diff --git a/cloudinit/util.py b/cloudinit/util.py index 415ca374..ec68925e 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1751,8 +1751,12 @@ def write_file(filename, content, mode=0o644, omode="wb", copy_mode=False): else: content = decode_binary(content) write_type = 'characters' + try: + mode_r = "%o" % mode + except TypeError: + mode_r = "%r" % mode LOG.debug("Writing to %s - %s: [%s] %s %s", - filename, omode, mode, len(content), write_type) + filename, omode, mode_r, len(content), write_type) with SeLinuxGuard(path=filename): with open(filename, omode) as fh: fh.write(content) diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index e78abce2..569f1aef 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -97,11 +97,13 @@ class CiTestCase(TestCase): super(CiTestCase, self).setUp() if self.with_logs: # Create a log handler so unit tests can search expected logs. - logger = logging.getLogger() + self.logger = logging.getLogger() self.logs = six.StringIO() + formatter = logging.Formatter('%(levelname)s: %(message)s') handler = logging.StreamHandler(self.logs) - self.old_handlers = logger.handlers - logger.handlers = [handler] + handler.setFormatter(formatter) + self.old_handlers = self.logger.handlers + self.logger.handlers = [handler] def tearDown(self): if self.with_logs: diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 42f49e06..b17f389c 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -263,7 +263,8 @@ fdescfs /dev/fd fdescfs rw 0 0 {}, distro=None, paths=self.paths) self.assertFalse(dsrc.get_data()) self.assertEqual( - "Non-Azure DMI asset tag '{0}' discovered.\n".format(nonazure_tag), + "DEBUG: Non-Azure DMI asset tag '{0}' discovered.\n".format( + nonazure_tag), self.logs.getvalue()) def test_basic_seed_dir(self): diff --git a/tests/unittests/test_handler/test_handler_ntp.py b/tests/unittests/test_handler/test_handler_ntp.py index 3a9f7f7e..c4299d94 100644 --- a/tests/unittests/test_handler/test_handler_ntp.py +++ b/tests/unittests/test_handler/test_handler_ntp.py @@ -216,7 +216,8 @@ class TestNtp(FilesystemMockingTestCase): """When no ntp section is defined handler logs a warning and noops.""" cc_ntp.handle('cc_ntp', {}, None, None, []) self.assertEqual( - 'Skipping module named cc_ntp, not present or disabled by cfg\n', + 'DEBUG: Skipping module named cc_ntp, ' + 'not present or disabled by cfg\n', self.logs.getvalue()) def test_ntp_handler_schema_validation_allows_empty_ntp_config(self): diff --git a/tests/unittests/test_handler/test_handler_write_files.py b/tests/unittests/test_handler/test_handler_write_files.py index fb252d1d..88a4742b 100644 --- a/tests/unittests/test_handler/test_handler_write_files.py +++ b/tests/unittests/test_handler/test_handler_write_files.py @@ -1,10 +1,10 @@ # This file is part of cloud-init. See LICENSE file for license information. -from cloudinit.config.cc_write_files import write_files +from cloudinit.config.cc_write_files import write_files, decode_perms from cloudinit import log as logging from cloudinit import util -from ..helpers import FilesystemMockingTestCase +from ..helpers import CiTestCase, FilesystemMockingTestCase import base64 import gzip @@ -98,6 +98,33 @@ class TestWriteFiles(FilesystemMockingTestCase): self.assertEqual(len(expected), flen_expected) +class TestDecodePerms(CiTestCase): + + with_logs = True + + def test_none_returns_default(self): + """If None is passed as perms, then default should be returned.""" + default = object() + found = decode_perms(None, default, self.logger) + self.assertEqual(default, found) + + def test_integer(self): + """A valid integer should return itself.""" + found = decode_perms(0o755, None, self.logger) + self.assertEqual(0o755, found) + + def test_valid_octal_string(self): + """A string should be read as octal.""" + found = decode_perms("644", None, self.logger) + self.assertEqual(0o644, found) + + def test_invalid_octal_string_returns_default_and_warns(self): + """A string with invalid octal should warn and return default.""" + found = decode_perms("999", None, self.logger) + self.assertIsNone(found) + self.assertIn("WARNING: Undecodable", self.logs.getvalue()) + + def _gzip_bytes(data): buf = six.BytesIO() fp = None -- cgit v1.2.3 From 977c4cf42e795e35cf4ac31b5f000736c674502e Mon Sep 17 00:00:00 2001 From: Andrew Jorgensen Date: Wed, 5 Mar 2014 14:56:13 -0800 Subject: main: Don't use templater to format the welcome message Some versions of Cheetah returned everything as unicode by default (not utf-8 or ascii) and some varieties of syslog would choke on unicode. Jinja2 is probably fine, but Python's format() is perfectly adequate for a short message like the welcome message. Reviewed-by: Tom Kirchner Reviewed-by: Ben Cressey --- cloudinit/cmd/main.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py index 26cc2654..ce3c10dd 100644 --- a/cloudinit/cmd/main.py +++ b/cloudinit/cmd/main.py @@ -3,10 +3,12 @@ # Copyright (C) 2012 Canonical Ltd. # Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # Copyright (C) 2012 Yahoo! Inc. +# Copyright (C) 2017 Amazon.com, Inc. or its affiliates # # Author: Scott Moser # Author: Juerg Haefliger # Author: Joshua Harlow +# Author: Andrew Jorgensen # # This file is part of cloud-init. See LICENSE file for license information. @@ -25,7 +27,6 @@ from cloudinit import netinfo from cloudinit import signal_handler from cloudinit import sources from cloudinit import stages -from cloudinit import templater from cloudinit import url_helper from cloudinit import util from cloudinit import version @@ -42,9 +43,9 @@ from cloudinit import atomic_helper from cloudinit.dhclient_hook import LogDhclient -# Pretty little cheetah formatted welcome message template -WELCOME_MSG_TPL = ("Cloud-init v. ${version} running '${action}' at " - "${timestamp}. Up ${uptime} seconds.") +# Welcome message template +WELCOME_MSG_TPL = ("Cloud-init v. {version} running '{action}' at " + "{timestamp}. Up {uptime} seconds.") # Module section template MOD_SECTION_TPL = "cloud_%s_modules" @@ -88,13 +89,11 @@ def welcome(action, msg=None): def welcome_format(action): - tpl_params = { - 'version': version.version_string(), - 'uptime': util.uptime(), - 'timestamp': util.time_rfc2822(), - 'action': action, - } - return templater.render_string(WELCOME_MSG_TPL, tpl_params) + return WELCOME_MSG_TPL.format( + version=version.version_string(), + uptime=util.uptime(), + timestamp=util.time_rfc2822(), + action=action) def extract_fns(args): -- cgit v1.2.3 From 8a06a1244c8ee20902db050e142c5a0b2fd777a9 Mon Sep 17 00:00:00 2001 From: Hongjiang Zhang Date: Wed, 7 Jun 2017 13:58:51 +0800 Subject: FreeBSD: fix cdrom mounting failure if /mnt/cdrom/secure did not exist. The current method is to attempt to mount the cdrom (/dev/cd0), if it is successful, /dev/cd0 is configured, otherwise, it is not configured. The problem is it forgets to check whether the mounting destination folder is created or not. As a result, mounting attempt failed even if cdrom is ready. LP: #1696295 --- cloudinit/sources/DataSourceAzure.py | 15 +++++++-------- tests/unittests/test_datasource/test_azure.py | 12 +++++++++++- 2 files changed, 18 insertions(+), 9 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index a0b9eaef..a8bad90b 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -799,18 +799,17 @@ def encrypt_pass(password, salt_id="$6$"): def list_possible_azure_ds_devs(): - # return a sorted list of devices that might have a azure datasource devlist = [] if util.is_FreeBSD(): + # add '/dev/cd0' to devlist if it is configured + # here wants to test whether '/dev/cd0' is available cdrom_dev = "/dev/cd0" try: - util.subp(["mount", "-o", "ro", "-t", "udf", cdrom_dev, - "/mnt/cdrom/secure"]) - except util.ProcessExecutionError: - LOG.debug("Fail to mount cd") - return devlist - util.subp(["umount", "/mnt/cdrom/secure"]) - devlist.append(cdrom_dev) + with open(cdrom_dev) as fp: + fp.read(1024) + devlist.append(cdrom_dev) + except IOError: + LOG.debug("cdrom (%s) is not configured", cdrom_dev) else: for fstype in ("iso9660", "udf"): devlist.extend(util.find_devs_with("TYPE=%s" % fstype)) diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index b17f389c..114b1a5d 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -7,7 +7,8 @@ from cloudinit.util import find_freebsd_part from cloudinit.util import get_path_dev_freebsd from ..helpers import (CiTestCase, TestCase, populate_dir, mock, - ExitStack, PY26, SkipTest) + ExitStack, PY26, PY3, SkipTest) +from mock import patch, mock_open import crypt import os @@ -543,6 +544,15 @@ fdescfs /dev/fd fdescfs rw 0 0 ds.get_data() self.assertEqual(self.instance_id, ds.metadata['instance-id']) + def test_list_possible_azure_ds_devs(self): + devlist = [] + with patch('platform.platform', + mock.MagicMock(return_value="FreeBSD")): + name = 'builtins.open' if PY3 else '__builtin__.open' + with patch(name, mock_open(read_data="data")): + devlist.extend(dsaz.list_possible_azure_ds_devs()) + self.assertEqual(devlist, ['/dev/cd0']) + class TestAzureBounce(TestCase): -- cgit v1.2.3 From ea0a534d93544837e44d03e3394233d28c247f7d Mon Sep 17 00:00:00 2001 From: Hongjiang Zhang Date: Tue, 13 Jun 2017 16:21:02 +0800 Subject: FreeBSD: replace ifdown/ifup with "ifconfig down" and "ifconfig up". Fix the issue caused by different commands on Linux and FreeBSD. On Linux, we used ifdown and ifup to enable and disable a NIC, but on FreeBSD, the counterpart is "ifconfig down" and "ifconfig up". LP: #1697815 --- cloudinit/sources/DataSourceAzure.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index a8bad90b..ebb53d0a 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -177,6 +177,11 @@ if util.is_FreeBSD(): RESOURCE_DISK_PATH = "/dev/" + res_disk else: LOG.debug("resource disk is None") + BOUNCE_COMMAND = [ + 'sh', '-xc', + ("i=$interface; x=0; ifconfig down $i || x=$?; " + "ifconfig up $i || x=$?; exit $x") + ] BUILTIN_DS_CONFIG = { 'agent_command': AGENT_START_BUILTIN, -- cgit v1.2.3 From 9ccb8f5e2ab262ee04bb9c103c1302479f7c81d3 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 15 Jun 2017 16:39:50 -0400 Subject: FreeBSD: fix test failure The previous commit caused test failure. This separates out _check_freebsd_cdrom and mocks it in a test rather than patching open. --- cloudinit/sources/DataSourceAzure.py | 21 +++++++++++++-------- tests/unittests/test_datasource/test_azure.py | 21 +++++++++++---------- 2 files changed, 24 insertions(+), 18 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index ebb53d0a..71e7c55c 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -803,18 +803,23 @@ def encrypt_pass(password, salt_id="$6$"): return crypt.crypt(password, salt_id + util.rand_str(strlen=16)) +def _check_freebsd_cdrom(cdrom_dev): + """Return boolean indicating path to cdrom device has content.""" + try: + with open(cdrom_dev) as fp: + fp.read(1024) + return True + except IOError: + LOG.debug("cdrom (%s) is not configured", cdrom_dev) + return False + + def list_possible_azure_ds_devs(): devlist = [] if util.is_FreeBSD(): - # add '/dev/cd0' to devlist if it is configured - # here wants to test whether '/dev/cd0' is available cdrom_dev = "/dev/cd0" - try: - with open(cdrom_dev) as fp: - fp.read(1024) - devlist.append(cdrom_dev) - except IOError: - LOG.debug("cdrom (%s) is not configured", cdrom_dev) + if _check_freebsd_cdrom(cdrom_dev): + return [cdrom_dev] else: for fstype in ("iso9660", "udf"): devlist.extend(util.find_devs_with("TYPE=%s" % fstype)) diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 114b1a5d..7d33daf7 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -7,8 +7,7 @@ from cloudinit.util import find_freebsd_part from cloudinit.util import get_path_dev_freebsd from ..helpers import (CiTestCase, TestCase, populate_dir, mock, - ExitStack, PY26, PY3, SkipTest) -from mock import patch, mock_open + ExitStack, PY26, SkipTest) import crypt import os @@ -544,14 +543,16 @@ fdescfs /dev/fd fdescfs rw 0 0 ds.get_data() self.assertEqual(self.instance_id, ds.metadata['instance-id']) - def test_list_possible_azure_ds_devs(self): - devlist = [] - with patch('platform.platform', - mock.MagicMock(return_value="FreeBSD")): - name = 'builtins.open' if PY3 else '__builtin__.open' - with patch(name, mock_open(read_data="data")): - devlist.extend(dsaz.list_possible_azure_ds_devs()) - self.assertEqual(devlist, ['/dev/cd0']) + @mock.patch("cloudinit.sources.DataSourceAzure.util.is_FreeBSD") + @mock.patch("cloudinit.sources.DataSourceAzure._check_freebsd_cdrom") + def test_list_possible_azure_ds_devs(self, m_check_fbsd_cdrom, + m_is_FreeBSD): + """On FreeBSD, possible devs should show /dev/cd0.""" + m_is_FreeBSD.return_value = True + m_check_fbsd_cdrom.return_value = True + self.assertEqual(dsaz.list_possible_azure_ds_devs(), ['/dev/cd0']) + self.assertEqual( + [mock.call("/dev/cd0")], m_check_fbsd_cdrom.call_args_list) class TestAzureBounce(TestCase): -- cgit v1.2.3 From ecb408afa1104fe49ce6eb1dc5708be56abd5cb2 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 15 Jun 2017 10:03:45 -0400 Subject: FreeBSD: Make freebsd a variant, fix unittests and tools/build-on-freebsd. - Simplify the logic of 'variant' in util.system_info much of the data from https://github.com/hpcugent/easybuild/wiki/OS_flavor_name_version - fix get_resource_disk_on_freebsd when running on a system without an Azure resource disk. - fix tools/build-on-freebsd to replace oauth with oauthlib and add bash which is a dependency for tests. - update a fiew places that were checking for freebsd but not using the util.is_FreeBSD() --- cloudinit/config/cc_growpart.py | 2 +- cloudinit/config/cc_power_state_change.py | 2 +- cloudinit/sources/DataSourceAzure.py | 2 +- cloudinit/util.py | 46 ++++++++++-------------- config/cloud.cfg.tmpl | 20 +++++------ tests/unittests/test_handler/test_handler_ntp.py | 2 +- tests/unittests/test_util.py | 9 +++-- tools/build-on-freebsd | 6 ++-- 8 files changed, 40 insertions(+), 49 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index d2bc6e6c..bafca9d8 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -214,7 +214,7 @@ def device_part_info(devpath): # FreeBSD doesn't know of sysfs so just get everything we need from # the device, like /dev/vtbd0p2. - if util.system_info()["platform"].startswith('FreeBSD'): + if util.is_FreeBSD(): m = re.search('^(/dev/.+)p([0-9])$', devpath) return (m.group(1), m.group(2)) diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py index c1c6fe7e..eba58b02 100644 --- a/cloudinit/config/cc_power_state_change.py +++ b/cloudinit/config/cc_power_state_change.py @@ -71,7 +71,7 @@ def givecmdline(pid): # Example output from procstat -c 1 # PID COMM ARGS # 1 init /bin/init -- - if util.system_info()["platform"].startswith('FreeBSD'): + if util.is_FreeBSD(): (output, _err) = util.subp(['procstat', '-c', str(pid)]) line = output.splitlines()[1] m = re.search('\d+ (\w|\.|-)+\s+(/\w.+)', line) diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 71e7c55c..4fe0d635 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -101,7 +101,7 @@ def get_dev_storvsc_sysctl(): sysctl_out, err = util.subp(['sysctl', 'dev.storvsc']) except util.ProcessExecutionError: LOG.debug("Fail to execute sysctl dev.storvsc") - return None + sysctl_out = "" return sysctl_out diff --git a/cloudinit/util.py b/cloudinit/util.py index ec68925e..c93b6d7e 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -573,7 +573,7 @@ def is_ipv4(instr): def is_FreeBSD(): - return system_info()['platform'].startswith('FreeBSD') + return system_info()['variant'] == "freebsd" def get_cfg_option_bool(yobj, key, default=False): @@ -598,37 +598,29 @@ def get_cfg_option_int(yobj, key, default=0): def system_info(): info = { 'platform': platform.platform(), + 'system': platform.system(), 'release': platform.release(), 'python': platform.python_version(), 'uname': platform.uname(), - 'dist': platform.linux_distribution(), # pylint: disable=W1505 + 'dist': platform.dist(), # pylint: disable=W1505 } - plat = info['platform'].lower() - # Try to get more info about what it actually is, in a format - # that we can easily use across linux and variants... - if plat.startswith('darwin'): - info['variant'] = 'darwin' - elif plat.endswith("bsd"): - info['variant'] = 'bsd' - elif plat.startswith('win'): - info['variant'] = 'windows' - elif 'linux' in plat: - # Try to get a single string out of these... - linux_dist, _version, _id = info['dist'] - linux_dist = linux_dist.lower() - if linux_dist in ('ubuntu', 'linuxmint', 'mint'): - info['variant'] = 'ubuntu' + system = info['system'].lower() + var = 'unknown' + if system == "linux": + linux_dist = info['dist'][0].lower() + if linux_dist in ('centos', 'fedora', 'debian'): + var = linux_dist + elif linux_dist in ('ubuntu', 'linuxmint', 'mint'): + var = 'ubuntu' + elif linux_dist == 'redhat': + var = 'rhel' else: - for prefix, variant in [('redhat', 'rhel'), - ('centos', 'centos'), - ('fedora', 'fedora'), - ('debian', 'debian')]: - if linux_dist.startswith(prefix): - info['variant'] = variant - if 'variant' not in info: - info['variant'] = 'linux' - if 'variant' not in info: - info['variant'] = 'unknown' + var = 'linux' + elif system in ('windows', 'darwin', "freebsd"): + var = system + + info['variant'] = var + return info diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl index 5af2a88f..f4b9069b 100644 --- a/config/cloud.cfg.tmpl +++ b/config/cloud.cfg.tmpl @@ -2,7 +2,7 @@ # The top level settings are used as module # and system configuration. -{% if variant in ["bsd"] %} +{% if variant in ["freebsd"] %} syslog_fix_perms: root:wheel {% endif %} # A set of users which may be applied and/or used by various modules @@ -13,7 +13,7 @@ users: # If this is set, 'root' will not be able to ssh in and they # will get a message to login instead as the default $user -{% if variant in ["bsd"] %} +{% if variant in ["freebsd"] %} disable_root: false {% else %} disable_root: true @@ -30,7 +30,7 @@ ssh_pwauth: 0 # This will cause the set+update hostname module to not operate (if true) preserve_hostname: false -{% if variant in ["bsd"] %} +{% if variant in ["freebsd"] %} # This should not be required, but leave it in place until the real cause of # not beeing able to find -any- datasources is resolved. datasource_list: ['ConfigDrive', 'Azure', 'OpenStack', 'Ec2'] @@ -53,13 +53,13 @@ cloud_init_modules: - write-files - growpart - resizefs -{% if variant not in ["bsd"] %} +{% if variant not in ["freebsd"] %} - disk_setup - mounts {% endif %} - set_hostname - update_hostname -{% if variant not in ["bsd"] %} +{% if variant not in ["freebsd"] %} - update_etc_hosts - ca-certs - rsyslog @@ -87,7 +87,7 @@ cloud_config_modules: - apt-pipelining - apt-configure {% endif %} -{% if variant not in ["bsd"] %} +{% if variant not in ["freebsd"] %} - ntp {% endif %} - timezone @@ -108,7 +108,7 @@ cloud_final_modules: - landscape - lxd {% endif %} -{% if variant not in ["bsd"] %} +{% if variant not in ["freebsd"] %} - puppet - chef - salt-minion @@ -130,10 +130,8 @@ cloud_final_modules: # (not accessible to handlers/transforms) system_info: # This will affect which distro class gets used -{% if variant in ["centos", "debian", "fedora", "rhel", "ubuntu"] %} +{% if variant in ["centos", "debian", "fedora", "rhel", "ubuntu", "freebsd"] %} distro: {{ variant }} -{% elif variant in ["bsd"] %} - distro: freebsd {% else %} # Unknown/fallback distro. distro: ubuntu @@ -182,7 +180,7 @@ system_info: cloud_dir: /var/lib/cloud/ templates_dir: /etc/cloud/templates/ ssh_svcname: sshd -{% elif variant in ["bsd"] %} +{% elif variant in ["freebsd"] %} # Default user name + that default users groups (if added/used) default_user: name: freebsd diff --git a/tests/unittests/test_handler/test_handler_ntp.py b/tests/unittests/test_handler/test_handler_ntp.py index c4299d94..7f278646 100644 --- a/tests/unittests/test_handler/test_handler_ntp.py +++ b/tests/unittests/test_handler/test_handler_ntp.py @@ -62,7 +62,7 @@ class TestNtp(FilesystemMockingTestCase): def test_ntp_rename_ntp_conf(self): """When NTP_CONF exists, rename_ntp moves it.""" ntpconf = self.tmp_path("ntp.conf", self.new_root) - os.mknod(ntpconf) + util.write_file(ntpconf, "") with mock.patch("cloudinit.config.cc_ntp.NTP_CONF", ntpconf): cc_ntp.rename_ntp_conf() self.assertFalse(os.path.exists(ntpconf)) diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 014aa6a3..a73fd26a 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -20,6 +20,9 @@ except ImportError: import mock +BASH = util.which('bash') + + class FakeSelinux(object): def __init__(self, match_what): @@ -544,17 +547,17 @@ class TestReadSeeded(helpers.TestCase): class TestSubp(helpers.TestCase): - stdin2err = ['bash', '-c', 'cat >&2'] + stdin2err = [BASH, '-c', 'cat >&2'] stdin2out = ['cat'] utf8_invalid = b'ab\xaadef' utf8_valid = b'start \xc3\xa9 end' utf8_valid_2 = b'd\xc3\xa9j\xc8\xa7' - printenv = ['bash', '-c', 'for n in "$@"; do echo "$n=${!n}"; done', '--'] + printenv = [BASH, '-c', 'for n in "$@"; do echo "$n=${!n}"; done', '--'] def printf_cmd(self, *args): # bash's printf supports \xaa. So does /usr/bin/printf # but by using bash, we remove dependency on another program. - return(['bash', '-c', 'printf "$@"', 'printf'] + list(args)) + return([BASH, '-c', 'printf "$@"', 'printf'] + list(args)) def test_subp_handles_utf8(self): # The given bytes contain utf-8 accented characters as seen in e.g. diff --git a/tools/build-on-freebsd b/tools/build-on-freebsd index ccc10b40..ff9153ad 100755 --- a/tools/build-on-freebsd +++ b/tools/build-on-freebsd @@ -8,6 +8,7 @@ fail() { echo "FAILED:" "$@" 1>&2; exit 1; } # Check dependencies: depschecked=/tmp/c-i.dependencieschecked pkgs=" + bash dmidecode e2fsprogs py27-Jinja2 @@ -16,7 +17,7 @@ pkgs=" py27-configobj py27-jsonpatch py27-jsonpointer - py27-oauth + py27-oauthlib py27-prettytable py27-requests py27-serial @@ -35,9 +36,6 @@ touch $depschecked python setup.py build python setup.py install -O1 --skip-build --prefix /usr/local/ --init-system sysvinit_freebsd -# Install the correct config file: -cp config/cloud.cfg-freebsd /etc/cloud/cloud.cfg - # Enable cloud-init in /etc/rc.conf: sed -i.bak -e "/cloudinit_enable=.*/d" /etc/rc.conf echo 'cloudinit_enable="YES"' >> /etc/rc.conf -- cgit v1.2.3 From ebc9ecbc8a76bdf511a456fb72339a7eb4c20568 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Tue, 20 Jun 2017 17:06:43 -0500 Subject: Azure: Add network-config, Refactor net layer to handle duplicate macs. On systems with network devices with duplicate mac addresses, cloud-init will fail to rename the devices according to the specified network configuration. Refactor net layer to search by device driver and device id if available. Azure systems may have duplicate mac addresses by design. Update Azure datasource to run at init-local time and let Azure datasource generate a fallback networking config to handle advanced networking configurations. Lastly, add a 'setup' method to the datasources that is called before userdata/vendordata is processed but after networking is up. That is used here on Azure to interact with the 'fabric'. --- cloudinit/cmd/main.py | 3 + cloudinit/net/__init__.py | 181 ++++++++-- cloudinit/net/eni.py | 2 + cloudinit/net/renderer.py | 4 +- cloudinit/net/udev.py | 7 +- cloudinit/sources/DataSourceAzure.py | 114 +++++- cloudinit/sources/__init__.py | 15 +- cloudinit/stages.py | 5 + tests/unittests/test_datasource/test_azure.py | 174 +++++++-- tests/unittests/test_datasource/test_common.py | 2 +- tests/unittests/test_net.py | 478 ++++++++++++++++++++++++- 11 files changed, 887 insertions(+), 98 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py index ce3c10dd..139e03b3 100644 --- a/cloudinit/cmd/main.py +++ b/cloudinit/cmd/main.py @@ -372,6 +372,9 @@ def main_init(name, args): LOG.debug("[%s] %s is in local mode, will apply init modules now.", mode, init.datasource) + # Give the datasource a chance to use network resources. + # This is used on Azure to communicate with the fabric over network. + init.setup_datasource() # update fully realizes user-data (pulling in #include if necessary) init.update() # Stage 7 diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 65accbb0..cba991a5 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -97,6 +97,10 @@ def is_bridge(devname): return os.path.exists(sys_dev_path(devname, "bridge")) +def is_bond(devname): + return os.path.exists(sys_dev_path(devname, "bonding")) + + def is_vlan(devname): uevent = str(read_sys_net_safe(devname, "uevent")) return 'DEVTYPE=vlan' in uevent.splitlines() @@ -124,6 +128,26 @@ def is_present(devname): return os.path.exists(sys_dev_path(devname)) +def device_driver(devname): + """Return the device driver for net device named 'devname'.""" + driver = None + driver_path = sys_dev_path(devname, "device/driver") + # driver is a symlink to the driver *dir* + if os.path.islink(driver_path): + driver = os.path.basename(os.readlink(driver_path)) + + return driver + + +def device_devid(devname): + """Return the device id string for net device named 'devname'.""" + dev_id = read_sys_net_safe(devname, "device/device") + if dev_id is False: + return None + + return dev_id + + def get_devicelist(): return os.listdir(SYS_CLASS_NET) @@ -138,12 +162,21 @@ def is_disabled_cfg(cfg): return cfg.get('config') == "disabled" -def generate_fallback_config(): +def generate_fallback_config(blacklist_drivers=None, config_driver=None): """Determine which attached net dev is most likely to have a connection and generate network state to run dhcp on that interface""" + + if not config_driver: + config_driver = False + + if not blacklist_drivers: + blacklist_drivers = [] + # get list of interfaces that could have connections invalid_interfaces = set(['lo']) - potential_interfaces = set(get_devicelist()) + potential_interfaces = set([device for device in get_devicelist() + if device_driver(device) not in + blacklist_drivers]) potential_interfaces = potential_interfaces.difference(invalid_interfaces) # sort into interfaces with carrier, interfaces which could have carrier, # and ignore interfaces that are definitely disconnected @@ -155,6 +188,9 @@ def generate_fallback_config(): if is_bridge(interface): # skip any bridges continue + if is_bond(interface): + # skip any bonds + continue carrier = read_sys_net_int(interface, 'carrier') if carrier: connected.append(interface) @@ -194,9 +230,18 @@ def generate_fallback_config(): break if target_mac and target_name: nconf = {'config': [], 'version': 1} - nconf['config'].append( - {'type': 'physical', 'name': target_name, - 'mac_address': target_mac, 'subnets': [{'type': 'dhcp'}]}) + cfg = {'type': 'physical', 'name': target_name, + 'mac_address': target_mac, 'subnets': [{'type': 'dhcp'}]} + # inject the device driver name, dev_id into config if enabled and + # device has a valid device driver value + if config_driver: + driver = device_driver(target_name) + if driver: + cfg['params'] = { + 'driver': driver, + 'device_id': device_devid(target_name), + } + nconf['config'].append(cfg) return nconf else: # can't read any interfaces addresses (or there are none); give up @@ -217,10 +262,16 @@ def apply_network_config_names(netcfg, strict_present=True, strict_busy=True): if ent.get('type') != 'physical': continue mac = ent.get('mac_address') - name = ent.get('name') if not mac: continue - renames.append([mac, name]) + name = ent.get('name') + driver = ent.get('params', {}).get('driver') + device_id = ent.get('params', {}).get('device_id') + if not driver: + driver = device_driver(name) + if not device_id: + device_id = device_devid(name) + renames.append([mac, name, driver, device_id]) return _rename_interfaces(renames) @@ -245,15 +296,27 @@ def _get_current_rename_info(check_downable=True): """Collect information necessary for rename_interfaces. returns a dictionary by mac address like: - {mac: - {'name': name - 'up': boolean: is_up(name), + {name: + { 'downable': None or boolean indicating that the - device has only automatically assigned ip addrs.}} + device has only automatically assigned ip addrs. + 'device_id': Device id value (if it has one) + 'driver': Device driver (if it has one) + 'mac': mac address + 'name': name + 'up': boolean: is_up(name) + }} """ - bymac = {} - for mac, name in get_interfaces_by_mac().items(): - bymac[mac] = {'name': name, 'up': is_up(name), 'downable': None} + cur_info = {} + for (name, mac, driver, device_id) in get_interfaces(): + cur_info[name] = { + 'downable': None, + 'device_id': device_id, + 'driver': driver, + 'mac': mac, + 'name': name, + 'up': is_up(name), + } if check_downable: nmatch = re.compile(r"[0-9]+:\s+(\w+)[@:]") @@ -265,11 +328,11 @@ def _get_current_rename_info(check_downable=True): for bytes_out in (ipv6, ipv4): nics_with_addresses.update(nmatch.findall(bytes_out)) - for d in bymac.values(): + for d in cur_info.values(): d['downable'] = (d['up'] is False or d['name'] not in nics_with_addresses) - return bymac + return cur_info def _rename_interfaces(renames, strict_present=True, strict_busy=True, @@ -282,15 +345,15 @@ def _rename_interfaces(renames, strict_present=True, strict_busy=True, if current_info is None: current_info = _get_current_rename_info() - cur_bymac = {} - for mac, data in current_info.items(): + cur_info = {} + for name, data in current_info.items(): cur = data.copy() - cur['mac'] = mac - cur_bymac[mac] = cur + cur['name'] = name + cur_info[name] = cur def update_byname(bymac): return dict((data['name'], data) - for data in bymac.values()) + for data in cur_info.values()) def rename(cur, new): util.subp(["ip", "link", "set", cur, "name", new], capture=True) @@ -304,14 +367,48 @@ def _rename_interfaces(renames, strict_present=True, strict_busy=True, ops = [] errors = [] ups = [] - cur_byname = update_byname(cur_bymac) + cur_byname = update_byname(cur_info) tmpname_fmt = "cirename%d" tmpi = -1 - for mac, new_name in renames: - cur = cur_bymac.get(mac, {}) - cur_name = cur.get('name') + def entry_match(data, mac, driver, device_id): + """match if set and in data""" + if mac and driver and device_id: + return (data['mac'] == mac and + data['driver'] == driver and + data['device_id'] == device_id) + elif mac and driver: + return (data['mac'] == mac and + data['driver'] == driver) + elif mac: + return (data['mac'] == mac) + + return False + + def find_entry(mac, driver, device_id): + match = [data for data in cur_info.values() + if entry_match(data, mac, driver, device_id)] + if len(match): + if len(match) > 1: + msg = ('Failed to match a single device. Matched devices "%s"' + ' with search values "(mac:%s driver:%s device_id:%s)"' + % (match, mac, driver, device_id)) + raise ValueError(msg) + return match[0] + + return None + + for mac, new_name, driver, device_id in renames: cur_ops = [] + cur = find_entry(mac, driver, device_id) + if not cur: + if strict_present: + errors.append( + "[nic not present] Cannot rename mac=%s to %s" + ", not available." % (mac, new_name)) + continue + + cur_name = cur.get('name') if cur_name == new_name: # nothing to do continue @@ -351,13 +448,13 @@ def _rename_interfaces(renames, strict_present=True, strict_busy=True, cur_ops.append(("rename", mac, new_name, (new_name, tmp_name))) target['name'] = tmp_name - cur_byname = update_byname(cur_bymac) + cur_byname = update_byname(cur_info) if target['up']: ups.append(("up", mac, new_name, (tmp_name,))) cur_ops.append(("rename", mac, new_name, (cur['name'], new_name))) cur['name'] = new_name - cur_byname = update_byname(cur_bymac) + cur_byname = update_byname(cur_info) ops += cur_ops opmap = {'rename': rename, 'down': down, 'up': up} @@ -426,6 +523,36 @@ def get_interfaces_by_mac(): return ret +def get_interfaces(): + """Return list of interface tuples (name, mac, driver, device_id) + + Bridges and any devices that have a 'stolen' mac are excluded.""" + try: + devs = get_devicelist() + except OSError as e: + if e.errno == errno.ENOENT: + devs = [] + else: + raise + ret = [] + empty_mac = '00:00:00:00:00:00' + for name in devs: + if not interface_has_own_mac(name): + continue + if is_bridge(name): + continue + if is_vlan(name): + continue + mac = get_interface_mac(name) + # some devices may not have a mac (tun0) + if not mac: + continue + if mac == empty_mac and name != 'lo': + continue + ret.append((name, mac, device_driver(name), device_devid(name))) + return ret + + class RendererNotFoundError(RuntimeError): pass diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index 98ce01e4..b707146c 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -72,6 +72,8 @@ def _iface_add_attrs(iface, index): content = [] ignore_map = [ 'control', + 'device_id', + 'driver', 'index', 'inet', 'mode', diff --git a/cloudinit/net/renderer.py b/cloudinit/net/renderer.py index c68658dc..bba139e5 100644 --- a/cloudinit/net/renderer.py +++ b/cloudinit/net/renderer.py @@ -34,8 +34,10 @@ class Renderer(object): for iface in network_state.iter_interfaces(filter_by_physical): # for physical interfaces write out a persist net udev rule if 'name' in iface and iface.get('mac_address'): + driver = iface.get('driver', None) content.write(generate_udev_rule(iface['name'], - iface['mac_address'])) + iface['mac_address'], + driver=driver)) return content.getvalue() @abc.abstractmethod diff --git a/cloudinit/net/udev.py b/cloudinit/net/udev.py index fd2fd8c7..58c0a708 100644 --- a/cloudinit/net/udev.py +++ b/cloudinit/net/udev.py @@ -23,7 +23,7 @@ def compose_udev_setting(key, value): return '%s="%s"' % (key, value) -def generate_udev_rule(interface, mac): +def generate_udev_rule(interface, mac, driver=None): """Return a udev rule to set the name of network interface with `mac`. The rule ends up as a single line looking something like: @@ -31,10 +31,13 @@ def generate_udev_rule(interface, mac): SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}="ff:ee:dd:cc:bb:aa", NAME="eth0" """ + if not driver: + driver = '?*' + rule = ', '.join([ compose_udev_equality('SUBSYSTEM', 'net'), compose_udev_equality('ACTION', 'add'), - compose_udev_equality('DRIVERS', '?*'), + compose_udev_equality('DRIVERS', driver), compose_udev_attr_equality('address', mac), compose_udev_setting('NAME', interface), ]) diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 4fe0d635..b5a95a1f 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -16,6 +16,7 @@ from xml.dom import minidom import xml.etree.ElementTree as ET from cloudinit import log as logging +from cloudinit import net from cloudinit import sources from cloudinit.sources.helpers.azure import get_metadata_from_fabric from cloudinit import util @@ -245,7 +246,9 @@ def temporary_hostname(temp_hostname, cfg, hostname_command='hostname'): set_hostname(previous_hostname, hostname_command) -class DataSourceAzureNet(sources.DataSource): +class DataSourceAzure(sources.DataSource): + _negotiated = False + def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) self.seed_dir = os.path.join(paths.seed_dir, 'azure') @@ -255,6 +258,7 @@ class DataSourceAzureNet(sources.DataSource): util.get_cfg_by_path(sys_cfg, DS_CFG_PATH, {}), BUILTIN_DS_CONFIG]) self.dhclient_lease_file = self.ds_cfg.get('dhclient_lease_file') + self._network_config = None def __str__(self): root = sources.DataSource.__str__(self) @@ -331,6 +335,7 @@ class DataSourceAzureNet(sources.DataSource): if asset_tag != AZURE_CHASSIS_ASSET_TAG: LOG.debug("Non-Azure DMI asset tag '%s' discovered.", asset_tag) return False + ddir = self.ds_cfg['data_dir'] candidates = [self.seed_dir] @@ -375,13 +380,14 @@ class DataSourceAzureNet(sources.DataSource): LOG.debug("using files cached in %s", ddir) # azure / hyper-v provides random data here + # TODO. find the seed on FreeBSD platform + # now update ds_cfg to reflect contents pass in config if not util.is_FreeBSD(): seed = util.load_file("/sys/firmware/acpi/tables/OEM0", quiet=True, decode=False) if seed: self.metadata['random_seed'] = seed - # TODO. find the seed on FreeBSD platform - # now update ds_cfg to reflect contents pass in config + user_ds_cfg = util.get_cfg_by_path(self.cfg, DS_CFG_PATH, {}) self.ds_cfg = util.mergemanydict([user_ds_cfg, self.ds_cfg]) @@ -389,6 +395,40 @@ class DataSourceAzureNet(sources.DataSource): # the directory to be protected. write_files(ddir, files, dirmode=0o700) + self.metadata['instance-id'] = util.read_dmi_data('system-uuid') + + return True + + def device_name_to_device(self, name): + return self.ds_cfg['disk_aliases'].get(name) + + def get_config_obj(self): + return self.cfg + + def check_instance_id(self, sys_cfg): + # quickly (local check only) if self.instance_id is still valid + return sources.instance_id_matches_system_uuid(self.get_instance_id()) + + def setup(self, is_new_instance): + if self._negotiated is False: + LOG.debug("negotiating for %s (new_instance=%s)", + self.get_instance_id(), is_new_instance) + fabric_data = self._negotiate() + LOG.debug("negotiating returned %s", fabric_data) + if fabric_data: + self.metadata.update(fabric_data) + self._negotiated = True + else: + LOG.debug("negotiating already done for %s", + self.get_instance_id()) + + def _negotiate(self): + """Negotiate with fabric and return data from it. + + On success, returns a dictionary including 'public_keys'. + On failure, returns False. + """ + if self.ds_cfg['agent_command'] == AGENT_START_BUILTIN: self.bounce_network_with_azure_hostname() @@ -398,31 +438,64 @@ class DataSourceAzureNet(sources.DataSource): else: metadata_func = self.get_metadata_from_agent + LOG.debug("negotiating with fabric via agent command %s", + self.ds_cfg['agent_command']) try: fabric_data = metadata_func() except Exception as exc: - LOG.info("Error communicating with Azure fabric; assume we aren't" - " on Azure.", exc_info=True) + LOG.warning( + "Error communicating with Azure fabric; You may experience." + "connectivity issues.", exc_info=True) return False - self.metadata['instance-id'] = util.read_dmi_data('system-uuid') - self.metadata.update(fabric_data) - - return True - def device_name_to_device(self, name): - return self.ds_cfg['disk_aliases'].get(name) - - def get_config_obj(self): - return self.cfg - - def check_instance_id(self, sys_cfg): - # quickly (local check only) if self.instance_id is still valid - return sources.instance_id_matches_system_uuid(self.get_instance_id()) + return fabric_data def activate(self, cfg, is_new_instance): address_ephemeral_resize(is_new_instance=is_new_instance) return + @property + def network_config(self): + """Generate a network config like net.generate_fallback_network() with + the following execptions. + + 1. Probe the drivers of the net-devices present and inject them in + the network configuration under params: driver: value + 2. If the driver value is 'mlx4_core', the control mode should be + set to manual. The device will be later used to build a bond, + for now we want to ensure the device gets named but does not + break any network configuration + """ + blacklist = ['mlx4_core'] + if not self._network_config: + LOG.debug('Azure: generating fallback configuration') + # generate a network config, blacklist picking any mlx4_core devs + netconfig = net.generate_fallback_config( + blacklist_drivers=blacklist, config_driver=True) + + # if we have any blacklisted devices, update the network_config to + # include the device, mac, and driver values, but with no ip + # config; this ensures udev rules are generated but won't affect + # ip configuration + bl_found = 0 + for bl_dev in [dev for dev in net.get_devicelist() + if net.device_driver(dev) in blacklist]: + bl_found += 1 + cfg = { + 'type': 'physical', + 'name': 'vf%d' % bl_found, + 'mac_address': net.get_interface_mac(bl_dev), + 'params': { + 'driver': net.device_driver(bl_dev), + 'device_id': net.device_devid(bl_dev), + }, + } + netconfig['config'].append(cfg) + + self._network_config = netconfig + + return self._network_config + def _partitions_on_device(devpath, maxnum=16): # return a list of tuples (ptnum, path) for each part on devpath @@ -849,9 +922,12 @@ class NonAzureDataSource(Exception): pass +# Legacy: Must be present in case we load an old pkl object +DataSourceAzureNet = DataSourceAzure + # Used to match classes to dependencies datasources = [ - (DataSourceAzureNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), + (DataSourceAzure, (sources.DEP_FILESYSTEM, )), ] diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index c3ce36d6..952caf35 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -251,10 +251,23 @@ class DataSource(object): def first_instance_boot(self): return + def setup(self, is_new_instance): + """setup(is_new_instance) + + This is called before user-data and vendor-data have been processed. + + Unless the datasource has set mode to 'local', then networking + per 'fallback' or per 'network_config' will have been written and + brought up the OS at this point. + """ + return + def activate(self, cfg, is_new_instance): """activate(cfg, is_new_instance) - This is called before the init_modules will be called. + This is called before the init_modules will be called but after + the user-data and vendor-data have been fully processed. + The cfg is fully up to date config, it contains a merged view of system config, datasource config, user config, vendor config. It should be used rather than the sys_cfg passed to __init__. diff --git a/cloudinit/stages.py b/cloudinit/stages.py index ad557827..a1c4a517 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -362,6 +362,11 @@ class Init(object): self._store_userdata() self._store_vendordata() + def setup_datasource(self): + if self.datasource is None: + raise RuntimeError("Datasource is None, cannot setup.") + self.datasource.setup(is_new_instance=self.is_new_instance()) + def activate_datasource(self): if self.datasource is None: raise RuntimeError("Datasource is None, cannot activate.") diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 7d33daf7..20e70fb7 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -181,13 +181,19 @@ scbus-1 on xpt0 bus 0 side_effect=_dmi_mocks)), ]) - dsrc = dsaz.DataSourceAzureNet( + dsrc = dsaz.DataSourceAzure( data.get('sys_cfg', {}), distro=None, paths=self.paths) if agent_command is not None: dsrc.ds_cfg['agent_command'] = agent_command return dsrc + def _get_and_setup(self, dsrc): + ret = dsrc.get_data() + if ret: + dsrc.setup(True) + return ret + def xml_equals(self, oxml, nxml): """Compare two sets of XML to make sure they are equal""" @@ -259,7 +265,7 @@ fdescfs /dev/fd fdescfs rw 0 0 # Return a non-matching asset tag value nonazure_tag = dsaz.AZURE_CHASSIS_ASSET_TAG + 'X' m_read_dmi_data.return_value = nonazure_tag - dsrc = dsaz.DataSourceAzureNet( + dsrc = dsaz.DataSourceAzure( {}, distro=None, paths=self.paths) self.assertFalse(dsrc.get_data()) self.assertEqual( @@ -299,7 +305,7 @@ fdescfs /dev/fd fdescfs rw 0 0 data = {'ovfcontent': construct_valid_ovf_env(data=odata)} dsrc = self._get_ds(data) - ret = dsrc.get_data() + ret = self._get_and_setup(dsrc) self.assertTrue(ret) self.assertEqual(data['agent_invoked'], cfg['agent_command']) @@ -312,7 +318,7 @@ fdescfs /dev/fd fdescfs rw 0 0 data = {'ovfcontent': construct_valid_ovf_env(data=odata)} dsrc = self._get_ds(data) - ret = dsrc.get_data() + ret = self._get_and_setup(dsrc) self.assertTrue(ret) self.assertEqual(data['agent_invoked'], cfg['agent_command']) @@ -322,7 +328,7 @@ fdescfs /dev/fd fdescfs rw 0 0 'sys_cfg': sys_cfg} dsrc = self._get_ds(data) - ret = dsrc.get_data() + ret = self._get_and_setup(dsrc) self.assertTrue(ret) self.assertEqual(data['agent_invoked'], '_COMMAND') @@ -394,7 +400,7 @@ fdescfs /dev/fd fdescfs rw 0 0 pubkeys=pubkeys)} dsrc = self._get_ds(data, agent_command=['not', '__builtin__']) - ret = dsrc.get_data() + ret = self._get_and_setup(dsrc) self.assertTrue(ret) for mypk in mypklist: self.assertIn(mypk, dsrc.cfg['_pubkeys']) @@ -409,7 +415,7 @@ fdescfs /dev/fd fdescfs rw 0 0 pubkeys=pubkeys)} dsrc = self._get_ds(data, agent_command=['not', '__builtin__']) - ret = dsrc.get_data() + ret = self._get_and_setup(dsrc) self.assertTrue(ret) for mypk in mypklist: @@ -425,7 +431,7 @@ fdescfs /dev/fd fdescfs rw 0 0 pubkeys=pubkeys)} dsrc = self._get_ds(data, agent_command=['not', '__builtin__']) - ret = dsrc.get_data() + ret = self._get_and_setup(dsrc) self.assertTrue(ret) for mypk in mypklist: @@ -519,18 +525,20 @@ fdescfs /dev/fd fdescfs rw 0 0 dsrc.get_data() def test_exception_fetching_fabric_data_doesnt_propagate(self): - ds = self._get_ds({'ovfcontent': construct_valid_ovf_env()}) - ds.ds_cfg['agent_command'] = '__builtin__' + """Errors communicating with fabric should warn, but return True.""" + dsrc = self._get_ds({'ovfcontent': construct_valid_ovf_env()}) + dsrc.ds_cfg['agent_command'] = '__builtin__' self.get_metadata_from_fabric.side_effect = Exception - self.assertFalse(ds.get_data()) + ret = self._get_and_setup(dsrc) + self.assertTrue(ret) def test_fabric_data_included_in_metadata(self): - ds = self._get_ds({'ovfcontent': construct_valid_ovf_env()}) - ds.ds_cfg['agent_command'] = '__builtin__' + dsrc = self._get_ds({'ovfcontent': construct_valid_ovf_env()}) + dsrc.ds_cfg['agent_command'] = '__builtin__' self.get_metadata_from_fabric.return_value = {'test': 'value'} - ret = ds.get_data() + ret = self._get_and_setup(dsrc) self.assertTrue(ret) - self.assertEqual('value', ds.metadata['test']) + self.assertEqual('value', dsrc.metadata['test']) def test_instance_id_from_dmidecode_used(self): ds = self._get_ds({'ovfcontent': construct_valid_ovf_env()}) @@ -554,6 +562,84 @@ fdescfs /dev/fd fdescfs rw 0 0 self.assertEqual( [mock.call("/dev/cd0")], m_check_fbsd_cdrom.call_args_list) + @mock.patch('cloudinit.net.get_interface_mac') + @mock.patch('cloudinit.net.get_devicelist') + @mock.patch('cloudinit.net.device_driver') + @mock.patch('cloudinit.net.generate_fallback_config') + def test_network_config(self, mock_fallback, mock_dd, + mock_devlist, mock_get_mac): + odata = {'HostName': "myhost", 'UserName': "myuser"} + data = {'ovfcontent': construct_valid_ovf_env(data=odata), + 'sys_cfg': {}} + + fallback_config = { + 'version': 1, + 'config': [{ + 'type': 'physical', 'name': 'eth0', + 'mac_address': '00:11:22:33:44:55', + 'params': {'driver': 'hv_netsvc'}, + 'subnets': [{'type': 'dhcp'}], + }] + } + mock_fallback.return_value = fallback_config + + mock_devlist.return_value = ['eth0'] + mock_dd.return_value = ['hv_netsvc'] + mock_get_mac.return_value = '00:11:22:33:44:55' + + dsrc = self._get_ds(data) + ret = dsrc.get_data() + self.assertTrue(ret) + + netconfig = dsrc.network_config + self.assertEqual(netconfig, fallback_config) + mock_fallback.assert_called_with(blacklist_drivers=['mlx4_core'], + config_driver=True) + + @mock.patch('cloudinit.net.get_interface_mac') + @mock.patch('cloudinit.net.get_devicelist') + @mock.patch('cloudinit.net.device_driver') + @mock.patch('cloudinit.net.generate_fallback_config') + def test_network_config_blacklist(self, mock_fallback, mock_dd, + mock_devlist, mock_get_mac): + odata = {'HostName': "myhost", 'UserName': "myuser"} + data = {'ovfcontent': construct_valid_ovf_env(data=odata), + 'sys_cfg': {}} + + fallback_config = { + 'version': 1, + 'config': [{ + 'type': 'physical', 'name': 'eth0', + 'mac_address': '00:11:22:33:44:55', + 'params': {'driver': 'hv_netsvc'}, + 'subnets': [{'type': 'dhcp'}], + }] + } + blacklist_config = { + 'type': 'physical', + 'name': 'eth1', + 'mac_address': '00:11:22:33:44:55', + 'params': {'driver': 'mlx4_core'} + } + mock_fallback.return_value = fallback_config + + mock_devlist.return_value = ['eth0', 'eth1'] + mock_dd.side_effect = [ + 'hv_netsvc', # list composition, skipped + 'mlx4_core', # list composition, match + 'mlx4_core', # config get driver name + ] + mock_get_mac.return_value = '00:11:22:33:44:55' + + dsrc = self._get_ds(data) + ret = dsrc.get_data() + self.assertTrue(ret) + + netconfig = dsrc.network_config + expected_config = fallback_config + expected_config['config'].append(blacklist_config) + self.assertEqual(netconfig, expected_config) + class TestAzureBounce(TestCase): @@ -603,12 +689,18 @@ class TestAzureBounce(TestCase): if ovfcontent is not None: populate_dir(os.path.join(self.paths.seed_dir, "azure"), {'ovf-env.xml': ovfcontent}) - dsrc = dsaz.DataSourceAzureNet( + dsrc = dsaz.DataSourceAzure( {}, distro=None, paths=self.paths) if agent_command is not None: dsrc.ds_cfg['agent_command'] = agent_command return dsrc + def _get_and_setup(self, dsrc): + ret = dsrc.get_data() + if ret: + dsrc.setup(True) + return ret + def get_ovf_env_with_dscfg(self, hostname, cfg): odata = { 'HostName': hostname, @@ -652,17 +744,20 @@ class TestAzureBounce(TestCase): host_name = 'unchanged-host-name' self.get_hostname.return_value = host_name cfg = {'hostname_bounce': {'policy': 'force'}} - self._get_ds(self.get_ovf_env_with_dscfg(host_name, cfg), - agent_command=['not', '__builtin__']).get_data() + dsrc = self._get_ds(self.get_ovf_env_with_dscfg(host_name, cfg), + agent_command=['not', '__builtin__']) + ret = self._get_and_setup(dsrc) + self.assertTrue(ret) self.assertEqual(1, perform_hostname_bounce.call_count) def test_different_hostnames_sets_hostname(self): expected_hostname = 'azure-expected-host-name' self.get_hostname.return_value = 'default-host-name' - self._get_ds( + dsrc = self._get_ds( self.get_ovf_env_with_dscfg(expected_hostname, {}), - agent_command=['not', '__builtin__'], - ).get_data() + agent_command=['not', '__builtin__']) + ret = self._get_and_setup(dsrc) + self.assertTrue(ret) self.assertEqual(expected_hostname, self.set_hostname.call_args_list[0][0][0]) @@ -671,19 +766,21 @@ class TestAzureBounce(TestCase): self, perform_hostname_bounce): expected_hostname = 'azure-expected-host-name' self.get_hostname.return_value = 'default-host-name' - self._get_ds( + dsrc = self._get_ds( self.get_ovf_env_with_dscfg(expected_hostname, {}), - agent_command=['not', '__builtin__'], - ).get_data() + agent_command=['not', '__builtin__']) + ret = self._get_and_setup(dsrc) + self.assertTrue(ret) self.assertEqual(1, perform_hostname_bounce.call_count) def test_different_hostnames_sets_hostname_back(self): initial_host_name = 'default-host-name' self.get_hostname.return_value = initial_host_name - self._get_ds( + dsrc = self._get_ds( self.get_ovf_env_with_dscfg('some-host-name', {}), - agent_command=['not', '__builtin__'], - ).get_data() + agent_command=['not', '__builtin__']) + ret = self._get_and_setup(dsrc) + self.assertTrue(ret) self.assertEqual(initial_host_name, self.set_hostname.call_args_list[-1][0][0]) @@ -693,10 +790,11 @@ class TestAzureBounce(TestCase): perform_hostname_bounce.side_effect = Exception initial_host_name = 'default-host-name' self.get_hostname.return_value = initial_host_name - self._get_ds( + dsrc = self._get_ds( self.get_ovf_env_with_dscfg('some-host-name', {}), - agent_command=['not', '__builtin__'], - ).get_data() + agent_command=['not', '__builtin__']) + ret = self._get_and_setup(dsrc) + self.assertTrue(ret) self.assertEqual(initial_host_name, self.set_hostname.call_args_list[-1][0][0]) @@ -707,7 +805,9 @@ class TestAzureBounce(TestCase): self.get_hostname.return_value = old_hostname cfg = {'hostname_bounce': {'interface': interface, 'policy': 'force'}} data = self.get_ovf_env_with_dscfg(hostname, cfg) - self._get_ds(data, agent_command=['not', '__builtin__']).get_data() + dsrc = self._get_ds(data, agent_command=['not', '__builtin__']) + ret = self._get_and_setup(dsrc) + self.assertTrue(ret) self.assertEqual(1, self.subp.call_count) bounce_env = self.subp.call_args[1]['env'] self.assertEqual(interface, bounce_env['interface']) @@ -719,7 +819,9 @@ class TestAzureBounce(TestCase): dsaz.BUILTIN_DS_CONFIG['hostname_bounce']['command'] = cmd cfg = {'hostname_bounce': {'policy': 'force'}} data = self.get_ovf_env_with_dscfg('some-hostname', cfg) - self._get_ds(data, agent_command=['not', '__builtin__']).get_data() + dsrc = self._get_ds(data, agent_command=['not', '__builtin__']) + ret = self._get_and_setup(dsrc) + self.assertTrue(ret) self.assertEqual(1, self.subp.call_count) bounce_args = self.subp.call_args[1]['args'] self.assertEqual(cmd, bounce_args) @@ -975,4 +1077,12 @@ class TestCanDevBeReformatted(CiTestCase): self.assertEqual(False, value) self.assertIn("3 or more", msg.lower()) + +class TestAzureNetExists(CiTestCase): + def test_azure_net_must_exist_for_legacy_objpkl(self): + """DataSourceAzureNet must exist for old obj.pkl files + that reference it.""" + self.assertTrue(hasattr(dsaz, "DataSourceAzureNet")) + + # vi: ts=4 expandtab diff --git a/tests/unittests/test_datasource/test_common.py b/tests/unittests/test_datasource/test_common.py index 7649b9ae..2ff1d9df 100644 --- a/tests/unittests/test_datasource/test_common.py +++ b/tests/unittests/test_datasource/test_common.py @@ -26,6 +26,7 @@ from cloudinit.sources import DataSourceNone as DSNone from .. import helpers as test_helpers DEFAULT_LOCAL = [ + Azure.DataSourceAzure, CloudSigma.DataSourceCloudSigma, ConfigDrive.DataSourceConfigDrive, DigitalOcean.DataSourceDigitalOcean, @@ -38,7 +39,6 @@ DEFAULT_LOCAL = [ DEFAULT_NETWORK = [ AliYun.DataSourceAliYun, AltCloud.DataSourceAltCloud, - Azure.DataSourceAzureNet, Bigstep.DataSourceBigstep, CloudStack.DataSourceCloudStack, DSNone.DataSourceNone, diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 8edc0b89..06e8f094 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -836,38 +836,176 @@ CONFIG_V1_EXPLICIT_LOOPBACK = { 'subnets': [{'control': 'auto', 'type': 'loopback'}]}, ]} +DEFAULT_DEV_ATTRS = { + 'eth1000': { + "bridge": False, + "carrier": False, + "dormant": False, + "operstate": "down", + "address": "07-1C-C6-75-A4-BE", + "device/driver": None, + "device/device": None, + } +} + def _setup_test(tmp_dir, mock_get_devicelist, mock_read_sys_net, - mock_sys_dev_path): - mock_get_devicelist.return_value = ['eth1000'] - dev_characteristics = { - 'eth1000': { - "bridge": False, - "carrier": False, - "dormant": False, - "operstate": "down", - "address": "07-1C-C6-75-A4-BE", - } - } + mock_sys_dev_path, dev_attrs=None): + if not dev_attrs: + dev_attrs = DEFAULT_DEV_ATTRS + + mock_get_devicelist.return_value = dev_attrs.keys() def fake_read(devname, path, translate=None, on_enoent=None, on_keyerror=None, on_einval=None): - return dev_characteristics[devname][path] + return dev_attrs[devname][path] mock_read_sys_net.side_effect = fake_read def sys_dev_path(devname, path=""): - return tmp_dir + devname + "/" + path + return tmp_dir + "/" + devname + "/" + path - for dev in dev_characteristics: + for dev in dev_attrs: os.makedirs(os.path.join(tmp_dir, dev)) with open(os.path.join(tmp_dir, dev, 'operstate'), 'w') as fh: - fh.write("down") + fh.write(dev_attrs[dev]['operstate']) + os.makedirs(os.path.join(tmp_dir, dev, "device")) + for key in ['device/driver']: + if key in dev_attrs[dev] and dev_attrs[dev][key]: + target = dev_attrs[dev][key] + link = os.path.join(tmp_dir, dev, key) + print('symlink %s -> %s' % (link, target)) + os.symlink(target, link) mock_sys_dev_path.side_effect = sys_dev_path +class TestGenerateFallbackConfig(CiTestCase): + + @mock.patch("cloudinit.net.sys_dev_path") + @mock.patch("cloudinit.net.read_sys_net") + @mock.patch("cloudinit.net.get_devicelist") + def test_device_driver(self, mock_get_devicelist, mock_read_sys_net, + mock_sys_dev_path): + devices = { + 'eth0': { + 'bridge': False, 'carrier': False, 'dormant': False, + 'operstate': 'down', 'address': '00:11:22:33:44:55', + 'device/driver': 'hv_netsvc', 'device/device': '0x3'}, + 'eth1': { + 'bridge': False, 'carrier': False, 'dormant': False, + 'operstate': 'down', 'address': '00:11:22:33:44:55', + 'device/driver': 'mlx4_core', 'device/device': '0x7'}, + } + + tmp_dir = self.tmp_dir() + _setup_test(tmp_dir, mock_get_devicelist, + mock_read_sys_net, mock_sys_dev_path, + dev_attrs=devices) + + network_cfg = net.generate_fallback_config(config_driver=True) + ns = network_state.parse_net_config_data(network_cfg, + skip_broken=False) + + render_dir = os.path.join(tmp_dir, "render") + os.makedirs(render_dir) + + # don't set rulepath so eni writes them + renderer = eni.Renderer( + {'eni_path': 'interfaces', 'netrules_path': 'netrules'}) + renderer.render_network_state(ns, render_dir) + + self.assertTrue(os.path.exists(os.path.join(render_dir, + 'interfaces'))) + with open(os.path.join(render_dir, 'interfaces')) as fh: + contents = fh.read() + print(contents) + expected = """ +auto lo +iface lo inet loopback + +auto eth0 +iface eth0 inet dhcp +""" + self.assertEqual(expected.lstrip(), contents.lstrip()) + + self.assertTrue(os.path.exists(os.path.join(render_dir, 'netrules'))) + with open(os.path.join(render_dir, 'netrules')) as fh: + contents = fh.read() + print(contents) + expected_rule = [ + 'SUBSYSTEM=="net"', + 'ACTION=="add"', + 'DRIVERS=="hv_netsvc"', + 'ATTR{address}=="00:11:22:33:44:55"', + 'NAME="eth0"', + ] + self.assertEqual(", ".join(expected_rule) + '\n', contents.lstrip()) + + @mock.patch("cloudinit.net.sys_dev_path") + @mock.patch("cloudinit.net.read_sys_net") + @mock.patch("cloudinit.net.get_devicelist") + def test_device_driver_blacklist(self, mock_get_devicelist, + mock_read_sys_net, mock_sys_dev_path): + devices = { + 'eth1': { + 'bridge': False, 'carrier': False, 'dormant': False, + 'operstate': 'down', 'address': '00:11:22:33:44:55', + 'device/driver': 'hv_netsvc', 'device/device': '0x3'}, + 'eth0': { + 'bridge': False, 'carrier': False, 'dormant': False, + 'operstate': 'down', 'address': '00:11:22:33:44:55', + 'device/driver': 'mlx4_core', 'device/device': '0x7'}, + } + + tmp_dir = self.tmp_dir() + _setup_test(tmp_dir, mock_get_devicelist, + mock_read_sys_net, mock_sys_dev_path, + dev_attrs=devices) + + blacklist = ['mlx4_core'] + network_cfg = net.generate_fallback_config(blacklist_drivers=blacklist, + config_driver=True) + ns = network_state.parse_net_config_data(network_cfg, + skip_broken=False) + + render_dir = os.path.join(tmp_dir, "render") + os.makedirs(render_dir) + + # don't set rulepath so eni writes them + renderer = eni.Renderer( + {'eni_path': 'interfaces', 'netrules_path': 'netrules'}) + renderer.render_network_state(ns, render_dir) + + self.assertTrue(os.path.exists(os.path.join(render_dir, + 'interfaces'))) + with open(os.path.join(render_dir, 'interfaces')) as fh: + contents = fh.read() + print(contents) + expected = """ +auto lo +iface lo inet loopback + +auto eth1 +iface eth1 inet dhcp +""" + self.assertEqual(expected.lstrip(), contents.lstrip()) + + self.assertTrue(os.path.exists(os.path.join(render_dir, 'netrules'))) + with open(os.path.join(render_dir, 'netrules')) as fh: + contents = fh.read() + print(contents) + expected_rule = [ + 'SUBSYSTEM=="net"', + 'ACTION=="add"', + 'DRIVERS=="hv_netsvc"', + 'ATTR{address}=="00:11:22:33:44:55"', + 'NAME="eth1"', + ] + self.assertEqual(", ".join(expected_rule) + '\n', contents.lstrip()) + + class TestSysConfigRendering(CiTestCase): @mock.patch("cloudinit.net.sys_dev_path") @@ -1560,6 +1698,118 @@ class TestNetRenderers(CiTestCase): priority=['sysconfig', 'eni']) +class TestGetInterfaces(CiTestCase): + _data = {'bonds': ['bond1'], + 'bridges': ['bridge1'], + 'vlans': ['bond1.101'], + 'own_macs': ['enp0s1', 'enp0s2', 'bridge1-nic', 'bridge1', + 'bond1.101', 'lo', 'eth1'], + 'macs': {'enp0s1': 'aa:aa:aa:aa:aa:01', + 'enp0s2': 'aa:aa:aa:aa:aa:02', + 'bond1': 'aa:aa:aa:aa:aa:01', + 'bond1.101': 'aa:aa:aa:aa:aa:01', + 'bridge1': 'aa:aa:aa:aa:aa:03', + 'bridge1-nic': 'aa:aa:aa:aa:aa:03', + 'lo': '00:00:00:00:00:00', + 'greptap0': '00:00:00:00:00:00', + 'eth1': 'aa:aa:aa:aa:aa:01', + 'tun0': None}, + 'drivers': {'enp0s1': 'virtio_net', + 'enp0s2': 'e1000', + 'bond1': None, + 'bond1.101': None, + 'bridge1': None, + 'bridge1-nic': None, + 'lo': None, + 'greptap0': None, + 'eth1': 'mlx4_core', + 'tun0': None}} + data = {} + + def _se_get_devicelist(self): + return list(self.data['devices']) + + def _se_device_driver(self, name): + return self.data['drivers'][name] + + def _se_device_devid(self, name): + return '0x%s' % sorted(list(self.data['drivers'].keys())).index(name) + + def _se_get_interface_mac(self, name): + return self.data['macs'][name] + + def _se_is_bridge(self, name): + return name in self.data['bridges'] + + def _se_is_vlan(self, name): + return name in self.data['vlans'] + + def _se_interface_has_own_mac(self, name): + return name in self.data['own_macs'] + + def _mock_setup(self): + self.data = copy.deepcopy(self._data) + self.data['devices'] = set(list(self.data['macs'].keys())) + mocks = ('get_devicelist', 'get_interface_mac', 'is_bridge', + 'interface_has_own_mac', 'is_vlan', 'device_driver', + 'device_devid') + self.mocks = {} + for n in mocks: + m = mock.patch('cloudinit.net.' + n, + side_effect=getattr(self, '_se_' + n)) + self.addCleanup(m.stop) + self.mocks[n] = m.start() + + def test_gi_includes_duplicate_macs(self): + self._mock_setup() + ret = net.get_interfaces() + + self.assertIn('enp0s1', self._se_get_devicelist()) + self.assertIn('eth1', self._se_get_devicelist()) + found = [ent for ent in ret if 'aa:aa:aa:aa:aa:01' in ent] + self.assertEqual(len(found), 2) + + def test_gi_excludes_any_without_mac_address(self): + self._mock_setup() + ret = net.get_interfaces() + + self.assertIn('tun0', self._se_get_devicelist()) + found = [ent for ent in ret if 'tun0' in ent] + self.assertEqual(len(found), 0) + + def test_gi_excludes_stolen_macs(self): + self._mock_setup() + ret = net.get_interfaces() + self.mocks['interface_has_own_mac'].assert_has_calls( + [mock.call('enp0s1'), mock.call('bond1')], any_order=True) + expected = [ + ('enp0s2', 'aa:aa:aa:aa:aa:02', 'e1000', '0x5'), + ('enp0s1', 'aa:aa:aa:aa:aa:01', 'virtio_net', '0x4'), + ('eth1', 'aa:aa:aa:aa:aa:01', 'mlx4_core', '0x6'), + ('lo', '00:00:00:00:00:00', None, '0x8'), + ('bridge1-nic', 'aa:aa:aa:aa:aa:03', None, '0x3'), + ] + self.assertEqual(sorted(expected), sorted(ret)) + + def test_gi_excludes_bridges(self): + self._mock_setup() + # add a device 'b1', make all return they have their "own mac", + # set everything other than 'b1' to be a bridge. + # then expect b1 is the only thing left. + self.data['macs']['b1'] = 'aa:aa:aa:aa:aa:b1' + self.data['drivers']['b1'] = None + self.data['devices'].add('b1') + self.data['bonds'] = [] + self.data['own_macs'] = self.data['devices'] + self.data['bridges'] = [f for f in self.data['devices'] if f != "b1"] + ret = net.get_interfaces() + self.assertEqual([('b1', 'aa:aa:aa:aa:aa:b1', None, '0x0')], ret) + self.mocks['is_bridge'].assert_has_calls( + [mock.call('bridge1'), mock.call('enp0s1'), mock.call('bond1'), + mock.call('b1')], + any_order=True) + + class TestGetInterfacesByMac(CiTestCase): _data = {'bonds': ['bond1'], 'bridges': ['bridge1'], @@ -1691,4 +1941,202 @@ def _gzip_data(data): gzfp.close() return iobuf.getvalue() + +class TestRenameInterfaces(CiTestCase): + + @mock.patch('cloudinit.util.subp') + def test_rename_all(self, mock_subp): + renames = [ + ('00:11:22:33:44:55', 'interface0', 'virtio_net', '0x3'), + ('00:11:22:33:44:aa', 'interface2', 'virtio_net', '0x5'), + ] + current_info = { + 'ens3': { + 'downable': True, + 'device_id': '0x3', + 'driver': 'virtio_net', + 'mac': '00:11:22:33:44:55', + 'name': 'ens3', + 'up': False}, + 'ens5': { + 'downable': True, + 'device_id': '0x5', + 'driver': 'virtio_net', + 'mac': '00:11:22:33:44:aa', + 'name': 'ens5', + 'up': False}, + } + net._rename_interfaces(renames, current_info=current_info) + print(mock_subp.call_args_list) + mock_subp.assert_has_calls([ + mock.call(['ip', 'link', 'set', 'ens3', 'name', 'interface0'], + capture=True), + mock.call(['ip', 'link', 'set', 'ens5', 'name', 'interface2'], + capture=True), + ]) + + @mock.patch('cloudinit.util.subp') + def test_rename_no_driver_no_device_id(self, mock_subp): + renames = [ + ('00:11:22:33:44:55', 'interface0', None, None), + ('00:11:22:33:44:aa', 'interface1', None, None), + ] + current_info = { + 'eth0': { + 'downable': True, + 'device_id': None, + 'driver': None, + 'mac': '00:11:22:33:44:55', + 'name': 'eth0', + 'up': False}, + 'eth1': { + 'downable': True, + 'device_id': None, + 'driver': None, + 'mac': '00:11:22:33:44:aa', + 'name': 'eth1', + 'up': False}, + } + net._rename_interfaces(renames, current_info=current_info) + print(mock_subp.call_args_list) + mock_subp.assert_has_calls([ + mock.call(['ip', 'link', 'set', 'eth0', 'name', 'interface0'], + capture=True), + mock.call(['ip', 'link', 'set', 'eth1', 'name', 'interface1'], + capture=True), + ]) + + @mock.patch('cloudinit.util.subp') + def test_rename_all_bounce(self, mock_subp): + renames = [ + ('00:11:22:33:44:55', 'interface0', 'virtio_net', '0x3'), + ('00:11:22:33:44:aa', 'interface2', 'virtio_net', '0x5'), + ] + current_info = { + 'ens3': { + 'downable': True, + 'device_id': '0x3', + 'driver': 'virtio_net', + 'mac': '00:11:22:33:44:55', + 'name': 'ens3', + 'up': True}, + 'ens5': { + 'downable': True, + 'device_id': '0x5', + 'driver': 'virtio_net', + 'mac': '00:11:22:33:44:aa', + 'name': 'ens5', + 'up': True}, + } + net._rename_interfaces(renames, current_info=current_info) + print(mock_subp.call_args_list) + mock_subp.assert_has_calls([ + mock.call(['ip', 'link', 'set', 'ens3', 'down'], capture=True), + mock.call(['ip', 'link', 'set', 'ens3', 'name', 'interface0'], + capture=True), + mock.call(['ip', 'link', 'set', 'ens5', 'down'], capture=True), + mock.call(['ip', 'link', 'set', 'ens5', 'name', 'interface2'], + capture=True), + mock.call(['ip', 'link', 'set', 'interface0', 'up'], capture=True), + mock.call(['ip', 'link', 'set', 'interface2', 'up'], capture=True) + ]) + + @mock.patch('cloudinit.util.subp') + def test_rename_duplicate_macs(self, mock_subp): + renames = [ + ('00:11:22:33:44:55', 'eth0', 'hv_netsvc', '0x3'), + ('00:11:22:33:44:55', 'vf1', 'mlx4_core', '0x5'), + ] + current_info = { + 'eth0': { + 'downable': True, + 'device_id': '0x3', + 'driver': 'hv_netsvc', + 'mac': '00:11:22:33:44:55', + 'name': 'eth0', + 'up': False}, + 'eth1': { + 'downable': True, + 'device_id': '0x5', + 'driver': 'mlx4_core', + 'mac': '00:11:22:33:44:55', + 'name': 'eth1', + 'up': False}, + } + net._rename_interfaces(renames, current_info=current_info) + print(mock_subp.call_args_list) + mock_subp.assert_has_calls([ + mock.call(['ip', 'link', 'set', 'eth1', 'name', 'vf1'], + capture=True), + ]) + + @mock.patch('cloudinit.util.subp') + def test_rename_duplicate_macs_driver_no_devid(self, mock_subp): + renames = [ + ('00:11:22:33:44:55', 'eth0', 'hv_netsvc', None), + ('00:11:22:33:44:55', 'vf1', 'mlx4_core', None), + ] + current_info = { + 'eth0': { + 'downable': True, + 'device_id': '0x3', + 'driver': 'hv_netsvc', + 'mac': '00:11:22:33:44:55', + 'name': 'eth0', + 'up': False}, + 'eth1': { + 'downable': True, + 'device_id': '0x5', + 'driver': 'mlx4_core', + 'mac': '00:11:22:33:44:55', + 'name': 'eth1', + 'up': False}, + } + net._rename_interfaces(renames, current_info=current_info) + print(mock_subp.call_args_list) + mock_subp.assert_has_calls([ + mock.call(['ip', 'link', 'set', 'eth1', 'name', 'vf1'], + capture=True), + ]) + + @mock.patch('cloudinit.util.subp') + def test_rename_multi_mac_dups(self, mock_subp): + renames = [ + ('00:11:22:33:44:55', 'eth0', 'hv_netsvc', '0x3'), + ('00:11:22:33:44:55', 'vf1', 'mlx4_core', '0x5'), + ('00:11:22:33:44:55', 'vf2', 'mlx4_core', '0x7'), + ] + current_info = { + 'eth0': { + 'downable': True, + 'device_id': '0x3', + 'driver': 'hv_netsvc', + 'mac': '00:11:22:33:44:55', + 'name': 'eth0', + 'up': False}, + 'eth1': { + 'downable': True, + 'device_id': '0x5', + 'driver': 'mlx4_core', + 'mac': '00:11:22:33:44:55', + 'name': 'eth1', + 'up': False}, + 'eth2': { + 'downable': True, + 'device_id': '0x7', + 'driver': 'mlx4_core', + 'mac': '00:11:22:33:44:55', + 'name': 'eth2', + 'up': False}, + } + net._rename_interfaces(renames, current_info=current_info) + print(mock_subp.call_args_list) + mock_subp.assert_has_calls([ + mock.call(['ip', 'link', 'set', 'eth1', 'name', 'vf1'], + capture=True), + mock.call(['ip', 'link', 'set', 'eth2', 'name', 'vf2'], + capture=True), + ]) + + # vi: ts=4 expandtab -- cgit v1.2.3 From 4d9f24f5c385cb7fa21d87a097ccd9a297613a75 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 29 Jun 2017 13:32:15 -0400 Subject: read_dmi_data: always return None when inside a container. This fixes stacktrace and warning message that would be printed to the log if running inside a container and read_dmi_data tried to access a key that was not present. In a container, the /sys/class/dmi/id data is not relevant to the but to the host. Additionally an unpriviledged container might see strange behavior: # cd /sys/class/dmi/id/ # id -u 0 # ls -l chassis_serial -r-------- 1 nobody nogroup 4096 Jun 29 16:49 chassis_serial # cat chassis_serial cat: /sys/class/dmi/id/chassis_serial: Permission denied The solution here is to just always return None when running in a container. LP: #1701325 --- cloudinit/util.py | 7 +++++++ tests/unittests/test_util.py | 23 +++++++++++++++++++++++ 2 files changed, 30 insertions(+) (limited to 'cloudinit') diff --git a/cloudinit/util.py b/cloudinit/util.py index c93b6d7e..b486e182 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2397,6 +2397,10 @@ def read_dmi_data(key): """ Wrapper for reading DMI data. + If running in a container return None. This is because DMI data is + assumed to be not useful in a container as it does not represent the + container but rather the host. + This will do the following (returning the first that produces a result): 1) Use a mapping to translate `key` from dmidecode naming to @@ -2407,6 +2411,9 @@ def read_dmi_data(key): If all of the above fail to find a value, None will be returned. """ + if is_container(): + return None + syspath_value = _read_dmi_syspath(key) if syspath_value is not None: return syspath_value diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index a73fd26a..65035be2 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -365,6 +365,9 @@ class TestReadDMIData(helpers.FilesystemMockingTestCase): self.addCleanup(shutil.rmtree, self.new_root) self.patchOS(self.new_root) self.patchUtils(self.new_root) + p = mock.patch("cloudinit.util.is_container", return_value=False) + self.addCleanup(p.stop) + self._m_is_container = p.start() def _create_sysfs_parent_directory(self): util.ensure_dir(os.path.join('sys', 'class', 'dmi', 'id')) @@ -453,6 +456,26 @@ class TestReadDMIData(helpers.FilesystemMockingTestCase): self._create_sysfs_file(sysfs_key, dmi_value) self.assertEqual(expected, util.read_dmi_data(dmi_key)) + def test_container_returns_none(self): + """In a container read_dmi_data should always return None.""" + + # first verify we get the value if not in container + self._m_is_container.return_value = False + key, val = ("system-product-name", "my_product") + self._create_sysfs_file('product_name', val) + self.assertEqual(val, util.read_dmi_data(key)) + + # then verify in container returns None + self._m_is_container.return_value = True + self.assertIsNone(util.read_dmi_data(key)) + + def test_container_returns_none_on_unknown(self): + """In a container even bogus keys return None.""" + self._m_is_container.return_value = True + self._create_sysfs_file('product_name', "should-be-ignored") + self.assertIsNone(util.read_dmi_data("bogus")) + self.assertIsNone(util.read_dmi_data("system-product-name")) + class TestMultiLog(helpers.FilesystemMockingTestCase): -- cgit v1.2.3 From 4330a98161a5c514c7f85a76a58d057c85b00174 Mon Sep 17 00:00:00 2001 From: Andrew Jorgensen Date: Fri, 16 Jun 2017 18:17:36 +0000 Subject: write_files: Remove log from helper function signatures. Instead of passing around a 'log' reference to functions, just import logging and use that. This is the pattern that is now more common in cloud-init. --- cloudinit/config/cc_write_files.py | 27 ++++++++++++---------- .../test_handler/test_handler_write_files.py | 14 +++++------ 2 files changed, 22 insertions(+), 19 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_write_files.py b/cloudinit/config/cc_write_files.py index 1835a31b..54ae3a68 100644 --- a/cloudinit/config/cc_write_files.py +++ b/cloudinit/config/cc_write_files.py @@ -50,6 +50,7 @@ import base64 import os import six +from cloudinit import log as logging from cloudinit.settings import PER_INSTANCE from cloudinit import util @@ -60,6 +61,8 @@ DEFAULT_OWNER = "root:root" DEFAULT_PERMS = 0o644 UNKNOWN_ENC = 'text/plain' +LOG = logging.getLogger(__name__) + def handle(name, cfg, _cloud, log, _args): files = cfg.get('write_files') @@ -67,10 +70,10 @@ def handle(name, cfg, _cloud, log, _args): log.debug(("Skipping module named %s," " no/empty 'write_files' key in configuration"), name) return - write_files(name, files, log) + write_files(name, files) -def canonicalize_extraction(encoding_type, log): +def canonicalize_extraction(encoding_type): if not encoding_type: encoding_type = '' encoding_type = encoding_type.lower().strip() @@ -85,31 +88,31 @@ def canonicalize_extraction(encoding_type, log): if encoding_type in ['b64', 'base64']: return ['application/base64'] if encoding_type: - log.warn("Unknown encoding type %s, assuming %s", - encoding_type, UNKNOWN_ENC) + LOG.warning("Unknown encoding type %s, assuming %s", + encoding_type, UNKNOWN_ENC) return [UNKNOWN_ENC] -def write_files(name, files, log): +def write_files(name, files): if not files: return for (i, f_info) in enumerate(files): path = f_info.get('path') if not path: - log.warn("No path provided to write for entry %s in module %s", - i + 1, name) + LOG.warning("No path provided to write for entry %s in module %s", + i + 1, name) continue path = os.path.abspath(path) - extractions = canonicalize_extraction(f_info.get('encoding'), log) + extractions = canonicalize_extraction(f_info.get('encoding')) contents = extract_contents(f_info.get('content', ''), extractions) (u, g) = util.extract_usergroup(f_info.get('owner', DEFAULT_OWNER)) - perms = decode_perms(f_info.get('permissions'), DEFAULT_PERMS, log) + perms = decode_perms(f_info.get('permissions'), DEFAULT_PERMS) util.write_file(path, contents, mode=perms) util.chownbyname(path, u, g) -def decode_perms(perm, default, log): +def decode_perms(perm, default): if perm is None: return default try: @@ -126,8 +129,8 @@ def decode_perms(perm, default, log): reps.append("%o" % r) except TypeError: reps.append("%r" % r) - log.warning( - "Undecodable permissions {0}, returning default {1}".format(*reps)) + LOG.warning( + "Undecodable permissions %s, returning default %s", *reps) return default diff --git a/tests/unittests/test_handler/test_handler_write_files.py b/tests/unittests/test_handler/test_handler_write_files.py index 88a4742b..1129e77d 100644 --- a/tests/unittests/test_handler/test_handler_write_files.py +++ b/tests/unittests/test_handler/test_handler_write_files.py @@ -49,13 +49,13 @@ class TestWriteFiles(FilesystemMockingTestCase): expected = "hello world\n" filename = "/tmp/my.file" write_files( - "test_simple", [{"content": expected, "path": filename}], LOG) + "test_simple", [{"content": expected, "path": filename}]) self.assertEqual(util.load_file(filename), expected) def test_yaml_binary(self): self.patchUtils(self.tmp) data = util.load_yaml(YAML_TEXT) - write_files("testname", data['write_files'], LOG) + write_files("testname", data['write_files']) for path, content in YAML_CONTENT_EXPECTED.items(): self.assertEqual(util.load_file(path), content) @@ -87,7 +87,7 @@ class TestWriteFiles(FilesystemMockingTestCase): files.append(cur) expected.append((cur['path'], data)) - write_files("test_decoding", files, LOG) + write_files("test_decoding", files) for path, content in expected: self.assertEqual(util.load_file(path, decode=False), content) @@ -105,22 +105,22 @@ class TestDecodePerms(CiTestCase): def test_none_returns_default(self): """If None is passed as perms, then default should be returned.""" default = object() - found = decode_perms(None, default, self.logger) + found = decode_perms(None, default) self.assertEqual(default, found) def test_integer(self): """A valid integer should return itself.""" - found = decode_perms(0o755, None, self.logger) + found = decode_perms(0o755, None) self.assertEqual(0o755, found) def test_valid_octal_string(self): """A string should be read as octal.""" - found = decode_perms("644", None, self.logger) + found = decode_perms("644", None) self.assertEqual(0o644, found) def test_invalid_octal_string_returns_default_and_warns(self): """A string with invalid octal should warn and return default.""" - found = decode_perms("999", None, self.logger) + found = decode_perms("999", None) self.assertIsNone(found) self.assertIn("WARNING: Undecodable", self.logs.getvalue()) -- cgit v1.2.3 From be8e84b3639d41202a4e1ce7626b2053498fecdb Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 17 Jul 2017 10:20:40 -0400 Subject: Support comments in content read by load_shell_content. load_shell_content previously would not allow shell comment characters in the content being parsed. If comments=True is not passed then an exception would previously be raised as the line would not be guaranteed to have an '=' in it. --- cloudinit/util.py | 2 +- tests/unittests/test_util.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/util.py b/cloudinit/util.py index b486e182..f570b9d3 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2529,7 +2529,7 @@ def load_shell_content(content, add_empty=False, empty_val=None): if PY26 and isinstance(blob, six.text_type): # Older versions don't support unicode input blob = blob.encode("utf8") - return shlex.split(blob) + return shlex.split(blob, comments=True) data = {} for line in _shlex_split(content): diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 65035be2..f38a664c 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -807,4 +807,20 @@ class TestSystemIsSnappy(helpers.FilesystemMockingTestCase): self.reRoot(root_d) self.assertTrue(util.system_is_snappy()) + +class TestLoadShellContent(helpers.TestCase): + def test_comments_handled_correctly(self): + """Shell comments should be allowed in the content.""" + self.assertEqual( + {'key1': 'val1', 'key2': 'val2', 'key3': 'val3 #tricky'}, + util.load_shell_content('\n'.join([ + "#top of file comment", + "key1=val1 #this is a comment", + "# second comment", + 'key2="val2" # inlin comment' + '#badkey=wark', + 'key3="val3 #tricky"', + '']))) + + # vi: ts=4 expandtab -- cgit v1.2.3 From e80517ae6aea49c9ab3bd622a33fee44014f485f Mon Sep 17 00:00:00 2001 From: Julien Castets Date: Tue, 25 Apr 2017 09:06:13 +0000 Subject: Scaleway: add datasource with user and vendor data for Scaleway. Here we add and enable by default a datasource for Scaleway cloud. The datasource quickly exits unless one of three things: a.) 'Scaleway' found as the system vendor b.) 'scaleway' found on the kernel command line. c.) the directory /var/run/scaleway exists (this is currently created by the scaleway initramfs module). One interesting bit of this particular datasource is that it requires the source port of the http request to be < 1024. --- cloudinit/settings.py | 1 + cloudinit/sources/DataSourceScaleway.py | 234 ++++++++++++++++++++ cloudinit/url_helper.py | 10 +- tests/unittests/test_datasource/test_common.py | 2 + tests/unittests/test_datasource/test_scaleway.py | 262 +++++++++++++++++++++++ tools/ds-identify | 18 +- 6 files changed, 524 insertions(+), 3 deletions(-) create mode 100644 cloudinit/sources/DataSourceScaleway.py create mode 100644 tests/unittests/test_datasource/test_scaleway.py (limited to 'cloudinit') diff --git a/cloudinit/settings.py b/cloudinit/settings.py index 0abd8a4a..c120498f 100644 --- a/cloudinit/settings.py +++ b/cloudinit/settings.py @@ -35,6 +35,7 @@ CFG_BUILTIN = { 'CloudStack', 'SmartOS', 'Bigstep', + 'Scaleway', # At the end to act as a 'catch' when none of the above work... 'None', ], diff --git a/cloudinit/sources/DataSourceScaleway.py b/cloudinit/sources/DataSourceScaleway.py new file mode 100644 index 00000000..3a8a8e8f --- /dev/null +++ b/cloudinit/sources/DataSourceScaleway.py @@ -0,0 +1,234 @@ +# Author: Julien Castets +# +# This file is part of cloud-init. See LICENSE file for license information. + +# Scaleway API: +# https://developer.scaleway.com/#metadata + +import json +import os +import socket +import time + +import requests + +# pylint fails to import the two modules below. +# These are imported via requests.packages rather than urllib3 because: +# a.) the provider of the requests package should ensure that urllib3 +# contained in it is consistent/correct. +# b.) cloud-init does not specifically have a dependency on urllib3 +# +# For future reference, see: +# https://github.com/kennethreitz/requests/pull/2375 +# https://github.com/requests/requests/issues/4104 +# pylint: disable=E0401 +from requests.packages.urllib3.connection import HTTPConnection +from requests.packages.urllib3.poolmanager import PoolManager + +from cloudinit import log as logging +from cloudinit import sources +from cloudinit import url_helper +from cloudinit import util + + +LOG = logging.getLogger(__name__) + +DS_BASE_URL = 'http://169.254.42.42' + +BUILTIN_DS_CONFIG = { + 'metadata_url': DS_BASE_URL + '/conf?format=json', + 'userdata_url': DS_BASE_URL + '/user_data/cloud-init', + 'vendordata_url': DS_BASE_URL + '/vendor_data/cloud-init' +} + +DEF_MD_RETRIES = 5 +DEF_MD_TIMEOUT = 10 + + +def on_scaleway(): + """ + There are three ways to detect if you are on Scaleway: + + * check DMI data: not yet implemented by Scaleway, but the check is made to + be future-proof. + * the initrd created the file /var/run/scaleway. + * "scaleway" is in the kernel cmdline. + """ + vendor_name = util.read_dmi_data('system-manufacturer') + if vendor_name == 'Scaleway': + return True + + if os.path.exists('/var/run/scaleway'): + return True + + cmdline = util.get_cmdline() + if 'scaleway' in cmdline: + return True + + return False + + +class SourceAddressAdapter(requests.adapters.HTTPAdapter): + """ + Adapter for requests to choose the local address to bind to. + """ + def __init__(self, source_address, **kwargs): + self.source_address = source_address + super(SourceAddressAdapter, self).__init__(**kwargs) + + def init_poolmanager(self, connections, maxsize, block=False): + socket_options = HTTPConnection.default_socket_options + [ + (socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + ] + self.poolmanager = PoolManager(num_pools=connections, + maxsize=maxsize, + block=block, + source_address=self.source_address, + socket_options=socket_options) + + +def query_data_api_once(api_address, timeout, requests_session): + """ + Retrieve user data or vendor data. + + Scaleway user/vendor data API returns HTTP/404 if user/vendor data is not + set. + + This function calls `url_helper.readurl` but instead of considering + HTTP/404 as an error that requires a retry, it considers it as empty + user/vendor data. + + Also, be aware the user data/vendor API requires the source port to be + below 1024 to ensure the client is root (since non-root users can't bind + ports below 1024). If requests raises ConnectionError (EADDRINUSE), the + caller should retry to call this function on an other port. + """ + try: + resp = url_helper.readurl( + api_address, + data=None, + timeout=timeout, + # It's the caller's responsability to recall this function in case + # of exception. Don't let url_helper.readurl() retry by itself. + retries=0, + session=requests_session, + # If the error is a HTTP/404 or a ConnectionError, go into raise + # block below. + exception_cb=lambda _, exc: exc.code == 404 or ( + isinstance(exc.cause, requests.exceptions.ConnectionError) + ) + ) + return util.decode_binary(resp.contents) + except url_helper.UrlError as exc: + # Empty user data. + if exc.code == 404: + return None + raise + + +def query_data_api(api_type, api_address, retries, timeout): + """Get user or vendor data. + + Handle the retrying logic in case the source port is used. + + Scaleway metadata service requires the source port of the client to + be a privileged port (<1024). This is done to ensure that only a + privileged user on the system can access the metadata service. + """ + # Query user/vendor data. Try to make a request on the first privileged + # port available. + for port in range(1, max(retries, 2)): + try: + LOG.debug( + 'Trying to get %s data (bind on port %d)...', + api_type, port + ) + requests_session = requests.Session() + requests_session.mount( + 'http://', + SourceAddressAdapter(source_address=('0.0.0.0', port)) + ) + data = query_data_api_once( + api_address, + timeout=timeout, + requests_session=requests_session + ) + LOG.debug('%s-data downloaded', api_type) + return data + + except url_helper.UrlError as exc: + # Local port already in use or HTTP/429. + LOG.warning('Error while trying to get %s data: %s', api_type, exc) + time.sleep(5) + last_exc = exc + continue + + # Max number of retries reached. + raise last_exc + + +class DataSourceScaleway(sources.DataSource): + + def __init__(self, sys_cfg, distro, paths): + super(DataSourceScaleway, self).__init__(sys_cfg, distro, paths) + + self.ds_cfg = util.mergemanydict([ + util.get_cfg_by_path(sys_cfg, ["datasource", "Scaleway"], {}), + BUILTIN_DS_CONFIG + ]) + + self.metadata_address = self.ds_cfg['metadata_url'] + self.userdata_address = self.ds_cfg['userdata_url'] + self.vendordata_address = self.ds_cfg['vendordata_url'] + + self.retries = int(self.ds_cfg.get('retries', DEF_MD_RETRIES)) + self.timeout = int(self.ds_cfg.get('timeout', DEF_MD_TIMEOUT)) + + def get_data(self): + if not on_scaleway(): + return False + + resp = url_helper.readurl(self.metadata_address, + timeout=self.timeout, + retries=self.retries) + self.metadata = json.loads(util.decode_binary(resp.contents)) + + self.userdata_raw = query_data_api( + 'user-data', self.userdata_address, + self.retries, self.timeout + ) + self.vendordata_raw = query_data_api( + 'vendor-data', self.vendordata_address, + self.retries, self.timeout + ) + return True + + @property + def launch_index(self): + return None + + def get_instance_id(self): + return self.metadata['id'] + + def get_public_ssh_keys(self): + return [key['key'] for key in self.metadata['ssh_public_keys']] + + def get_hostname(self, fqdn=False, resolve_ip=False): + return self.metadata['hostname'] + + @property + def availability_zone(self): + return None + + @property + def region(self): + return None + + +datasources = [ + (DataSourceScaleway, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), +] + + +def get_datasource_list(depends): + return sources.list_from_depends(depends, datasources) diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index d2b92e6a..7cf76aae 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -172,7 +172,8 @@ def _get_ssl_args(url, ssl_details): def readurl(url, data=None, timeout=None, retries=0, sec_between=1, headers=None, headers_cb=None, ssl_details=None, - check_status=True, allow_redirects=True, exception_cb=None): + check_status=True, allow_redirects=True, exception_cb=None, + session=None): url = _cleanurl(url) req_args = { 'url': url, @@ -231,7 +232,12 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, LOG.debug("[%s/%s] open '%s' with %s configuration", i, manual_tries, url, filtered_req_args) - r = requests.request(**req_args) + if session is None: + session = requests.Session() + + with session as sess: + r = sess.request(**req_args) + if check_status: r.raise_for_status() LOG.debug("Read from %s (%s, %sb) after %s attempts", url, diff --git a/tests/unittests/test_datasource/test_common.py b/tests/unittests/test_datasource/test_common.py index 2ff1d9df..413e87ac 100644 --- a/tests/unittests/test_datasource/test_common.py +++ b/tests/unittests/test_datasource/test_common.py @@ -19,6 +19,7 @@ from cloudinit.sources import ( DataSourceOpenNebula as OpenNebula, DataSourceOpenStack as OpenStack, DataSourceOVF as OVF, + DataSourceScaleway as Scaleway, DataSourceSmartOS as SmartOS, ) from cloudinit.sources import DataSourceNone as DSNone @@ -48,6 +49,7 @@ DEFAULT_NETWORK = [ NoCloud.DataSourceNoCloudNet, OpenStack.DataSourceOpenStack, OVF.DataSourceOVFNet, + Scaleway.DataSourceScaleway, ] diff --git a/tests/unittests/test_datasource/test_scaleway.py b/tests/unittests/test_datasource/test_scaleway.py new file mode 100644 index 00000000..65d83ad7 --- /dev/null +++ b/tests/unittests/test_datasource/test_scaleway.py @@ -0,0 +1,262 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +import json + +import httpretty +import requests + +from cloudinit import helpers +from cloudinit import settings +from cloudinit.sources import DataSourceScaleway + +from ..helpers import mock, HttprettyTestCase, TestCase + + +class DataResponses(object): + """ + Possible responses of the API endpoint + 169.254.42.42/user_data/cloud-init and + 169.254.42.42/vendor_data/cloud-init. + """ + + FAKE_USER_DATA = '#!/bin/bash\necho "user-data"' + + @staticmethod + def rate_limited(method, uri, headers): + return 429, headers, '' + + @staticmethod + def api_error(method, uri, headers): + return 500, headers, '' + + @classmethod + def get_ok(cls, method, uri, headers): + return 200, headers, cls.FAKE_USER_DATA + + @staticmethod + def empty(method, uri, headers): + """ + No user data for this server. + """ + return 404, headers, '' + + +class MetadataResponses(object): + """ + Possible responses of the metadata API. + """ + + FAKE_METADATA = { + 'id': '00000000-0000-0000-0000-000000000000', + 'hostname': 'scaleway.host', + 'ssh_public_keys': [{ + 'key': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABA', + 'fingerprint': '2048 06:ae:... login (RSA)' + }, { + 'key': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABCCCCC', + 'fingerprint': '2048 06:ff:... login2 (RSA)' + }] + } + + @classmethod + def get_ok(cls, method, uri, headers): + return 200, headers, json.dumps(cls.FAKE_METADATA) + + +class TestOnScaleway(TestCase): + + def install_mocks(self, fake_dmi, fake_file_exists, fake_cmdline): + mock, faked = fake_dmi + mock.return_value = 'Scaleway' if faked else 'Whatever' + + mock, faked = fake_file_exists + mock.return_value = faked + + mock, faked = fake_cmdline + mock.return_value = \ + 'initrd=initrd showopts scaleway nousb' if faked \ + else 'BOOT_IMAGE=/vmlinuz-3.11.0-26-generic' + + @mock.patch('cloudinit.util.get_cmdline') + @mock.patch('os.path.exists') + @mock.patch('cloudinit.util.read_dmi_data') + def test_not_on_scaleway(self, m_read_dmi_data, m_file_exists, + m_get_cmdline): + self.install_mocks( + fake_dmi=(m_read_dmi_data, False), + fake_file_exists=(m_file_exists, False), + fake_cmdline=(m_get_cmdline, False) + ) + self.assertFalse(DataSourceScaleway.on_scaleway()) + + # When not on Scaleway, get_data() returns False. + datasource = DataSourceScaleway.DataSourceScaleway( + settings.CFG_BUILTIN, None, helpers.Paths({}) + ) + self.assertFalse(datasource.get_data()) + + @mock.patch('cloudinit.util.get_cmdline') + @mock.patch('os.path.exists') + @mock.patch('cloudinit.util.read_dmi_data') + def test_on_scaleway_dmi(self, m_read_dmi_data, m_file_exists, + m_get_cmdline): + """ + dmidecode returns "Scaleway". + """ + # dmidecode returns "Scaleway" + self.install_mocks( + fake_dmi=(m_read_dmi_data, True), + fake_file_exists=(m_file_exists, False), + fake_cmdline=(m_get_cmdline, False) + ) + self.assertTrue(DataSourceScaleway.on_scaleway()) + + @mock.patch('cloudinit.util.get_cmdline') + @mock.patch('os.path.exists') + @mock.patch('cloudinit.util.read_dmi_data') + def test_on_scaleway_var_run_scaleway(self, m_read_dmi_data, m_file_exists, + m_get_cmdline): + """ + /var/run/scaleway exists. + """ + self.install_mocks( + fake_dmi=(m_read_dmi_data, False), + fake_file_exists=(m_file_exists, True), + fake_cmdline=(m_get_cmdline, False) + ) + self.assertTrue(DataSourceScaleway.on_scaleway()) + + @mock.patch('cloudinit.util.get_cmdline') + @mock.patch('os.path.exists') + @mock.patch('cloudinit.util.read_dmi_data') + def test_on_scaleway_cmdline(self, m_read_dmi_data, m_file_exists, + m_get_cmdline): + """ + "scaleway" in /proc/cmdline. + """ + self.install_mocks( + fake_dmi=(m_read_dmi_data, False), + fake_file_exists=(m_file_exists, False), + fake_cmdline=(m_get_cmdline, True) + ) + self.assertTrue(DataSourceScaleway.on_scaleway()) + + +def get_source_address_adapter(*args, **kwargs): + """ + Scaleway user/vendor data API requires to be called with a privileged port. + + If the unittests are run as non-root, the user doesn't have the permission + to bind on ports below 1024. + + This function removes the bind on a privileged address, since anyway the + HTTP call is mocked by httpretty. + """ + kwargs.pop('source_address') + return requests.adapters.HTTPAdapter(*args, **kwargs) + + +class TestDataSourceScaleway(HttprettyTestCase): + + def setUp(self): + self.datasource = DataSourceScaleway.DataSourceScaleway( + settings.CFG_BUILTIN, None, helpers.Paths({}) + ) + super(TestDataSourceScaleway, self).setUp() + + self.metadata_url = \ + DataSourceScaleway.BUILTIN_DS_CONFIG['metadata_url'] + self.userdata_url = \ + DataSourceScaleway.BUILTIN_DS_CONFIG['userdata_url'] + self.vendordata_url = \ + DataSourceScaleway.BUILTIN_DS_CONFIG['vendordata_url'] + + @httpretty.activate + @mock.patch('cloudinit.sources.DataSourceScaleway.SourceAddressAdapter', + get_source_address_adapter) + @mock.patch('cloudinit.util.get_cmdline') + @mock.patch('time.sleep', return_value=None) + def test_metadata_ok(self, sleep, m_get_cmdline): + """ + get_data() returns metadata, user data and vendor data. + """ + m_get_cmdline.return_value = 'scaleway' + + # Make user data API return a valid response + httpretty.register_uri(httpretty.GET, self.metadata_url, + body=MetadataResponses.get_ok) + httpretty.register_uri(httpretty.GET, self.userdata_url, + body=DataResponses.get_ok) + httpretty.register_uri(httpretty.GET, self.vendordata_url, + body=DataResponses.get_ok) + self.datasource.get_data() + + self.assertEqual(self.datasource.get_instance_id(), + MetadataResponses.FAKE_METADATA['id']) + self.assertEqual(self.datasource.get_public_ssh_keys(), [ + elem['key'] for elem in + MetadataResponses.FAKE_METADATA['ssh_public_keys'] + ]) + self.assertEqual(self.datasource.get_hostname(), + MetadataResponses.FAKE_METADATA['hostname']) + self.assertEqual(self.datasource.get_userdata_raw(), + DataResponses.FAKE_USER_DATA) + self.assertEqual(self.datasource.get_vendordata_raw(), + DataResponses.FAKE_USER_DATA) + self.assertIsNone(self.datasource.availability_zone) + self.assertIsNone(self.datasource.region) + self.assertEqual(sleep.call_count, 0) + + @httpretty.activate + @mock.patch('cloudinit.sources.DataSourceScaleway.SourceAddressAdapter', + get_source_address_adapter) + @mock.patch('cloudinit.util.get_cmdline') + @mock.patch('time.sleep', return_value=None) + def test_metadata_404(self, sleep, m_get_cmdline): + """ + get_data() returns metadata, but no user data nor vendor data. + """ + m_get_cmdline.return_value = 'scaleway' + + # Make user and vendor data APIs return HTTP/404, which means there is + # no user / vendor data for the server. + httpretty.register_uri(httpretty.GET, self.metadata_url, + body=MetadataResponses.get_ok) + httpretty.register_uri(httpretty.GET, self.userdata_url, + body=DataResponses.empty) + httpretty.register_uri(httpretty.GET, self.vendordata_url, + body=DataResponses.empty) + self.datasource.get_data() + self.assertIsNone(self.datasource.get_userdata_raw()) + self.assertIsNone(self.datasource.get_vendordata_raw()) + self.assertEqual(sleep.call_count, 0) + + @httpretty.activate + @mock.patch('cloudinit.sources.DataSourceScaleway.SourceAddressAdapter', + get_source_address_adapter) + @mock.patch('cloudinit.util.get_cmdline') + @mock.patch('time.sleep', return_value=None) + def test_metadata_rate_limit(self, sleep, m_get_cmdline): + """ + get_data() is rate limited two times by the metadata API when fetching + user data. + """ + m_get_cmdline.return_value = 'scaleway' + + httpretty.register_uri(httpretty.GET, self.metadata_url, + body=MetadataResponses.get_ok) + httpretty.register_uri(httpretty.GET, self.vendordata_url, + body=DataResponses.empty) + + httpretty.register_uri( + httpretty.GET, self.userdata_url, + responses=[ + httpretty.Response(body=DataResponses.rate_limited), + httpretty.Response(body=DataResponses.rate_limited), + httpretty.Response(body=DataResponses.get_ok), + ] + ) + self.datasource.get_data() + self.assertEqual(self.datasource.get_userdata_raw(), + DataResponses.FAKE_USER_DATA) + self.assertEqual(sleep.call_count, 2) diff --git a/tools/ds-identify b/tools/ds-identify index 7c8b144b..33bd2991 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -112,7 +112,7 @@ DI_DSNAME="" # be searched if there is no setting found in config. DI_DSLIST_DEFAULT="MAAS ConfigDrive NoCloud AltCloud Azure Bigstep \ CloudSigma CloudStack DigitalOcean AliYun Ec2 GCE OpenNebula OpenStack \ -OVF SmartOS" +OVF SmartOS Scaleway" DI_DSLIST="" DI_MODE="" DI_ON_FOUND="" @@ -896,6 +896,22 @@ dscheck_None() { return ${DS_NOT_FOUND} } +dscheck_Scaleway() { + if [ "${DI_DMI_SYS_VENDOR}" = "Scaleway" ]; then + return $DS_FOUND + fi + + case " ${DI_KERNEL_CMDLINE} " in + *\ scaleway\ *) return ${DS_FOUND};; + esac + + if [ -f ${PATH_ROOT}/var/run/scaleway ]; then + return ${DS_FOUND} + fi + + return ${DS_NOT_FOUND} +} + collect_info() { read_virt read_pid1_product_name -- cgit v1.2.3 From d1e8eb73aca6a3f5cee415774dcf540e934ec250 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Fri, 9 Jun 2017 12:35:11 -0500 Subject: sysconfig: include GATEWAY value if set in subnet Render the GATEWAY= value in interface files which have a gateway in the subnet configuration. LP: #1686856 --- cloudinit/net/sysconfig.py | 3 ++ tests/unittests/test_distros/test_netconfig.py | 2 ++ tests/unittests/test_net.py | 38 ++++++++++++++++++++++++++ 3 files changed, 43 insertions(+) (limited to 'cloudinit') diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index 7ed11d1e..ad8c268e 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -329,6 +329,9 @@ class Renderer(renderer.Renderer): iface_cfg['NETMASK' + suff] = \ net_prefix_to_ipv4_mask(subnet['prefix']) + if 'gateway' in subnet: + iface_cfg['GATEWAY'] = subnet['gateway'] + @classmethod def _render_subnet_routes(cls, iface_cfg, route_cfg, subnets): for i, subnet in enumerate(subnets, start=len(iface_cfg.children)): diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py index 83580cc0..dffe1781 100644 --- a/tests/unittests/test_distros/test_netconfig.py +++ b/tests/unittests/test_distros/test_netconfig.py @@ -479,6 +479,7 @@ BOOTPROTO=none DEVICE=eth0 IPADDR=192.168.1.5 NETMASK=255.255.255.0 +GATEWAY=192.168.1.254 NM_CONTROLLED=no ONBOOT=yes TYPE=Ethernet @@ -627,6 +628,7 @@ IPV6_AUTOCONF=no BOOTPROTO=none DEVICE=eth0 IPV6ADDR=2607:f0d0:1002:0011::2/64 +GATEWAY=2607:f0d0:1002:0011::1 IPV6INIT=yes NM_CONTROLLED=no ONBOOT=yes diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 06e8f094..71c9c457 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -828,6 +828,7 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true } } + CONFIG_V1_EXPLICIT_LOOPBACK = { 'version': 1, 'config': [{'name': 'eth0', 'type': 'physical', @@ -836,6 +837,18 @@ CONFIG_V1_EXPLICIT_LOOPBACK = { 'subnets': [{'control': 'auto', 'type': 'loopback'}]}, ]} + +CONFIG_V1_SIMPLE_SUBNET = { + 'version': 1, + 'config': [{'mac_address': '52:54:00:12:34:00', + 'name': 'interface0', + 'subnets': [{'address': '10.0.2.15', + 'gateway': '10.0.2.2', + 'netmask': '255.255.255.0', + 'type': 'static'}], + 'type': 'physical'}]} + + DEFAULT_DEV_ATTRS = { 'eth1000': { "bridge": False, @@ -1135,6 +1148,31 @@ USERCTL=no with open(os.path.join(render_dir, fn)) as fh: self.assertEqual(expected_content, fh.read()) + def test_network_config_v1_samples(self): + ns = network_state.parse_net_config_data(CONFIG_V1_SIMPLE_SUBNET) + render_dir = self.tmp_path("render") + os.makedirs(render_dir) + renderer = sysconfig.Renderer() + renderer.render_network_state(ns, render_dir) + found = dir2dict(render_dir) + nspath = '/etc/sysconfig/network-scripts/' + self.assertNotIn(nspath + 'ifcfg-lo', found.keys()) + expected = """\ +# Created by cloud-init on instance boot automatically, do not edit. +# +BOOTPROTO=none +DEVICE=interface0 +GATEWAY=10.0.2.2 +HWADDR=52:54:00:12:34:00 +IPADDR=10.0.2.15 +NETMASK=255.255.255.0 +NM_CONTROLLED=no +ONBOOT=yes +TYPE=Ethernet +USERCTL=no +""" + self.assertEqual(expected, found[nspath + 'ifcfg-interface0']) + def test_config_with_explicit_loopback(self): ns = network_state.parse_net_config_data(CONFIG_V1_EXPLICIT_LOOPBACK) render_dir = self.tmp_path("render") -- cgit v1.2.3 From 865e941f3f88c7daeafbf1eab856e02ce2b6a5f7 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 12 Jul 2017 17:16:35 -0700 Subject: tests: fixes for issues uncovered when moving to python 3.6. This includes a few fixes found when testing with python 3.6. - fix eni renderer when target is None This just uses the util.target_path() in the event that target is None. - change test cases to not rely on the cached result of util.get_cmdline() and other cached globals. Update the base TestCase to unset that cache. - mock calls to system_is_snappy from the create_users test cases. - drop unused _pp_root in test_simple_run.py LP: #1703697 --- cloudinit/net/netplan.py | 3 ++- tests/unittests/helpers.py | 21 +++++++++++++++- tests/unittests/test_distros/test_create_users.py | 30 +++++++++-------------- tests/unittests/test_runs/test_simple_run.py | 18 -------------- 4 files changed, 34 insertions(+), 38 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/net/netplan.py b/cloudinit/net/netplan.py index 67543305..9f35b72b 100644 --- a/cloudinit/net/netplan.py +++ b/cloudinit/net/netplan.py @@ -209,7 +209,8 @@ class Renderer(renderer.Renderer): # check network state for version # if v2, then extract network_state.config # else render_v2_from_state - fpnplan = os.path.join(target, self.netplan_path) + fpnplan = os.path.join(util.target_path(target), self.netplan_path) + util.ensure_dir(os.path.dirname(fpnplan)) header = self.netplan_header if self.netplan_header else "" diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 569f1aef..6691cf82 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -82,7 +82,26 @@ def retarget_many_wrapper(new_base, am, old_func): class TestCase(unittest2.TestCase): - pass + def reset_global_state(self): + """Reset any global state to its original settings. + + cloudinit caches some values in cloudinit.util. Unit tests that + involved those cached paths were then subject to failure if the order + of invocation changed (LP: #1703697). + + This function resets any of these global state variables to their + initial state. + + In the future this should really be done with some registry that + can then be cleaned in a more obvious way. + """ + util.PROC_CMDLINE = None + util._DNS_REDIRECT_IP = None + util._LSB_RELEASE = {} + + def setUp(self): + super(unittest2.TestCase, self).setUp() + self.reset_global_state() class CiTestCase(TestCase): diff --git a/tests/unittests/test_distros/test_create_users.py b/tests/unittests/test_distros/test_create_users.py index 9ded4f6c..1d02f7bd 100644 --- a/tests/unittests/test_distros/test_create_users.py +++ b/tests/unittests/test_distros/test_create_users.py @@ -38,6 +38,8 @@ class MyBaseDistro(distros.Distro): raise NotImplementedError() +@mock.patch("cloudinit.distros.util.system_is_snappy", return_value=False) +@mock.patch("cloudinit.distros.util.subp") class TestCreateUser(TestCase): def setUp(self): super(TestCase, self).setUp() @@ -53,8 +55,7 @@ class TestCreateUser(TestCase): logcmd[i + 1] = 'REDACTED' return mock.call(args, logstring=logcmd) - @mock.patch("cloudinit.distros.util.subp") - def test_basic(self, m_subp): + def test_basic(self, m_subp, m_is_snappy): user = 'foouser' self.dist.create_user(user) self.assertEqual( @@ -62,8 +63,7 @@ class TestCreateUser(TestCase): [self._useradd2call([user, '-m']), mock.call(['passwd', '-l', user])]) - @mock.patch("cloudinit.distros.util.subp") - def test_no_home(self, m_subp): + def test_no_home(self, m_subp, m_is_snappy): user = 'foouser' self.dist.create_user(user, no_create_home=True) self.assertEqual( @@ -71,8 +71,7 @@ class TestCreateUser(TestCase): [self._useradd2call([user, '-M']), mock.call(['passwd', '-l', user])]) - @mock.patch("cloudinit.distros.util.subp") - def test_system_user(self, m_subp): + def test_system_user(self, m_subp, m_is_snappy): # system user should have no home and get --system user = 'foouser' self.dist.create_user(user, system=True) @@ -81,8 +80,7 @@ class TestCreateUser(TestCase): [self._useradd2call([user, '--system', '-M']), mock.call(['passwd', '-l', user])]) - @mock.patch("cloudinit.distros.util.subp") - def test_explicit_no_home_false(self, m_subp): + def test_explicit_no_home_false(self, m_subp, m_is_snappy): user = 'foouser' self.dist.create_user(user, no_create_home=False) self.assertEqual( @@ -90,16 +88,14 @@ class TestCreateUser(TestCase): [self._useradd2call([user, '-m']), mock.call(['passwd', '-l', user])]) - @mock.patch("cloudinit.distros.util.subp") - def test_unlocked(self, m_subp): + def test_unlocked(self, m_subp, m_is_snappy): user = 'foouser' self.dist.create_user(user, lock_passwd=False) self.assertEqual( m_subp.call_args_list, [self._useradd2call([user, '-m'])]) - @mock.patch("cloudinit.distros.util.subp") - def test_set_password(self, m_subp): + def test_set_password(self, m_subp, m_is_snappy): user = 'foouser' password = 'passfoo' self.dist.create_user(user, passwd=password) @@ -109,8 +105,7 @@ class TestCreateUser(TestCase): mock.call(['passwd', '-l', user])]) @mock.patch("cloudinit.distros.util.is_group") - @mock.patch("cloudinit.distros.util.subp") - def test_group_added(self, m_subp, m_is_group): + def test_group_added(self, m_is_group, m_subp, m_is_snappy): m_is_group.return_value = False user = 'foouser' self.dist.create_user(user, groups=['group1']) @@ -121,8 +116,7 @@ class TestCreateUser(TestCase): self.assertEqual(m_subp.call_args_list, expected) @mock.patch("cloudinit.distros.util.is_group") - @mock.patch("cloudinit.distros.util.subp") - def test_only_new_group_added(self, m_subp, m_is_group): + def test_only_new_group_added(self, m_is_group, m_subp, m_is_snappy): ex_groups = ['existing_group'] groups = ['group1', ex_groups[0]] m_is_group.side_effect = lambda m: m in ex_groups @@ -135,8 +129,8 @@ class TestCreateUser(TestCase): self.assertEqual(m_subp.call_args_list, expected) @mock.patch("cloudinit.distros.util.is_group") - @mock.patch("cloudinit.distros.util.subp") - def test_create_groups_with_whitespace_string(self, m_subp, m_is_group): + def test_create_groups_with_whitespace_string( + self, m_is_group, m_subp, m_is_snappy): # groups supported as a comma delimeted string even with white space m_is_group.return_value = False user = 'foouser' diff --git a/tests/unittests/test_runs/test_simple_run.py b/tests/unittests/test_runs/test_simple_run.py index 31324204..55f15b55 100644 --- a/tests/unittests/test_runs/test_simple_run.py +++ b/tests/unittests/test_runs/test_simple_run.py @@ -16,24 +16,6 @@ class TestSimpleRun(helpers.FilesystemMockingTestCase): self.patchOS(root) self.patchUtils(root) - def _pp_root(self, root, repatch=True): - for (dirpath, dirnames, filenames) in os.walk(root): - print(dirpath) - for f in filenames: - joined = os.path.join(dirpath, f) - if os.path.islink(joined): - print("f %s - (symlink)" % (f)) - else: - print("f %s" % (f)) - for d in dirnames: - joined = os.path.join(dirpath, d) - if os.path.islink(joined): - print("d %s - (symlink)" % (d)) - else: - print("d %s" % (d)) - if repatch: - self._patchIn(root) - def test_none_ds(self): new_root = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, new_root) -- cgit v1.2.3 From c0060fe4892197179a5cfbfd3239cf3b6c3e5029 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 19 Jul 2017 09:28:52 -0400 Subject: net: fix renaming of nics to support mac addresses written in upper case. The network device renaming code previously required the case of the mac address input to match that of the data read from the system. For example, if user provided network config with mac address in upper case, then cloud-init would not rename the device correctly as /sys/class/net/address stores lower case values. The fix here is to always compare lower case mac addresses. LP: #1705147 --- cloudinit/net/__init__.py | 8 ++++++-- tests/unittests/test_net.py | 27 +++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index cba991a5..d1740e56 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -302,7 +302,7 @@ def _get_current_rename_info(check_downable=True): device has only automatically assigned ip addrs. 'device_id': Device id value (if it has one) 'driver': Device driver (if it has one) - 'mac': mac address + 'mac': mac address (in lower case) 'name': name 'up': boolean: is_up(name) }} @@ -313,7 +313,7 @@ def _get_current_rename_info(check_downable=True): 'downable': None, 'device_id': device_id, 'driver': driver, - 'mac': mac, + 'mac': mac.lower(), 'name': name, 'up': is_up(name), } @@ -348,6 +348,8 @@ def _rename_interfaces(renames, strict_present=True, strict_busy=True, cur_info = {} for name, data in current_info.items(): cur = data.copy() + if cur.get('mac'): + cur['mac'] = cur['mac'].lower() cur['name'] = name cur_info[name] = cur @@ -399,6 +401,8 @@ def _rename_interfaces(renames, strict_present=True, strict_busy=True, return None for mac, new_name, driver, device_id in renames: + if mac: + mac = mac.lower() cur_ops = [] cur = find_entry(mac, driver, device_id) if not cur: diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 71c9c457..76721bab 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -2176,5 +2176,32 @@ class TestRenameInterfaces(CiTestCase): capture=True), ]) + @mock.patch('cloudinit.util.subp') + def test_rename_macs_case_insensitive(self, mock_subp): + """_rename_interfaces must support upper or lower case macs.""" + renames = [ + ('aa:aa:aa:aa:aa:aa', 'en0', None, None), + ('BB:BB:BB:BB:BB:BB', 'en1', None, None), + ('cc:cc:cc:cc:cc:cc', 'en2', None, None), + ('DD:DD:DD:DD:DD:DD', 'en3', None, None), + ] + current_info = { + 'eth0': {'downable': True, 'mac': 'AA:AA:AA:AA:AA:AA', + 'name': 'eth0', 'up': False}, + 'eth1': {'downable': True, 'mac': 'bb:bb:bb:bb:bb:bb', + 'name': 'eth1', 'up': False}, + 'eth2': {'downable': True, 'mac': 'cc:cc:cc:cc:cc:cc', + 'name': 'eth2', 'up': False}, + 'eth3': {'downable': True, 'mac': 'DD:DD:DD:DD:DD:DD', + 'name': 'eth3', 'up': False}, + } + net._rename_interfaces(renames, current_info=current_info) + + expected = [ + mock.call(['ip', 'link', 'set', 'eth%d' % i, 'name', 'en%d' % i], + capture=True) + for i in range(len(renames))] + mock_subp.assert_has_calls(expected) + # vi: ts=4 expandtab -- cgit v1.2.3 From 97abd83513bee191b58f095f4d683b18acce0b49 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Fri, 9 Jun 2017 15:33:37 -0500 Subject: sysconfig: ipv6 and default gateway fixes. With this change, entries in IPV6ADDR and IPV6ADDR_SECONDARIES will now always be in format addr/prefix. When a subnet has a gateway will be written. If the gateway is ipv6, use the key IPV6_DEFAULTGW rather than GATEWAY. LP: #1704872 --- cloudinit/net/sysconfig.py | 20 +++++++++----------- tests/unittests/test_distros/test_netconfig.py | 6 ++++-- tests/unittests/test_net.py | 3 ++- 3 files changed, 15 insertions(+), 14 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index ad8c268e..de6601af 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -10,7 +10,8 @@ from cloudinit.distros.parsers import resolv_conf from cloudinit import util from . import renderer -from .network_state import subnet_is_ipv6, net_prefix_to_ipv4_mask +from .network_state import ( + is_ipv6_addr, net_prefix_to_ipv4_mask, subnet_is_ipv6) def _make_header(sep='#'): @@ -308,20 +309,13 @@ class Renderer(renderer.Renderer): elif subnet_type == 'static': if subnet_is_ipv6(subnet): ipv6_index = ipv6_index + 1 - if 'netmask' in subnet and str(subnet['netmask']) != "": - ipv6_cidr = (subnet['address'] + - '/' + - str(subnet['netmask'])) - else: - ipv6_cidr = subnet['address'] + ipv6_cidr = "%s/%s" % (subnet['address'], subnet['prefix']) if ipv6_index == 0: iface_cfg['IPV6ADDR'] = ipv6_cidr elif ipv6_index == 1: iface_cfg['IPV6ADDR_SECONDARIES'] = ipv6_cidr else: - iface_cfg['IPV6ADDR_SECONDARIES'] = ( - iface_cfg['IPV6ADDR_SECONDARIES'] + - " " + ipv6_cidr) + iface_cfg['IPV6ADDR_SECONDARIES'] += " " + ipv6_cidr else: ipv4_index = ipv4_index + 1 suff = "" if ipv4_index == 0 else str(ipv4_index) @@ -330,7 +324,11 @@ class Renderer(renderer.Renderer): net_prefix_to_ipv4_mask(subnet['prefix']) if 'gateway' in subnet: - iface_cfg['GATEWAY'] = subnet['gateway'] + iface_cfg['DEFROUTE'] = True + if is_ipv6_addr(subnet['gateway']): + iface_cfg['IPV6_DEFAULTGW'] = subnet['gateway'] + else: + iface_cfg['GATEWAY'] = subnet['gateway'] @classmethod def _render_subnet_routes(cls, iface_cfg, route_cfg, subnets): diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py index dffe1781..2f505d93 100644 --- a/tests/unittests/test_distros/test_netconfig.py +++ b/tests/unittests/test_distros/test_netconfig.py @@ -476,10 +476,11 @@ NETWORKING=yes # Created by cloud-init on instance boot automatically, do not edit. # BOOTPROTO=none +DEFROUTE=yes DEVICE=eth0 +GATEWAY=192.168.1.254 IPADDR=192.168.1.5 NETMASK=255.255.255.0 -GATEWAY=192.168.1.254 NM_CONTROLLED=no ONBOOT=yes TYPE=Ethernet @@ -626,10 +627,11 @@ IPV6_AUTOCONF=no # Created by cloud-init on instance boot automatically, do not edit. # BOOTPROTO=none +DEFROUTE=yes DEVICE=eth0 IPV6ADDR=2607:f0d0:1002:0011::2/64 -GATEWAY=2607:f0d0:1002:0011::1 IPV6INIT=yes +IPV6_DEFAULTGW=2607:f0d0:1002:0011::1 NM_CONTROLLED=no ONBOOT=yes TYPE=Ethernet diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 76721bab..22242717 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -298,7 +298,7 @@ DEVICE=eth0 GATEWAY=172.19.3.254 HWADDR=fa:16:3e:ed:9a:59 IPADDR=172.19.1.34 -IPV6ADDR=2001:DB8::10 +IPV6ADDR=2001:DB8::10/64 IPV6ADDR_SECONDARIES="2001:DB9::10/64 2001:DB10::10/64" IPV6INIT=yes IPV6_DEFAULTGW=2001:DB8::1 @@ -1161,6 +1161,7 @@ USERCTL=no # Created by cloud-init on instance boot automatically, do not edit. # BOOTPROTO=none +DEFROUTE=yes DEVICE=interface0 GATEWAY=10.0.2.2 HWADDR=52:54:00:12:34:00 -- cgit v1.2.3 From 51febf7363692d7947fe17a4fbfcb85058168ccb Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Wed, 14 Jun 2017 12:58:40 -0500 Subject: sysconfig: fix rendering of bond, bridge and vlan types. Previously, virtual types (bond, bridge, vlan) were almost completely broken. They would not get any network configuration (ip addresses or dhcp config) and or routes rendered. This fixes those issues. For bonds we now correctly render BONDING_SLAVE entries. Also add tests for simple bond, bridge and vlan. LP: #1695092 --- cloudinit/net/renderer.py | 4 + cloudinit/net/sysconfig.py | 43 +++++-- tests/unittests/test_net.py | 266 +++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 304 insertions(+), 9 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/net/renderer.py b/cloudinit/net/renderer.py index bba139e5..57652e27 100644 --- a/cloudinit/net/renderer.py +++ b/cloudinit/net/renderer.py @@ -20,6 +20,10 @@ def filter_by_name(match_name): return lambda iface: match_name == iface['name'] +def filter_by_attr(match_name): + return lambda iface: (match_name in iface and iface[match_name]) + + filter_by_physical = filter_by_type('physical') diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index de6601af..eb3c91d2 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -407,24 +407,41 @@ class Renderer(renderer.Renderer): @classmethod def _render_bond_interfaces(cls, network_state, iface_contents): bond_filter = renderer.filter_by_type('bond') + slave_filter = renderer.filter_by_attr('bond-master') for iface in network_state.iter_interfaces(bond_filter): iface_name = iface['name'] iface_cfg = iface_contents[iface_name] cls._render_bonding_opts(iface_cfg, iface) - iface_master_name = iface['bond-master'] - iface_cfg['MASTER'] = iface_master_name - iface_cfg['SLAVE'] = True + # Ensure that the master interface (and any of its children) # are actually marked as being bond types... - master_cfg = iface_contents[iface_master_name] - master_cfgs = [master_cfg] - master_cfgs.extend(master_cfg.children) + master_cfgs = [iface_cfg] + master_cfgs.extend(iface_cfg.children) for master_cfg in master_cfgs: master_cfg['BONDING_MASTER'] = True master_cfg.kind = 'bond' - @staticmethod - def _render_vlan_interfaces(network_state, iface_contents): + iface_subnets = iface.get("subnets", []) + route_cfg = iface_cfg.routes + cls._render_subnets(iface_cfg, iface_subnets) + cls._render_subnet_routes(iface_cfg, route_cfg, iface_subnets) + + # iter_interfaces on network-state is not sorted to produce + # consistent numbers we need to sort. + bond_slaves = sorted( + [slave_iface['name'] for slave_iface in + network_state.iter_interfaces(slave_filter) + if slave_iface['bond-master'] == iface_name]) + for index, bond_slave in enumerate(bond_slaves): + slavestr = 'BONDING_SLAVE%s' % index + iface_cfg[slavestr] = bond_slave + + slave_cfg = iface_contents[bond_slave] + slave_cfg['MASTER'] = iface_name + slave_cfg['SLAVE'] = True + + @classmethod + def _render_vlan_interfaces(cls, network_state, iface_contents): vlan_filter = renderer.filter_by_type('vlan') for iface in network_state.iter_interfaces(vlan_filter): iface_name = iface['name'] @@ -432,6 +449,11 @@ class Renderer(renderer.Renderer): iface_cfg['VLAN'] = True iface_cfg['PHYSDEV'] = iface_name[:iface_name.rfind('.')] + iface_subnets = iface.get("subnets", []) + route_cfg = iface_cfg.routes + cls._render_subnets(iface_cfg, iface_subnets) + cls._render_subnet_routes(iface_cfg, route_cfg, iface_subnets) + @staticmethod def _render_dns(network_state, existing_dns_path=None): content = resolv_conf.ResolvConf("") @@ -478,6 +500,11 @@ class Renderer(renderer.Renderer): for bridge_cfg in bridged_cfgs: bridge_cfg['BRIDGE'] = iface_name + iface_subnets = iface.get("subnets", []) + route_cfg = iface_cfg.routes + cls._render_subnets(iface_cfg, iface_subnets) + cls._render_subnet_routes(iface_cfg, route_cfg, iface_subnets) + @classmethod def _render_sysconfig(cls, base_sysconf_dir, network_state): '''Given state, return /etc/sysconfig files + contents''' diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 22242717..f786eea0 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -825,7 +825,211 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true gateway: 11.0.0.1 metric: 3 """).lstrip(), - } + }, + 'bond': { + 'yaml': textwrap.dedent(""" + version: 1 + config: + - type: physical + name: bond0s0 + mac_address: "aa:bb:cc:dd:e8:00" + - type: physical + name: bond0s1 + mac_address: "aa:bb:cc:dd:e8:01" + - type: bond + name: bond0 + mac_address: "aa:bb:cc:dd:e8:ff" + bond_interfaces: + - bond0s0 + - bond0s1 + params: + bond-mode: active-backup + bond_miimon: 100 + bond-xmit-hash-policy: "layer3+4" + subnets: + - type: static + address: 192.168.0.2/24 + gateway: 192.168.0.1 + routes: + - gateway: 192.168.0.3 + netmask: 255.255.255.0 + network: 10.1.3.0 + - type: static + address: 192.168.1.2/24 + - type: static + address: 2001:1::1/92 + """), + 'expected_sysconfig': { + 'ifcfg-bond0': textwrap.dedent("""\ + BONDING_MASTER=yes + BONDING_OPTS="mode=active-backup xmit_hash_policy=layer3+4 miimon=100" + BONDING_SLAVE0=bond0s0 + BONDING_SLAVE1=bond0s1 + BOOTPROTO=none + DEFROUTE=yes + DEVICE=bond0 + GATEWAY=192.168.0.1 + HWADDR=aa:bb:cc:dd:e8:ff + IPADDR=192.168.0.2 + IPADDR1=192.168.1.2 + IPV6ADDR=2001:1::1/92 + IPV6INIT=yes + NETMASK=255.255.255.0 + NETMASK1=255.255.255.0 + NM_CONTROLLED=no + ONBOOT=yes + TYPE=Bond + USERCTL=no + """), + 'ifcfg-bond0s0': textwrap.dedent("""\ + BOOTPROTO=none + DEVICE=bond0s0 + HWADDR=aa:bb:cc:dd:e8:00 + MASTER=bond0 + NM_CONTROLLED=no + ONBOOT=yes + SLAVE=yes + TYPE=Ethernet + USERCTL=no + """), + 'route6-bond0': textwrap.dedent("""\ + """), + 'route-bond0': textwrap.dedent("""\ + ADDRESS0=10.1.3.0 + GATEWAY0=192.168.0.3 + NETMASK0=255.255.255.0 + """), + 'ifcfg-bond0s1': textwrap.dedent("""\ + BOOTPROTO=none + DEVICE=bond0s1 + HWADDR=aa:bb:cc:dd:e8:01 + MASTER=bond0 + NM_CONTROLLED=no + ONBOOT=yes + SLAVE=yes + TYPE=Ethernet + USERCTL=no + """), + }, + }, + 'vlan': { + 'yaml': textwrap.dedent(""" + version: 1 + config: + - type: physical + name: en0 + mac_address: "aa:bb:cc:dd:e8:00" + - type: vlan + name: en0.99 + vlan_link: en0 + vlan_id: 99 + subnets: + - type: static + address: '192.168.2.2/24' + - type: static + address: '192.168.1.2/24' + gateway: 192.168.1.1 + - type: static + address: 2001:1::bbbb/96 + routes: + - gateway: 2001:1::1 + netmask: '::' + network: '::' + """), + 'expected_sysconfig': { + 'ifcfg-en0': textwrap.dedent("""\ + BOOTPROTO=none + DEVICE=en0 + HWADDR=aa:bb:cc:dd:e8:00 + NM_CONTROLLED=no + ONBOOT=yes + TYPE=Ethernet + USERCTL=no"""), + 'ifcfg-en0.99': textwrap.dedent("""\ + BOOTPROTO=none + DEFROUTE=yes + DEVICE=en0.99 + GATEWAY=2001:1::1 + IPADDR=192.168.2.2 + IPADDR1=192.168.1.2 + IPV6ADDR=2001:1::bbbb/96 + IPV6INIT=yes + NETMASK=255.255.255.0 + NETMASK1=255.255.255.0 + NM_CONTROLLED=no + ONBOOT=yes + PHYSDEV=en0 + TYPE=Ethernet + USERCTL=no + VLAN=yes"""), + }, + }, + 'bridge': { + 'yaml': textwrap.dedent(""" + version: 1 + config: + - type: physical + name: eth0 + mac_address: "52:54:00:12:34:00" + subnets: + - type: static + address: 2001:1::100/96 + - type: physical + name: eth1 + mac_address: "52:54:00:12:34:01" + subnets: + - type: static + address: 2001:1::101/96 + - type: bridge + name: br0 + bridge_interfaces: + - eth0 + - eth1 + params: + bridge_stp: 'off' + bridge_bridgeprio: 22 + subnets: + - type: static + address: 192.168.2.2/24"""), + 'expected_sysconfig': { + 'ifcfg-br0': textwrap.dedent("""\ + BOOTPROTO=none + DEVICE=br0 + IPADDR=192.168.2.2 + NETMASK=255.255.255.0 + NM_CONTROLLED=no + ONBOOT=yes + PRIO=22 + STP=off + TYPE=Bridge + USERCTL=no + """), + 'ifcfg-eth0': textwrap.dedent("""\ + BOOTPROTO=none + BRIDGE=br0 + DEVICE=eth0 + HWADDR=52:54:00:12:34:00 + IPV6ADDR=2001:1::100/96 + IPV6INIT=yes + NM_CONTROLLED=no + ONBOOT=yes + TYPE=Ethernet + USERCTL=no + """), + 'ifcfg-eth1': textwrap.dedent("""\ + BOOTPROTO=none + BRIDGE=br0 + DEVICE=eth1 + HWADDR=52:54:00:12:34:01 + IPV6ADDR=2001:1::101/96 + IPV6INIT=yes + NM_CONTROLLED=no + ONBOOT=yes + TYPE=Ethernet + USERCTL=no + """), + }, + }, } @@ -1021,6 +1225,48 @@ iface eth1 inet dhcp class TestSysConfigRendering(CiTestCase): + scripts_dir = '/etc/sysconfig/network-scripts' + header = ('# Created by cloud-init on instance boot automatically, ' + 'do not edit.\n#\n') + + def _render_and_read(self, network_config=None, state=None, dir=None): + if dir is None: + dir = self.tmp_dir() + + if network_config: + ns = network_state.parse_net_config_data(network_config) + elif state: + ns = state + else: + raise ValueError("Expected data or state, got neither") + + renderer = sysconfig.Renderer() + renderer.render_network_state(ns, dir) + return dir2dict(dir) + + def _compare_files_to_expected(self, expected, found): + orig_maxdiff = self.maxDiff + expected_d = dict( + (os.path.join(self.scripts_dir, k), util.load_shell_content(v)) + for k, v in expected.items()) + + # only compare the files in scripts_dir + scripts_found = dict( + (k, util.load_shell_content(v)) for k, v in found.items() + if k.startswith(self.scripts_dir)) + try: + self.maxDiff = None + self.assertEqual(expected_d, scripts_found) + finally: + self.maxDiff = orig_maxdiff + + def _assert_headers(self, found): + missing = [f for f in found + if (f.startswith(self.scripts_dir) and + not found[f].startswith(self.header))] + if missing: + raise AssertionError("Missing headers in: %s" % missing) + @mock.patch("cloudinit.net.sys_dev_path") @mock.patch("cloudinit.net.read_sys_net") @mock.patch("cloudinit.net.get_devicelist") @@ -1195,6 +1441,24 @@ USERCTL=no """ self.assertEqual(expected, found[nspath + 'ifcfg-eth0']) + def test_bond_config(self): + entry = NETWORK_CONFIGS['bond'] + found = self._render_and_read(network_config=yaml.load(entry['yaml'])) + self._compare_files_to_expected(entry['expected_sysconfig'], found) + self._assert_headers(found) + + def test_vlan_config(self): + entry = NETWORK_CONFIGS['vlan'] + found = self._render_and_read(network_config=yaml.load(entry['yaml'])) + self._compare_files_to_expected(entry['expected_sysconfig'], found) + self._assert_headers(found) + + def test_bridge_config(self): + entry = NETWORK_CONFIGS['bridge'] + found = self._render_and_read(network_config=yaml.load(entry['yaml'])) + self._compare_files_to_expected(entry['expected_sysconfig'], found) + self._assert_headers(found) + class TestEniNetRendering(CiTestCase): -- cgit v1.2.3 From 31fa6f9d0f945868349c033fa049d2467ddcd478 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Thu, 15 Jun 2017 11:51:54 -0500 Subject: sysconfig: fix ipv6 gateway routes Currently only the subnet is checked for 'ipv6' setting, however, the routes array may include a mix of v4 or v6 configurations, in particular, the gateway in a route may be ipv6, and if so, should export the value via IPV6_DEFAULTGW in the ifcfg-XXXX file. Additionally, if the route is v6, it should rendering a routes6-XXXX file; this is present but missing the 'dev ' scoping. LP: #1694801 --- cloudinit/net/sysconfig.py | 11 ++++++----- tests/unittests/test_net.py | 3 ++- 2 files changed, 8 insertions(+), 6 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index eb3c91d2..abdd4dee 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -152,9 +152,10 @@ class Route(ConfigMap): elif proto == "ipv6" and self.is_ipv6_route(address_value): netmask_value = str(self._conf['NETMASK' + index]) gateway_value = str(self._conf['GATEWAY' + index]) - buf.write("%s/%s via %s\n" % (address_value, - netmask_value, - gateway_value)) + buf.write("%s/%s via %s dev %s\n" % (address_value, + netmask_value, + gateway_value, + self._route_name)) return buf.getvalue() @@ -334,7 +335,7 @@ class Renderer(renderer.Renderer): def _render_subnet_routes(cls, iface_cfg, route_cfg, subnets): for i, subnet in enumerate(subnets, start=len(iface_cfg.children)): for route in subnet.get('routes', []): - is_ipv6 = subnet.get('ipv6') + is_ipv6 = subnet.get('ipv6') or is_ipv6_addr(route['gateway']) if _is_default_route(route): if ( @@ -356,7 +357,7 @@ class Renderer(renderer.Renderer): # also provided the default route? iface_cfg['DEFROUTE'] = True if 'gateway' in route: - if is_ipv6: + if is_ipv6 or is_ipv6_addr(route['gateway']): iface_cfg['IPV6_DEFAULTGW'] = route['gateway'] route_cfg.has_set_default_ipv6 = True else: diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index f786eea0..c012600f 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -949,11 +949,12 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true BOOTPROTO=none DEFROUTE=yes DEVICE=en0.99 - GATEWAY=2001:1::1 + GATEWAY=192.168.1.1 IPADDR=192.168.2.2 IPADDR1=192.168.1.2 IPV6ADDR=2001:1::bbbb/96 IPV6INIT=yes + IPV6_DEFAULTGW=2001:1::1 NETMASK=255.255.255.0 NETMASK1=255.255.255.0 NM_CONTROLLED=no -- cgit v1.2.3 From 8317bcab7cd08f1dcd96095c0cb746b57cb27234 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Thu, 15 Jun 2017 13:12:03 -0500 Subject: sysconfig: handle manual type subnets Implement manual control for sysconfig by using ONBOOT=N. This allows an interface to be configured but not brought up. Note that ONBOOT is per-interface not per address. LP: #1687725 --- cloudinit/net/sysconfig.py | 3 +++ tests/unittests/test_net.py | 46 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) (limited to 'cloudinit') diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index abdd4dee..b0f2ccf5 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -298,6 +298,9 @@ class Renderer(renderer.Renderer): " for interface '%s'" % (subnet_type, iface_cfg.name)) + if subnet.get('control') == 'manual': + iface_cfg['ONBOOT'] = False + # set IPv4 and IPv6 static addresses ipv4_index = -1 ipv6_index = -1 diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index c012600f..e625934f 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -1031,6 +1031,39 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true """), }, }, + 'manual': { + 'yaml': textwrap.dedent(""" + version: 1 + config: + - type: physical + name: eth0 + mac_address: "52:54:00:12:34:00" + subnets: + - type: static + address: 192.168.1.2/24 + control: manual"""), + 'expected_eni': textwrap.dedent("""\ + auto lo + iface lo inet loopback + + # control-manual eth0 + iface eth0 inet static + address 192.168.1.2/24 + """), + 'expected_sysconfig': { + 'ifcfg-eth0': textwrap.dedent("""\ + BOOTPROTO=none + DEVICE=eth0 + HWADDR=52:54:00:12:34:00 + IPADDR=192.168.1.2 + NETMASK=255.255.255.0 + NM_CONTROLLED=no + ONBOOT=no + TYPE=Ethernet + USERCTL=no + """), + }, + }, } @@ -1460,6 +1493,12 @@ USERCTL=no self._compare_files_to_expected(entry['expected_sysconfig'], found) self._assert_headers(found) + def test_manual_config(self): + entry = NETWORK_CONFIGS['manual'] + found = self._render_and_read(network_config=yaml.load(entry['yaml'])) + self._compare_files_to_expected(entry['expected_sysconfig'], found) + self._assert_headers(found) + class TestEniNetRendering(CiTestCase): @@ -1911,6 +1950,13 @@ class TestEniRoundTrip(CiTestCase): entry['expected_eni'].splitlines(), files['/etc/network/interfaces'].splitlines()) + def testsimple_render_manual(self): + entry = NETWORK_CONFIGS['manual'] + files = self._render_and_read(network_config=yaml.load(entry['yaml'])) + self.assertEqual( + entry['expected_eni'].splitlines(), + files['/etc/network/interfaces'].splitlines()) + def test_routes_rendered(self): # as reported in bug 1649652 conf = [ -- cgit v1.2.3 From 353c6902fb94899d410eb2f8bcc0bb81916e0e9e Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Thu, 15 Jun 2017 15:09:17 -0500 Subject: sysconfig: enable mtu set per subnet, including ipv6 mtu Render MTU values if present in subnet and route configurations for v4 and v6. LP: #1702513 --- cloudinit/net/sysconfig.py | 5 +++ tests/unittests/test_net.py | 76 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) (limited to 'cloudinit') diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index b0f2ccf5..c7df36c0 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -273,6 +273,7 @@ class Renderer(renderer.Renderer): # modifying base values according to subnets for i, subnet in enumerate(subnets, start=len(iface_cfg.children)): + mtu_key = 'MTU' subnet_type = subnet.get('type') if subnet_type == 'dhcp6': iface_cfg['IPV6INIT'] = True @@ -292,7 +293,11 @@ class Renderer(renderer.Renderer): # if iface_cfg['BOOTPROTO'] == 'none': # iface_cfg['BOOTPROTO'] = 'static' if subnet_is_ipv6(subnet): + mtu_key = 'IPV6_MTU' iface_cfg['IPV6INIT'] = True + + if 'mtu' in subnet: + iface_cfg[mtu_key] = subnet['mtu'] else: raise ValueError("Unknown subnet type '%s' found" " for interface '%s'" % (subnet_type, diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index e625934f..a9261ebd 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -482,6 +482,62 @@ NETWORK_CONFIGS = { - {'type': 'dhcp6'} """).rstrip(' '), }, + 'v4_and_v6_static': { + 'expected_eni': textwrap.dedent("""\ + auto lo + iface lo inet loopback + + auto iface0 + iface iface0 inet static + address 192.168.14.2/24 + mtu 9000 + + # control-alias iface0 + iface iface0 inet6 static + address 2001:1::1/64 + mtu 1500 + """).rstrip(' '), + 'expected_netplan': textwrap.dedent(""" + network: + version: 2 + ethernets: + iface0: + addresses: + - 192.168.14.2/24 + - 2001:1::1/64 + mtu: 9000 + mtu6: 1500 + """).rstrip(' '), + 'yaml': textwrap.dedent("""\ + version: 1 + config: + - type: 'physical' + name: 'iface0' + subnets: + - type: static + address: 192.168.14.2/24 + mtu: 9000 + - type: static + address: 2001:1::1/64 + mtu: 1500 + """).rstrip(' '), + 'expected_sysconfig': { + 'ifcfg-iface0': textwrap.dedent("""\ + BOOTPROTO=none + DEVICE=iface0 + IPADDR=192.168.14.2 + IPV6ADDR=2001:1::1/64 + IPV6INIT=yes + NETMASK=255.255.255.0 + NM_CONTROLLED=no + ONBOOT=yes + TYPE=Ethernet + USERCTL=no + MTU=9000 + IPV6_MTU=1500 + """), + }, + }, 'all': { 'expected_eni': ("""\ auto lo @@ -1499,6 +1555,12 @@ USERCTL=no self._compare_files_to_expected(entry['expected_sysconfig'], found) self._assert_headers(found) + def test_v4_and_v6_static_config(self): + entry = NETWORK_CONFIGS['v4_and_v6_static'] + found = self._render_and_read(network_config=yaml.load(entry['yaml'])) + self._compare_files_to_expected(entry['expected_sysconfig'], found) + self._assert_headers(found) + class TestEniNetRendering(CiTestCase): @@ -1892,6 +1954,13 @@ class TestNetplanRoundTrip(CiTestCase): entry['expected_netplan'].splitlines(), files['/etc/netplan/50-cloud-init.yaml'].splitlines()) + def testsimple_render_v4_and_v6_static(self): + entry = NETWORK_CONFIGS['v4_and_v6_static'] + files = self._render_and_read(network_config=yaml.load(entry['yaml'])) + self.assertEqual( + entry['expected_netplan'].splitlines(), + files['/etc/netplan/50-cloud-init.yaml'].splitlines()) + def testsimple_render_all(self): entry = NETWORK_CONFIGS['all'] files = self._render_and_read(network_config=yaml.load(entry['yaml'])) @@ -1950,6 +2019,13 @@ class TestEniRoundTrip(CiTestCase): entry['expected_eni'].splitlines(), files['/etc/network/interfaces'].splitlines()) + def testsimple_render_v4_and_v6_static(self): + entry = NETWORK_CONFIGS['v4_and_v6_static'] + files = self._render_and_read(network_config=yaml.load(entry['yaml'])) + self.assertEqual( + entry['expected_eni'].splitlines(), + files['/etc/network/interfaces'].splitlines()) + def testsimple_render_manual(self): entry = NETWORK_CONFIGS['manual'] files = self._render_and_read(network_config=yaml.load(entry['yaml'])) -- cgit v1.2.3 From 811ce49d74afb158b2626a201ef5c714d5c9b059 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Wed, 28 Jun 2017 16:29:18 -0500 Subject: net: eni route rendering missed ipv6 default route config In some network configurations a network value of '::' and a netmask value of '::' were used to indicate a default IPV6 gateway. Commit d00da2d5 removed ipv6 'netmask' attributes and calculate a prefix length value instead. The eni route rendering failed to update the check to use prefix value of 0 to indicate the presence of an IPV6 default route. A broken ipv6 default route rendered like: post-up route add -net :: netmask :: gw 2001:4800:78ff:1b::1 || true And with this patch, it now renders like: post-up route add -A inet6 default gw 2001:4800:78ff:1b::1 || true LP: #1701097 --- cloudinit/net/eni.py | 2 +- tests/unittests/test_net.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index b707146c..bb80ec02 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -355,7 +355,7 @@ class Renderer(renderer.Renderer): default_gw = " default gw %s" % route['gateway'] content.append(up + default_gw + or_true) content.append(down + default_gw + or_true) - elif route['network'] == '::' and route['netmask'] == 0: + elif route['network'] == '::' and route['prefix'] == 0: # ipv6! default_gw = " -A inet6 default gw %s" % route['gateway'] content.append(up + default_gw + or_true) diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index a9261ebd..d15cd1f4 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -597,6 +597,8 @@ iface br0 inet static # control-alias br0 iface br0 inet6 static address 2001:1::1/64 + post-up route add -A inet6 default gw 2001:4800:78ff:1b::1 || true + pre-down route del -A inet6 default gw 2001:4800:78ff:1b::1 || true auto bond0.200 iface bond0.200 inet dhcp @@ -731,6 +733,9 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true eth3: 50 eth4: 75 priority: 22 + routes: + - to: ::/0 + via: 2001:4800:78ff:1b::1 vlans: bond0.200: dhcp4: true @@ -863,6 +868,10 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true address: 192.168.14.2/24 - type: static address: 2001:1::1/64 # default to /64 + routes: + - gateway: 2001:4800:78ff:1b::1 + netmask: '::' + network: '::' # A global nameserver. - type: nameserver address: 8.8.8.8 -- cgit v1.2.3 From 7e41b2a773b81452f14a18ec8c4f3316a66d3f5e Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Thu, 20 Jul 2017 14:46:30 -0500 Subject: sysconfig: use MACADDR on bonds/bridges to configure mac_address Previously, sysconfig rendered HWADDR for all interface types, but that value is only used to identify physical devices. Instead use MACADDR to configure the MAC on virtual devices, like bonds and bridges. - Sort bond slave list to ensure consistent ordering in sysconfig rendered files. - Add unittests for sysconfig rendering of bonds/bridges with mac_address LP: #1701417 --- cloudinit/net/sysconfig.py | 11 ++++ tests/unittests/test_net.py | 149 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 159 insertions(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index c7df36c0..9184bce6 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -264,6 +264,9 @@ class Renderer(renderer.Renderer): for (old_key, new_key) in [('mac_address', 'HWADDR'), ('mtu', 'MTU')]: old_value = iface.get(old_key) if old_value is not None: + # only set HWADDR on physical interfaces + if old_key == 'mac_address' and iface['type'] != 'physical': + continue iface_cfg[new_key] = old_value @classmethod @@ -430,6 +433,9 @@ class Renderer(renderer.Renderer): master_cfg['BONDING_MASTER'] = True master_cfg.kind = 'bond' + if iface.get('mac_address'): + iface_cfg['MACADDR'] = iface.get('mac_address') + iface_subnets = iface.get("subnets", []) route_cfg = iface_cfg.routes cls._render_subnets(iface_cfg, iface_subnets) @@ -441,6 +447,7 @@ class Renderer(renderer.Renderer): [slave_iface['name'] for slave_iface in network_state.iter_interfaces(slave_filter) if slave_iface['bond-master'] == iface_name]) + for index, bond_slave in enumerate(bond_slaves): slavestr = 'BONDING_SLAVE%s' % index iface_cfg[slavestr] = bond_slave @@ -499,6 +506,10 @@ class Renderer(renderer.Renderer): for old_key, new_key in cls.bridge_opts_keys: if old_key in iface: iface_cfg[new_key] = iface[old_key] + + if iface.get('mac_address'): + iface_cfg['MACADDR'] = iface.get('mac_address') + # Is this the right key to get all the connected interfaces? for bridged_iface_name in iface.get('bridge_ports', []): # Ensure all bridged interfaces are correctly tagged diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index d15cd1f4..caf31342 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -422,6 +422,28 @@ NETWORK_CONFIGS = { via: 65.61.151.37 set-name: eth99 """).rstrip(' '), + 'expected_sysconfig': { + 'ifcfg-eth1': textwrap.dedent("""\ + BOOTPROTO=none + DEVICE=eth1 + HWADDR=cf:d6:af:48:e8:80 + NM_CONTROLLED=no + ONBOOT=yes + TYPE=Ethernet + USERCTL=no"""), + 'ifcfg-eth99': textwrap.dedent("""\ + BOOTPROTO=dhcp + DEFROUTE=yes + DEVICE=eth99 + GATEWAY=65.61.151.37 + HWADDR=c0:d6:9f:2c:e8:80 + IPADDR=192.168.21.3 + NETMASK=255.255.255.0 + NM_CONTROLLED=no + ONBOOT=yes + TYPE=Ethernet + USERCTL=no"""), + }, 'yaml': textwrap.dedent(""" version: 1 config: @@ -758,6 +780,119 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true - sacchromyces.maas - brettanomyces.maas """).rstrip(' '), + 'expected_sysconfig': { + 'ifcfg-bond0': textwrap.dedent("""\ + BONDING_MASTER=yes + BONDING_OPTS="mode=active-backup """ + """xmit_hash_policy=layer3+4 """ + """miimon=100" + BONDING_SLAVE0=eth1 + BONDING_SLAVE1=eth2 + BOOTPROTO=dhcp + DEVICE=bond0 + DHCPV6C=yes + IPV6INIT=yes + MACADDR=aa:bb:cc:dd:ee:ff + NM_CONTROLLED=no + ONBOOT=yes + TYPE=Bond + USERCTL=no"""), + 'ifcfg-bond0.200': textwrap.dedent("""\ + BOOTPROTO=dhcp + DEVICE=bond0.200 + NM_CONTROLLED=no + ONBOOT=yes + PHYSDEV=bond0 + TYPE=Ethernet + USERCTL=no + VLAN=yes"""), + 'ifcfg-br0': textwrap.dedent("""\ + AGEING=250 + BOOTPROTO=none + DEFROUTE=yes + DEVICE=br0 + IPADDR=192.168.14.2 + IPV6ADDR=2001:1::1/64 + IPV6INIT=yes + IPV6_DEFAULTGW=2001:4800:78ff:1b::1 + NETMASK=255.255.255.0 + NM_CONTROLLED=no + ONBOOT=yes + PRIO=22 + STP=off + TYPE=Bridge + USERCTL=no"""), + 'ifcfg-eth0': textwrap.dedent("""\ + BOOTPROTO=none + DEVICE=eth0 + HWADDR=c0:d6:9f:2c:e8:80 + NM_CONTROLLED=no + ONBOOT=yes + TYPE=Ethernet + USERCTL=no"""), + 'ifcfg-eth0.101': textwrap.dedent("""\ + BOOTPROTO=none + DEFROUTE=yes + DEVICE=eth0.101 + GATEWAY=192.168.0.1 + IPADDR=192.168.0.2 + IPADDR1=192.168.2.10 + MTU=1500 + NETMASK=255.255.255.0 + NETMASK1=255.255.255.0 + NM_CONTROLLED=no + ONBOOT=yes + PHYSDEV=eth0 + TYPE=Ethernet + USERCTL=no + VLAN=yes"""), + 'ifcfg-eth1': textwrap.dedent("""\ + BOOTPROTO=none + DEVICE=eth1 + HWADDR=aa:d6:9f:2c:e8:80 + MASTER=bond0 + NM_CONTROLLED=no + ONBOOT=yes + SLAVE=yes + TYPE=Ethernet + USERCTL=no"""), + 'ifcfg-eth2': textwrap.dedent("""\ + BOOTPROTO=none + DEVICE=eth2 + HWADDR=c0:bb:9f:2c:e8:80 + MASTER=bond0 + NM_CONTROLLED=no + ONBOOT=yes + SLAVE=yes + TYPE=Ethernet + USERCTL=no"""), + 'ifcfg-eth3': textwrap.dedent("""\ + BOOTPROTO=none + BRIDGE=br0 + DEVICE=eth3 + HWADDR=66:bb:9f:2c:e8:80 + NM_CONTROLLED=no + ONBOOT=yes + TYPE=Ethernet + USERCTL=no"""), + 'ifcfg-eth4': textwrap.dedent("""\ + BOOTPROTO=none + BRIDGE=br0 + DEVICE=eth4 + HWADDR=98:bb:9f:2c:e8:80 + NM_CONTROLLED=no + ONBOOT=yes + TYPE=Ethernet + USERCTL=no"""), + 'ifcfg-eth5': textwrap.dedent("""\ + BOOTPROTO=dhcp + DEVICE=eth5 + HWADDR=98:bb:9f:2c:e8:8a + NM_CONTROLLED=no + ONBOOT=no + TYPE=Ethernet + USERCTL=no""") + }, 'yaml': textwrap.dedent(""" version: 1 config: @@ -934,7 +1069,7 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true DEFROUTE=yes DEVICE=bond0 GATEWAY=192.168.0.1 - HWADDR=aa:bb:cc:dd:e8:ff + MACADDR=aa:bb:cc:dd:e8:ff IPADDR=192.168.0.2 IPADDR1=192.168.1.2 IPV6ADDR=2001:1::1/92 @@ -1564,6 +1699,18 @@ USERCTL=no self._compare_files_to_expected(entry['expected_sysconfig'], found) self._assert_headers(found) + def test_all_config(self): + entry = NETWORK_CONFIGS['all'] + found = self._render_and_read(network_config=yaml.load(entry['yaml'])) + self._compare_files_to_expected(entry['expected_sysconfig'], found) + self._assert_headers(found) + + def test_small_config(self): + entry = NETWORK_CONFIGS['small'] + found = self._render_and_read(network_config=yaml.load(entry['yaml'])) + self._compare_files_to_expected(entry['expected_sysconfig'], found) + self._assert_headers(found) + def test_v4_and_v6_static_config(self): entry = NETWORK_CONFIGS['v4_and_v6_static'] found = self._render_and_read(network_config=yaml.load(entry['yaml'])) -- cgit v1.2.3 From 42a7b34a12be7b0c43cfe8b94b397794d3e24c94 Mon Sep 17 00:00:00 2001 From: Bob Aman Date: Wed, 21 Jun 2017 11:30:58 -0700 Subject: Drop rand_str() usage in DNS redirection detection Making lots of random invalid DNS queries interferes with the ability of security teams to identify malicious or anomalous behavior from DNS logs. The same goal should be achievable with a consistent query for a name that is disallowed. LP: #1088611 --- cloudinit/util.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/util.py b/cloudinit/util.py index f570b9d3..ce2c6034 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1128,14 +1128,14 @@ def is_resolvable(name): we have to append '.'. The top level 'invalid' domain is invalid per RFC. And example.com - should also not exist. The random entry will be resolved inside - the search list. + should also not exist. The '__cloud_init_expected_not_found__' entry will + be resolved inside the search list. """ global _DNS_REDIRECT_IP if _DNS_REDIRECT_IP is None: badips = set() badnames = ("does-not-exist.example.com.", "example.invalid.", - rand_str()) + "__cloud_init_expected_not_found__") badresults = {} for iname in badnames: try: -- cgit v1.2.3 From 681baff19dc16ef2efdfb8499ad74aea0efbe467 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 21 Jul 2017 16:50:27 -0400 Subject: sysconfig: support subnet type of 'manual'. The subnet type 'manual' was used as a way to declare a device and set an MTU for it but not assign network addresses. This updates the manual example config to handle that case and provides expected rendered output for sysconfig, eni, and netplan. --- cloudinit/net/sysconfig.py | 9 ++++-- tests/unittests/test_net.py | 76 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 82 insertions(+), 3 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index 9184bce6..a550f97c 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -61,6 +61,9 @@ class ConfigMap(object): def __getitem__(self, key): return self._conf[key] + def __contains__(self, key): + return key in self._conf + def drop(self, key): self._conf.pop(key, None) @@ -298,14 +301,16 @@ class Renderer(renderer.Renderer): if subnet_is_ipv6(subnet): mtu_key = 'IPV6_MTU' iface_cfg['IPV6INIT'] = True - if 'mtu' in subnet: iface_cfg[mtu_key] = subnet['mtu'] + elif subnet_type == 'manual': + # If the subnet has an MTU setting, then ONBOOT=True + # to apply the setting + iface_cfg['ONBOOT'] = mtu_key in iface_cfg else: raise ValueError("Unknown subnet type '%s' found" " for interface '%s'" % (subnet_type, iface_cfg.name)) - if subnet.get('control') == 'manual': iface_cfg['ONBOOT'] = False diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index caf31342..e49abcc4 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -1241,7 +1241,20 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true subnets: - type: static address: 192.168.1.2/24 - control: manual"""), + control: manual + - type: physical + name: eth1 + mtu: 1480 + mac_address: "52:54:00:12:34:aa" + subnets: + - type: manual + - type: physical + name: eth2 + mac_address: "52:54:00:12:34:ff" + subnets: + - type: manual + control: manual + """), 'expected_eni': textwrap.dedent("""\ auto lo iface lo inet loopback @@ -1249,6 +1262,34 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true # control-manual eth0 iface eth0 inet static address 192.168.1.2/24 + + auto eth1 + iface eth1 inet manual + mtu 1480 + + # control-manual eth2 + iface eth2 inet manual + """), + 'expected_netplan': textwrap.dedent("""\ + + network: + version: 2 + ethernets: + eth0: + addresses: + - 192.168.1.2/24 + match: + macaddress: '52:54:00:12:34:00' + set-name: eth0 + eth1: + match: + macaddress: 52:54:00:12:34:aa + mtu: 1480 + set-name: eth1 + eth2: + match: + macaddress: 52:54:00:12:34:ff + set-name: eth2 """), 'expected_sysconfig': { 'ifcfg-eth0': textwrap.dedent("""\ @@ -1262,6 +1303,25 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true TYPE=Ethernet USERCTL=no """), + 'ifcfg-eth1': textwrap.dedent("""\ + BOOTPROTO=none + DEVICE=eth1 + HWADDR=52:54:00:12:34:aa + MTU=1480 + NM_CONTROLLED=no + ONBOOT=yes + TYPE=Ethernet + USERCTL=no + """), + 'ifcfg-eth2': textwrap.dedent("""\ + BOOTPROTO=none + DEVICE=eth2 + HWADDR=52:54:00:12:34:ff + NM_CONTROLLED=no + ONBOOT=no + TYPE=Ethernet + USERCTL=no + """), }, }, } @@ -2124,6 +2184,13 @@ class TestNetplanRoundTrip(CiTestCase): entry['expected_netplan'].splitlines(), files['/etc/netplan/50-cloud-init.yaml'].splitlines()) + def testsimple_render_manual(self): + entry = NETWORK_CONFIGS['manual'] + files = self._render_and_read(network_config=yaml.load(entry['yaml'])) + self.assertEqual( + entry['expected_netplan'].splitlines(), + files['/etc/netplan/50-cloud-init.yaml'].splitlines()) + class TestEniRoundTrip(CiTestCase): def _render_and_read(self, network_config=None, state=None, eni_path=None, @@ -2183,6 +2250,13 @@ class TestEniRoundTrip(CiTestCase): files['/etc/network/interfaces'].splitlines()) def testsimple_render_manual(self): + """Test rendering of 'manual' for 'type' and 'control'. + + 'type: manual' in a subnet is odd, but it is the way that was used + to declare that a network device should get a mtu set on it even + if there were no addresses to configure. Also strange is the fact + that in order to apply that MTU the ifupdown device must be set + to 'auto', or the MTU would not be set.""" entry = NETWORK_CONFIGS['manual'] files = self._render_and_read(network_config=yaml.load(entry['yaml'])) self.assertEqual( -- cgit v1.2.3 From 85c984c3c74b0a8751698e20915b89756580b09f Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Fri, 21 Jul 2017 11:56:50 -0700 Subject: archlinux: fix set hostname usage of write_file. cloud-init fails to set the hostname on Arch Linux because that _write_hostname passes conf instead of str(conf) to util.write_file. LP: #1705306 --- cloudinit/distros/arch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py index 75d46201..b4c0ba72 100644 --- a/cloudinit/distros/arch.py +++ b/cloudinit/distros/arch.py @@ -119,7 +119,7 @@ class Distro(distros.Distro): if not conf: conf = HostnameConf('') conf.set_hostname(your_hostname) - util.write_file(out_fn, conf, 0o644) + util.write_file(out_fn, str(conf), omode="w", mode=0o644) def _read_system_hostname(self): sys_hostname = self._read_hostname(self.hostname_conf_fn) -- cgit v1.2.3 From 0ef61b289472665f4e3059a24a8b9b91246f06ee Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 8 Jun 2017 15:42:12 -0500 Subject: locale: Do not re-run locale-gen if provided locale is system default. If the system configure default in /etc/default/locale is set to the same value that is provided for cloud-init's "locale" setting, then do not re-run locale-gen. This allows images built with a locale already generated to not re-run locale-gen (which can be very heavy). Also here is a fix to invoke update-locale correctly and remove the internal writing of /etc/default/locale. We were calling update-locale This ends up having no affect. The more correct invocation is: update-locale LANG= Also added some support here should we ever want to change setting LANG to setting LC_ALL (or any other key). Lastly, a test change to allow us to use assert_not_called from mock. Versions of mock in CentOS 6 do not have assert_not_called. --- cloudinit/distros/debian.py | 48 +++++++++++++---- tests/unittests/helpers.py | 12 +++++ tests/unittests/test_distros/test_debian.py | 82 +++++++++++++++++++++++++++++ 3 files changed, 131 insertions(+), 11 deletions(-) create mode 100644 tests/unittests/test_distros/test_debian.py (limited to 'cloudinit') diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index d06d46a6..abfb81f4 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -37,11 +37,11 @@ ENI_HEADER = """# This file is generated from information provided by """ NETWORK_CONF_FN = "/etc/network/interfaces.d/50-cloud-init.cfg" +LOCALE_CONF_FN = "/etc/default/locale" class Distro(distros.Distro): hostname_conf_fn = "/etc/hostname" - locale_conf_fn = "/etc/default/locale" network_conf_fn = { "eni": "/etc/network/interfaces.d/50-cloud-init.cfg", "netplan": "/etc/netplan/50-cloud-init.yaml" @@ -64,16 +64,8 @@ class Distro(distros.Distro): def apply_locale(self, locale, out_fn=None): if not out_fn: - out_fn = self.locale_conf_fn - util.subp(['locale-gen', locale], capture=False) - util.subp(['update-locale', locale], capture=False) - # "" provides trailing newline during join - lines = [ - util.make_header(), - 'LANG="%s"' % (locale), - "", - ] - util.write_file(out_fn, "\n".join(lines)) + out_fn = LOCALE_CONF_FN + apply_locale(locale, out_fn) def install_packages(self, pkglist): self.update_package_sources() @@ -225,4 +217,38 @@ def _maybe_remove_legacy_eth0(path="/etc/network/interfaces.d/eth0.cfg"): LOG.warning(msg) + +def apply_locale(locale, sys_path=LOCALE_CONF_FN, keyname='LANG'): + """Apply the locale. + + Run locale-gen for the provided locale and set the default + system variable `keyname` appropriately in the provided `sys_path`. + + If sys_path indicates that `keyname` is already set to `locale` + then no changes will be made and locale-gen not called. + This allows images built with a locale already generated to not re-run + locale-gen which can be very heavy. + """ + if not locale: + raise ValueError('Failed to provide locale value.') + + if not sys_path: + raise ValueError('Invalid path: %s' % sys_path) + + if os.path.exists(sys_path): + locale_content = util.load_file(sys_path) + # if LANG isn't present, regen + sys_defaults = util.load_shell_content(locale_content) + sys_val = sys_defaults.get(keyname, "") + if sys_val.lower() == locale.lower(): + LOG.debug( + "System has '%s=%s' requested '%s', skipping regeneration.", + keyname, sys_val, locale) + return + + util.subp(['locale-gen', locale], capture=False) + util.subp( + ['update-locale', '--locale-file=' + sys_path, + '%s=%s' % (keyname, locale)], capture=False) + # vi: ts=4 expandtab diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 6691cf82..08c5c469 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -376,4 +376,16 @@ except AttributeError: return wrapper return decorator + +# older versions of mock do not have the useful 'assert_not_called' +if not hasattr(mock.Mock, 'assert_not_called'): + def __mock_assert_not_called(mmock): + if mmock.call_count != 0: + msg = ("[citest] Expected '%s' to not have been called. " + "Called %s times." % + (mmock._mock_name or 'mock', mmock.call_count)) + raise AssertionError(msg) + mock.Mock.assert_not_called = __mock_assert_not_called + + # vi: ts=4 expandtab diff --git a/tests/unittests/test_distros/test_debian.py b/tests/unittests/test_distros/test_debian.py new file mode 100644 index 00000000..2330ad52 --- /dev/null +++ b/tests/unittests/test_distros/test_debian.py @@ -0,0 +1,82 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from ..helpers import (CiTestCase, mock) + +from cloudinit.distros.debian import apply_locale +from cloudinit import util + + +@mock.patch("cloudinit.distros.debian.util.subp") +class TestDebianApplyLocale(CiTestCase): + def test_no_rerun(self, m_subp): + """If system has defined locale, no re-run is expected.""" + spath = self.tmp_path("default-locale") + m_subp.return_value = (None, None) + locale = 'en_US.UTF-8' + util.write_file(spath, 'LANG=%s\n' % locale, omode="w") + apply_locale(locale, sys_path=spath) + m_subp.assert_not_called() + + def test_rerun_if_different(self, m_subp): + """If system has different locale, locale-gen should be called.""" + spath = self.tmp_path("default-locale") + m_subp.return_value = (None, None) + locale = 'en_US.UTF-8' + util.write_file(spath, 'LANG=fr_FR.UTF-8', omode="w") + apply_locale(locale, sys_path=spath) + self.assertEqual( + [['locale-gen', locale], + ['update-locale', '--locale-file=' + spath, 'LANG=%s' % locale]], + [p[0][0] for p in m_subp.call_args_list]) + + def test_rerun_if_no_file(self, m_subp): + """If system has no locale file, locale-gen should be called.""" + spath = self.tmp_path("default-locale") + m_subp.return_value = (None, None) + locale = 'en_US.UTF-8' + apply_locale(locale, sys_path=spath) + self.assertEqual( + [['locale-gen', locale], + ['update-locale', '--locale-file=' + spath, 'LANG=%s' % locale]], + [p[0][0] for p in m_subp.call_args_list]) + + def test_rerun_on_unset_system_locale(self, m_subp): + """If system has unset locale, locale-gen should be called.""" + m_subp.return_value = (None, None) + spath = self.tmp_path("default-locale") + locale = 'en_US.UTF-8' + util.write_file(spath, 'LANG=', omode="w") + apply_locale(locale, sys_path=spath) + self.assertEqual( + [['locale-gen', locale], + ['update-locale', '--locale-file=' + spath, 'LANG=%s' % locale]], + [p[0][0] for p in m_subp.call_args_list]) + + def test_rerun_on_mismatched_keys(self, m_subp): + """If key is LC_ALL and system has only LANG, rerun is expected.""" + m_subp.return_value = (None, None) + spath = self.tmp_path("default-locale") + locale = 'en_US.UTF-8' + util.write_file(spath, 'LANG=', omode="w") + apply_locale(locale, sys_path=spath, keyname='LC_ALL') + self.assertEqual( + [['locale-gen', locale], + ['update-locale', '--locale-file=' + spath, + 'LC_ALL=%s' % locale]], + [p[0][0] for p in m_subp.call_args_list]) + + def test_falseish_locale_raises_valueerror(self, m_subp): + """locale as None or "" is invalid and should raise ValueError.""" + + with self.assertRaises(ValueError) as ctext_m: + apply_locale(None) + m_subp.assert_not_called() + + self.assertEqual( + 'Failed to provide locale value.', str(ctext_m.exception)) + + with self.assertRaises(ValueError) as ctext_m: + apply_locale("") + m_subp.assert_not_called() + self.assertEqual( + 'Failed to provide locale value.', str(ctext_m.exception)) -- cgit v1.2.3 From ebdbf30c0274f078f7a66f6dc9efc8a22a220757 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 17 Jul 2017 13:48:19 -0400 Subject: tests: Add initial tests for EC2 and improve a docstring. EC2 was the original, but this adds some initial tests for that datasource. Also updates a docstring for an internal method. --- cloudinit/sources/DataSourceEc2.py | 14 +- tests/unittests/test_datasource/test_ec2.py | 202 ++++++++++++++++++++++++++++ 2 files changed, 212 insertions(+), 4 deletions(-) create mode 100644 tests/unittests/test_datasource/test_ec2.py (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 9e2fdc0a..4ec9592f 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -316,10 +316,16 @@ def identify_platform(): def _collect_platform_data(): - # returns a dictionary with all lower case values: - # uuid: system-uuid from dmi or /sys/hypervisor - # uuid_source: 'hypervisor' (/sys/hypervisor/uuid) or 'dmi' - # serial: dmi 'system-serial-number' (/sys/.../product_serial) + """Returns a dictionary of platform info from dmi or /sys/hypervisor. + + Keys in the dictionary are as follows: + uuid: system-uuid from dmi or /sys/hypervisor + uuid_source: 'hypervisor' (/sys/hypervisor/uuid) or 'dmi' + serial: dmi 'system-serial-number' (/sys/.../product_serial) + + On Ec2 instances experimentation is that product_serial is upper case, + and product_uuid is lower case. This returns lower case values for both. + """ data = {} try: uuid = util.load_file("/sys/hypervisor/uuid").strip() diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py new file mode 100644 index 00000000..12230ae2 --- /dev/null +++ b/tests/unittests/test_datasource/test_ec2.py @@ -0,0 +1,202 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +import httpretty +import mock + +from .. import helpers as test_helpers +from cloudinit import helpers +from cloudinit.sources import DataSourceEc2 as ec2 + + +# collected from api version 2009-04-04/ with +# python3 -c 'import json +# from cloudinit.ec2_utils import get_instance_metadata as gm +# print(json.dumps(gm("2009-04-04"), indent=1, sort_keys=True))' +DEFAULT_METADATA = { + "ami-id": "ami-80861296", + "ami-launch-index": "0", + "ami-manifest-path": "(unknown)", + "block-device-mapping": {"ami": "/dev/sda1", "root": "/dev/sda1"}, + "hostname": "ip-10-0-0-149", + "instance-action": "none", + "instance-id": "i-0052913950685138c", + "instance-type": "t2.micro", + "local-hostname": "ip-10-0-0-149", + "local-ipv4": "10.0.0.149", + "placement": {"availability-zone": "us-east-1b"}, + "profile": "default-hvm", + "public-hostname": "", + "public-ipv4": "107.23.188.247", + "public-keys": {"brickies": ["ssh-rsa AAAAB3Nz....w== brickies"]}, + "reservation-id": "r-00a2c173fb5782a08", + "security-groups": "wide-open" +} + + +def _register_ssh_keys(rfunc, base_url, keys_data): + """handle ssh key inconsistencies. + + public-keys in the ec2 metadata is inconsistently formatted compared + to other entries. + Given keys_data of {name1: pubkey1, name2: pubkey2} + + This registers the following urls: + base_url 0={name1}\n1={name2} # (for each name) + base_url/ 0={name1}\n1={name2} # (for each name) + base_url/0 openssh-key + base_url/0/ openssh-key + base_url/0/openssh-key {pubkey1} + base_url/0/openssh-key/ {pubkey1} + ... + """ + + base_url = base_url.rstrip("/") + odd_index = '\n'.join( + ["{0}={1}".format(n, name) + for n, name in enumerate(sorted(keys_data))]) + + rfunc(base_url, odd_index) + rfunc(base_url + "/", odd_index) + + for n, name in enumerate(sorted(keys_data)): + val = keys_data[name] + if isinstance(val, list): + val = '\n'.join(val) + burl = base_url + "/%s" % n + rfunc(burl, "openssh-key") + rfunc(burl + "/", "openssh-key") + rfunc(burl + "/%s/openssh-key" % name, val) + rfunc(burl + "/%s/openssh-key/" % name, val) + + +def register_mock_metaserver(base_url, data): + """Register with httpretty a ec2 metadata like service serving 'data'. + + If given a dictionary, it will populate urls under base_url for + that dictionary. For example, input of + {"instance-id": "i-abc", "mac": "00:16:3e:00:00:00"} + populates + base_url with 'instance-id\nmac' + base_url/ with 'instance-id\nmac' + base_url/instance-id with i-abc + base_url/mac with 00:16:3e:00:00:00 + In the index, references to lists or dictionaries have a trailing /. + """ + def register_helper(register, base_url, body): + base_url = base_url.rstrip("/") + if isinstance(body, str): + register(base_url, body) + elif isinstance(body, list): + register(base_url, '\n'.join(body) + '\n') + register(base_url + '/', '\n'.join(body) + '\n') + elif isinstance(body, dict): + vals = [] + for k, v in body.items(): + if k == 'public-keys': + _register_ssh_keys( + register, base_url + '/public-keys/', v) + continue + suffix = k.rstrip("/") + if not isinstance(v, (str, list)): + suffix += "/" + vals.append(suffix) + url = base_url + '/' + suffix + register_helper(register, url, v) + register(base_url, '\n'.join(vals) + '\n') + register(base_url + '/', '\n'.join(vals) + '\n') + elif body is None: + register(base_url, 'not found', status_code=404) + + def myreg(*argc, **kwargs): + # print("register_url(%s, %s)" % (argc, kwargs)) + return httpretty.register_uri(httpretty.GET, *argc, **kwargs) + + register_helper(myreg, base_url, data) + + +class TestEc2(test_helpers.HttprettyTestCase): + valid_platform_data = { + 'uuid': 'ec212f79-87d1-2f1d-588f-d86dc0fd5412', + 'uuid_source': 'dmi', + 'serial': 'ec212f79-87d1-2f1d-588f-d86dc0fd5412', + } + + def setUp(self): + super(TestEc2, self).setUp() + self.metadata_addr = ec2.DataSourceEc2.metadata_urls[0] + self.api_ver = '2009-04-04' + + @property + def metadata_url(self): + return '/'.join([self.metadata_addr, self.api_ver, 'meta-data', '']) + + @property + def userdata_url(self): + return '/'.join([self.metadata_addr, self.api_ver, 'user-data']) + + def _patch_add_cleanup(self, mpath, *args, **kwargs): + p = mock.patch(mpath, *args, **kwargs) + p.start() + self.addCleanup(p.stop) + + def _setup_ds(self, sys_cfg, platform_data, md, ud=None): + distro = {} + paths = helpers.Paths({}) + if sys_cfg is None: + sys_cfg = {} + ds = ec2.DataSourceEc2(sys_cfg=sys_cfg, distro=distro, paths=paths) + if platform_data is not None: + self._patch_add_cleanup( + "cloudinit.sources.DataSourceEc2._collect_platform_data", + return_value=platform_data) + + if md: + register_mock_metaserver(self.metadata_url, md) + register_mock_metaserver(self.userdata_url, ud) + + return ds + + @httpretty.activate + def test_valid_platform_with_strict_true(self): + """Valid platform data should return true with strict_id true.""" + ds = self._setup_ds( + platform_data=self.valid_platform_data, + sys_cfg={'datasource': {'Ec2': {'strict_id': True}}}, + md=DEFAULT_METADATA) + ret = ds.get_data() + self.assertEqual(True, ret) + + @httpretty.activate + def test_valid_platform_with_strict_false(self): + """Valid platform data should return true with strict_id false.""" + ds = self._setup_ds( + platform_data=self.valid_platform_data, + sys_cfg={'datasource': {'Ec2': {'strict_id': False}}}, + md=DEFAULT_METADATA) + ret = ds.get_data() + self.assertEqual(True, ret) + + @httpretty.activate + def test_unknown_platform_with_strict_true(self): + """Unknown platform data with strict_id true should return False.""" + uuid = 'ab439480-72bf-11d3-91fc-b8aded755F9a' + ds = self._setup_ds( + platform_data={'uuid': uuid, 'uuid_source': 'dmi', 'serial': ''}, + sys_cfg={'datasource': {'Ec2': {'strict_id': True}}}, + md=DEFAULT_METADATA) + ret = ds.get_data() + self.assertEqual(False, ret) + + @httpretty.activate + def test_unknown_platform_with_strict_false(self): + """Unknown platform data with strict_id false should return True.""" + uuid = 'ab439480-72bf-11d3-91fc-b8aded755F9a' + ds = self._setup_ds( + platform_data={'uuid': uuid, 'uuid_source': 'dmi', 'serial': ''}, + sys_cfg={'datasource': {'Ec2': {'strict_id': False}}}, + md=DEFAULT_METADATA) + ret = ds.get_data() + self.assertEqual(True, ret) + + +# vi: ts=4 expandtab -- cgit v1.2.3 From e586fe35a692b7519000005c8024ebd2bcbc82e0 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Fri, 28 Jul 2017 14:28:47 -0600 Subject: cloudinit.net: add initialize_network_device function and tests This is not yet called, but will be called in a subsequent Ec2-related branch to manually initialize a network interface with the responses using dhcp discovery without any dhcp-script side-effects. The functionality has been tested on Ec2 ubuntu and CentOS vms to ensure that network interface initialization works in both OS-types. Since there was poor unit test coverage for the cloudinit.net.__init__ module, this branch adds a bunch of coverage to the functions in cloudinit.net.__init. We can also now have unit tests local to the cloudinit modules. The benefits of having unittests under cloudinit module: - Proximity of unittest to cloudinit module makes it easier for ongoing devs to know where to augment unit tests. The tests.unittest directory is organizated such that it - Allows for 1 to 1 name mapping module -> tests/test_module.py - Improved test and module isolation, if we find unit tests have to import from a number of modules besides the module under test, it will better prompt resturcturing of the module. This also branch touches: - tox.ini to run unit tests found in cloudinit as well as include all test-requirements for pylint since we now have unit tests living within cloudinit package - setup.py to exclude any test modules under cloudinit when packaging --- cloudinit/net/__init__.py | 131 ++++++++-- cloudinit/net/tests/__init__.py | 0 cloudinit/net/tests/test_init.py | 522 +++++++++++++++++++++++++++++++++++++++ setup.py | 2 +- tox.ini | 14 +- 5 files changed, 648 insertions(+), 21 deletions(-) create mode 100644 cloudinit/net/tests/__init__.py create mode 100644 cloudinit/net/tests/test_init.py (limited to 'cloudinit') diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index d1740e56..46cb9c85 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -10,6 +10,7 @@ import logging import os import re +from cloudinit.net.network_state import mask_to_net_prefix from cloudinit import util LOG = logging.getLogger(__name__) @@ -28,8 +29,13 @@ def _natural_sort_key(s, _nsre=re.compile('([0-9]+)')): for text in re.split(_nsre, s)] +def get_sys_class_path(): + """Simple function to return the global SYS_CLASS_NET.""" + return SYS_CLASS_NET + + def sys_dev_path(devname, path=""): - return SYS_CLASS_NET + devname + "/" + path + return get_sys_class_path() + devname + "/" + path def read_sys_net(devname, path, translate=None, @@ -77,7 +83,7 @@ def read_sys_net_int(iface, field): return None try: return int(val) - except TypeError: + except ValueError: return None @@ -149,7 +155,14 @@ def device_devid(devname): def get_devicelist(): - return os.listdir(SYS_CLASS_NET) + try: + devs = os.listdir(get_sys_class_path()) + except OSError as e: + if e.errno == errno.ENOENT: + devs = [] + else: + raise + return devs class ParserError(Exception): @@ -497,14 +510,8 @@ def get_interfaces_by_mac(): """Build a dictionary of tuples {mac: name}. Bridges and any devices that have a 'stolen' mac are excluded.""" - try: - devs = get_devicelist() - except OSError as e: - if e.errno == errno.ENOENT: - devs = [] - else: - raise ret = {} + devs = get_devicelist() empty_mac = '00:00:00:00:00:00' for name in devs: if not interface_has_own_mac(name): @@ -531,14 +538,8 @@ def get_interfaces(): """Return list of interface tuples (name, mac, driver, device_id) Bridges and any devices that have a 'stolen' mac are excluded.""" - try: - devs = get_devicelist() - except OSError as e: - if e.errno == errno.ENOENT: - devs = [] - else: - raise ret = [] + devs = get_devicelist() empty_mac = '00:00:00:00:00:00' for name in devs: if not interface_has_own_mac(name): @@ -557,6 +558,102 @@ def get_interfaces(): return ret +class EphemeralIPv4Network(object): + """Context manager which sets up temporary static network configuration. + + No operations are performed if the provided interface is already connected. + If unconnected, bring up the interface with valid ip, prefix and broadcast. + If router is provided setup a default route for that interface. Upon + context exit, clean up the interface leaving no configuration behind. + """ + + def __init__(self, interface, ip, prefix_or_mask, broadcast, router=None): + """Setup context manager and validate call signature. + + @param interface: Name of the network interface to bring up. + @param ip: IP address to assign to the interface. + @param prefix_or_mask: Either netmask of the format X.X.X.X or an int + prefix. + @param broadcast: Broadcast address for the IPv4 network. + @param router: Optionally the default gateway IP. + """ + if not all([interface, ip, prefix_or_mask, broadcast]): + raise ValueError( + 'Cannot init network on {0} with {1}/{2} and bcast {3}'.format( + interface, ip, prefix_or_mask, broadcast)) + try: + self.prefix = mask_to_net_prefix(prefix_or_mask) + except ValueError as e: + raise ValueError( + 'Cannot setup network: {0}'.format(e)) + self.interface = interface + self.ip = ip + self.broadcast = broadcast + self.router = router + self.cleanup_cmds = [] # List of commands to run to cleanup state. + + def __enter__(self): + """Perform ephemeral network setup if interface is not connected.""" + self._bringup_device() + if self.router: + self._bringup_router() + + def __exit__(self, excp_type, excp_value, excp_traceback): + for cmd in self.cleanup_cmds: + util.subp(cmd, capture=True) + + def _delete_address(self, address, prefix): + """Perform the ip command to remove the specified address.""" + util.subp( + ['ip', '-family', 'inet', 'addr', 'del', + '%s/%s' % (address, prefix), 'dev', self.interface], + capture=True) + + def _bringup_device(self): + """Perform the ip comands to fully setup the device.""" + cidr = '{0}/{1}'.format(self.ip, self.prefix) + LOG.debug( + 'Attempting setup of ephemeral network on %s with %s brd %s', + self.interface, cidr, self.broadcast) + try: + util.subp( + ['ip', '-family', 'inet', 'addr', 'add', cidr, 'broadcast', + self.broadcast, 'dev', self.interface], + capture=True, update_env={'LANG': 'C'}) + except util.ProcessExecutionError as e: + if "File exists" not in e.stderr: + raise + LOG.debug( + 'Skip ephemeral network setup, %s already has address %s', + self.interface, self.ip) + else: + # Address creation success, bring up device and queue cleanup + util.subp( + ['ip', '-family', 'inet', 'link', 'set', 'dev', self.interface, + 'up'], capture=True) + self.cleanup_cmds.append( + ['ip', '-family', 'inet', 'link', 'set', 'dev', self.interface, + 'down']) + self.cleanup_cmds.append( + ['ip', '-family', 'inet', 'addr', 'del', cidr, 'dev', + self.interface]) + + def _bringup_router(self): + """Perform the ip commands to fully setup the router if needed.""" + # Check if a default route exists and exit if it does + out, _ = util.subp(['ip', 'route', 'show', '0.0.0.0/0'], capture=True) + if 'default' in out: + LOG.debug( + 'Skip ephemeral route setup. %s already has default route: %s', + self.interface, out.strip()) + return + util.subp( + ['ip', '-4', 'route', 'add', 'default', 'via', self.router, + 'dev', self.interface], capture=True) + self.cleanup_cmds.insert( + 0, ['ip', '-4', 'route', 'del', 'default', 'dev', self.interface]) + + class RendererNotFoundError(RuntimeError): pass diff --git a/cloudinit/net/tests/__init__.py b/cloudinit/net/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cloudinit/net/tests/test_init.py b/cloudinit/net/tests/test_init.py new file mode 100644 index 00000000..272a6ebd --- /dev/null +++ b/cloudinit/net/tests/test_init.py @@ -0,0 +1,522 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +import copy +import errno +import mock +import os + +import cloudinit.net as net +from cloudinit.util import ensure_file, write_file, ProcessExecutionError +from tests.unittests.helpers import CiTestCase + + +class TestSysDevPath(CiTestCase): + + def test_sys_dev_path(self): + """sys_dev_path returns a path under SYS_CLASS_NET for a device.""" + dev = 'something' + path = 'attribute' + expected = net.SYS_CLASS_NET + dev + '/' + path + self.assertEqual(expected, net.sys_dev_path(dev, path)) + + def test_sys_dev_path_without_path(self): + """When path param isn't provided it defaults to empty string.""" + dev = 'something' + expected = net.SYS_CLASS_NET + dev + '/' + self.assertEqual(expected, net.sys_dev_path(dev)) + + +class TestReadSysNet(CiTestCase): + with_logs = True + + def setUp(self): + super(TestReadSysNet, self).setUp() + sys_mock = mock.patch('cloudinit.net.get_sys_class_path') + self.m_sys_path = sys_mock.start() + self.sysdir = self.tmp_dir() + '/' + self.m_sys_path.return_value = self.sysdir + self.addCleanup(sys_mock.stop) + + def test_read_sys_net_strips_contents_of_sys_path(self): + """read_sys_net strips whitespace from the contents of a sys file.""" + content = 'some stuff with trailing whitespace\t\r\n' + write_file(os.path.join(self.sysdir, 'dev', 'attr'), content) + self.assertEqual(content.strip(), net.read_sys_net('dev', 'attr')) + + def test_read_sys_net_reraises_oserror(self): + """read_sys_net raises OSError/IOError when file doesn't exist.""" + # Non-specific Exception because versions of python OSError vs IOError. + with self.assertRaises(Exception) as context_manager: # noqa: H202 + net.read_sys_net('dev', 'attr') + error = context_manager.exception + self.assertIn('No such file or directory', str(error)) + + def test_read_sys_net_handles_error_with_on_enoent(self): + """read_sys_net handles OSError/IOError with on_enoent if provided.""" + handled_errors = [] + + def on_enoent(e): + handled_errors.append(e) + + net.read_sys_net('dev', 'attr', on_enoent=on_enoent) + error = handled_errors[0] + self.assertIsInstance(error, Exception) + self.assertIn('No such file or directory', str(error)) + + def test_read_sys_net_translates_content(self): + """read_sys_net translates content when translate dict is provided.""" + content = "you're welcome\n" + write_file(os.path.join(self.sysdir, 'dev', 'attr'), content) + translate = {"you're welcome": 'de nada'} + self.assertEqual( + 'de nada', + net.read_sys_net('dev', 'attr', translate=translate)) + + def test_read_sys_net_errors_on_translation_failures(self): + """read_sys_net raises a KeyError and logs details on failure.""" + content = "you're welcome\n" + write_file(os.path.join(self.sysdir, 'dev', 'attr'), content) + with self.assertRaises(KeyError) as context_manager: + net.read_sys_net('dev', 'attr', translate={}) + error = context_manager.exception + self.assertEqual('"you\'re welcome"', str(error)) + self.assertIn( + "Found unexpected (not translatable) value 'you're welcome' in " + "'{0}dev/attr".format(self.sysdir), + self.logs.getvalue()) + + def test_read_sys_net_handles_handles_with_onkeyerror(self): + """read_sys_net handles translation errors calling on_keyerror.""" + content = "you're welcome\n" + write_file(os.path.join(self.sysdir, 'dev', 'attr'), content) + handled_errors = [] + + def on_keyerror(e): + handled_errors.append(e) + + net.read_sys_net('dev', 'attr', translate={}, on_keyerror=on_keyerror) + error = handled_errors[0] + self.assertIsInstance(error, KeyError) + self.assertEqual('"you\'re welcome"', str(error)) + + def test_read_sys_net_safe_false_on_translate_failure(self): + """read_sys_net_safe returns False on translation failures.""" + content = "you're welcome\n" + write_file(os.path.join(self.sysdir, 'dev', 'attr'), content) + self.assertFalse(net.read_sys_net_safe('dev', 'attr', translate={})) + + def test_read_sys_net_safe_returns_false_on_noent_failure(self): + """read_sys_net_safe returns False on file not found failures.""" + self.assertFalse(net.read_sys_net_safe('dev', 'attr')) + + def test_read_sys_net_int_returns_none_on_error(self): + """read_sys_net_safe returns None on failures.""" + self.assertFalse(net.read_sys_net_int('dev', 'attr')) + + def test_read_sys_net_int_returns_none_on_valueerror(self): + """read_sys_net_safe returns None when content is not an int.""" + write_file(os.path.join(self.sysdir, 'dev', 'attr'), 'NOTINT\n') + self.assertFalse(net.read_sys_net_int('dev', 'attr')) + + def test_read_sys_net_int_returns_integer_from_content(self): + """read_sys_net_safe returns None on failures.""" + write_file(os.path.join(self.sysdir, 'dev', 'attr'), '1\n') + self.assertEqual(1, net.read_sys_net_int('dev', 'attr')) + + def test_is_up_true(self): + """is_up is True if sys/net/devname/operstate is 'up' or 'unknown'.""" + for state in ['up', 'unknown']: + write_file(os.path.join(self.sysdir, 'eth0', 'operstate'), state) + self.assertTrue(net.is_up('eth0')) + + def test_is_up_false(self): + """is_up is False if sys/net/devname/operstate is 'down' or invalid.""" + for state in ['down', 'incomprehensible']: + write_file(os.path.join(self.sysdir, 'eth0', 'operstate'), state) + self.assertFalse(net.is_up('eth0')) + + def test_is_wireless(self): + """is_wireless is True when /sys/net/devname/wireless exists.""" + self.assertFalse(net.is_wireless('eth0')) + ensure_file(os.path.join(self.sysdir, 'eth0', 'wireless')) + self.assertTrue(net.is_wireless('eth0')) + + def test_is_bridge(self): + """is_bridge is True when /sys/net/devname/bridge exists.""" + self.assertFalse(net.is_bridge('eth0')) + ensure_file(os.path.join(self.sysdir, 'eth0', 'bridge')) + self.assertTrue(net.is_bridge('eth0')) + + def test_is_bond(self): + """is_bond is True when /sys/net/devname/bonding exists.""" + self.assertFalse(net.is_bond('eth0')) + ensure_file(os.path.join(self.sysdir, 'eth0', 'bonding')) + self.assertTrue(net.is_bond('eth0')) + + def test_is_vlan(self): + """is_vlan is True when /sys/net/devname/uevent has DEVTYPE=vlan.""" + ensure_file(os.path.join(self.sysdir, 'eth0', 'uevent')) + self.assertFalse(net.is_vlan('eth0')) + content = 'junk\nDEVTYPE=vlan\njunk\n' + write_file(os.path.join(self.sysdir, 'eth0', 'uevent'), content) + self.assertTrue(net.is_vlan('eth0')) + + def test_is_connected_when_physically_connected(self): + """is_connected is True when /sys/net/devname/iflink reports 2.""" + self.assertFalse(net.is_connected('eth0')) + write_file(os.path.join(self.sysdir, 'eth0', 'iflink'), "2") + self.assertTrue(net.is_connected('eth0')) + + def test_is_connected_when_wireless_and_carrier_active(self): + """is_connected is True if wireless /sys/net/devname/carrier is 1.""" + self.assertFalse(net.is_connected('eth0')) + ensure_file(os.path.join(self.sysdir, 'eth0', 'wireless')) + self.assertFalse(net.is_connected('eth0')) + write_file(os.path.join(self.sysdir, 'eth0', 'carrier'), "1") + self.assertTrue(net.is_connected('eth0')) + + def test_is_physical(self): + """is_physical is True when /sys/net/devname/device exists.""" + self.assertFalse(net.is_physical('eth0')) + ensure_file(os.path.join(self.sysdir, 'eth0', 'device')) + self.assertTrue(net.is_physical('eth0')) + + def test_is_present(self): + """is_present is True when /sys/net/devname exists.""" + self.assertFalse(net.is_present('eth0')) + ensure_file(os.path.join(self.sysdir, 'eth0', 'device')) + self.assertTrue(net.is_present('eth0')) + + +class TestGenerateFallbackConfig(CiTestCase): + + def setUp(self): + super(TestGenerateFallbackConfig, self).setUp() + sys_mock = mock.patch('cloudinit.net.get_sys_class_path') + self.m_sys_path = sys_mock.start() + self.sysdir = self.tmp_dir() + '/' + self.m_sys_path.return_value = self.sysdir + self.addCleanup(sys_mock.stop) + + def test_generate_fallback_finds_connected_eth_with_mac(self): + """generate_fallback_config finds any connected device with a mac.""" + write_file(os.path.join(self.sysdir, 'eth0', 'carrier'), '1') + write_file(os.path.join(self.sysdir, 'eth1', 'carrier'), '1') + mac = 'aa:bb:cc:aa:bb:cc' + write_file(os.path.join(self.sysdir, 'eth1', 'address'), mac) + expected = { + 'config': [{'type': 'physical', 'mac_address': mac, + 'name': 'eth1', 'subnets': [{'type': 'dhcp'}]}], + 'version': 1} + self.assertEqual(expected, net.generate_fallback_config()) + + def test_generate_fallback_finds_dormant_eth_with_mac(self): + """generate_fallback_config finds any dormant device with a mac.""" + write_file(os.path.join(self.sysdir, 'eth0', 'dormant'), '1') + mac = 'aa:bb:cc:aa:bb:cc' + write_file(os.path.join(self.sysdir, 'eth0', 'address'), mac) + expected = { + 'config': [{'type': 'physical', 'mac_address': mac, + 'name': 'eth0', 'subnets': [{'type': 'dhcp'}]}], + 'version': 1} + self.assertEqual(expected, net.generate_fallback_config()) + + def test_generate_fallback_finds_eth_by_operstate(self): + """generate_fallback_config finds any dormant device with a mac.""" + mac = 'aa:bb:cc:aa:bb:cc' + write_file(os.path.join(self.sysdir, 'eth0', 'address'), mac) + expected = { + 'config': [{'type': 'physical', 'mac_address': mac, + 'name': 'eth0', 'subnets': [{'type': 'dhcp'}]}], + 'version': 1} + valid_operstates = ['dormant', 'down', 'lowerlayerdown', 'unknown'] + for state in valid_operstates: + write_file(os.path.join(self.sysdir, 'eth0', 'operstate'), state) + self.assertEqual(expected, net.generate_fallback_config()) + write_file(os.path.join(self.sysdir, 'eth0', 'operstate'), 'noworky') + self.assertIsNone(net.generate_fallback_config()) + + def test_generate_fallback_config_skips_veth(self): + """generate_fallback_config will skip any veth interfaces.""" + # A connected veth which gets ignored + write_file(os.path.join(self.sysdir, 'veth0', 'carrier'), '1') + self.assertIsNone(net.generate_fallback_config()) + + def test_generate_fallback_config_skips_bridges(self): + """generate_fallback_config will skip any bridges interfaces.""" + # A connected veth which gets ignored + write_file(os.path.join(self.sysdir, 'eth0', 'carrier'), '1') + mac = 'aa:bb:cc:aa:bb:cc' + write_file(os.path.join(self.sysdir, 'eth0', 'address'), mac) + ensure_file(os.path.join(self.sysdir, 'eth0', 'bridge')) + self.assertIsNone(net.generate_fallback_config()) + + def test_generate_fallback_config_skips_bonds(self): + """generate_fallback_config will skip any bonded interfaces.""" + # A connected veth which gets ignored + write_file(os.path.join(self.sysdir, 'eth0', 'carrier'), '1') + mac = 'aa:bb:cc:aa:bb:cc' + write_file(os.path.join(self.sysdir, 'eth0', 'address'), mac) + ensure_file(os.path.join(self.sysdir, 'eth0', 'bonding')) + self.assertIsNone(net.generate_fallback_config()) + + +class TestGetDeviceList(CiTestCase): + + def setUp(self): + super(TestGetDeviceList, self).setUp() + sys_mock = mock.patch('cloudinit.net.get_sys_class_path') + self.m_sys_path = sys_mock.start() + self.sysdir = self.tmp_dir() + '/' + self.m_sys_path.return_value = self.sysdir + self.addCleanup(sys_mock.stop) + + def test_get_devicelist_raise_oserror(self): + """get_devicelist raise any non-ENOENT OSerror.""" + error = OSError('Can not do it') + error.errno = errno.EPERM # Set non-ENOENT + self.m_sys_path.side_effect = error + with self.assertRaises(OSError) as context_manager: + net.get_devicelist() + exception = context_manager.exception + self.assertEqual('Can not do it', str(exception)) + + def test_get_devicelist_empty_without_sys_net(self): + """get_devicelist returns empty list when missing SYS_CLASS_NET.""" + self.m_sys_path.return_value = 'idontexist' + self.assertEqual([], net.get_devicelist()) + + def test_get_devicelist_empty_with_no_devices_in_sys_net(self): + """get_devicelist returns empty directoty listing for SYS_CLASS_NET.""" + self.assertEqual([], net.get_devicelist()) + + def test_get_devicelist_lists_any_subdirectories_in_sys_net(self): + """get_devicelist returns a directory listing for SYS_CLASS_NET.""" + write_file(os.path.join(self.sysdir, 'eth0', 'operstate'), 'up') + write_file(os.path.join(self.sysdir, 'eth1', 'operstate'), 'up') + self.assertItemsEqual(['eth0', 'eth1'], net.get_devicelist()) + + +class TestGetInterfaceMAC(CiTestCase): + + def setUp(self): + super(TestGetInterfaceMAC, self).setUp() + sys_mock = mock.patch('cloudinit.net.get_sys_class_path') + self.m_sys_path = sys_mock.start() + self.sysdir = self.tmp_dir() + '/' + self.m_sys_path.return_value = self.sysdir + self.addCleanup(sys_mock.stop) + + def test_get_interface_mac_false_with_no_mac(self): + """get_device_list returns False when no mac is reported.""" + ensure_file(os.path.join(self.sysdir, 'eth0', 'bonding')) + mac_path = os.path.join(self.sysdir, 'eth0', 'address') + self.assertFalse(os.path.exists(mac_path)) + self.assertFalse(net.get_interface_mac('eth0')) + + def test_get_interface_mac(self): + """get_interfaces returns the mac from SYS_CLASS_NET/dev/address.""" + mac = 'aa:bb:cc:aa:bb:cc' + write_file(os.path.join(self.sysdir, 'eth1', 'address'), mac) + self.assertEqual(mac, net.get_interface_mac('eth1')) + + def test_get_interface_mac_grabs_bonding_address(self): + """get_interfaces returns the source device mac for bonded devices.""" + source_dev_mac = 'aa:bb:cc:aa:bb:cc' + bonded_mac = 'dd:ee:ff:dd:ee:ff' + write_file(os.path.join(self.sysdir, 'eth1', 'address'), bonded_mac) + write_file( + os.path.join(self.sysdir, 'eth1', 'bonding_slave', 'perm_hwaddr'), + source_dev_mac) + self.assertEqual(source_dev_mac, net.get_interface_mac('eth1')) + + def test_get_interfaces_empty_list_without_sys_net(self): + """get_interfaces returns an empty list when missing SYS_CLASS_NET.""" + self.m_sys_path.return_value = 'idontexist' + self.assertEqual([], net.get_interfaces()) + + def test_get_interfaces_by_mac_skips_empty_mac(self): + """Ignore 00:00:00:00:00:00 addresses from get_interfaces_by_mac.""" + empty_mac = '00:00:00:00:00:00' + mac = 'aa:bb:cc:aa:bb:cc' + write_file(os.path.join(self.sysdir, 'eth1', 'address'), empty_mac) + write_file(os.path.join(self.sysdir, 'eth1', 'addr_assign_type'), '0') + write_file(os.path.join(self.sysdir, 'eth2', 'addr_assign_type'), '0') + write_file(os.path.join(self.sysdir, 'eth2', 'address'), mac) + expected = [('eth2', 'aa:bb:cc:aa:bb:cc', None, None)] + self.assertEqual(expected, net.get_interfaces()) + + def test_get_interfaces_by_mac_skips_missing_mac(self): + """Ignore interfaces without an address from get_interfaces_by_mac.""" + write_file(os.path.join(self.sysdir, 'eth1', 'addr_assign_type'), '0') + address_path = os.path.join(self.sysdir, 'eth1', 'address') + self.assertFalse(os.path.exists(address_path)) + mac = 'aa:bb:cc:aa:bb:cc' + write_file(os.path.join(self.sysdir, 'eth2', 'addr_assign_type'), '0') + write_file(os.path.join(self.sysdir, 'eth2', 'address'), mac) + expected = [('eth2', 'aa:bb:cc:aa:bb:cc', None, None)] + self.assertEqual(expected, net.get_interfaces()) + + +class TestInterfaceHasOwnMAC(CiTestCase): + + def setUp(self): + super(TestInterfaceHasOwnMAC, self).setUp() + sys_mock = mock.patch('cloudinit.net.get_sys_class_path') + self.m_sys_path = sys_mock.start() + self.sysdir = self.tmp_dir() + '/' + self.m_sys_path.return_value = self.sysdir + self.addCleanup(sys_mock.stop) + + def test_interface_has_own_mac_false_when_stolen(self): + """Return False from interface_has_own_mac when address is stolen.""" + write_file(os.path.join(self.sysdir, 'eth1', 'addr_assign_type'), '2') + self.assertFalse(net.interface_has_own_mac('eth1')) + + def test_interface_has_own_mac_true_when_not_stolen(self): + """Return False from interface_has_own_mac when mac isn't stolen.""" + valid_assign_types = ['0', '1', '3'] + assign_path = os.path.join(self.sysdir, 'eth1', 'addr_assign_type') + for _type in valid_assign_types: + write_file(assign_path, _type) + self.assertTrue(net.interface_has_own_mac('eth1')) + + def test_interface_has_own_mac_strict_errors_on_absent_assign_type(self): + """When addr_assign_type is absent, interface_has_own_mac errors.""" + with self.assertRaises(ValueError): + net.interface_has_own_mac('eth1', strict=True) + + +@mock.patch('cloudinit.net.util.subp') +class TestEphemeralIPV4Network(CiTestCase): + + with_logs = True + + def setUp(self): + super(TestEphemeralIPV4Network, self).setUp() + sys_mock = mock.patch('cloudinit.net.get_sys_class_path') + self.m_sys_path = sys_mock.start() + self.sysdir = self.tmp_dir() + '/' + self.m_sys_path.return_value = self.sysdir + self.addCleanup(sys_mock.stop) + + def test_ephemeral_ipv4_network_errors_on_missing_params(self, m_subp): + """No required params for EphemeralIPv4Network can be None.""" + required_params = { + 'interface': 'eth0', 'ip': '192.168.2.2', + 'prefix_or_mask': '255.255.255.0', 'broadcast': '192.168.2.255'} + for key in required_params.keys(): + params = copy.deepcopy(required_params) + params[key] = None + with self.assertRaises(ValueError) as context_manager: + net.EphemeralIPv4Network(**params) + error = context_manager.exception + self.assertIn('Cannot init network on', str(error)) + self.assertEqual(0, m_subp.call_count) + + def test_ephemeral_ipv4_network_errors_invalid_mask(self, m_subp): + """Raise an error when prefix_or_mask is not a netmask or prefix.""" + params = { + 'interface': 'eth0', 'ip': '192.168.2.2', + 'broadcast': '192.168.2.255'} + invalid_masks = ('invalid', 'invalid.', '123.123.123') + for error_val in invalid_masks: + params['prefix_or_mask'] = error_val + with self.assertRaises(ValueError) as context_manager: + with net.EphemeralIPv4Network(**params): + pass + error = context_manager.exception + self.assertIn('Cannot setup network: netmask', str(error)) + self.assertEqual(0, m_subp.call_count) + + def test_ephemeral_ipv4_network_performs_teardown(self, m_subp): + """EphemeralIPv4Network performs teardown on the device if setup.""" + expected_setup_calls = [ + mock.call( + ['ip', '-family', 'inet', 'addr', 'add', '192.168.2.2/24', + 'broadcast', '192.168.2.255', 'dev', 'eth0'], + capture=True, update_env={'LANG': 'C'}), + mock.call( + ['ip', '-family', 'inet', 'link', 'set', 'dev', 'eth0', 'up'], + capture=True)] + expected_teardown_calls = [ + mock.call( + ['ip', '-family', 'inet', 'link', 'set', 'dev', 'eth0', + 'down'], capture=True), + mock.call( + ['ip', '-family', 'inet', 'addr', 'del', '192.168.2.2/24', + 'dev', 'eth0'], capture=True)] + params = { + 'interface': 'eth0', 'ip': '192.168.2.2', + 'prefix_or_mask': '255.255.255.0', 'broadcast': '192.168.2.255'} + with net.EphemeralIPv4Network(**params): + self.assertEqual(expected_setup_calls, m_subp.call_args_list) + m_subp.assert_has_calls(expected_teardown_calls) + + def test_ephemeral_ipv4_network_noop_when_configured(self, m_subp): + """EphemeralIPv4Network handles exception when address is setup. + + It performs no cleanup as the interface was already setup. + """ + params = { + 'interface': 'eth0', 'ip': '192.168.2.2', + 'prefix_or_mask': '255.255.255.0', 'broadcast': '192.168.2.255'} + m_subp.side_effect = ProcessExecutionError( + '', 'RTNETLINK answers: File exists', 2) + expected_calls = [ + mock.call( + ['ip', '-family', 'inet', 'addr', 'add', '192.168.2.2/24', + 'broadcast', '192.168.2.255', 'dev', 'eth0'], + capture=True, update_env={'LANG': 'C'})] + with net.EphemeralIPv4Network(**params): + pass + self.assertEqual(expected_calls, m_subp.call_args_list) + self.assertIn( + 'Skip ephemeral network setup, eth0 already has address', + self.logs.getvalue()) + + def test_ephemeral_ipv4_network_with_prefix(self, m_subp): + """EphemeralIPv4Network takes a valid prefix to setup the network.""" + params = { + 'interface': 'eth0', 'ip': '192.168.2.2', + 'prefix_or_mask': '24', 'broadcast': '192.168.2.255'} + for prefix_val in ['24', 16]: # prefix can be int or string + params['prefix_or_mask'] = prefix_val + with net.EphemeralIPv4Network(**params): + pass + m_subp.assert_has_calls([mock.call( + ['ip', '-family', 'inet', 'addr', 'add', '192.168.2.2/24', + 'broadcast', '192.168.2.255', 'dev', 'eth0'], + capture=True, update_env={'LANG': 'C'})]) + m_subp.assert_has_calls([mock.call( + ['ip', '-family', 'inet', 'addr', 'add', '192.168.2.2/16', + 'broadcast', '192.168.2.255', 'dev', 'eth0'], + capture=True, update_env={'LANG': 'C'})]) + + def test_ephemeral_ipv4_network_with_new_default_route(self, m_subp): + """Add the route when router is set and no default route exists.""" + params = { + 'interface': 'eth0', 'ip': '192.168.2.2', + 'prefix_or_mask': '255.255.255.0', 'broadcast': '192.168.2.255', + 'router': '192.168.2.1'} + m_subp.return_value = '', '' # Empty response from ip route gw check + expected_setup_calls = [ + mock.call( + ['ip', '-family', 'inet', 'addr', 'add', '192.168.2.2/24', + 'broadcast', '192.168.2.255', 'dev', 'eth0'], + capture=True, update_env={'LANG': 'C'}), + mock.call( + ['ip', '-family', 'inet', 'link', 'set', 'dev', 'eth0', 'up'], + capture=True), + mock.call( + ['ip', 'route', 'show', '0.0.0.0/0'], capture=True), + mock.call( + ['ip', '-4', 'route', 'add', 'default', 'via', + '192.168.2.1', 'dev', 'eth0'], capture=True)] + expected_teardown_calls = [mock.call( + ['ip', '-4', 'route', 'del', 'default', 'dev', 'eth0'], + capture=True)] + + with net.EphemeralIPv4Network(**params): + self.assertEqual(expected_setup_calls, m_subp.call_args_list) + m_subp.assert_has_calls(expected_teardown_calls) diff --git a/setup.py b/setup.py index b1bde43b..5c65c7fe 100755 --- a/setup.py +++ b/setup.py @@ -240,7 +240,7 @@ setuptools.setup( author='Scott Moser', author_email='scott.moser@canonical.com', url='http://launchpad.net/cloud-init/', - packages=setuptools.find_packages(exclude=['tests']), + packages=setuptools.find_packages(exclude=['tests.*', '*.tests', 'tests']), scripts=['tools/cloud-init-per'], license='Dual-licensed under GPLv3 or Apache 2.0', data_files=data_files, diff --git a/tox.ini b/tox.ini index 1140f9b2..ef768847 100644 --- a/tox.ini +++ b/tox.ini @@ -21,7 +21,11 @@ setenv = LC_ALL = en_US.utf-8 [testenv:pylint] -deps = pylint==1.7.1 +deps = + # requirements + pylint==1.7.1 + # test-requirements because unit tests are now present in cloudinit tree + -r{toxinidir}/test-requirements.txt commands = {envpython} -m pylint {posargs:cloudinit} [testenv:py3] @@ -29,7 +33,7 @@ basepython = python3 deps = -r{toxinidir}/test-requirements.txt commands = {envpython} -m nose {posargs:--with-coverage \ --cover-erase --cover-branches --cover-inclusive \ - --cover-package=cloudinit tests/unittests} + --cover-package=cloudinit tests/unittests cloudinit} [testenv:py27] basepython = python2.7 @@ -98,7 +102,11 @@ deps = pyflakes [testenv:tip-pylint] commands = {envpython} -m pylint {posargs:cloudinit} -deps = pylint +deps = + # requirements + pylint + # test-requirements + -r{toxinidir}/test-requirements.txt [testenv:citest] basepython = python3 -- cgit v1.2.3