From c12f4dd835077e3aa39e9056c25f136ff6a6655b Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Thu, 20 Oct 2016 15:49:59 -0600 Subject: Add coverage collection to tox unit tests. First step in increasing coverage is knowing what coverage is currently at. By default, tox only runs coverage on py3 as it is slower to run with coverage. --- .gitignore | 1 + test-requirements.txt | 1 + tox.ini | 3 +++ 3 files changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index 77eb9c74..865cac15 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ dist *.pyc __pycache__ .tox +.coverage diff --git a/test-requirements.txt b/test-requirements.txt index 6bf38940..0e7fc8fb 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,6 +3,7 @@ httpretty>=0.7.1 mock nose unittest2 +coverage # Only needed if you want to know the test times # nose-timer diff --git a/tox.ini b/tox.ini index 277858ed..08318a9f 100644 --- a/tox.ini +++ b/tox.ini @@ -19,6 +19,9 @@ setenv = [testenv:py3] basepython = python3 +commands = {envpython} -m nose \ + {posargs:--with-coverage --cover-erase \ + --cover-branches --cover-package=cloudinit --cover-inclusive} [testenv:py26] commands = nosetests {posargs:tests} -- cgit v1.2.3 From 4f8ceffb2e3a9feefcb718bda7a7f0f21ef7ab7c Mon Sep 17 00:00:00 2001 From: "kaihuan.pkh" Date: Thu, 13 Oct 2016 20:31:49 +0800 Subject: AliYun: Add new datasource for Ali-Cloud ECS Support AliYun(Ali-Cloud ECS). This datasource inherits from EC2, the main difference is the meta-server address is changed to 100.100.100.200. The datasource behaves similarly to EC2 and relies on network polling. As such, it is not enabled by default. --- cloudinit/sources/DataSourceAliYun.py | 49 ++++++++ cloudinit/sources/DataSourceEc2.py | 18 ++- tests/unittests/test_datasource/test_aliyun.py | 148 +++++++++++++++++++++++++ 3 files changed, 205 insertions(+), 10 deletions(-) create mode 100644 cloudinit/sources/DataSourceAliYun.py create mode 100644 tests/unittests/test_datasource/test_aliyun.py diff --git a/cloudinit/sources/DataSourceAliYun.py b/cloudinit/sources/DataSourceAliYun.py new file mode 100644 index 00000000..19957212 --- /dev/null +++ b/cloudinit/sources/DataSourceAliYun.py @@ -0,0 +1,49 @@ +# vi: ts=4 expandtab + +import os + +from cloudinit import sources +from cloudinit.sources import DataSourceEc2 as EC2 + +DEF_MD_VERSION = "2016-01-01" + + +class DataSourceAliYun(EC2.DataSourceEc2): + metadata_urls = ["http://100.100.100.200"] + + def __init__(self, sys_cfg, distro, paths): + super(DataSourceAliYun, self).__init__(sys_cfg, distro, paths) + self.seed_dir = os.path.join(paths.seed_dir, "AliYun") + self.api_ver = DEF_MD_VERSION + + def get_hostname(self, fqdn=False, _resolve_ip=False): + return self.metadata.get('hostname', 'localhost.localdomain') + + def get_public_ssh_keys(self): + return parse_public_keys(self.metadata.get('public-keys', {})) + + +def parse_public_keys(public_keys): + keys = [] + for key_id, key_body in public_keys.items(): + if isinstance(key_body, str): + keys.append(key_body.strip()) + elif isinstance(key_body, list): + keys.extend(key_body) + elif isinstance(key_body, dict): + key = key_body.get('openssh-key', []) + if isinstance(key, str): + keys.append(key.strip()) + elif isinstance(key, list): + keys.extend(key) + return keys + +# Used to match classes to dependencies +datasources = [ + (DataSourceAliYun, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), +] + + +# Return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return sources.list_from_depends(depends, datasources) diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 6fe2a0bb..bc84ef5d 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -31,21 +31,19 @@ from cloudinit import util LOG = logging.getLogger(__name__) -DEF_MD_URL = "http://169.254.169.254" - # Which version we are requesting of the ec2 metadata apis DEF_MD_VERSION = '2009-04-04' -# Default metadata urls that will be used if none are provided -# They will be checked for 'resolveability' and some of the -# following may be discarded if they do not resolve -DEF_MD_URLS = [DEF_MD_URL, "http://instance-data.:8773"] - class DataSourceEc2(sources.DataSource): + # Default metadata urls that will be used if none are provided + # They will be checked for 'resolveability' and some of the + # following may be discarded if they do not resolve + metadata_urls = ["http://169.254.169.254", "http://instance-data.:8773"] + def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) - self.metadata_address = DEF_MD_URL + self.metadata_address = None self.seed_dir = os.path.join(paths.seed_dir, "ec2") self.api_ver = DEF_MD_VERSION @@ -106,7 +104,7 @@ class DataSourceEc2(sources.DataSource): return False # Remove addresses from the list that wont resolve. - mdurls = mcfg.get("metadata_urls", DEF_MD_URLS) + mdurls = mcfg.get("metadata_urls", self.metadata_urls) filtered = [x for x in mdurls if util.is_resolvable_url(x)] if set(filtered) != set(mdurls): @@ -117,7 +115,7 @@ class DataSourceEc2(sources.DataSource): mdurls = filtered else: LOG.warn("Empty metadata url list! using default list") - mdurls = DEF_MD_URLS + mdurls = self.metadata_urls urls = [] url2base = {} diff --git a/tests/unittests/test_datasource/test_aliyun.py b/tests/unittests/test_datasource/test_aliyun.py new file mode 100644 index 00000000..6f1de072 --- /dev/null +++ b/tests/unittests/test_datasource/test_aliyun.py @@ -0,0 +1,148 @@ +import functools +import httpretty +import os + +from .. import helpers as test_helpers +from cloudinit import helpers +from cloudinit.sources import DataSourceAliYun as ay + +DEFAULT_METADATA = { + 'instance-id': 'aliyun-test-vm-00', + 'eipv4': '10.0.0.1', + 'hostname': 'test-hostname', + 'image-id': 'm-test', + 'launch-index': '0', + 'mac': '00:16:3e:00:00:00', + 'network-type': 'vpc', + 'private-ipv4': '192.168.0.1', + 'serial-number': 'test-string', + 'vpc-cidr-block': '192.168.0.0/16', + 'vpc-id': 'test-vpc', + 'vswitch-id': 'test-vpc', + 'vswitch-cidr-block': '192.168.0.0/16', + 'zone-id': 'test-zone-1', + 'ntp-conf': {'ntp_servers': [ + 'ntp1.aliyun.com', + 'ntp2.aliyun.com', + 'ntp3.aliyun.com']}, + 'source-address': ['http://mirrors.aliyun.com', + 'http://mirrors.aliyuncs.com'], + 'public-keys': {'key-pair-1': {'openssh-key': 'ssh-rsa AAAAB3...'}, + 'key-pair-2': {'openssh-key': 'ssh-rsa AAAAB3...'}} +} + +DEFAULT_USERDATA = """\ +#cloud-config + +hostname: localhost""" + + +def register_mock_metaserver(base_url, data): + def register_helper(register, base_url, body): + if isinstance(body, str): + register(base_url, body) + elif isinstance(body, list): + register(base_url.rstrip('/'), '\n'.join(body) + '\n') + elif isinstance(body, dict): + vals = [] + for k, v in body.items(): + if isinstance(v, (str, list)): + suffix = k.rstrip('/') + else: + suffix = k.rstrip('/') + '/' + vals.append(suffix) + url = base_url.rstrip('/') + '/' + suffix + register_helper(register, url, v) + register(base_url, '\n'.join(vals) + '\n') + + register = functools.partial(httpretty.register_uri, httpretty.GET) + register_helper(register, base_url, data) + + +class TestAliYunDatasource(test_helpers.HttprettyTestCase): + def setUp(self): + super(TestAliYunDatasource, self).setUp() + cfg = {'datasource': {'AliYun': {'timeout': '1', 'max_wait': '1'}}} + distro = {} + paths = helpers.Paths({}) + self.ds = ay.DataSourceAliYun(cfg, distro, paths) + self.metadata_address = self.ds.metadata_urls[0] + self.api_ver = self.ds.api_ver + + @property + def default_metadata(self): + return DEFAULT_METADATA + + @property + def default_userdata(self): + return DEFAULT_USERDATA + + @property + def metadata_url(self): + return os.path.join(self.metadata_address, + self.api_ver, 'meta-data') + '/' + + @property + def userdata_url(self): + return os.path.join(self.metadata_address, + self.api_ver, 'user-data') + + def regist_default_server(self): + register_mock_metaserver(self.metadata_url, self.default_metadata) + register_mock_metaserver(self.userdata_url, self.default_userdata) + + def _test_get_data(self): + self.assertEqual(self.ds.metadata, self.default_metadata) + self.assertEqual(self.ds.userdata_raw, + self.default_userdata.encode('utf8')) + + def _test_get_sshkey(self): + pub_keys = [v['openssh-key'] for (_, v) in + self.default_metadata['public-keys'].items()] + self.assertEqual(self.ds.get_public_ssh_keys(), pub_keys) + + def _test_get_iid(self): + self.assertEqual(self.default_metadata['instance-id'], + self.ds.get_instance_id()) + + def _test_host_name(self): + self.assertEqual(self.default_metadata['hostname'], + self.ds.get_hostname()) + + @httpretty.activate + def test_with_mock_server(self): + self.regist_default_server() + self.ds.get_data() + self._test_get_data() + self._test_get_sshkey() + self._test_get_iid() + self._test_host_name() + + def test_parse_public_keys(self): + public_keys = {} + self.assertEqual(ay.parse_public_keys(public_keys), []) + + public_keys = {'key-pair-0': 'ssh-key-0'} + self.assertEqual(ay.parse_public_keys(public_keys), + [public_keys['key-pair-0']]) + + public_keys = {'key-pair-0': 'ssh-key-0', 'key-pair-1': 'ssh-key-1'} + self.assertEqual(set(ay.parse_public_keys(public_keys)), + set([public_keys['key-pair-0'], + public_keys['key-pair-1']])) + + public_keys = {'key-pair-0': ['ssh-key-0', 'ssh-key-1']} + self.assertEqual(ay.parse_public_keys(public_keys), + public_keys['key-pair-0']) + + public_keys = {'key-pair-0': {'openssh-key': []}} + self.assertEqual(ay.parse_public_keys(public_keys), []) + + public_keys = {'key-pair-0': {'openssh-key': 'ssh-key-0'}} + self.assertEqual(ay.parse_public_keys(public_keys), + [public_keys['key-pair-0']['openssh-key']]) + + public_keys = {'key-pair-0': {'openssh-key': ['ssh-key-0', + 'ssh-key-1']}} + self.assertEqual(ay.parse_public_keys(public_keys), + public_keys['key-pair-0']['openssh-key']) -- cgit v1.2.3 From 3416e2ee7f65defdb15aab861a85767d13e8c34c Mon Sep 17 00:00:00 2001 From: Robert Schweikert Date: Sat, 29 Oct 2016 09:29:53 -0400 Subject: dmidecode: Allow dmidecode to be used on aarch64 aarch64 systems have functional dmidecode, so allow that to be used. - aarch64 has support for dmidecode as well --- cloudinit/util.py | 3 ++- tests/unittests/test_util.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cloudinit/util.py b/cloudinit/util.py index 4b3fd0cb..9a3d3cd7 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2342,7 +2342,8 @@ def read_dmi_data(key): # running dmidecode can be problematic on some arches (LP: #1243287) uname_arch = os.uname()[4] if not (uname_arch == "x86_64" or - (uname_arch.startswith("i") and uname_arch[2:] == "86")): + (uname_arch.startswith("i") and uname_arch[2:] == "86") or + uname_arch == 'aarch64'): LOG.debug("dmidata is not supported on %s", uname_arch) return None diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 881509aa..f6a8ab75 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -386,7 +386,7 @@ class TestReadDMIData(helpers.FilesystemMockingTestCase): dmi_name = 'use-dmidecode' self._configure_dmidecode_return(dmi_name, dmi_val) - expected = {'armel': None, 'aarch64': None, 'x86_64': dmi_val} + expected = {'armel': None, 'aarch64': dmi_val, 'x86_64': dmi_val} found = {} # we do not run the 'dmi-decode' binary on some arches # verify that anything requested that is not in the sysfs dir -- cgit v1.2.3 From 0efae9c20b8230accfa96febffe6fc0ffb43a969 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 28 Oct 2016 16:22:48 -0400 Subject: doc: improve HACKING.rst file Put a bit more information and a few style fixes in HACKING.rst. --- HACKING.rst | 85 ++++++++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 56 insertions(+), 29 deletions(-) diff --git a/HACKING.rst b/HACKING.rst index 63a5bde0..4072d0fd 100644 --- a/HACKING.rst +++ b/HACKING.rst @@ -3,28 +3,35 @@ Hacking on cloud-init ===================== This document describes how to contribute changes to cloud-init. +It assumes you have a `Launchpad`_ account, and refers to your launchpad user +as ``LP_USER`` throughout. Do these things once -------------------- -* If you have not already, be sure to sign the CCA: +* To contribute, you must sign the Canonical `contributor license agreement`_ - - `Canonical Contributor Agreement`_ + If you have already signed it as an individual, your Launchpad user will be listed in the `contributor-agreement-canonical`_ group. Unfortunately there is no easy way to check if an organization or company you are doing work for has signed. If you are unsure or have questions, email `Scott Moser `_ or ping smoser in ``#cloud-init`` channel via freenode. -* Clone the `LaunchPad`_ repository: +* Clone the upstream `repository`_ on Launchpad:: - git clone YOUR_USERNAME@git.launchpad.net:cloud-init + git clone https://git.launchpad.net/cloud-init cd cloud-init - If you would prefer a bzr style `git clone lp:cloud-init`, see - the `Instructions on LaunchPad`_ for more information. + There is more information on Launchpad as a git hosting site in + `Launchpad git documentation`_. -* Create a new remote pointing to your personal LaunchPad - repository:: +* Create a new remote pointing to your personal Launchpad repository - git remote add YOUR_USERNAME YOUR_USERNAME@git.launchpad.net:~YOUR_USERNAME/cloud-init + This is equivalent to 'fork' on github:: -.. _Canonical Contributor Agreement: http://www.canonical.com/contributors + git remote add LP_USER git+ssh://LP_USER@git.launchpad.net/~LP_USER/cloud-init + git push LP_USER master + +.. _repository: https://git.launchpad.net/cloud-init +.. _contributor license agreement: http://www.canonical.com/contributors +.. _contributor-agreement-canonical: https://launchpad.net/%7Econtributor-agreement-canonical/+members +.. _Launchpad git documentation: https://help.launchpad.net/Code/Git Do these things for each feature or bug --------------------------------------- @@ -33,40 +40,60 @@ Do these things for each feature or bug git checkout -b my-topic-branch -.. _Instructions on launchpad: https://help.launchpad.net/Code/Git - * Make and commit your changes (note, you can make multiple commits, fixes, more commits.):: git commit -* Check pep8 and test, and address any issues:: +* Run unit tests and lint/formatting checks with `tox`_:: - make test pep8 + tox -* Push your changes to your personal LaunchPad repository:: +* Push your changes to your personal Launchpad repository:: - git push -u YOUR_USERNAME my-topic-branch + git push -u LP_USER my-topic-branch * Use your browser to create a merge request: - - Open the branch on `LaunchPad`_ + - Open the branch on Launchpad. + + - You can see a web view of your repository and navigate to the branch at: + + ``https://code.launchpad.net/~LP_USER/cloud-init/`` + + - It will typically be at: + + ``https://code.launchpad.net/~LP_USER/cloud-init/+git/cloud-init/+ref/BRANCHNAME`` + + for example, here is larsks move-to-git branch: https://code.launchpad.net/~larsks/cloud-init/+git/cloud-init/+ref/feature/move-to-git + + - Click 'Propose for merging' + - Select 'lp:cloud-init' as the target repository + - Type '``master``' as the Target reference path + - Click 'Propose Merge' + - On the next page, hit 'Set commit message' and type a git combined git style commit message like:: + + Activate the frobnicator. + + The frobnicator was previously inactive and now runs by default. + This may save the world some day. Then, list the bugs you fixed + as footers with syntax as shown here. + + The commit message should be one summary line of less than + 74 characters followed by a blank line, and then one or more + paragraphs describing the change and why it was needed. - - It will typically be at - ``https://code.launchpad.net/~YOUR_USERNAME/cloud-init/+git/cloud-init/+ref/BRANCHNAME`` - for example - https://code.launchpad.net/~larsks/cloud-init/+git/cloud-init/+ref/feature/move-to-git + This is the message that will be used on the commit when it + is sqaushed and merged into trunk. - - Click 'Propose for merging` - - Select ``cloud-init`` as the target repository - - Select ``master`` as the target reference path + LP: #1 -Then, someone on cloud-init-dev (currently `Scott Moser`_ and `Joshua -Harlow`_) will review your changes and follow up in the merge request. +Then, someone in the `cloud-init-dev`_ group will review your changes and +follow up in the merge request. -Feel free to ping and/or join ``#cloud-init`` on freenode (irc) if you +Feel free to ping and/or join ``#cloud-init`` on freenode irc if you have any questions. +.. _tox: https://tox.readthedocs.io/en/latest/ .. _Launchpad: https://launchpad.net -.. _Scott Moser: https://launchpad.net/~smoser -.. _Joshua Harlow: https://launchpad.net/~harlowja +.. _cloud-init-dev: https://launchpad.net/~cloud-init-dev/+members#active -- cgit v1.2.3 From 59d4ba565718dba1a6c7d61cb5e8d5aa505101fa Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 3 Nov 2016 09:27:54 -0400 Subject: Add coverage dependency to bddeb to fix package build. When we added coverage to test-requirements, we need to add the mapping to package name. Without it there, bddeb complains that it cannot translate the dependency. Note, though, that the Makefile does not invoke nose with coverage. So we don't actually use that dependency. --- packages/bddeb | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/bddeb b/packages/bddeb index abb7b607..79ac9768 100755 --- a/packages/bddeb +++ b/packages/bddeb @@ -29,6 +29,7 @@ if "avoid-pep8-E402-import-not-top-of-file": # file pypi package name to a debian/ubuntu package name. STD_NAMED_PACKAGES = [ 'configobj', + 'coverage', 'jinja2', 'jsonpatch', 'oauthlib', -- cgit v1.2.3 From a1cdebdea65ccd827060c823146992bba9debe19 Mon Sep 17 00:00:00 2001 From: LaMont Jones Date: Mon, 31 Oct 2016 21:48:44 -0600 Subject: net/cmdline: Further adjustments to ipv6 support The implementation to add ipv6 support to Ubuntu initramfs changed (see bug 1621507). The changes here adjust to handle the new path. Now, the ipv6 route includes using the variable 'DEVICE6' in net6-DEVICE.conf files. LP: #1621615 --- cloudinit/net/cmdline.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py index 933317d5..4075a279 100644 --- a/cloudinit/net/cmdline.py +++ b/cloudinit/net/cmdline.py @@ -76,12 +76,13 @@ def _klibc_to_config_entry(content, mac_addrs=None): data = _load_shell_content(content) try: - name = data['DEVICE'] + name = data['DEVICE'] if 'DEVICE' in data else data['DEVICE6'] except KeyError: - raise ValueError("no 'DEVICE' entry in data") + raise ValueError("no 'DEVICE' or 'DEVICE6' entry in data") # ipconfig on precise does not write PROTO - proto = data.get('PROTO') + # IPv6 config gives us IPV6PROTO, not PROTO. + proto = data.get('PROTO', data.get('IPV6PROTO')) if not proto: if data.get('filename'): proto = 'dhcp' -- cgit v1.2.3