From 5b065316113b97aadb43e63cc31bb8639f6a6376 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 14 Dec 2018 03:24:26 +0000 Subject: Update to pylint 2.2.2. The tip-pylint tox target correctly reported the invalid use of string formatting. The change here is to: a.) Fix the error that was caught. b.) move to pylint 2.2.2 for the default 'pylint' target. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tox.ini') diff --git a/tox.ini b/tox.ini index 2fb3209d..d983348b 100644 --- a/tox.ini +++ b/tox.ini @@ -21,7 +21,7 @@ setenv = basepython = python3 deps = # requirements - pylint==1.8.1 + pylint==2.2.2 # test-requirements because unit tests are now present in cloudinit tree -r{toxinidir}/test-requirements.txt commands = {envpython} -m pylint {posargs:cloudinit tests tools} -- cgit v1.2.3 From c7248059dd2faaaadfbcef5c83e8e8ea166d6767 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Fri, 25 Jan 2019 22:35:40 +0000 Subject: tox: fix disco httpretty dependencies for py37 LP: #1813361 --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tox.ini') diff --git a/tox.ini b/tox.ini index d983348b..d3717200 100644 --- a/tox.ini +++ b/tox.ini @@ -75,7 +75,7 @@ deps = jsonpatch==1.16 six==1.10.0 # test-requirements - httpretty==0.8.6 + httpretty==0.9.6 mock==1.3.0 nose==1.3.7 unittest2==1.1.0 -- cgit v1.2.3 From 200b0ac1fc1709f6c06bb963beb3080a5b5c6fb1 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Mon, 18 Mar 2019 17:10:13 +0000 Subject: tox: bump pylint version to latest (2.3.1) The previous version was emitting errors due to an incompatibility with one of its dependencies. (We could have pinned the dependency instead, but staying current on pylint is a worthy goal in and of itself.) --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tox.ini') diff --git a/tox.ini b/tox.ini index d3717200..967321f8 100644 --- a/tox.ini +++ b/tox.ini @@ -21,7 +21,7 @@ setenv = basepython = python3 deps = # requirements - pylint==2.2.2 + pylint==2.3.1 # test-requirements because unit tests are now present in cloudinit tree -r{toxinidir}/test-requirements.txt commands = {envpython} -m pylint {posargs:cloudinit tests tools} -- cgit v1.2.3 From dfe50e300882e3affcb02e686578807aea921b99 Mon Sep 17 00:00:00 2001 From: Thomas Bechtold Date: Thu, 21 Mar 2019 16:22:29 +0000 Subject: tox: Update testenv for openSUSE Leap to 15.0 Use the requirements for the openSUSE Leap 15.0 release. --- tox.ini | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) (limited to 'tox.ini') diff --git a/tox.ini b/tox.ini index 967321f8..1f01eb76 100644 --- a/tox.ini +++ b/tox.ini @@ -96,19 +96,18 @@ deps = six==1.9.0 -r{toxinidir}/test-requirements.txt -[testenv:opensusel42] +[testenv:opensusel150] basepython = python2.7 commands = nosetests {posargs:tests/unittests cloudinit} deps = # requirements - argparse==1.3.0 - jinja2==2.8 - PyYAML==3.11 - oauthlib==0.7.2 + jinja2==2.10 + PyYAML==3.12 + oauthlib==2.0.6 configobj==5.0.6 - requests==2.11.1 - jsonpatch==1.11 - six==1.9.0 + requests==2.18.4 + jsonpatch==1.16 + six==1.11.0 -r{toxinidir}/test-requirements.txt [testenv:tip-pycodestyle] -- cgit v1.2.3 From 385232dc8da6532b54342cd11e6d822ff7cd3e5a Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Sat, 7 Sep 2019 02:27:45 +0000 Subject: doc: document doc, create makefile and tox target MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Create makefile and tox targets for documentation building and testing to better replicate the live web docs using the same theme. * Created docs.rst to explain how to build and contribute to documentation with style guide and tips. * doc/rtd/conf.py:     * Add copyright to rtd config     * Use Sphinx's RTD theme to replicate actual docs --- Makefile | 4 ++- doc/rtd/conf.py | 13 ++------ doc/rtd/index.rst | 9 ------ doc/rtd/topics/docs.rst | 84 +++++++++++++++++++++++++++++++++++++++++++++++++ tox.ini | 9 ++++-- 5 files changed, 97 insertions(+), 22 deletions(-) create mode 100644 doc/rtd/topics/docs.rst (limited to 'tox.ini') diff --git a/Makefile b/Makefile index 4ace2270..2c6d0c80 100644 --- a/Makefile +++ b/Makefile @@ -106,7 +106,9 @@ deb-src: echo sudo apt-get install devscripts; exit 1; } $(PYVER) ./packages/bddeb -S -d +doc: + tox -e doc .PHONY: test pyflakes pyflakes3 clean pep8 rpm srpm deb deb-src yaml .PHONY: check_version pip-test-requirements pip-requirements clean_pyc -.PHONY: unittest unittest3 style-check +.PHONY: unittest unittest3 style-check doc diff --git a/doc/rtd/conf.py b/doc/rtd/conf.py index 4174477c..9b274843 100644 --- a/doc/rtd/conf.py +++ b/doc/rtd/conf.py @@ -17,7 +17,8 @@ from cloudinit.config.schema import get_schema_doc # ] # General information about the project. -project = 'Cloud-Init' +project = 'cloud-init' +copyright = '2019, Canonical Ltd.' # -- General configuration ---------------------------------------------------- @@ -59,15 +60,7 @@ show_authors = False # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'default' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -html_theme_options = { - "bodyfont": "Ubuntu, Arial, sans-serif", - "headfont": "Ubuntu, Arial, sans-serif" -} +html_theme = 'sphinx_rtd_theme' # The name of an image file (relative to this directory) to place at the top # of the sidebar. diff --git a/doc/rtd/index.rst b/doc/rtd/index.rst index 20a99a30..c670b20e 100644 --- a/doc/rtd/index.rst +++ b/doc/rtd/index.rst @@ -1,14 +1,5 @@ .. _index: -.. http://thomas-cokelaer.info/tutorials/sphinx/rest_syntax.html -.. As suggested at link above for headings use: -.. # with overline, for parts -.. * with overline, for chapters -.. =, for sections -.. -, for subsections -.. ^, for subsubsections -.. “, for paragraphs - ############# Documentation ############# diff --git a/doc/rtd/topics/docs.rst b/doc/rtd/topics/docs.rst new file mode 100644 index 00000000..1b15377e --- /dev/null +++ b/doc/rtd/topics/docs.rst @@ -0,0 +1,84 @@ +.. _docs: + +Docs +**** + +These docs are hosted on Read the Docs. The following will explain how to +contribute to and build these docs locally. + +The documentation is primarily written in reStructuredText. + + +Building +======== + +There is a makefile target to build the documentation for you: + +.. code-block:: shell-session + + $ tox -e doc + +This will do two things: + +- Build the documentation using sphinx +- Run doc8 against the documentation source code + +Once build the HTML files will be viewable in ``doc/rtd_html``. Use your +web browser to open ``index.html`` to view and navigate the site. + +Style Guide +=========== + +Headings +-------- +The headings used across the documentation use the following hierarchy: + +- ``*****``: used once atop of a new page +- ``=====``: each sections on the page +- ``-----``: subsections +- ``^^^^^``: sub-subsections +- ``"""""``: paragraphs + +The top level header ``######`` is reserved for the first page. + +If under and overline are used, their length must be identical. The length of +the underline must be at least as long as the title itself + +Line Length +----------- +Please keep the line lengths to a maximum of **79** characters. This ensures +that the pages and tables do not get too wide that side scrolling is required. + +Header +------ +Adding a link at the top of the page allows for the page to be referenced by +other pages. For example for the FAQ page this would be: + +.. code-block:: rst + + .. _faq: + +Footer +------ +The footer should include the textwidth + +.. code-block:: rst + + .. vi: textwidth=79 + +Vertical Whitespace +------------------- +One newline between each section helps ensure readability of the documentation +source code. + +Common Words +------------ +There are some common words that should follow specific usage: + +- ``cloud-init``: always lower case with a hyphen, unless starting a sentence + in which case only the 'C' is capitalized (e.g. ``Cloud-init``). +- ``metadata``: one word +- ``user data``: two words, not to be combined +- ``vendor data``: like user data, it is two words + +.. vi: textwidth=79 diff --git a/tox.ini b/tox.ini index 1f01eb76..f5baf328 100644 --- a/tox.ini +++ b/tox.ini @@ -53,8 +53,13 @@ exclude = .venv,.tox,dist,doc,*egg,.git,build,tools [testenv:doc] basepython = python3 -deps = sphinx -commands = {envpython} -m sphinx {posargs:doc/rtd doc/rtd_html} +deps = + doc8 + sphinx + sphinx_rtd_theme +commands = + {envpython} -m sphinx {posargs:doc/rtd doc/rtd_html} + doc8 doc/rtd [testenv:xenial] commands = -- cgit v1.2.3 From aa3e4961ceae5a5c5b5cf13221b5f6721991fe75 Mon Sep 17 00:00:00 2001 From: ahosmanmsft Date: Tue, 26 Nov 2019 11:36:00 -0700 Subject: cloud_tests: add azure platform support to integration tests Added Azure to cloud tests supporting upstream integration testing. Implement the inherited platform classes, Azure configurations to release/platform, and docs on how to run Azure CI. --- .pylintrc | 2 +- doc/rtd/topics/tests.rst | 52 +++++ integration-requirements.txt | 9 + tests/cloud_tests/platforms.yaml | 6 + tests/cloud_tests/platforms/__init__.py | 2 + tests/cloud_tests/platforms/azurecloud/__init__.py | 0 tests/cloud_tests/platforms/azurecloud/image.py | 108 +++++++++ tests/cloud_tests/platforms/azurecloud/instance.py | 243 +++++++++++++++++++++ tests/cloud_tests/platforms/azurecloud/platform.py | 232 ++++++++++++++++++++ .../cloud_tests/platforms/azurecloud/regions.json | 42 ++++ tests/cloud_tests/platforms/azurecloud/snapshot.py | 58 +++++ tests/cloud_tests/platforms/ec2/image.py | 1 + tests/cloud_tests/platforms/ec2/platform.py | 3 +- tests/cloud_tests/releases.yaml | 2 + tox.ini | 2 + 15 files changed, 760 insertions(+), 2 deletions(-) create mode 100644 tests/cloud_tests/platforms/azurecloud/__init__.py create mode 100644 tests/cloud_tests/platforms/azurecloud/image.py create mode 100644 tests/cloud_tests/platforms/azurecloud/instance.py create mode 100644 tests/cloud_tests/platforms/azurecloud/platform.py create mode 100644 tests/cloud_tests/platforms/azurecloud/regions.json create mode 100644 tests/cloud_tests/platforms/azurecloud/snapshot.py (limited to 'tox.ini') diff --git a/.pylintrc b/.pylintrc index 365c8c8b..c83546a6 100644 --- a/.pylintrc +++ b/.pylintrc @@ -62,7 +62,7 @@ ignored-modules= # for classes with dynamically set attributes). This supports the use of # qualified names. # argparse.Namespace from https://github.com/PyCQA/pylint/issues/2413 -ignored-classes=argparse.Namespace,optparse.Values,thread._local +ignored-classes=argparse.Namespace,optparse.Values,thread._local,ImageManager,ContainerManager # List of members which are set dynamically and missed by pylint inference # system, and so shouldn't trigger E1101 when accessed. Python regular diff --git a/doc/rtd/topics/tests.rst b/doc/rtd/topics/tests.rst index a2c703a5..3b27f805 100644 --- a/doc/rtd/topics/tests.rst +++ b/doc/rtd/topics/tests.rst @@ -423,6 +423,58 @@ generated when running ``aws configure``: region = us-west-2 +Azure Cloud +----------- + +To run on Azure Cloud platform users login with Service Principal and export +credentials file. Region is defaulted and can be set in ``tests/cloud_tests/platforms.yaml``. +The Service Principal credentials are the standard authentication for Azure SDK +to interact with Azure Services: + +Create Service Principal account or login + +.. code-block:: shell-session + + $ az ad sp create-for-rbac --name "APP_ID" --password "STRONG-SECRET-PASSWORD" + +.. code-block:: shell-session + + $ az login --service-principal --username "APP_ID" --password "STRONG-SECRET-PASSWORD" + +Export credentials + +.. code-block:: shell-session + + $ az ad sp create-for-rbac --sdk-auth > $HOME/.azure/credentials.json + +.. code-block:: json + + { + "clientId": "", + "clientSecret": "", + "subscriptionId": "", + "tenantId": "", + "activeDirectoryEndpointUrl": "https://login.microsoftonline.com", + "resourceManagerEndpointUrl": "https://management.azure.com/", + "activeDirectoryGraphResourceId": "https://graph.windows.net/", + "sqlManagementEndpointUrl": "https://management.core.windows.net:8443/", + "galleryEndpointUrl": "https://gallery.azure.com/", + "managementEndpointUrl": "https://management.core.windows.net/" + } + +Set region in platforms.yaml + +.. code-block:: yaml + :emphasize-lines: 3 + + azurecloud: + enabled: true + region: West US 2 + vm_size: Standard_DS1_v2 + storage_sku: standard_lrs + tag: ci + + Architecture ============ diff --git a/integration-requirements.txt b/integration-requirements.txt index fe5ad45d..897d6110 100644 --- a/integration-requirements.txt +++ b/integration-requirements.txt @@ -20,3 +20,12 @@ git+https://github.com/lxc/pylxd.git@4b8ab1802f9aee4eb29cf7b119dae0aa47150779 # finds latest image information git+https://git.launchpad.net/simplestreams + +# azure backend +azure-storage==0.36.0 +msrestazure==0.6.1 +azure-common==1.1.23 +azure-mgmt-compute==7.0.0 +azure-mgmt-network==5.0.0 +azure-mgmt-resource==4.0.0 +azure-mgmt-storage==6.0.0 diff --git a/tests/cloud_tests/platforms.yaml b/tests/cloud_tests/platforms.yaml index 652a7051..eaaa0a71 100644 --- a/tests/cloud_tests/platforms.yaml +++ b/tests/cloud_tests/platforms.yaml @@ -67,5 +67,11 @@ platforms: nocloud-kvm: enabled: true cache_mode: cache=none,aio=native + azurecloud: + enabled: true + region: West US 2 + vm_size: Standard_DS1_v2 + storage_sku: standard_lrs + tag: ci # vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/__init__.py b/tests/cloud_tests/platforms/__init__.py index a01e51ac..6a410b84 100644 --- a/tests/cloud_tests/platforms/__init__.py +++ b/tests/cloud_tests/platforms/__init__.py @@ -5,11 +5,13 @@ from .ec2 import platform as ec2 from .lxd import platform as lxd from .nocloudkvm import platform as nocloudkvm +from .azurecloud import platform as azurecloud PLATFORMS = { 'ec2': ec2.EC2Platform, 'nocloud-kvm': nocloudkvm.NoCloudKVMPlatform, 'lxd': lxd.LXDPlatform, + 'azurecloud': azurecloud.AzureCloudPlatform, } diff --git a/tests/cloud_tests/platforms/azurecloud/__init__.py b/tests/cloud_tests/platforms/azurecloud/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cloud_tests/platforms/azurecloud/image.py b/tests/cloud_tests/platforms/azurecloud/image.py new file mode 100644 index 00000000..96a946f3 --- /dev/null +++ b/tests/cloud_tests/platforms/azurecloud/image.py @@ -0,0 +1,108 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Azure Cloud image Base class.""" + +from tests.cloud_tests import LOG + +from ..images import Image +from .snapshot import AzureCloudSnapshot + + +class AzureCloudImage(Image): + """Azure Cloud backed image.""" + + platform_name = 'azurecloud' + + def __init__(self, platform, config, image_id): + """Set up image. + + @param platform: platform object + @param config: image configuration + @param image_id: image id used to boot instance + """ + super(AzureCloudImage, self).__init__(platform, config) + self.image_id = image_id + self._img_instance = None + + @property + def _instance(self): + """Internal use only, returns a running instance""" + LOG.debug('creating instance') + if not self._img_instance: + self._img_instance = self.platform.create_instance( + self.properties, self.config, self.features, + self.image_id, user_data=None) + return self._img_instance + + def destroy(self): + """Delete the instance used to create a custom image.""" + LOG.debug('deleting VM that was used to create image') + if self._img_instance: + LOG.debug('Deleting instance %s', self._img_instance.name) + delete_vm = self.platform.compute_client.virtual_machines.delete( + self.platform.resource_group.name, self.image_id) + delete_vm.wait() + + super(AzureCloudImage, self).destroy() + + def _execute(self, *args, **kwargs): + """Execute command in image, modifying image.""" + LOG.debug('executing commands on image') + self._instance.start() + return self._instance._execute(*args, **kwargs) + + def push_file(self, local_path, remote_path): + """Copy file at 'local_path' to instance at 'remote_path'.""" + LOG.debug('pushing file to image') + return self._instance.push_file(local_path, remote_path) + + def run_script(self, *args, **kwargs): + """Run script in image, modifying image. + + @return_value: script output + """ + LOG.debug('running script on image') + self._instance.start() + return self._instance.run_script(*args, **kwargs) + + def snapshot(self): + """ Create snapshot (image) of instance, wait until done. + + If no instance has been booted, base image is returned. + Otherwise runs the clean script, deallocates, generalizes + and creates custom image from instance. + """ + LOG.debug('creating image from VM') + if not self._img_instance: + return AzureCloudSnapshot(self.platform, self.properties, + self.config, self.features, + self.image_id, delete_on_destroy=False) + + if self.config.get('boot_clean_script'): + self._img_instance.run_script(self.config.get('boot_clean_script')) + + deallocate = self.platform.compute_client.virtual_machines.deallocate( + self.platform.resource_group.name, self.image_id) + deallocate.wait() + + self.platform.compute_client.virtual_machines.generalize( + self.platform.resource_group.name, self.image_id) + + image_params = { + "location": self.platform.location, + "properties": { + "sourceVirtualMachine": { + "id": self._img_instance.instance.id + } + } + } + self.platform.compute_client.images.create_or_update( + self.platform.resource_group.name, self.image_id, + image_params) + + self.destroy() + + return AzureCloudSnapshot(self.platform, self.properties, self.config, + self.features, self.image_id) + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/azurecloud/instance.py b/tests/cloud_tests/platforms/azurecloud/instance.py new file mode 100644 index 00000000..3d77a1a7 --- /dev/null +++ b/tests/cloud_tests/platforms/azurecloud/instance.py @@ -0,0 +1,243 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Base Azure Cloud instance.""" + +from datetime import datetime, timedelta +from urllib.parse import urlparse +from time import sleep +import traceback +import os + + +# pylint: disable=no-name-in-module +from azure.storage.blob import BlockBlobService, BlobPermissions +from msrestazure.azure_exceptions import CloudError + +from tests.cloud_tests import LOG + +from ..instances import Instance + + +class AzureCloudInstance(Instance): + """Azure Cloud backed instance.""" + + platform_name = 'azurecloud' + + def __init__(self, platform, properties, config, + features, image_id, user_data=None): + """Set up instance. + + @param platform: platform object + @param properties: dictionary of properties + @param config: dictionary of configuration values + @param features: dictionary of supported feature flags + @param image_id: image to find and/or use + @param user_data: test user-data to pass to instance + """ + super(AzureCloudInstance, self).__init__( + platform, image_id, properties, config, features) + + self.ssh_port = 22 + self.ssh_ip = None + self.instance = None + self.image_id = image_id + self.user_data = user_data + self.ssh_key_file = os.path.join( + platform.config['data_dir'], platform.config['private_key']) + self.ssh_pubkey_file = os.path.join( + platform.config['data_dir'], platform.config['public_key']) + self.blob_client, self.container, self.blob = None, None, None + + def start(self, wait=True, wait_for_cloud_init=False): + """Start instance with the platforms NIC.""" + if self.instance: + return + data = self.image_id.split('-') + release, support = data[2].replace('_', '.'), data[3] + sku = '%s-%s' % (release, support) if support == 'LTS' else release + image_resource_id = '/subscriptions/%s' \ + '/resourceGroups/%s' \ + '/providers/Microsoft.Compute/images/%s' % ( + self.platform.subscription_id, + self.platform.resource_group.name, + self.image_id) + storage_uri = "http://%s.blob.core.windows.net" \ + % self.platform.storage.name + with open(self.ssh_pubkey_file, 'r') as key: + ssh_pub_keydata = key.read() + + image_exists = False + try: + LOG.debug('finding image in resource group using image_id') + self.platform.compute_client.images.get( + self.platform.resource_group.name, + self.image_id + ) + image_exists = True + LOG.debug('image found, launching instance') + except CloudError: + LOG.debug( + 'image not found, launching instance with base image') + pass + + vm_params = { + 'location': self.platform.location, + 'os_profile': { + 'computer_name': 'CI', + 'admin_username': self.ssh_username, + "customData": self.user_data, + "linuxConfiguration": { + "disable_password_authentication": True, + "ssh": { + "public_keys": [{ + "path": "/home/%s/.ssh/authorized_keys" % + self.ssh_username, + "keyData": ssh_pub_keydata + }] + } + } + }, + "diagnosticsProfile": { + "bootDiagnostics": { + "storageUri": storage_uri, + "enabled": True + } + }, + 'hardware_profile': { + 'vm_size': self.platform.vm_size + }, + 'storage_profile': { + 'image_reference': { + 'id': image_resource_id + } if image_exists else { + 'publisher': 'Canonical', + 'offer': 'UbuntuServer', + 'sku': sku, + 'version': 'latest' + } + }, + 'network_profile': { + 'network_interfaces': [{ + 'id': self.platform.nic.id + }] + }, + 'tags': { + 'Name': self.platform.tag, + } + } + + try: + self.instance = self.platform.compute_client.virtual_machines.\ + create_or_update(self.platform.resource_group.name, + self.image_id, vm_params) + except CloudError: + raise RuntimeError('failed creating instance:\n{}'.format( + traceback.format_exc())) + + if wait: + self.instance.wait() + self.ssh_ip = self.platform.network_client.\ + public_ip_addresses.get( + self.platform.resource_group.name, + self.platform.public_ip.name + ).ip_address + self._wait_for_system(wait_for_cloud_init) + + self.instance = self.instance.result() + self.blob_client, self.container, self.blob =\ + self._get_blob_client() + + def shutdown(self, wait=True): + """Finds console log then stopping/deallocates VM""" + LOG.debug('waiting on console log before stopping') + attempts, exists = 5, False + while not exists and attempts: + try: + attempts -= 1 + exists = self.blob_client.get_blob_to_bytes( + self.container, self.blob) + LOG.debug('found console log') + except Exception as e: + if attempts: + LOG.debug('Unable to find console log, ' + '%s attempts remaining', attempts) + sleep(15) + else: + LOG.warning('Could not find console log: %s', e) + pass + + LOG.debug('stopping instance %s', self.image_id) + vm_deallocate = \ + self.platform.compute_client.virtual_machines.deallocate( + self.platform.resource_group.name, self.image_id) + if wait: + vm_deallocate.wait() + + def destroy(self): + """Delete VM and close all connections""" + if self.instance: + LOG.debug('destroying instance: %s', self.image_id) + vm_delete = self.platform.compute_client.virtual_machines.delete( + self.platform.resource_group.name, self.image_id) + vm_delete.wait() + + self._ssh_close() + + super(AzureCloudInstance, self).destroy() + + def _execute(self, command, stdin=None, env=None): + """Execute command on instance.""" + env_args = [] + if env: + env_args = ['env'] + ["%s=%s" for k, v in env.items()] + + return self._ssh(['sudo'] + env_args + list(command), stdin=stdin) + + def _get_blob_client(self): + """ + Use VM details to retrieve container and blob name. + Then Create blob service client for sas token to + retrieve console log. + + :return: blob service, container name, blob name + """ + LOG.debug('creating blob service for console log') + storage = self.platform.storage_client.storage_accounts.get_properties( + self.platform.resource_group.name, self.platform.storage.name) + + keys = self.platform.storage_client.storage_accounts.list_keys( + self.platform.resource_group.name, self.platform.storage.name + ).keys[0].value + + virtual_machine = self.platform.compute_client.virtual_machines.get( + self.platform.resource_group.name, self.instance.name, + expand='instanceView') + + blob_uri = virtual_machine.instance_view.boot_diagnostics.\ + serial_console_log_blob_uri + + container, blob = urlparse(blob_uri).path.split('/')[-2:] + + blob_client = BlockBlobService( + account_name=storage.name, + account_key=keys) + + sas = blob_client.generate_blob_shared_access_signature( + container_name=container, blob_name=blob, protocol='https', + expiry=datetime.utcnow() + timedelta(hours=1), + permission=BlobPermissions.READ) + + blob_client = BlockBlobService( + account_name=storage.name, + sas_token=sas) + + return blob_client, container, blob + + def console_log(self): + """Instance console. + + @return_value: bytes of this instance’s console + """ + boot_diagnostics = self.blob_client.get_blob_to_bytes( + self.container, self.blob) + return boot_diagnostics.content diff --git a/tests/cloud_tests/platforms/azurecloud/platform.py b/tests/cloud_tests/platforms/azurecloud/platform.py new file mode 100644 index 00000000..77f159eb --- /dev/null +++ b/tests/cloud_tests/platforms/azurecloud/platform.py @@ -0,0 +1,232 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Base Azure Cloud class.""" + +import os +import base64 +import traceback +from datetime import datetime +from tests.cloud_tests import LOG + +# pylint: disable=no-name-in-module +from azure.common.credentials import ServicePrincipalCredentials +# pylint: disable=no-name-in-module +from azure.mgmt.resource import ResourceManagementClient +# pylint: disable=no-name-in-module +from azure.mgmt.network import NetworkManagementClient +# pylint: disable=no-name-in-module +from azure.mgmt.compute import ComputeManagementClient +# pylint: disable=no-name-in-module +from azure.mgmt.storage import StorageManagementClient +from msrestazure.azure_exceptions import CloudError + +from .image import AzureCloudImage +from .instance import AzureCloudInstance +from ..platforms import Platform + +from cloudinit import util as c_util + + +class AzureCloudPlatform(Platform): + """Azure Cloud test platforms.""" + + platform_name = 'azurecloud' + + def __init__(self, config): + """Set up platform.""" + super(AzureCloudPlatform, self).__init__(config) + self.tag = '%s-%s' % ( + config['tag'], datetime.now().strftime('%Y%m%d%H%M%S')) + self.storage_sku = config['storage_sku'] + self.vm_size = config['vm_size'] + self.location = config['region'] + + try: + self.credentials, self.subscription_id = self._get_credentials() + + self.resource_client = ResourceManagementClient( + self.credentials, self.subscription_id) + self.compute_client = ComputeManagementClient( + self.credentials, self.subscription_id) + self.network_client = NetworkManagementClient( + self.credentials, self.subscription_id) + self.storage_client = StorageManagementClient( + self.credentials, self.subscription_id) + + self.resource_group = self._create_resource_group() + self.public_ip = self._create_public_ip_address() + self.storage = self._create_storage_account(config) + self.vnet = self._create_vnet() + self.subnet = self._create_subnet() + self.nic = self._create_nic() + except CloudError: + raise RuntimeError('failed creating a resource:\n{}'.format( + traceback.format_exc())) + + def create_instance(self, properties, config, features, + image_id, user_data=None): + """Create an instance + + @param properties: image properties + @param config: image configuration + @param features: image features + @param image_id: string of image id + @param user_data: test user-data to pass to instance + @return_value: cloud_tests.instances instance + """ + user_data = str(base64.b64encode( + user_data.encode('utf-8')), 'utf-8') + + return AzureCloudInstance(self, properties, config, features, + image_id, user_data) + + def get_image(self, img_conf): + """Get image using specified image configuration. + + @param img_conf: configuration for image + @return_value: cloud_tests.images instance + """ + ss_region = self.azure_location_to_simplestreams_region() + + filters = [ + 'arch=%s' % 'amd64', + 'endpoint=https://management.core.windows.net/', + 'region=%s' % ss_region, + 'release=%s' % img_conf['release'] + ] + + LOG.debug('finding image using streams') + image = self._query_streams(img_conf, filters) + + try: + image_id = image['id'] + LOG.debug('found image: %s', image_id) + if image_id.find('__') > 0: + image_id = image_id.split('__')[1] + LOG.debug('image_id shortened to %s', image_id) + except KeyError: + raise RuntimeError('no images found for %s' % img_conf['release']) + + return AzureCloudImage(self, img_conf, image_id) + + def destroy(self): + """Delete all resources in resource group.""" + LOG.debug("Deleting resource group: %s", self.resource_group.name) + delete = self.resource_client.resource_groups.delete( + self.resource_group.name) + delete.wait() + + def azure_location_to_simplestreams_region(self): + """Convert location to simplestreams region""" + location = self.location.lower().replace(' ', '') + LOG.debug('finding location %s using simple streams', location) + regions_file = os.path.join( + os.path.dirname(os.path.abspath(__file__)), 'regions.json') + region_simplestreams_map = c_util.load_json( + c_util.load_file(regions_file)) + return region_simplestreams_map.get(location, location) + + def _get_credentials(self): + """Get credentials from environment""" + LOG.debug('getting credentials from environment') + cred_file = os.path.expanduser('~/.azure/credentials.json') + try: + azure_creds = c_util.load_json( + c_util.load_file(cred_file)) + subscription_id = azure_creds['subscriptionId'] + credentials = ServicePrincipalCredentials( + client_id=azure_creds['clientId'], + secret=azure_creds['clientSecret'], + tenant=azure_creds['tenantId']) + return credentials, subscription_id + except KeyError: + raise RuntimeError('Please configure Azure service principal' + ' credentials in %s' % cred_file) + + def _create_resource_group(self): + """Create resource group""" + LOG.debug('creating resource group') + resource_group_name = self.tag + resource_group_params = { + 'location': self.location + } + resource_group = self.resource_client.resource_groups.create_or_update( + resource_group_name, resource_group_params) + return resource_group + + def _create_storage_account(self, config): + LOG.debug('creating storage account') + storage_account_name = 'storage%s' % datetime.now().\ + strftime('%Y%m%d%H%M%S') + storage_params = { + 'sku': { + 'name': config['storage_sku'] + }, + 'kind': "Storage", + 'location': self.location + } + storage_account = self.storage_client.storage_accounts.create( + self.resource_group.name, storage_account_name, storage_params) + return storage_account.result() + + def _create_public_ip_address(self): + """Create public ip address""" + LOG.debug('creating public ip address') + public_ip_name = '%s-ip' % self.resource_group.name + public_ip_params = { + 'location': self.location, + 'public_ip_allocation_method': 'Dynamic' + } + ip = self.network_client.public_ip_addresses.create_or_update( + self.resource_group.name, public_ip_name, public_ip_params) + return ip.result() + + def _create_vnet(self): + """create virtual network""" + LOG.debug('creating vnet') + vnet_name = '%s-vnet' % self.resource_group.name + vnet_params = { + 'location': self.location, + 'address_space': { + 'address_prefixes': ['10.0.0.0/16'] + } + } + vnet = self.network_client.virtual_networks.create_or_update( + self.resource_group.name, vnet_name, vnet_params) + return vnet.result() + + def _create_subnet(self): + """create sub-network""" + LOG.debug('creating subnet') + subnet_name = '%s-subnet' % self.resource_group.name + subnet_params = { + 'address_prefix': '10.0.0.0/24' + } + subnet = self.network_client.subnets.create_or_update( + self.resource_group.name, self.vnet.name, + subnet_name, subnet_params) + return subnet.result() + + def _create_nic(self): + """Create network interface controller""" + LOG.debug('creating nic') + nic_name = '%s-nic' % self.resource_group.name + nic_params = { + 'location': self.location, + 'ip_configurations': [{ + 'name': 'ipconfig', + 'subnet': { + 'id': self.subnet.id + }, + 'publicIpAddress': { + 'id': "/subscriptions/%s" + "/resourceGroups/%s/providers/Microsoft.Network" + "/publicIPAddresses/%s" % ( + self.subscription_id, self.resource_group.name, + self.public_ip.name), + } + }] + } + nic = self.network_client.network_interfaces.create_or_update( + self.resource_group.name, nic_name, nic_params) + return nic.result() diff --git a/tests/cloud_tests/platforms/azurecloud/regions.json b/tests/cloud_tests/platforms/azurecloud/regions.json new file mode 100644 index 00000000..c1b4da20 --- /dev/null +++ b/tests/cloud_tests/platforms/azurecloud/regions.json @@ -0,0 +1,42 @@ +{ + "eastasia": "East Asia", + "southeastasia": "Southeast Asia", + "centralus": "Central US", + "eastus": "East US", + "eastus2": "East US 2", + "westus": "West US", + "northcentralus": "North Central US", + "southcentralus": "South Central US", + "northeurope": "North Europe", + "westeurope": "West Europe", + "japanwest": "Japan West", + "japaneast": "Japan East", + "brazilsouth": "Brazil South", + "australiaeast": "Australia East", + "australiasoutheast": "Australia Southeast", + "southindia": "South India", + "centralindia": "Central India", + "westindia": "West India", + "canadacentral": "Canada Central", + "canadaeast": "Canada East", + "uksouth": "UK South", + "ukwest": "UK West", + "westcentralus": "West Central US", + "westus2": "West US 2", + "koreacentral": "Korea Central", + "koreasouth": "Korea South", + "francecentral": "France Central", + "francesouth": "France South", + "australiacentral": "Australia Central", + "australiacentral2": "Australia Central 2", + "uaecentral": "UAE Central", + "uaenorth": "UAE North", + "southafricanorth": "South Africa North", + "southafricawest": "South Africa West", + "switzerlandnorth": "Switzerland North", + "switzerlandwest": "Switzerland West", + "germanynorth": "Germany North", + "germanywestcentral": "Germany West Central", + "norwaywest": "Norway West", + "norwayeast": "Norway East" +} diff --git a/tests/cloud_tests/platforms/azurecloud/snapshot.py b/tests/cloud_tests/platforms/azurecloud/snapshot.py new file mode 100644 index 00000000..580cc596 --- /dev/null +++ b/tests/cloud_tests/platforms/azurecloud/snapshot.py @@ -0,0 +1,58 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Base Azure Cloud snapshot.""" + +from ..snapshots import Snapshot + +from tests.cloud_tests import LOG + + +class AzureCloudSnapshot(Snapshot): + """Azure Cloud image copy backed snapshot.""" + + platform_name = 'azurecloud' + + def __init__(self, platform, properties, config, features, image_id, + delete_on_destroy=True): + """Set up snapshot. + + @param platform: platform object + @param properties: image properties + @param config: image config + @param features: supported feature flags + """ + super(AzureCloudSnapshot, self).__init__( + platform, properties, config, features) + + self.image_id = image_id + self.delete_on_destroy = delete_on_destroy + + def launch(self, user_data, meta_data=None, block=True, start=True, + use_desc=None): + """Launch instance. + + @param user_data: user-data for the instance + @param meta_data: meta_data for the instance + @param block: wait until instance is created + @param start: start instance and wait until fully started + @param use_desc: description of snapshot instance use + @return_value: an Instance + """ + if meta_data is not None: + raise ValueError("metadata not supported on Azure Cloud tests") + + instance = self.platform.create_instance( + self.properties, self.config, self.features, + self.image_id, user_data) + + return instance + + def destroy(self): + """Clean up snapshot data.""" + LOG.debug('destroying image %s', self.image_id) + if self.delete_on_destroy: + self.platform.compute_client.images.delete( + self.platform.resource_group.name, + self.image_id) + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/ec2/image.py b/tests/cloud_tests/platforms/ec2/image.py index 7bedf59d..d7b2c908 100644 --- a/tests/cloud_tests/platforms/ec2/image.py +++ b/tests/cloud_tests/platforms/ec2/image.py @@ -4,6 +4,7 @@ from ..images import Image from .snapshot import EC2Snapshot + from tests.cloud_tests import LOG diff --git a/tests/cloud_tests/platforms/ec2/platform.py b/tests/cloud_tests/platforms/ec2/platform.py index f188c27b..7a3d0fe0 100644 --- a/tests/cloud_tests/platforms/ec2/platform.py +++ b/tests/cloud_tests/platforms/ec2/platform.py @@ -135,6 +135,7 @@ class EC2Platform(Platform): def _create_internet_gateway(self): """Create Internet Gateway and assign to VPC.""" LOG.debug('creating internet gateway') + # pylint: disable=no-member internet_gateway = self.ec2_resource.create_internet_gateway() internet_gateway.attach_to_vpc(VpcId=self.vpc.id) self._tag_resource(internet_gateway) @@ -190,7 +191,7 @@ class EC2Platform(Platform): """Setup AWS EC2 VPC or return existing VPC.""" LOG.debug('creating new vpc') try: - vpc = self.ec2_resource.create_vpc( + vpc = self.ec2_resource.create_vpc( # pylint: disable=no-member CidrBlock=self.ipv4_cidr, AmazonProvidedIpv6CidrBlock=True) except botocore.exceptions.ClientError as e: diff --git a/tests/cloud_tests/releases.yaml b/tests/cloud_tests/releases.yaml index 924ad956..7ddc5b85 100644 --- a/tests/cloud_tests/releases.yaml +++ b/tests/cloud_tests/releases.yaml @@ -55,6 +55,8 @@ default_release_config: # cloud-init, so must pull cloud-init in from repo using # setup_image.upgrade upgrade: true + azurecloud: + boot_timeout: 300 features: # all currently supported feature flags diff --git a/tox.ini b/tox.ini index f5baf328..042346bb 100644 --- a/tox.ini +++ b/tox.ini @@ -24,6 +24,7 @@ deps = pylint==2.3.1 # test-requirements because unit tests are now present in cloudinit tree -r{toxinidir}/test-requirements.txt + -r{toxinidir}/integration-requirements.txt commands = {envpython} -m pylint {posargs:cloudinit tests tools} [testenv:py3] @@ -135,6 +136,7 @@ deps = pylint # test-requirements -r{toxinidir}/test-requirements.txt + -r{toxinidir}/integration-requirements.txt [testenv:citest] basepython = python3 -- cgit v1.2.3 From bbd9ef9f5dc9b3728d0d10534bf5d8b372bd1641 Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Fri, 6 Dec 2019 12:41:40 -0800 Subject: docs: Add security.md to readthedocs * docs: Add security.md to readthedocs This enables the ability to show the security policy on both GitHub and on the readthedocs site. To do this, enable the ability to import Markdown based files and translate them to rst. * Add doc-requirements.txt and update tox to use Also removes the extra, uncessary extension addition of .md --- SECURITY.md | 4 ++-- doc-requirements.txt | 4 ++++ doc/rtd/conf.py | 1 + doc/rtd/index.rst | 1 + doc/rtd/topics/security.rst | 5 +++++ tox.ini | 4 +--- 6 files changed, 14 insertions(+), 5 deletions(-) create mode 100644 doc-requirements.txt create mode 100644 doc/rtd/topics/security.rst (limited to 'tox.ini') diff --git a/SECURITY.md b/SECURITY.md index a09b1503..69360bb7 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -50,8 +50,8 @@ determined time for disclosure has arrived the following will occur: * An email is sent to the [public cloud-init mailing list](https://lists.launchpad.net/cloud-init/) The disclosure timeframe is coordinated with the reporter and members of the - cloud-init-security list. This depends on a number of factors: - +cloud-init-security list. This depends on a number of factors: + * The reporter might have their own disclosure timeline (e.g. Google Project Zero and many others use a 90-days after initial report OR when a fix becomes public) diff --git a/doc-requirements.txt b/doc-requirements.txt new file mode 100644 index 00000000..2d4ca7be --- /dev/null +++ b/doc-requirements.txt @@ -0,0 +1,4 @@ +doc8 +m2r +sphinx +sphinx_rtd_theme \ No newline at end of file diff --git a/doc/rtd/conf.py b/doc/rtd/conf.py index 9b274843..86441986 100644 --- a/doc/rtd/conf.py +++ b/doc/rtd/conf.py @@ -28,6 +28,7 @@ copyright = '2019, Canonical Ltd.' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ + 'm2r', 'sphinx.ext.autodoc', 'sphinx.ext.autosectionlabel', 'sphinx.ext.viewcode', diff --git a/doc/rtd/index.rst b/doc/rtd/index.rst index 826e8c48..d2662edf 100644 --- a/doc/rtd/index.rst +++ b/doc/rtd/index.rst @@ -67,6 +67,7 @@ Having trouble? We would like to help! :caption: Development topics/hacking.rst + topics/security.rst topics/debugging.rst topics/logging.rst topics/dir_layout.rst diff --git a/doc/rtd/topics/security.rst b/doc/rtd/topics/security.rst new file mode 100644 index 00000000..b8386843 --- /dev/null +++ b/doc/rtd/topics/security.rst @@ -0,0 +1,5 @@ +.. _security: + +.. mdinclude:: ../../../SECURITY.md + +.. vi: textwidth=78 diff --git a/tox.ini b/tox.ini index 042346bb..fef9643b 100644 --- a/tox.ini +++ b/tox.ini @@ -55,9 +55,7 @@ exclude = .venv,.tox,dist,doc,*egg,.git,build,tools [testenv:doc] basepython = python3 deps = - doc8 - sphinx - sphinx_rtd_theme + -r{toxinidir}/doc-requirements.txt commands = {envpython} -m sphinx {posargs:doc/rtd doc/rtd_html} doc8 doc/rtd -- cgit v1.2.3 From 8c96cbc1b6ec4c862e1fa36ef25ce56a00a1bfa6 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Wed, 18 Dec 2019 12:36:58 -0500 Subject: ci: emit names of tests run in Travis (#120) This makes it easier to debug differences in test behaviour between Travis and local developer environments. --- .travis.yml | 12 +++++++++--- tox.ini | 2 ++ 2 files changed, 11 insertions(+), 3 deletions(-) (limited to 'tox.ini') diff --git a/.travis.yml b/.travis.yml index 834a5681..9efaad14 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,9 @@ matrix: fast_finish: true include: - python: 3.6 - env: TOXENV=py3 + env: + TOXENV=py3 + NOSE_VERBOSE=2 # List all tests run by nose - install: - git fetch --unshallow - sudo apt-get build-dep -y cloud-init @@ -42,9 +44,13 @@ matrix: # Ubuntu LTS: Integration - sg lxd -c 'tox -e citest -- run --verbose --preserve-data --data-dir results --os-name xenial --test modules/apt_configure_sources_list.yaml --test modules/ntp_servers --test modules/set_password_list --test modules/user_groups --deb cloud-init_*_all.deb' - python: 2.7 - env: TOXENV=py27 + env: + TOXENV=py27 + NOSE_VERBOSE=2 # List all tests run by nose - python: 3.4 - env: TOXENV=xenial + env: + TOXENV=xenial + NOSE_VERBOSE=2 # List all tests run by nose # Travis doesn't support Python 3.4 on bionic, so use xenial dist: xenial - python: 3.6 diff --git a/tox.ini b/tox.ini index fef9643b..846e7e3f 100644 --- a/tox.ini +++ b/tox.ini @@ -6,6 +6,8 @@ recreate = True commands = python -m nose {posargs:tests/unittests cloudinit} setenv = LC_ALL = en_US.utf-8 +passenv= + NOSE_VERBOSE [testenv:pycodestyle] basepython = python3 -- cgit v1.2.3 From 2c77a287a89e8356697fc2c03522e10aa523a512 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 20 Dec 2019 15:49:09 -0500 Subject: ci: remove Python 2.7 from CI runs (#137) Specifically, drop it from the default list of environments that tox will run, and from Travis. (We retain the configuration in tox.ini for now, for any remaining Python 2.7 needs.) --- .travis.yml | 4 ---- tox.ini | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) (limited to 'tox.ini') diff --git a/.travis.yml b/.travis.yml index 9efaad14..15157b86 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,10 +43,6 @@ matrix: - sudo -E su $USER -c 'sbuild --nolog --verbose --dist=xenial cloud-init_*.dsc' # Ubuntu LTS: Integration - sg lxd -c 'tox -e citest -- run --verbose --preserve-data --data-dir results --os-name xenial --test modules/apt_configure_sources_list.yaml --test modules/ntp_servers --test modules/set_password_list --test modules/user_groups --deb cloud-init_*_all.deb' - - python: 2.7 - env: - TOXENV=py27 - NOSE_VERBOSE=2 # List all tests run by nose - python: 3.4 env: TOXENV=xenial diff --git a/tox.ini b/tox.ini index 846e7e3f..8612f034 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27, py3, xenial, pycodestyle, pyflakes, pylint +envlist = py3, xenial, pycodestyle, pyflakes, pylint recreate = True [testenv] -- cgit v1.2.3