From 3984726a6a3b1c044a8a0b78719f5438c3546e08 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 8 Jan 2019 04:52:45 +0000 Subject: doc: clean up some datasource documentation. The change to datasources.rst here is obvious typo fix. The change to azure is to reduce the two 'Customization' sections to a single and clean up some other duplicate text. --- doc/rtd/topics/datasources.rst | 2 +- doc/rtd/topics/datasources/azure.rst | 61 +++++++++++++----------------------- 2 files changed, 22 insertions(+), 41 deletions(-) (limited to 'doc') diff --git a/doc/rtd/topics/datasources.rst b/doc/rtd/topics/datasources.rst index e34f145c..5abbaefd 100644 --- a/doc/rtd/topics/datasources.rst +++ b/doc/rtd/topics/datasources.rst @@ -18,7 +18,7 @@ single way to access the different cloud systems methods to provide this data through the typical usage of subclasses. Any metadata processed by cloud-init's datasources is persisted as -``/run/cloud0-init/instance-data.json``. Cloud-init provides tooling +``/run/cloud-init/instance-data.json``. Cloud-init provides tooling to quickly introspect some of that data. See :ref:`instance_metadata` for more information. diff --git a/doc/rtd/topics/datasources/azure.rst b/doc/rtd/topics/datasources/azure.rst index f73c3694..720a475c 100644 --- a/doc/rtd/topics/datasources/azure.rst +++ b/doc/rtd/topics/datasources/azure.rst @@ -23,18 +23,18 @@ information in json format to /run/cloud-init/dhclient.hook/.json. In order for cloud-init to leverage this method to find the endpoint, the cloud.cfg file must contain: -datasource: - Azure: - set_hostname: False - agent_command: __builtin__ +.. sourcecode:: yaml + + datasource: + Azure: + set_hostname: False + agent_command: __builtin__ If those files are not available, the fallback is to check the leases file for the endpoint server (again option 245). You can define the path to the lease file with the 'dhclient_lease_file' -configuration. The default value is /var/lib/dhcp/dhclient.eth0.leases. - - dhclient_lease_file: /var/lib/dhcp/dhclient.eth0.leases +configuration. walinuxagent ------------ @@ -60,7 +60,7 @@ in order to use waagent.conf with cloud-init, the following settings are recomme Configuration ------------- The following configuration can be set for the datasource in system -configuration (in `/etc/cloud/cloud.cfg` or `/etc/cloud/cloud.cfg.d/`). +configuration (in ``/etc/cloud/cloud.cfg`` or ``/etc/cloud/cloud.cfg.d/``). The settings that may be configured are: @@ -76,13 +76,25 @@ The settings that may be configured are: * **disk_aliases**: A dictionary defining which device paths should be interpreted as ephemeral images. See cc_disk_setup module for more info. * **hostname_bounce**: A dictionary Azure hostname bounce behavior to react to - metadata changes. + metadata changes. The '``hostname_bounce: command``' entry can be either + the literal string 'builtin' or a command to execute. The command will be + invoked after the hostname is set, and will have the 'interface' in its + environment. If ``set_hostname`` is not true, then ``hostname_bounce`` + will be ignored. An example might be: + + ``command: ["sh", "-c", "killall dhclient; dhclient $interface"]`` + * **hostname_bounce**: A dictionary Azure hostname bounce behavior to react to metadata changes. Azure will throttle ifup/down in some cases after metadata has been updated to inform dhcp server about updated hostnames. * **set_hostname**: Boolean set to True when we want Azure to set the hostname based on metadata. +Configuration for the datasource can also be read from a +``dscfg`` entry in the ``LinuxProvisioningConfigurationSet``. Content in +dscfg node is expected to be base64 encoded yaml content, and it will be +merged into the 'datasource: Azure' entry. + An example configuration with the default values is provided below: .. sourcecode:: yaml @@ -143,37 +155,6 @@ Example: -Configuration -------------- -Configuration for the datasource can be read from the system config's or set -via the `dscfg` entry in the `LinuxProvisioningConfigurationSet`. Content in -dscfg node is expected to be base64 encoded yaml content, and it will be -merged into the 'datasource: Azure' entry. - -The '``hostname_bounce: command``' entry can be either the literal string -'builtin' or a command to execute. The command will be invoked after the -hostname is set, and will have the 'interface' in its environment. If -``set_hostname`` is not true, then ``hostname_bounce`` will be ignored. - -An example might be: - command: ["sh", "-c", "killall dhclient; dhclient $interface"] - -.. code:: yaml - - datasource: - agent_command - Azure: - agent_command: [service, walinuxagent, start] - set_hostname: True - hostname_bounce: - # the name of the interface to bounce - interface: eth0 - # policy can be 'on', 'off' or 'force' - policy: on - # the method 'bounce' command. - command: "builtin" - hostname_command: "hostname" - hostname -------- When the user launches an instance, they provide a hostname for that instance. -- cgit v1.2.3 From 5f49ee0f3bdc9b3ebcc71b344b3918d4ef58c989 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 8 Jan 2019 18:46:44 +0000 Subject: Add documentation on adding a datasource. This adds documentation intended for a developer on how to add a new datasource to cloud-init. --- doc/rtd/topics/datasources.rst | 59 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) (limited to 'doc') diff --git a/doc/rtd/topics/datasources.rst b/doc/rtd/topics/datasources.rst index 5abbaefd..648c6068 100644 --- a/doc/rtd/topics/datasources.rst +++ b/doc/rtd/topics/datasources.rst @@ -80,6 +80,65 @@ The current interface that a datasource object must provide is the following: def get_package_mirror_info(self) +Adding a new Datasource +----------------------- +The datasource objects have a few touch points with cloud-init. If you +are interested in adding a new datasource for your cloud platform you'll +need to take care of the following items: + +* **Identify a mechanism for positive identification of the platform**: + It is good practice for a cloud platform to positively identify itself + to the guest. This allows the guest to make educated decisions based + on the platform on which it is running. On the x86 and arm64 architectures, + many clouds identify themselves through DMI data. For example, + Oracle's public cloud provides the string 'OracleCloud.com' in the + DMI chassis-asset field. + + cloud-init enabled images produce a log file with details about the + platform. Reading through this log in ``/run/cloud-init/ds-identify.log`` + may provide the information needed to uniquely identify the platform. + If the log is not present, you can generate it by running from source + ``./tools/ds-identify`` or the installed location + ``/usr/lib/cloud-init/ds-identify``. + + The mechanism used to identify the platform will be required for the + ds-identify and datasource module sections below. + +* **Add datasource module ``cloudinit/sources/DataSource.py``**: + It is suggested that you start by copying one of the simpler datasources + such as DataSourceHetzner. + +* **Add tests for datasource module**: + Add a new file with some tests for the module to + ``cloudinit/sources/test_.py``. For example see + ``cloudinit/sources/tests/test_oracle.py`` + +* **Update ds-identify**: In systemd systems, ds-identify is used to detect + which datasource should be enabled or if cloud-init should run at all. + You'll need to make changes to ``tools/ds-identify``. + +* **Add tests for ds-identify**: Add relevant tests in a new class to + ``tests/unittests/test_ds_identify.py``. You can use ``TestOracle`` as an + example. + +* **Add your datasource name to the builtin list of datasources:** Add + your datasource module name to the end of the ``datasource_list`` + entry in ``cloudinit/settings.py``. + +* **Add your your cloud platform to apport collection prompts:** Update the + list of cloud platforms in ``cloudinit/apport.py``. This list will be + provided to the user who invokes ``ubuntu-bug cloud-init``. + +* **Enable datasource by default in ubuntu packaging branches:** + Ubuntu packaging branches contain a template file + ``debian/cloud-init.templates`` that ultimately sets the default + datasource_list when installed via package. This file needs updating when + the commit gets into a package. + +* **Add documentation for your datasource**: You should add a new + file in ``doc/datasources/.rst`` + + Datasource Documentation ======================== The following is a list of the implemented datasources. -- cgit v1.2.3 From 3a897fbd792adc1a0031364bd46d64831ca91228 Mon Sep 17 00:00:00 2001 From: Dominic Schlegel Date: Wed, 6 Feb 2019 21:22:38 +0000 Subject: correct grammar issue in instance metadata documentation LP: #1802188 --- doc/rtd/topics/instancedata.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc') diff --git a/doc/rtd/topics/instancedata.rst b/doc/rtd/topics/instancedata.rst index 5d2dc948..231a008c 100644 --- a/doc/rtd/topics/instancedata.rst +++ b/doc/rtd/topics/instancedata.rst @@ -4,7 +4,7 @@ Instance Metadata ***************** -What is a instance data? +What is instance data? ======================== Instance data is the collection of all configuration data that cloud-init -- cgit v1.2.3 From 0bb4c74e7f2d008b015d5453a1be88ae807b1f9b Mon Sep 17 00:00:00 2001 From: "Guilherme G. Piccoli" Date: Thu, 14 Feb 2019 20:37:32 +0000 Subject: EC2: Rewrite network config on AWS Classic instances every boot AWS EC2 instances' network come in 2 basic flavors: Classic and VPC (Virtual Private Cloud). The former has an interesting behavior of having its MAC address changed whenever the instance is stopped/restarted. This behavior is not observed in VPC instances. In Ubuntu 18.04 (Bionic) the network "management" changed from ENI-style (etc/network/interfaces) to netplan, and when using netplan we observe the following block present in /etc/netplan/50-cloud-init.yaml: match: macaddress: aa:bb:cc:dd:ee:ff Jani Ollikainen noticed in Launchpad bug #1802073 that the EC2 Classic instances were booting without network access in Bionic after stop/restart procedure, due to their MAC address change behavior. It was narrowed down to the netplan MAC match block, that kept the old MAC address after stopping and restarting an instance, since the network configuration writing happens by default only once in EC2 instances, in the first boot. This patch changes the network configuration write to every boot in EC2 Classic instances, by checking against the "vpc-id" metadata information provided only in the VPC instances - if we don't have this metadata value, cloud-init will rewrite the network configuration file in every boot. This was tested in an EC2 Classic instance and proved to fix the issue; unit tests were also added for the new method is_classic_instance(). LP: #1802073 Reported-by: Jani Ollikainen Suggested-by: Ryan Harper Co-developed-by: Chad Smith Signed-off-by: Guilherme G. Piccoli --- cloudinit/sources/DataSourceEc2.py | 21 +++++++++++++++++++++ doc/rtd/topics/datasources/ec2.rst | 11 +++++++++++ tests/unittests/test_datasource/test_ec2.py | 24 ++++++++++++++++++++++++ 3 files changed, 56 insertions(+) (limited to 'doc') diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index eb6f27b2..4f2f6ccb 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -19,6 +19,7 @@ from cloudinit import sources from cloudinit import url_helper as uhelp from cloudinit import util from cloudinit import warnings +from cloudinit.event import EventType LOG = logging.getLogger(__name__) @@ -107,6 +108,19 @@ class DataSourceEc2(sources.DataSource): 'dynamic', {}).get('instance-identity', {}).get('document', {}) return True + def is_classic_instance(self): + """Report if this instance type is Ec2 Classic (non-vpc).""" + if not self.metadata: + # Can return False on inconclusive as we are also called in + # network_config where metadata will be present. + # Secondary call site is in packaging postinst script. + return False + ifaces_md = self.metadata.get('network', {}).get('interfaces', {}) + for _mac, mac_data in ifaces_md.get('macs', {}).items(): + if 'vpc-id' in mac_data: + return False + return True + @property def launch_index(self): if not self.metadata: @@ -320,6 +334,13 @@ class DataSourceEc2(sources.DataSource): if isinstance(net_md, dict): result = convert_ec2_metadata_network_config( net_md, macs_to_nics=macs_to_nics, fallback_nic=iface) + # RELEASE_BLOCKER: Xenial debian/postinst needs to add + # EventType.BOOT on upgrade path for classic. + + # Non-VPC (aka Classic) Ec2 instances need to rewrite the + # network config file every boot due to MAC address change. + if self.is_classic_instance(): + self.update_events['network'].add(EventType.BOOT) else: LOG.warning("Metadata 'network' key not valid: %s.", net_md) self._network_config = result diff --git a/doc/rtd/topics/datasources/ec2.rst b/doc/rtd/topics/datasources/ec2.rst index 64c325d8..76beca92 100644 --- a/doc/rtd/topics/datasources/ec2.rst +++ b/doc/rtd/topics/datasources/ec2.rst @@ -90,4 +90,15 @@ An example configuration with the default values is provided below: max_wait: 120 timeout: 50 +Notes +----- + * There are 2 types of EC2 instances network-wise: VPC ones (Virtual Private + Cloud) and Classic ones (also known as non-VPC). One major difference + between them is that Classic instances have their MAC address changed on + stop/restart operations, so cloud-init will recreate the network config + file for EC2 Classic instances every boot. On VPC instances this file is + generated only in the first boot of the instance. + The check for the instance type is performed by is_classic_instance() + method. + .. vi: textwidth=78 diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py index 1a5956d9..20d59bfd 100644 --- a/tests/unittests/test_datasource/test_ec2.py +++ b/tests/unittests/test_datasource/test_ec2.py @@ -401,6 +401,30 @@ class TestEc2(test_helpers.HttprettyTestCase): ds.metadata = DEFAULT_METADATA self.assertEqual('my-identity-id', ds.get_instance_id()) + def test_classic_instance_true(self): + """If no vpc-id in metadata, is_classic_instance must return true.""" + md_copy = copy.deepcopy(DEFAULT_METADATA) + ifaces_md = md_copy.get('network', {}).get('interfaces', {}) + for _mac, mac_data in ifaces_md.get('macs', {}).items(): + if 'vpc-id' in mac_data: + del mac_data['vpc-id'] + + ds = self._setup_ds( + platform_data=self.valid_platform_data, + sys_cfg={'datasource': {'Ec2': {'strict_id': False}}}, + md={'md': md_copy}) + self.assertTrue(ds.get_data()) + self.assertTrue(ds.is_classic_instance()) + + def test_classic_instance_false(self): + """If vpc-id in metadata, is_classic_instance must return false.""" + ds = self._setup_ds( + platform_data=self.valid_platform_data, + sys_cfg={'datasource': {'Ec2': {'strict_id': False}}}, + md={'md': DEFAULT_METADATA}) + self.assertTrue(ds.get_data()) + self.assertFalse(ds.is_classic_instance()) + @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery') def test_valid_platform_with_strict_true(self, m_dhcp): """Valid platform data should return true with strict_id true.""" -- cgit v1.2.3 From 79d40e6b7bce33af69572c6054b3398b8d8077c7 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Fri, 22 Feb 2019 09:56:03 +0000 Subject: doc: update merging doc with fixes and some additional details/examples Update config merging documentation with cloud-config syntax fix. Add an example showing how to merge two files with runcmd. --- doc/rtd/topics/merging.rst | 90 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 84 insertions(+), 6 deletions(-) (limited to 'doc') diff --git a/doc/rtd/topics/merging.rst b/doc/rtd/topics/merging.rst index c75ca59c..5f7ca18d 100644 --- a/doc/rtd/topics/merging.rst +++ b/doc/rtd/topics/merging.rst @@ -21,12 +21,12 @@ For example. .. code-block:: yaml #cloud-config (1) - run_cmd: + runcmd: - bash1 - bash2 #cloud-config (2) - run_cmd: + runcmd: - bash3 - bash4 @@ -36,7 +36,7 @@ cloud-config object that contains the following. .. code-block:: yaml #cloud-config (merged) - run_cmd: + runcmd: - bash3 - bash4 @@ -45,7 +45,7 @@ Typically this is not what users want; instead they would likely prefer: .. code-block:: yaml #cloud-config (merged) - run_cmd: + runcmd: - bash1 - bash2 - bash3 @@ -55,6 +55,45 @@ This way makes it easier to combine the various cloud-config objects you have into a more useful list, thus reducing duplication necessary to accomplish the same result with the previous method. + +Built-in Mergers +================ + +Cloud-init provides merging for the following built-in types: + +- Dict +- List +- String + +The ``Dict`` merger has the following options which control what is done with +values contained within the config. + +- ``allow_delete``: Existing values not present in the new value can be deleted, defaults to False +- ``no_replace``: Do not replace an existing value if one is already present, enabled by default. +- ``replace``: Overwrite existing values with new ones. + +The ``List`` merger has the following options which control what is done with +the values contained within the config. + +- ``append``: Add new value to the end of the list, defaults to False. +- ``prepend``: Add new values to the start of the list, defaults to False. +- ``no_replace``: Do not replace an existing value if one is already present, enabled by default. +- ``replace``: Overwrite existing values with new ones. + +The ``Str`` merger has the following options which control what is done with +the values contained within the config. + +- ``append``: Add new value to the end of the string, defaults to False. + +Common options for all merge types which control how recursive merging is +done on other types. + +- ``recurse_dict``: If True merge the new values of the dictionary, defaults to True. +- ``recurse_list``: If True merge the new values of the list, defaults to False. +- ``recurse_array``: Alias for ``recurse_list``. +- ``recurse_str``: If True merge the new values of the string, defaults to False. + + Customizability =============== @@ -164,8 +203,8 @@ string format (i.e. the second option above), for example: .. code-block:: python - {'merge_how': [{'name': 'list', 'settings': ['extend']}, - {'name': 'dict', 'settings': []}, + {'merge_how': [{'name': 'list', 'settings': ['append']}, + {'name': 'dict', 'settings': ['no_replace', 'recurse_list']}, {'name': 'str', 'settings': ['append']}]} This would be the equivalent format for default string format but in dictionary @@ -201,4 +240,43 @@ Note, however, that merge algorithms are not used *across* types of configuration. As was the case before merging was implemented, user-data will overwrite conf.d configuration without merging. +Example cloud-config +==================== + +A common request is to include multiple ``runcmd`` directives in different +files and merge all of the commands together. To achieve this, we must modify +the default merging to allow for dictionaries to join list values. + + +The first config + +.. code-block:: yaml + + #cloud-config + merge_how: + - name: list + settings: [append] + - name: dict + settings: [no_replace, recurse_list] + + runcmd: + - bash1 + - bash2 + +The second config + +.. code-block:: yaml + + #cloud-config + merge_how: + - name: list + settings: [append] + - name: dict + settings: [no_replace, recurse_list] + + runcmd: + - bash3 + - bash4 + + .. vi: textwidth=78 -- cgit v1.2.3 From 8cfcc28db1acc7594dbbf76b846f4964f40f9e63 Mon Sep 17 00:00:00 2001 From: Eric Williams Date: Mon, 25 Feb 2019 19:09:39 +0000 Subject: Enable encrypted_data_bag_secret support for Chef Encrypted data bags require a secrets file to be present to decrypt, and the location of the file must be configured the Chef client configuration file, client.rb. This update enables cloud-init's chef module to update that setting in client.rb. LP: #1817082 --- cloudinit/config/cc_chef.py | 3 +++ doc/examples/cloud-config-chef.txt | 3 +++ templates/chef_client.rb.tmpl | 5 ++++- tests/unittests/test_handler/test_handler_chef.py | 3 +++ 4 files changed, 13 insertions(+), 1 deletion(-) (limited to 'doc') diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 46abedd1..a6240306 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -51,6 +51,7 @@ file). chef: client_key: + encrypted_data_bag_secret: environment: file_backup_path: file_cache_path: @@ -114,6 +115,7 @@ CHEF_RB_TPL_DEFAULTS = { 'file_backup_path': "/var/backups/chef", 'pid_file': "/var/run/chef/client.pid", 'show_time': True, + 'encrypted_data_bag_secret': None, } CHEF_RB_TPL_BOOL_KEYS = frozenset(['show_time']) CHEF_RB_TPL_PATH_KEYS = frozenset([ @@ -124,6 +126,7 @@ CHEF_RB_TPL_PATH_KEYS = frozenset([ 'json_attribs', 'file_cache_path', 'pid_file', + 'encrypted_data_bag_secret', ]) CHEF_RB_TPL_KEYS = list(CHEF_RB_TPL_DEFAULTS.keys()) CHEF_RB_TPL_KEYS.extend(CHEF_RB_TPL_BOOL_KEYS) diff --git a/doc/examples/cloud-config-chef.txt b/doc/examples/cloud-config-chef.txt index defc5a54..2320e01a 100644 --- a/doc/examples/cloud-config-chef.txt +++ b/doc/examples/cloud-config-chef.txt @@ -98,6 +98,9 @@ chef: # to the install script omnibus_version: "12.3.0" + # If encrypted data bags are used, the client needs to have a secrets file + # configured to decrypt them + encrypted_data_bag_secret: "/etc/chef/encrypted_data_bag_secret" # Capture all subprocess output into a logfile # Useful for troubleshooting cloud-init issues diff --git a/templates/chef_client.rb.tmpl b/templates/chef_client.rb.tmpl index cbb6b15f..99978d3b 100644 --- a/templates/chef_client.rb.tmpl +++ b/templates/chef_client.rb.tmpl @@ -1,6 +1,6 @@ ## template:jinja {# -This file is only utilized if the module 'cc_chef' is enabled in +This file is only utilized if the module 'cc_chef' is enabled in cloud-config. Specifically, in order to enable it you need to add the following to config: chef: @@ -56,3 +56,6 @@ pid_file "{{pid_file}}" {% if show_time %} Chef::Log::Formatter.show_time = true {% endif %} +{% if encrypted_data_bag_secret %} +encrypted_data_bag_secret "{{encrypted_data_bag_secret}}" +{% endif %} diff --git a/tests/unittests/test_handler/test_handler_chef.py b/tests/unittests/test_handler/test_handler_chef.py index b16532ea..f4311268 100644 --- a/tests/unittests/test_handler/test_handler_chef.py +++ b/tests/unittests/test_handler/test_handler_chef.py @@ -145,6 +145,7 @@ class TestChef(FilesystemMockingTestCase): file_backup_path "/var/backups/chef" pid_file "/var/run/chef/client.pid" Chef::Log::Formatter.show_time = true + encrypted_data_bag_secret "/etc/chef/encrypted_data_bag_secret" """ tpl_file = util.load_file('templates/chef_client.rb.tmpl') self.patchUtils(self.tmp) @@ -157,6 +158,8 @@ class TestChef(FilesystemMockingTestCase): 'validation_name': 'bob', 'validation_key': "/etc/chef/vkey.pem", 'validation_cert': "this is my cert", + 'encrypted_data_bag_secret': + '/etc/chef/encrypted_data_bag_secret' }, } cc_chef.handle('chef', cfg, self.fetch_cloud('ubuntu'), LOG, []) -- cgit v1.2.3 From 43f5850767f2412c63a5e75d47598e5d0479fd25 Mon Sep 17 00:00:00 2001 From: Anton Olifir Date: Wed, 6 Mar 2019 00:39:02 +0000 Subject: Example for Microsoft Azure data disk added. --- doc/examples/cloud-config-disk-setup.txt | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) (limited to 'doc') diff --git a/doc/examples/cloud-config-disk-setup.txt b/doc/examples/cloud-config-disk-setup.txt index 43a62a26..89d9ff57 100644 --- a/doc/examples/cloud-config-disk-setup.txt +++ b/doc/examples/cloud-config-disk-setup.txt @@ -17,7 +17,7 @@ fs_setup: device: ephemeral0 partition: auto -# Default disk definitions for Windows Azure +# Default disk definitions for Microsoft Azure # ------------------------------------------ device_aliases: {'ephemeral0': '/dev/sdb'} @@ -34,6 +34,21 @@ fs_setup: replace_fs: ntfs +# Data disks definitions for Microsoft Azure +# ------------------------------------------ + +disk_setup: + /dev/disk/azure/scsi1/lun0: + table_type: gpt + layout: True + overwrite: True + +fs_setup: + - device: /dev/disk/azure/scsi1/lun0 + partition: 1 + filesystem: ext4 + + # Default disk definitions for SmartOS # ------------------------------------ @@ -242,7 +257,7 @@ fs_setup: # # "false": If an existing file system exists, skip the creation. # -# : This is a special directive, used for Windows Azure that +# : This is a special directive, used for Microsoft Azure that # instructs cloud-init to replace a file system of . NOTE: # unless you define a label, this requires the use of the 'any' partition # directive. -- cgit v1.2.3 From 6d58bd8a65e1e7723cd6019b0ceca39564c435fd Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Mon, 18 Mar 2019 18:09:43 +0000 Subject: doc: Refresh Azure walinuxagent docs - Remove outdated waagent.conf recommendations - Recommend using Provisioning.UseCloudInit - Reorganise sections so walinuxagent recommendations are easier to find --- doc/rtd/topics/datasources/azure.rst | 57 ++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 22 deletions(-) (limited to 'doc') diff --git a/doc/rtd/topics/datasources/azure.rst b/doc/rtd/topics/datasources/azure.rst index 720a475c..b41cddd9 100644 --- a/doc/rtd/topics/datasources/azure.rst +++ b/doc/rtd/topics/datasources/azure.rst @@ -5,9 +5,30 @@ Azure This datasource finds metadata and user-data from the Azure cloud platform. -Azure Platform --------------- -The azure cloud-platform provides initial data to an instance via an attached +walinuxagent +------------ +walinuxagent has several functions within images. For cloud-init +specifically, the relevant functionality it performs is to register the +instance with the Azure cloud platform at boot so networking will be +permitted. For more information about the other functionality of +walinuxagent, see `Azure's documentation +`_ for more details. +(Note, however, that only one of walinuxagent's provisioning and cloud-init +should be used to perform instance customisation.) + +If you are configuring walinuxagent yourself, you will want to ensure that you +have `Provisioning.UseCloudInit +`_ set to +``y``. + + +Builtin Agent +------------- +An alternative to using walinuxagent to register to the Azure cloud platform +is to use the ``__builtin__`` agent command. This section contains more +background on what that code path does, and how to enable it. + +The Azure cloud platform provides initial data to an instance via an attached CD formatted in UDF. That CD contains a 'ovf-env.xml' file that provides some information. Additional information is obtained via interaction with the "endpoint". @@ -36,25 +57,17 @@ for the endpoint server (again option 245). You can define the path to the lease file with the 'dhclient_lease_file' configuration. -walinuxagent ------------- -In order to operate correctly, cloud-init needs walinuxagent to provide much -of the interaction with azure. In addition to "provisioning" code, walinux -does the following on the agent is a long running daemon that handles the -following things: -- generate a x509 certificate and send that to the endpoint - -waagent.conf config -^^^^^^^^^^^^^^^^^^^ -in order to use waagent.conf with cloud-init, the following settings are recommended. Other values can be changed or set to the defaults. - - :: - - # disabling provisioning turns off all 'Provisioning.*' function - Provisioning.Enabled=n - # this is currently not handled by cloud-init, so let walinuxagent do it. - ResourceDisk.Format=y - ResourceDisk.MountPoint=/mnt + +IMDS +---- +Azure provides the `instance metadata service (IMDS) +`_ +which is a REST service on ``196.254.196.254`` providing additional +configuration information to the instance. Cloud-init uses the IMDS for: + +- network configuration for the instance which is applied per boot +- a preprovisioing gate which blocks instance configuration until Azure fabric + is ready to provision Configuration -- cgit v1.2.3 From 5e5894d68d21bf33649aca36973a0ef2fe72f01d Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 19 Mar 2019 14:24:37 +0000 Subject: Add ubuntu_drivers config module The ubuntu_drivers config module enables usage of the 'ubuntu-drivers' command. At this point it only serves as a way of installing NVIDIA drivers for general purpose graphics processing unit (GPGPU) functionality. Also, a small usability improvement to get_cfg_by_path to allow it to take a string for the key path "toplevel/second/mykey" in addition to the original: ("toplevel", "second", "mykey") --- cloudinit/config/cc_ubuntu_drivers.py | 112 +++++++++++++++++ cloudinit/config/tests/test_ubuntu_drivers.py | 174 ++++++++++++++++++++++++++ cloudinit/util.py | 15 +++ config/cloud.cfg.tmpl | 3 + doc/rtd/topics/modules.rst | 1 + tests/unittests/test_handler/test_schema.py | 1 + 6 files changed, 306 insertions(+) create mode 100644 cloudinit/config/cc_ubuntu_drivers.py create mode 100644 cloudinit/config/tests/test_ubuntu_drivers.py (limited to 'doc') diff --git a/cloudinit/config/cc_ubuntu_drivers.py b/cloudinit/config/cc_ubuntu_drivers.py new file mode 100644 index 00000000..91feb603 --- /dev/null +++ b/cloudinit/config/cc_ubuntu_drivers.py @@ -0,0 +1,112 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Ubuntu Drivers: Interact with third party drivers in Ubuntu.""" + +from textwrap import dedent + +from cloudinit.config.schema import ( + get_schema_doc, validate_cloudconfig_schema) +from cloudinit import log as logging +from cloudinit.settings import PER_INSTANCE +from cloudinit import type_utils +from cloudinit import util + +LOG = logging.getLogger(__name__) + +frequency = PER_INSTANCE +distros = ['ubuntu'] +schema = { + 'id': 'cc_ubuntu_drivers', + 'name': 'Ubuntu Drivers', + 'title': 'Interact with third party drivers in Ubuntu.', + 'description': dedent("""\ + This module interacts with the 'ubuntu-drivers' command to install + third party driver packages."""), + 'distros': distros, + 'examples': [dedent("""\ + drivers: + nvidia: + license-accepted: true + """)], + 'frequency': frequency, + 'type': 'object', + 'properties': { + 'drivers': { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'nvidia': { + 'type': 'object', + 'additionalProperties': False, + 'required': ['license-accepted'], + 'properties': { + 'license-accepted': { + 'type': 'boolean', + 'description': ("Do you accept the NVIDIA driver" + " license?"), + }, + 'version': { + 'type': 'string', + 'description': ( + 'The version of the driver to install (e.g.' + ' "390", "410"). Defaults to the latest' + ' version.'), + }, + }, + }, + }, + }, + }, +} +OLD_UBUNTU_DRIVERS_STDERR_NEEDLE = ( + "ubuntu-drivers: error: argument : invalid choice: 'install'") + +__doc__ = get_schema_doc(schema) # Supplement python help() + + +def install_drivers(cfg, pkg_install_func): + if not isinstance(cfg, dict): + raise TypeError( + "'drivers' config expected dict, found '%s': %s" % + (type_utils.obj_name(cfg), cfg)) + + cfgpath = 'nvidia/license-accepted' + # Call translate_bool to ensure that we treat string values like "yes" as + # acceptance and _don't_ treat string values like "nah" as acceptance + # because they're True-ish + nv_acc = util.translate_bool(util.get_cfg_by_path(cfg, cfgpath)) + if not nv_acc: + LOG.debug("Not installing NVIDIA drivers. %s=%s", cfgpath, nv_acc) + return + + if not util.which('ubuntu-drivers'): + LOG.debug("'ubuntu-drivers' command not available. " + "Installing ubuntu-drivers-common") + pkg_install_func(['ubuntu-drivers-common']) + + driver_arg = 'nvidia' + version_cfg = util.get_cfg_by_path(cfg, 'nvidia/version') + if version_cfg: + driver_arg += ':{}'.format(version_cfg) + + LOG.debug("Installing NVIDIA drivers (%s=%s, version=%s)", + cfgpath, nv_acc, version_cfg if version_cfg else 'latest') + + try: + util.subp(['ubuntu-drivers', 'install', '--gpgpu', driver_arg]) + except util.ProcessExecutionError as exc: + if OLD_UBUNTU_DRIVERS_STDERR_NEEDLE in exc.stderr: + LOG.warning('the available version of ubuntu-drivers is' + ' too old to perform requested driver installation') + elif 'No drivers found for installation.' in exc.stdout: + LOG.warning('ubuntu-drivers found no drivers for installation') + raise + + +def handle(name, cfg, cloud, log, _args): + if "drivers" not in cfg: + log.debug("Skipping module named %s, no 'drivers' key in config", name) + return + + validate_cloudconfig_schema(cfg, schema) + install_drivers(cfg['drivers'], cloud.distro.install_packages) diff --git a/cloudinit/config/tests/test_ubuntu_drivers.py b/cloudinit/config/tests/test_ubuntu_drivers.py new file mode 100644 index 00000000..efba4ce7 --- /dev/null +++ b/cloudinit/config/tests/test_ubuntu_drivers.py @@ -0,0 +1,174 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +import copy + +from cloudinit.tests.helpers import CiTestCase, skipUnlessJsonSchema, mock +from cloudinit.config.schema import ( + SchemaValidationError, validate_cloudconfig_schema) +from cloudinit.config import cc_ubuntu_drivers as drivers +from cloudinit.util import ProcessExecutionError + +MPATH = "cloudinit.config.cc_ubuntu_drivers." +OLD_UBUNTU_DRIVERS_ERROR_STDERR = ( + "ubuntu-drivers: error: argument : invalid choice: 'install' " + "(choose from 'list', 'autoinstall', 'devices', 'debug')\n") + + +class TestUbuntuDrivers(CiTestCase): + cfg_accepted = {'drivers': {'nvidia': {'license-accepted': True}}} + install_gpgpu = ['ubuntu-drivers', 'install', '--gpgpu', 'nvidia'] + + with_logs = True + + @skipUnlessJsonSchema() + def test_schema_requires_boolean_for_license_accepted(self): + with self.assertRaisesRegex( + SchemaValidationError, ".*license-accepted.*TRUE.*boolean"): + validate_cloudconfig_schema( + {'drivers': {'nvidia': {'license-accepted': "TRUE"}}}, + schema=drivers.schema, strict=True) + + @mock.patch(MPATH + "util.subp", return_value=('', '')) + @mock.patch(MPATH + "util.which", return_value=False) + def _assert_happy_path_taken(self, config, m_which, m_subp): + """Positive path test through handle. Package should be installed.""" + myCloud = mock.MagicMock() + drivers.handle('ubuntu_drivers', config, myCloud, None, None) + self.assertEqual([mock.call(['ubuntu-drivers-common'])], + myCloud.distro.install_packages.call_args_list) + self.assertEqual([mock.call(self.install_gpgpu)], + m_subp.call_args_list) + + def test_handle_does_package_install(self): + self._assert_happy_path_taken(self.cfg_accepted) + + def test_trueish_strings_are_considered_approval(self): + for true_value in ['yes', 'true', 'on', '1']: + new_config = copy.deepcopy(self.cfg_accepted) + new_config['drivers']['nvidia']['license-accepted'] = true_value + self._assert_happy_path_taken(new_config) + + @mock.patch(MPATH + "util.subp", side_effect=ProcessExecutionError( + stdout='No drivers found for installation.\n', exit_code=1)) + @mock.patch(MPATH + "util.which", return_value=False) + def test_handle_raises_error_if_no_drivers_found(self, m_which, m_subp): + """If ubuntu-drivers doesn't install any drivers, raise an error.""" + myCloud = mock.MagicMock() + with self.assertRaises(Exception): + drivers.handle( + 'ubuntu_drivers', self.cfg_accepted, myCloud, None, None) + self.assertEqual([mock.call(['ubuntu-drivers-common'])], + myCloud.distro.install_packages.call_args_list) + self.assertEqual([mock.call(self.install_gpgpu)], + m_subp.call_args_list) + self.assertIn('ubuntu-drivers found no drivers for installation', + self.logs.getvalue()) + + @mock.patch(MPATH + "util.subp", return_value=('', '')) + @mock.patch(MPATH + "util.which", return_value=False) + def _assert_inert_with_config(self, config, m_which, m_subp): + """Helper to reduce repetition when testing negative cases""" + myCloud = mock.MagicMock() + drivers.handle('ubuntu_drivers', config, myCloud, None, None) + self.assertEqual(0, myCloud.distro.install_packages.call_count) + self.assertEqual(0, m_subp.call_count) + + def test_handle_inert_if_license_not_accepted(self): + """Ensure we don't do anything if the license is rejected.""" + self._assert_inert_with_config( + {'drivers': {'nvidia': {'license-accepted': False}}}) + + def test_handle_inert_if_garbage_in_license_field(self): + """Ensure we don't do anything if unknown text is in license field.""" + self._assert_inert_with_config( + {'drivers': {'nvidia': {'license-accepted': 'garbage'}}}) + + def test_handle_inert_if_no_license_key(self): + """Ensure we don't do anything if no license key.""" + self._assert_inert_with_config({'drivers': {'nvidia': {}}}) + + def test_handle_inert_if_no_nvidia_key(self): + """Ensure we don't do anything if other license accepted.""" + self._assert_inert_with_config( + {'drivers': {'acme': {'license-accepted': True}}}) + + def test_handle_inert_if_string_given(self): + """Ensure we don't do anything if string refusal given.""" + for false_value in ['no', 'false', 'off', '0']: + self._assert_inert_with_config( + {'drivers': {'nvidia': {'license-accepted': false_value}}}) + + @mock.patch(MPATH + "install_drivers") + def test_handle_no_drivers_does_nothing(self, m_install_drivers): + """If no 'drivers' key in the config, nothing should be done.""" + myCloud = mock.MagicMock() + myLog = mock.MagicMock() + drivers.handle('ubuntu_drivers', {'foo': 'bzr'}, myCloud, myLog, None) + self.assertIn('Skipping module named', + myLog.debug.call_args_list[0][0][0]) + self.assertEqual(0, m_install_drivers.call_count) + + @mock.patch(MPATH + "util.subp", return_value=('', '')) + @mock.patch(MPATH + "util.which", return_value=True) + def test_install_drivers_no_install_if_present(self, m_which, m_subp): + """If 'ubuntu-drivers' is present, no package install should occur.""" + pkg_install = mock.MagicMock() + drivers.install_drivers(self.cfg_accepted['drivers'], + pkg_install_func=pkg_install) + self.assertEqual(0, pkg_install.call_count) + self.assertEqual([mock.call('ubuntu-drivers')], + m_which.call_args_list) + self.assertEqual([mock.call(self.install_gpgpu)], + m_subp.call_args_list) + + def test_install_drivers_rejects_invalid_config(self): + """install_drivers should raise TypeError if not given a config dict""" + pkg_install = mock.MagicMock() + with self.assertRaisesRegex(TypeError, ".*expected dict.*"): + drivers.install_drivers("mystring", pkg_install_func=pkg_install) + self.assertEqual(0, pkg_install.call_count) + + @mock.patch(MPATH + "util.subp", side_effect=ProcessExecutionError( + stderr=OLD_UBUNTU_DRIVERS_ERROR_STDERR, exit_code=2)) + @mock.patch(MPATH + "util.which", return_value=False) + def test_install_drivers_handles_old_ubuntu_drivers_gracefully( + self, m_which, m_subp): + """Older ubuntu-drivers versions should emit message and raise error""" + myCloud = mock.MagicMock() + with self.assertRaises(Exception): + drivers.handle( + 'ubuntu_drivers', self.cfg_accepted, myCloud, None, None) + self.assertEqual([mock.call(['ubuntu-drivers-common'])], + myCloud.distro.install_packages.call_args_list) + self.assertEqual([mock.call(self.install_gpgpu)], + m_subp.call_args_list) + self.assertIn('WARNING: the available version of ubuntu-drivers is' + ' too old to perform requested driver installation', + self.logs.getvalue()) + + +# Sub-class TestUbuntuDrivers to run the same test cases, but with a version +class TestUbuntuDriversWithVersion(TestUbuntuDrivers): + cfg_accepted = { + 'drivers': {'nvidia': {'license-accepted': True, 'version': '123'}}} + install_gpgpu = ['ubuntu-drivers', 'install', '--gpgpu', 'nvidia:123'] + + @mock.patch(MPATH + "util.subp", return_value=('', '')) + @mock.patch(MPATH + "util.which", return_value=False) + def test_version_none_uses_latest(self, m_which, m_subp): + myCloud = mock.MagicMock() + version_none_cfg = { + 'drivers': {'nvidia': {'license-accepted': True, 'version': None}}} + drivers.handle( + 'ubuntu_drivers', version_none_cfg, myCloud, None, None) + self.assertEqual( + [mock.call(['ubuntu-drivers', 'install', '--gpgpu', 'nvidia'])], + m_subp.call_args_list) + + def test_specifying_a_version_doesnt_override_license_acceptance(self): + self._assert_inert_with_config({ + 'drivers': {'nvidia': {'license-accepted': False, + 'version': '123'}} + }) + +# vi: ts=4 expandtab diff --git a/cloudinit/util.py b/cloudinit/util.py index a192091f..385f231c 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -703,6 +703,21 @@ def get_cfg_option_list(yobj, key, default=None): # get a cfg entry by its path array # for f['a']['b']: get_cfg_by_path(mycfg,('a','b')) def get_cfg_by_path(yobj, keyp, default=None): + """Return the value of the item at path C{keyp} in C{yobj}. + + example: + get_cfg_by_path({'a': {'b': {'num': 4}}}, 'a/b/num') == 4 + get_cfg_by_path({'a': {'b': {'num': 4}}}, 'c/d') == None + + @param yobj: A dictionary. + @param keyp: A path inside yobj. it can be a '/' delimited string, + or an iterable. + @param default: The default to return if the path does not exist. + @return: The value of the item at keyp." + is not found.""" + + if isinstance(keyp, six.string_types): + keyp = keyp.split("/") cur = yobj for tok in keyp: if tok not in cur: diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl index 7513176b..25db43e0 100644 --- a/config/cloud.cfg.tmpl +++ b/config/cloud.cfg.tmpl @@ -112,6 +112,9 @@ cloud_final_modules: - landscape - lxd {% endif %} +{% if variant in ["ubuntu", "unknown"] %} + - ubuntu-drivers +{% endif %} {% if variant not in ["freebsd"] %} - puppet - chef diff --git a/doc/rtd/topics/modules.rst b/doc/rtd/topics/modules.rst index d9720f6a..3dcdd3bc 100644 --- a/doc/rtd/topics/modules.rst +++ b/doc/rtd/topics/modules.rst @@ -54,6 +54,7 @@ Modules .. automodule:: cloudinit.config.cc_ssh_import_id .. automodule:: cloudinit.config.cc_timezone .. automodule:: cloudinit.config.cc_ubuntu_advantage +.. automodule:: cloudinit.config.cc_ubuntu_drivers .. automodule:: cloudinit.config.cc_update_etc_hosts .. automodule:: cloudinit.config.cc_update_hostname .. automodule:: cloudinit.config.cc_users_groups diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py index 1bad07f6..e69a47a9 100644 --- a/tests/unittests/test_handler/test_schema.py +++ b/tests/unittests/test_handler/test_schema.py @@ -28,6 +28,7 @@ class GetSchemaTest(CiTestCase): 'cc_runcmd', 'cc_snap', 'cc_ubuntu_advantage', + 'cc_ubuntu_drivers', 'cc_zypper_add_repo' ], [subschema['id'] for subschema in schema['allOf']]) -- cgit v1.2.3 From b76714c355a87416f9f07156b0f025aceaca7296 Mon Sep 17 00:00:00 2001 From: Risto Oikarinen Date: Tue, 9 Apr 2019 18:05:24 +0000 Subject: Change DataSourceNoCloud to ignore file system label's case. NoCloud data source now accepts both 'cidata' and 'CIDATA' as filesystem labels. This is similar to DataSourceConfigDrive's support for 'config-2' and 'CONFIG-2'. --- cloudinit/sources/DataSourceNoCloud.py | 4 ++- doc/rtd/topics/datasources/nocloud.rst | 2 +- tests/unittests/test_datasource/test_nocloud.py | 42 +++++++++++++++++++++++++ tests/unittests/test_ds_identify.py | 17 ++++++++++ tools/ds-identify | 7 +++-- 5 files changed, 67 insertions(+), 5 deletions(-) (limited to 'doc') diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 6860f0cc..fcf5d589 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -106,7 +106,9 @@ class DataSourceNoCloud(sources.DataSource): fslist = util.find_devs_with("TYPE=vfat") fslist.extend(util.find_devs_with("TYPE=iso9660")) - label_list = util.find_devs_with("LABEL=%s" % label) + label_list = util.find_devs_with("LABEL=%s" % label.upper()) + label_list.extend(util.find_devs_with("LABEL=%s" % label.lower())) + devlist = list(set(fslist) & set(label_list)) devlist.sort(reverse=True) diff --git a/doc/rtd/topics/datasources/nocloud.rst b/doc/rtd/topics/datasources/nocloud.rst index 08578e86..1c5cf961 100644 --- a/doc/rtd/topics/datasources/nocloud.rst +++ b/doc/rtd/topics/datasources/nocloud.rst @@ -9,7 +9,7 @@ network at all). You can provide meta-data and user-data to a local vm boot via files on a `vfat`_ or `iso9660`_ filesystem. The filesystem volume label must be -``cidata``. +``cidata`` or ``CIDATA``. Alternatively, you can provide meta-data via kernel command line or SMBIOS "serial number" option. The data must be passed in the form of a string: diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py index 3429272c..b785362f 100644 --- a/tests/unittests/test_datasource/test_nocloud.py +++ b/tests/unittests/test_datasource/test_nocloud.py @@ -32,6 +32,36 @@ class TestNoCloudDataSource(CiTestCase): self.mocks.enter_context( mock.patch.object(util, 'read_dmi_data', return_value=None)) + def _test_fs_config_is_read(self, fs_label, fs_label_to_search): + vfat_device = 'device-1' + + def m_mount_cb(device, callback, mtype): + if (device == vfat_device): + return {'meta-data': yaml.dump({'instance-id': 'IID'})} + else: + return {} + + def m_find_devs_with(query='', path=''): + if 'TYPE=vfat' == query: + return [vfat_device] + elif 'LABEL={}'.format(fs_label) == query: + return [vfat_device] + else: + return [] + + self.mocks.enter_context( + mock.patch.object(util, 'find_devs_with', + side_effect=m_find_devs_with)) + self.mocks.enter_context( + mock.patch.object(util, 'mount_cb', + side_effect=m_mount_cb)) + sys_cfg = {'datasource': {'NoCloud': {'fs_label': fs_label_to_search}}} + dsrc = dsNoCloud(sys_cfg=sys_cfg, distro=None, paths=self.paths) + ret = dsrc.get_data() + + self.assertEqual(dsrc.metadata.get('instance-id'), 'IID') + self.assertTrue(ret) + def test_nocloud_seed_dir_on_lxd(self, m_is_lxd): md = {'instance-id': 'IID', 'dsmode': 'local'} ud = b"USER_DATA_HERE" @@ -90,6 +120,18 @@ class TestNoCloudDataSource(CiTestCase): ret = dsrc.get_data() self.assertFalse(ret) + def test_fs_config_lowercase_label(self, m_is_lxd): + self._test_fs_config_is_read('cidata', 'cidata') + + def test_fs_config_uppercase_label(self, m_is_lxd): + self._test_fs_config_is_read('CIDATA', 'cidata') + + def test_fs_config_lowercase_label_search_uppercase(self, m_is_lxd): + self._test_fs_config_is_read('cidata', 'CIDATA') + + def test_fs_config_uppercase_label_search_uppercase(self, m_is_lxd): + self._test_fs_config_is_read('CIDATA', 'CIDATA') + def test_no_datasource_expected(self, m_is_lxd): # no source should be found if no cmdline, config, and fs_label=None sys_cfg = {'datasource': {'NoCloud': {'fs_label': None}}} diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py index d00c1b4b..8c18aa1a 100644 --- a/tests/unittests/test_ds_identify.py +++ b/tests/unittests/test_ds_identify.py @@ -520,6 +520,10 @@ class TestDsIdentify(DsIdentifyBase): """NoCloud is found with iso9660 filesystem on non-cdrom disk.""" self._test_ds_found('NoCloud') + def test_nocloud_upper(self): + """NoCloud is found with uppercase filesystem label.""" + self._test_ds_found('NoCloudUpper') + def test_nocloud_seed(self): """Nocloud seed directory.""" self._test_ds_found('NoCloud-seed') @@ -713,6 +717,19 @@ VALID_CFG = { 'dev/vdb': 'pretend iso content for cidata\n', } }, + 'NoCloudUpper': { + 'ds': 'NoCloud', + 'mocks': [ + MOCK_VIRT_IS_KVM, + {'name': 'blkid', 'ret': 0, + 'out': blkid_out( + BLKID_UEFI_UBUNTU + + [{'DEVNAME': 'vdb', 'TYPE': 'iso9660', 'LABEL': 'CIDATA'}])}, + ], + 'files': { + 'dev/vdb': 'pretend iso content for cidata\n', + } + }, 'NoCloud-seed': { 'ds': 'NoCloud', 'files': { diff --git a/tools/ds-identify b/tools/ds-identify index b78b2731..6518901e 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -620,7 +620,7 @@ dscheck_MAAS() { } dscheck_NoCloud() { - local fslabel="cidata" d="" + local fslabel="cidata CIDATA" d="" case " ${DI_KERNEL_CMDLINE} " in *\ ds=nocloud*) return ${DS_FOUND};; esac @@ -632,9 +632,10 @@ dscheck_NoCloud() { check_seed_dir "$d" meta-data user-data && return ${DS_FOUND} check_writable_seed_dir "$d" meta-data user-data && return ${DS_FOUND} done - if has_fs_with_label "${fslabel}"; then + if has_fs_with_label $fslabel; then return ${DS_FOUND} fi + return ${DS_NOT_FOUND} } @@ -762,7 +763,7 @@ is_cdrom_ovf() { # explicitly skip known labels of other types. rd_rdfe is azure. case "$label" in - config-2|CONFIG-2|rd_rdfe_stable*|cidata) return 1;; + config-2|CONFIG-2|rd_rdfe_stable*|cidata|CIDATA) return 1;; esac local idstr="http://schemas.dmtf.org/ovf/environment/1" -- cgit v1.2.3 From b993b0a308b2e8182beef965ff57e9a7974f5e17 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 25 Jun 2019 20:41:42 +0000 Subject: doc: indicate that netplan is default in Ubuntu now --- doc/rtd/topics/network-config-format-v2.rst | 2 +- doc/rtd/topics/network-config.rst | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) (limited to 'doc') diff --git a/doc/rtd/topics/network-config-format-v2.rst b/doc/rtd/topics/network-config-format-v2.rst index ea370ef5..50f5fa61 100644 --- a/doc/rtd/topics/network-config-format-v2.rst +++ b/doc/rtd/topics/network-config-format-v2.rst @@ -14,7 +14,7 @@ it must include ``version: 2`` and one or more of possible device Cloud-init will read this format from system config. For example the following could be present in -``/etc/cloud/cloud.cfg.d/custom-networking.cfg``: +``/etc/cloud/cloud.cfg.d/custom-networking.cfg``:: network: version: 2 diff --git a/doc/rtd/topics/network-config.rst b/doc/rtd/topics/network-config.rst index 1e994551..51ced4d1 100644 --- a/doc/rtd/topics/network-config.rst +++ b/doc/rtd/topics/network-config.rst @@ -163,10 +163,11 @@ found in Ubuntu and Debian. - **Netplan** -Since Ubuntu 16.10, codename Yakkety, the ``netplan`` project has been an -optional network configuration tool which consumes :ref:`network_config_v2` -input and renders network configuration for supported backends such as -``systemd-networkd`` and ``NetworkManager``. +Introduced in Ubuntu 16.10 (Yakkety Yak), `netplan `_ has +been the default network configuration tool in Ubuntu since 17.10 (Artful +Aardvark). netplan consumes :ref:`network_config_v2` input and renders +network configuration for supported backends such as ``systemd-networkd`` and +``NetworkManager``. - **Sysconfig** -- cgit v1.2.3 From a24550aee4c7282cd3624bf63f9501444e517678 Mon Sep 17 00:00:00 2001 From: Sam Gilson Date: Mon, 15 Jul 2019 21:50:33 +0000 Subject: Cloud-init analyze module: Added ability to analyze boot events. This branch introduces a new command line feature for cloud-init. Currently, the cloud-init module has the capability to analyze events in cloud-init.log in three ways: 'show', 'blame', 'dump'. These changes add a fourth capability, called 'boot'. Running the command 'cloud-init analyze boot' will provide the user three timestamps. 1) Timestamp for when the kernel starts initializing. 2) Timestamp for when the kernel finishes its initialization. 3) Timestamp for when systemd activates cloud-init. This feature enables cloud-init users to analyze different boot phases. This would aid in debugging performance issues related to cloud-init startup or tracking regression. --- cloudinit/analyze/__main__.py | 88 ++++++++++++++- cloudinit/analyze/show.py | 202 +++++++++++++++++++++++++++++++++-- cloudinit/analyze/tests/test_boot.py | 170 +++++++++++++++++++++++++++++ doc/rtd/topics/analyze.rst | 84 +++++++++++++++ doc/rtd/topics/capabilities.rst | 1 + doc/rtd/topics/debugging.rst | 13 +++ 6 files changed, 546 insertions(+), 12 deletions(-) create mode 100644 cloudinit/analyze/tests/test_boot.py create mode 100644 doc/rtd/topics/analyze.rst (limited to 'doc') diff --git a/cloudinit/analyze/__main__.py b/cloudinit/analyze/__main__.py index f8613656..99e5c203 100644 --- a/cloudinit/analyze/__main__.py +++ b/cloudinit/analyze/__main__.py @@ -7,7 +7,7 @@ import re import sys from cloudinit.util import json_dumps - +from datetime import datetime from . import dump from . import show @@ -52,9 +52,93 @@ def get_parser(parser=None): dest='outfile', default='-', help='specify where to write output. ') parser_dump.set_defaults(action=('dump', analyze_dump)) + parser_boot = subparsers.add_parser( + 'boot', help='Print list of boot times for kernel and cloud-init') + parser_boot.add_argument('-i', '--infile', action='store', + dest='infile', default='/var/log/cloud-init.log', + help='specify where to read input. ') + parser_boot.add_argument('-o', '--outfile', action='store', + dest='outfile', default='-', + help='specify where to write output.') + parser_boot.set_defaults(action=('boot', analyze_boot)) return parser +def analyze_boot(name, args): + """Report a list of how long different boot operations took. + + For Example: + -- Most Recent Boot Record -- + Kernel Started at: