diff options
32 files changed, 908 insertions, 50 deletions
@@ -20,6 +20,8 @@ all accounts would be locked unless 'system' was given (LP: #1096423). - Allow 'sr0' (or sr[0-9]) to be specified without /dev/ as a source for mounts. [Vlastimil Holer] + - allow config-drive-data to come from a CD device by more correctly + filtering out partitions. (LP: #1100545) 0.7.1: - sysvinit: fix missing dependency in cloud-init job for RHEL 5.6 - config-drive: map hostname to local-hostname (LP: #1061964) diff --git a/cloudinit/config/__init__.py b/cloudinit/config/__init__.py index 69a8cc68..d57453be 100644 --- a/cloudinit/config/__init__.py +++ b/cloudinit/config/__init__.py @@ -52,5 +52,7 @@ def fixup_module(mod, def_freq=PER_INSTANCE): if freq and freq not in FREQUENCIES: LOG.warn("Module %s has an unknown frequency %s", mod, freq) if not hasattr(mod, 'distros'): - setattr(mod, 'distros', None) + setattr(mod, 'distros', []) + if not hasattr(mod, 'osfamilies'): + setattr(mod, 'osfamilies', []) return mod diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index ddea8417..0db4aac7 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -35,6 +35,11 @@ from cloudinit import util from cloudinit.distros.parsers import hosts +OSFAMILIES = { + 'debian': ['debian', 'ubuntu'], + 'redhat': ['fedora', 'rhel'] +} + LOG = logging.getLogger(__name__) @@ -143,6 +148,16 @@ class Distro(object): def _select_hostname(self, hostname, fqdn): raise NotImplementedError() + @staticmethod + def expand_osfamily(family_list): + distros = [] + for family in family_list: + if not family in OSFAMILIES: + raise ValueError("No distibutions found for osfamily %s" + % (family)) + distros.extend(OSFAMILIES[family]) + return distros + def update_hostname(self, hostname, fqdn, prev_hostname_fn): applying_hostname = hostname diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 7422f4f0..49b73477 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -48,6 +48,7 @@ class Distro(distros.Distro): # calls from repeatly happening (when they # should only happen say once per instance...) self._runner = helpers.Runners(paths) + self.osfamily = 'debian' def apply_locale(self, locale, out_fn=None): if not out_fn: diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py index bc0877d5..e65be8d7 100644 --- a/cloudinit/distros/rhel.py +++ b/cloudinit/distros/rhel.py @@ -60,6 +60,7 @@ class Distro(distros.Distro): # calls from repeatly happening (when they # should only happen say once per instance...) self._runner = helpers.Runners(paths) + self.osfamily = 'redhat' def install_packages(self, pkglist): self.package_command('install', pkglist) diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index c7826851..ec016a1d 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -270,7 +270,7 @@ def find_candidate_devs(): combined = (by_label + [d for d in by_fstype if d not in by_label]) # We are looking for block device (sda, not sda1), ignore partitions - combined = [d for d in combined if d[-1] not in "0123456789"] + combined = [d for d in combined if not util.is_partition(d)] return combined diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 8d3213b4..d7d1dea0 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -529,11 +529,16 @@ class Modules(object): freq = mod.frequency if not freq in FREQUENCIES: freq = PER_INSTANCE - worked_distros = mod.distros + + worked_distros = set(mod.distros) + worked_distros.update( + distros.Distro.expand_osfamily(mod.osfamilies)) + if (worked_distros and d_name not in worked_distros): LOG.warn(("Module %s is verified on %s distros" " but not on %s distro. It may or may not work" - " correctly."), name, worked_distros, d_name) + " correctly."), name, list(worked_distros), + d_name) # Use the configs logger and not our own # TODO(harlowja): possibly check the module # for having a LOG attr and just give it back diff --git a/cloudinit/util.py b/cloudinit/util.py index ab918433..c0ea8d91 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1553,3 +1553,10 @@ def keyval_str_to_dict(kvstring): val = True ret[key] = val return ret + + +def is_partition(device): + if device.startswith("/dev/"): + device = device[5:] + + return os.path.isfile("/sys/class/block/%s/partition" % device) diff --git a/doc/examples/cloud-config-add-apt-repos.txt b/doc/examples/cloud-config-add-apt-repos.txt new file mode 100644 index 00000000..be9d5472 --- /dev/null +++ b/doc/examples/cloud-config-add-apt-repos.txt @@ -0,0 +1,34 @@ +#cloud-config + +# Add apt repositories +# +# Default: auto select based on cloud metadata +# in ec2, the default is <region>.archive.ubuntu.com +# apt_mirror: +# use the provided mirror +# apt_mirror_search: +# search the list for the first mirror. +# this is currently very limited, only verifying that +# the mirror is dns resolvable or an IP address +# +# if neither apt_mirror nor apt_mirror search is set (the default) +# then use the mirror provided by the DataSource found. +# In EC2, that means using <region>.ec2.archive.ubuntu.com +# +# if no mirror is provided by the DataSource, and 'apt_mirror_search_dns' is +# true, then search for dns names '<distro>-mirror' in each of +# - fqdn of this host per cloud metadata +# - localdomain +# - no domain (which would search domains listed in /etc/resolv.conf) +# If there is a dns entry for <distro>-mirror, then it is assumed that there +# is a distro mirror at http://<distro>-mirror.<domain>/<distro> +# +# That gives the cloud provider the opportunity to set mirrors of a distro +# up and expose them only by creating dns entries. +# +# if none of that is found, then the default distro mirror is used +apt_mirror: http://us.archive.ubuntu.com/ubuntu/ +apt_mirror_search: + - http://local-mirror.mydomain + - http://archive.ubuntu.com +apt_mirror_search_dns: False diff --git a/doc/examples/cloud-config-boot-cmds.txt b/doc/examples/cloud-config-boot-cmds.txt new file mode 100644 index 00000000..b281d327 --- /dev/null +++ b/doc/examples/cloud-config-boot-cmds.txt @@ -0,0 +1,15 @@ +#cloud-config + +# boot commands +# default: none +# this is very similar to runcmd, but commands run very early +# in the boot process, only slightly after a 'boothook' would run. +# bootcmd should really only be used for things that could not be +# done later in the boot process. bootcmd is very much like +# boothook, but possibly with more friendly. +# * bootcmd will run on every boot +# * the INSTANCE_ID variable will be set to the current instance id. +# * you can use 'cloud-init-boot-per' command to help only run once +bootcmd: + - echo 192.168.1.130 us.archive.ubuntu.com > /etc/hosts + - [ cloud-init-per, once, mymkfs, mkfs, /dev/vdb ] diff --git a/doc/examples/cloud-config-final-message.txt b/doc/examples/cloud-config-final-message.txt new file mode 100644 index 00000000..0ce31467 --- /dev/null +++ b/doc/examples/cloud-config-final-message.txt @@ -0,0 +1,7 @@ +#cloud-config + +# final_message +# default: cloud-init boot finished at $TIMESTAMP. Up $UPTIME seconds +# this message is written by cloud-final when the system is finished +# its first boot +final_message: "The system is finally up, after $UPTIME seconds" diff --git a/doc/examples/cloud-config-install-packages.txt b/doc/examples/cloud-config-install-packages.txt new file mode 100644 index 00000000..4984818f --- /dev/null +++ b/doc/examples/cloud-config-install-packages.txt @@ -0,0 +1,11 @@ +#cloud-config + +# Install additional packages on first boot +# +# Default: none +# +# if packages are specified, this apt_update will be set to true +# +packages: + - pwgen + - pastebinit diff --git a/doc/examples/cloud-config-mount-points.txt b/doc/examples/cloud-config-mount-points.txt new file mode 100644 index 00000000..416006db --- /dev/null +++ b/doc/examples/cloud-config-mount-points.txt @@ -0,0 +1,39 @@ +#cloud-config + +# set up mount points +# 'mounts' contains a list of lists +# the inner list are entries for an /etc/fstab line +# ie : [ fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno ] +# +# default: +# mounts: +# - [ ephemeral0, /mnt ] +# - [ swap, none, swap, sw, 0, 0 ] +# +# in order to remove a previously listed mount (ie, one from defaults) +# list only the fs_spec. For example, to override the default, of +# mounting swap: +# - [ swap ] +# or +# - [ swap, null ] +# +# - if a device does not exist at the time, an entry will still be +# written to /etc/fstab. +# - '/dev' can be ommitted for device names that begin with: xvd, sd, hd, vd +# - if an entry does not have all 6 fields, they will be filled in +# with values from 'mount_default_fields' below. +# +# Note, that you should set 'nobootwait' (see man fstab) for volumes that may +# not be attached at instance boot (or reboot) +# +mounts: + - [ ephemeral0, /mnt, auto, "defaults,noexec" ] + - [ sdc, /opt/data ] + - [ xvdh, /opt/data, "auto", "defaults,nobootwait", "0", "0" ] + - [ dd, /dev/zero ] + +# mount_default_fields +# These values are used to fill in any entries in 'mounts' that are not +# complete. This must be an array, and must have 7 fields. +mount_default_fields: [ None, None, "auto", "defaults,nobootwait", "0", "2" ] + diff --git a/doc/examples/cloud-config-phone-home.txt b/doc/examples/cloud-config-phone-home.txt new file mode 100644 index 00000000..7f2b69f7 --- /dev/null +++ b/doc/examples/cloud-config-phone-home.txt @@ -0,0 +1,14 @@ +#cloud-config + +# phone_home: if this dictionary is present, then the phone_home +# cloud-config module will post specified data back to the given +# url +# default: none +# phone_home: +# url: http://my.foo.bar/$INSTANCE/ +# post: all +# tries: 10 +# +phone_home: + url: http://my.example.com/$INSTANCE_ID/ + post: [ pub_key_dsa, pub_key_rsa, pub_key_ecdsa, instance_id ] diff --git a/doc/examples/cloud-config-power-state.txt b/doc/examples/cloud-config-power-state.txt new file mode 100644 index 00000000..59f062d0 --- /dev/null +++ b/doc/examples/cloud-config-power-state.txt @@ -0,0 +1,22 @@ +#cloud-config + +## poweroff or reboot system after finished +# default: none +# +# power_state can be used to make the system shutdown, reboot or +# halt after boot is finished. This same thing can be acheived by +# user-data scripts or by runcmd by simply invoking 'shutdown'. +# +# Doing it this way ensures that cloud-init is entirely finished with +# modules that would be executed, and avoids any error/log messages +# that may go to the console as a result of system services like +# syslog being taken down while cloud-init is running. +# +# delay: form accepted by shutdown. default is 'now'. other format +# accepted is +m (m in minutes) +# mode: required. must be one of 'poweroff', 'halt', 'reboot' +# message: provided as the message argument to 'shutdown'. default is none. +power_state: + delay: 30 + mode: poweroff + message: Bye Bye diff --git a/doc/examples/cloud-config-run-cmds.txt b/doc/examples/cloud-config-run-cmds.txt new file mode 100644 index 00000000..61b3bd63 --- /dev/null +++ b/doc/examples/cloud-config-run-cmds.txt @@ -0,0 +1,21 @@ +#cloud-config + +# run commands +# default: none +# runcmd contains a list of either lists or a string +# each item will be executed in order at rc.local like level with +# output to the console +# - if the item is a list, the items will be properly executed as if +# passed to execve(3) (with the first arg as the command). +# - if the item is a string, it will be simply written to the file and +# will be interpreted by 'sh' +# +# Note, that the list has to be proper yaml, so you have to escape +# any characters yaml would eat (':' can be problematic) +runcmd: + - [ ls, -l, / ] + - [ sh, -xc, "echo $(date) ': hello world!'" ] + - [ sh, -c, echo "=========hello world'=========" ] + - ls -l /root + - [ wget, "http://slashdot.org", -O, /tmp/index.html ] + diff --git a/doc/examples/cloud-config-ssh-keys.txt b/doc/examples/cloud-config-ssh-keys.txt new file mode 100644 index 00000000..235a114f --- /dev/null +++ b/doc/examples/cloud-config-ssh-keys.txt @@ -0,0 +1,46 @@ +#cloud-config + +# add each entry to ~/.ssh/authorized_keys for the configured user or the +# first user defined in the user definition directive. +ssh_authorized_keys: + - ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAGEA3FSyQwBI6Z+nCSjUUk8EEAnnkhXlukKoUPND/RRClWz2s5TCzIkd3Ou5+Cyz71X0XmazM3l5WgeErvtIwQMyT1KjNoMhoJMrJnWqQPOt5Q8zWd9qG7PBl9+eiH5qV7NZ mykey@host + - ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA3I7VUf2l5gSn5uavROsc5HRDpZdQueUq5ozemNSj8T7enqKHOEaFoU2VoPgGEWC9RyzSQVeyD6s7APMcE82EtmW4skVEgEGSbDc1pvxzxtchBj78hJP6Cf5TCMFSXw+Fz5rF1dR23QDbN1mkHs7adr8GW4kSWqU7Q7NDwfIrJJtO7Hi42GyXtvEONHbiRPOe8stqUly7MvUoN+5kfjBM8Qqpfl2+FNhTYWpMfYdPUnE7u536WqzFmsaqJctz3gBxH9Ex7dFtrxR4qiqEr9Qtlu3xGn7Bw07/+i1D+ey3ONkZLN+LQ714cgj8fRS4Hj29SCmXp5Kt5/82cD/VN3NtHw== smoser@brickies + +# Send pre-generated ssh private keys to the server +# If these are present, they will be written to /etc/ssh and +# new random keys will not be generated +# in addition to 'rsa' and 'dsa' as shown below, 'ecdsa' is also supported +ssh_keys: + rsa_private: | + -----BEGIN RSA PRIVATE KEY----- + MIIBxwIBAAJhAKD0YSHy73nUgysO13XsJmd4fHiFyQ+00R7VVu2iV9Qcon2LZS/x + 1cydPZ4pQpfjEha6WxZ6o8ci/Ea/w0n+0HGPwaxlEG2Z9inNtj3pgFrYcRztfECb + 1j6HCibZbAzYtwIBIwJgO8h72WjcmvcpZ8OvHSvTwAguO2TkR6mPgHsgSaKy6GJo + PUJnaZRWuba/HX0KGyhz19nPzLpzG5f0fYahlMJAyc13FV7K6kMBPXTRR6FxgHEg + L0MPC7cdqAwOVNcPY6A7AjEA1bNaIjOzFN2sfZX0j7OMhQuc4zP7r80zaGc5oy6W + p58hRAncFKEvnEq2CeL3vtuZAjEAwNBHpbNsBYTRPCHM7rZuG/iBtwp8Rxhc9I5w + ixvzMgi+HpGLWzUIBS+P/XhekIjPAjA285rVmEP+DR255Ls65QbgYhJmTzIXQ2T9 + luLvcmFBC6l35Uc4gTgg4ALsmXLn71MCMGMpSWspEvuGInayTCL+vEjmNBT+FAdO + W7D4zCpI43jRS9U06JVOeSc9CDk2lwiA3wIwCTB/6uc8Cq85D9YqpM10FuHjKpnP + REPPOyrAspdeOAV+6VKRavstea7+2DZmSUgE + -----END RSA PRIVATE KEY----- + + rsa_public: ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAGEAoPRhIfLvedSDKw7XdewmZ3h8eIXJD7TRHtVW7aJX1ByifYtlL/HVzJ09nilCl+MSFrpbFnqjxyL8Rr/DSf7QcY/BrGUQbZn2Kc22PemAWthxHO18QJvWPocKJtlsDNi3 smoser@localhost + + dsa_private: | + -----BEGIN DSA PRIVATE KEY----- + MIIBuwIBAAKBgQDP2HLu7pTExL89USyM0264RCyWX/CMLmukxX0Jdbm29ax8FBJT + pLrO8TIXVY5rPAJm1dTHnpuyJhOvU9G7M8tPUABtzSJh4GVSHlwaCfycwcpLv9TX + DgWIpSj+6EiHCyaRlB1/CBp9RiaB+10QcFbm+lapuET+/Au6vSDp9IRtlQIVAIMR + 8KucvUYbOEI+yv+5LW9u3z/BAoGBAI0q6JP+JvJmwZFaeCMMVxXUbqiSko/P1lsa + LNNBHZ5/8MOUIm8rB2FC6ziidfueJpqTMqeQmSAlEBCwnwreUnGfRrKoJpyPNENY + d15MG6N5J+z81sEcHFeprryZ+D3Ge9VjPq3Tf3NhKKwCDQ0240aPezbnjPeFm4mH + bYxxcZ9GAoGAXmLIFSQgiAPu459rCKxT46tHJtM0QfnNiEnQLbFluefZ/yiI4DI3 + 8UzTCOXLhUA7ybmZha+D/csj15Y9/BNFuO7unzVhikCQV9DTeXX46pG4s1o23JKC + /QaYWNMZ7kTRv+wWow9MhGiVdML4ZN4XnifuO5krqAybngIy66PMEoQCFEIsKKWv + 99iziAH0KBMVbxy03Trz + -----END DSA PRIVATE KEY----- + + dsa_public: ssh-dss AAAAB3NzaC1kc3MAAACBAM/Ycu7ulMTEvz1RLIzTbrhELJZf8Iwua6TFfQl1ubb1rHwUElOkus7xMhdVjms8AmbV1Meem7ImE69T0bszy09QAG3NImHgZVIeXBoJ/JzByku/1NcOBYilKP7oSIcLJpGUHX8IGn1GJoH7XRBwVub6Vqm4RP78C7q9IOn0hG2VAAAAFQCDEfCrnL1GGzhCPsr/uS1vbt8/wQAAAIEAjSrok/4m8mbBkVp4IwxXFdRuqJKSj8/WWxos00Ednn/ww5QibysHYULrOKJ1+54mmpMyp5CZICUQELCfCt5ScZ9GsqgmnI80Q1h3Xkwbo3kn7PzWwRwcV6muvJn4PcZ71WM+rdN/c2EorAINDTbjRo97NueM94WbiYdtjHFxn0YAAACAXmLIFSQgiAPu459rCKxT46tHJtM0QfnNiEnQLbFluefZ/yiI4DI38UzTCOXLhUA7ybmZha+D/csj15Y9/BNFuO7unzVhikCQV9DTeXX46pG4s1o23JKC/QaYWNMZ7kTRv+wWow9MhGiVdML4ZN4XnifuO5krqAybngIy66PMEoQ= smoser@localhost + + diff --git a/doc/examples/cloud-config-update-apt.txt b/doc/examples/cloud-config-update-apt.txt new file mode 100644 index 00000000..a83ce3f7 --- /dev/null +++ b/doc/examples/cloud-config-update-apt.txt @@ -0,0 +1,7 @@ +#cloud-config +# Update apt database on first boot +# (ie run apt-get update) +# +# Default: true +# Aliases: apt_update +package_update: false diff --git a/doc/examples/cloud-config-update-packages.txt b/doc/examples/cloud-config-update-packages.txt new file mode 100644 index 00000000..56b72c63 --- /dev/null +++ b/doc/examples/cloud-config-update-packages.txt @@ -0,0 +1,8 @@ +#cloud-config + +# Upgrade the instance on first boot +# (ie run apt-get upgrade) +# +# Default: false +# Aliases: apt_upgrade +package_upgrade: true diff --git a/doc/rtd/conf.py b/doc/rtd/conf.py new file mode 100644 index 00000000..56ec912f --- /dev/null +++ b/doc/rtd/conf.py @@ -0,0 +1,73 @@ +import sys, os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.insert(0, os.path.abspath('../../')) +sys.path.insert(0, os.path.abspath('../')) +sys.path.insert(0, os.path.abspath('./')) +sys.path.insert(0, os.path.abspath('.')) + +from cloudinit import version + +# Supress warnings for docs that aren't used yet +#unused_docs = [ +#] + +# General information about the project. +project = 'Cloud-Init' + +# -- General configuration ----------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = [ + 'sphinx.ext.intersphinx', +] + +intersphinx_mapping = { + 'sphinx': ('http://sphinx.pocoo.org', None) +} + +# The suffix of source filenames. +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +version = version.version_string() + +# Set the default Pygments syntax +highlight_language = 'python' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = [] + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +show_authors = False + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +html_theme_options = { + "bodyfont": "Arial, sans-serif", + "headfont": "Arial, sans-serif" +} + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +html_logo = 'logo.png' diff --git a/doc/rtd/index.rst b/doc/rtd/index.rst new file mode 100644 index 00000000..f878dbd4 --- /dev/null +++ b/doc/rtd/index.rst @@ -0,0 +1,29 @@ +.. _index: + +===================== +Documentation +===================== + +.. rubric:: Everything about cloud-init, a set of **python** scripts and utilities to make your cloud images be all they can be! + +Summary +----------------- + +`Cloud-init`_ is the *defacto* multi-distribution package that handles early initialization of a cloud instance. + + +---- + +.. toctree:: + :maxdepth: 2 + + topics/capabilities + topics/availability + topics/format + topics/dir_layout + topics/examples + topics/modules + topics/moreinfo + + +.. _Cloud-init: https://launchpad.net/cloud-init diff --git a/doc/rtd/logo.png b/doc/rtd/logo.png Binary files differnew file mode 100644 index 00000000..ed1b0a91 --- /dev/null +++ b/doc/rtd/logo.png diff --git a/doc/rtd/topics/availability.rst b/doc/rtd/topics/availability.rst new file mode 100644 index 00000000..2d58f808 --- /dev/null +++ b/doc/rtd/topics/availability.rst @@ -0,0 +1,20 @@ +============ +Availability +============ + +It is currently installed in the `Ubuntu Cloud Images`_ and also in the official `Ubuntu`_ images available on EC2. + +Versions for other systems can be (or have been) created for the following distributions: + +- Ubuntu +- Fedora +- Debian +- RHEL +- CentOS +- *and more...* + +So ask your distribution provider where you can obtain an image with it built-in if one is not already available ☺ + + +.. _Ubuntu Cloud Images: http://cloud-images.ubuntu.com/ +.. _Ubuntu: http://www.ubuntu.com/ diff --git a/doc/rtd/topics/capabilities.rst b/doc/rtd/topics/capabilities.rst new file mode 100644 index 00000000..63b34270 --- /dev/null +++ b/doc/rtd/topics/capabilities.rst @@ -0,0 +1,24 @@ +===================== +Capabilities +===================== + +- Setting a default locale +- Setting a instance hostname +- Generating instance ssh private keys +- Adding ssh keys to a users ``.ssh/authorized_keys`` so they can log in +- Setting up ephemeral mount points + +User configurability +-------------------- + +`Cloud-init`_ 's behavior can be configured via user-data. + + User-data can be given by the user at instance launch time. + +This is done via the ``--user-data`` or ``--user-data-file`` argument to ec2-run-instances for example. + +* Check your local clients documentation for how to provide a `user-data` string + or `user-data` file for usage by cloud-init on instance creation. + + +.. _Cloud-init: https://launchpad.net/cloud-init diff --git a/doc/rtd/topics/dir_layout.rst b/doc/rtd/topics/dir_layout.rst new file mode 100644 index 00000000..f072c585 --- /dev/null +++ b/doc/rtd/topics/dir_layout.rst @@ -0,0 +1,81 @@ +========= +Directory layout +========= + +Cloudinits's directory structure is somewhat different from a regular application:: + + /var/lib/cloud/ + - data/ + - instance-id + - previous-instance-id + - datasource + - previous-datasource + - previous-hostname + - handlers/ + - instance + - instances/ + i-00000XYZ/ + - boot-finished + - cloud-config.txt + - datasource + - handlers/ + - obj.pkl + - scripts/ + - sem/ + - user-data.txt + - user-data.txt.i + - scripts/ + - per-boot/ + - per-instance/ + - per-once/ + - seed/ + - sem/ + +``/var/lib/cloud`` + + The main directory containing the cloud-init specific subdirectories. + It is typically located at ``/var/lib`` but there are certain configuration + scenarios where this can be altered. + + TBD, describe this overriding more. + +``data/`` + + Contains information releated to instance ids, datasources and hostnames of the previous + and current instance if they are different. These can be examined as needed to + determine any information releated to a previous boot (if applicable). + +``handlers/`` + + Custom ``part-handlers`` code is written out here. Files that end up here are written + out with in the scheme of ``part-handler-XYZ`` where ``XYZ`` is the handler number (the + first handler found starts at 0). + + +``instance`` + + A symlink to the current ``instances/`` subdirectory that points to the currently + active instance (which is active is dependent on the datasource loaded). + +``instances/`` + + All instances that were created using this image end up with instance identifer + subdirectories (and corresponding data for each instance). The currently active + instance will be symlinked the the ``instance`` symlink file defined previously. + +``scripts/`` + + Scripts that are downloaded/created by the corresponding ``part-handler`` will end up + in one of these subdirectories. + +``seed/`` + + TBD + +``sem/`` + + Cloud-init has a concept of a module sempahore, which basically consists + of the module name and its frequency. These files are used to ensure a module + is only ran `per-once`, `per-instance`, `per-always`. This folder contains + sempaphore `files` which are only supposed to run `per-once` (not tied to the instance id). + diff --git a/doc/rtd/topics/examples.rst b/doc/rtd/topics/examples.rst new file mode 100644 index 00000000..9bbc33cc --- /dev/null +++ b/doc/rtd/topics/examples.rst @@ -0,0 +1,121 @@ +.. _yaml_examples: + +========= +Cloud config examples +========= + +Including users and groups +--------------------------- + +.. literalinclude:: ../../examples/cloud-config-user-groups.txt + :language: yaml + :linenos: + + +Writing out arbitrary files +--------------------------- + +.. literalinclude:: ../../examples/cloud-config-write-files.txt + :language: yaml + :linenos: + + +Adding a yum repository +--------------------------- + +.. literalinclude:: ../../examples/cloud-config-yum-repo.txt + :language: yaml + :linenos: + +Configure an instance's trusted CA certificates +------------------------------------------------------ + +.. literalinclude:: ../../examples/cloud-config-ca-certs.txt + :language: yaml + :linenos: + +Install and run `chef`_ recipes +------------------------------------------------------ + +.. literalinclude:: ../../examples/cloud-config-chef.txt + :language: yaml + :linenos: + +Setup and run `puppet`_ +------------------------------------------------------ + +.. literalinclude:: ../../examples/cloud-config-puppet.txt + :language: yaml + :linenos: + +Add apt repositories +--------------------------- + +.. literalinclude:: ../../examples/cloud-config-add-apt-repos.txt + :language: yaml + :linenos: + +Run commands on first boot +--------------------------- + +.. literalinclude:: ../../examples/cloud-config-boot-cmds.txt + :language: yaml + :linenos: + +.. literalinclude:: ../../examples/cloud-config-run-cmds.txt + :language: yaml + :linenos: + + +Alter the completion message +--------------------------- + +.. literalinclude:: ../../examples/cloud-config-final-message.txt + :language: yaml + :linenos: + +Install arbitrary packages +--------------------------- + +.. literalinclude:: ../../examples/cloud-config-install-packages.txt + :language: yaml + :linenos: + +Run apt or yum upgrade +--------------------------- + +.. literalinclude:: ../../examples/cloud-config-update-packages.txt + :language: yaml + :linenos: + +Adjust mount points mounted +--------------------------- + +.. literalinclude:: ../../examples/cloud-config-mount-points.txt + :language: yaml + :linenos: + +Call a url when finished +--------------------------- + +.. literalinclude:: ../../examples/cloud-config-phone-home.txt + :language: yaml + :linenos: + +Reboot/poweroff when finished +--------------------------- + +.. literalinclude:: ../../examples/cloud-config-power-state.txt + :language: yaml + :linenos: + +Configure instances ssh-keys +--------------------------- + +.. literalinclude:: ../../examples/cloud-config-ssh-keys.txt + :language: yaml + :linenos: + + +.. _chef: http://www.opscode.com/chef/ +.. _puppet: http://puppetlabs.com/ diff --git a/doc/rtd/topics/format.rst b/doc/rtd/topics/format.rst new file mode 100644 index 00000000..eba9533f --- /dev/null +++ b/doc/rtd/topics/format.rst @@ -0,0 +1,159 @@ +========= +Formats +========= + +User data that will be acted upon by cloud-init must be in one of the following types. + +Gzip Compressed Content +------------------------ + +Content found to be gzip compressed will be uncompressed. +The uncompressed data will then be used as if it were not compressed. +This is typically is useful because user-data is limited to ~16384 [#]_ bytes. + +Mime Multi Part Archive +------------------------ + +This list of rules is applied to each part of this multi-part file. +Using a mime-multi part file, the user can specify more than one type of data. + +For example, both a user data script and a cloud-config type could be specified. + +Supported content-types: + +- text/x-include-once-url +- text/x-include-url +- text/cloud-config-archive +- text/upstart-job +- text/cloud-config +- text/part-handler +- text/x-shellscript +- text/cloud-boothook + +Helper script to generate mime messages +~~~~~~~~~~~~~~~~ + +.. code-block:: python + + #!/usr/bin/python + + import sys + + from email.mime.multipart import MIMEMultipart + from email.mime.text import MIMEText + + if len(sys.argv) == 1: + print("%s input-file:type ..." % (sys.argv[0])) + sys.exit(1) + + combined_message = MIMEMultipart() + for i in sys.argv[1:]: + (filename, format_type) = i.split(":", 1) + with open(filename) as fh: + contents = fh.read() + sub_message = MIMEText(contents, format_type, sys.getdefaultencoding()) + sub_message.add_header('Content-Disposition', 'attachment; filename="%s"' % (filename)) + combined_message.attach(sub_message) + + print(combined_message) + + +User-Data Script +------------------------ + +Typically used by those who just want to execute a shell script. + +Begins with: ``#!`` or ``Content-Type: text/x-shellscript`` when using a MIME archive. + +Example +~~~~~~~ + +:: + + $ cat myscript.sh + + #!/bin/sh + echo "Hello World. The time is now $(date -R)!" | tee /root/output.txt + + $ euca-run-instances --key mykey --user-data-file myscript.sh ami-a07d95c9 + +Include File +------------ + +This content is a ``include`` file. + +The file contains a list of urls, one per line. +Each of the URLs will be read, and their content will be passed through this same set of rules. +Ie, the content read from the URL can be gzipped, mime-multi-part, or plain text. + +Begins with: ``#include`` or ``Content-Type: text/x-include-url`` when using a MIME archive. + +Cloud Config Data +----------------- + +Cloud-config is the simplest way to accomplish some things +via user-data. Using cloud-config syntax, the user can specify certain things in a human friendly format. + +These things include: + +- apt upgrade should be run on first boot +- a different apt mirror should be used +- additional apt sources should be added +- certain ssh keys should be imported +- *and many more...* + +**Note:** The file must be valid yaml syntax. + +See the :ref:`yaml_examples` section for a commented set of examples of supported cloud config formats. + +Begins with: ``#cloud-config`` or ``Content-Type: text/cloud-config`` when using a MIME archive. + +Upstart Job +----------- + +Content is placed into a file in ``/etc/init``, and will be consumed by upstart as any other upstart job. + +Begins with: ``#upstart-job`` or ``Content-Type: text/upstart-job`` when using a MIME archive. + +Cloud Boothook +-------------- + +This content is ``boothook`` data. It is stored in a file under ``/var/lib/cloud`` and then executed immediately. +This is the earliest ``hook`` available. Note, that there is no mechanism provided for running only once. The boothook must take care of this itself. +It is provided with the instance id in the environment variable ``INSTANCE_I``. This could be made use of to provide a 'once-per-instance' type of functionality. + +Begins with: ``#cloud-boothook`` or ``Content-Type: text/cloud-boothook`` when using a MIME archive. + +Part Handler +------------ + +This is a ``part-handler``. It will be written to a file in ``/var/lib/cloud/data`` based on its filename (which is generated). +This must be python code that contains a ``list_types`` method and a ``handle_type`` method. +Once the section is read the ``list_types`` method will be called. It must return a list of mime-types that this part-handler handles. + +The ``handle_type`` method must be like: + +.. code-block:: python + + def handle_part(data, ctype, filename, payload): + # data = the cloudinit object + # ctype = "__begin__", "__end__", or the mime-type of the part that is being handled. + # filename = the filename of the part (or a generated filename if none is present in mime data) + # payload = the parts' content + +Cloud-init will then call the ``handle_type`` method once at begin, once per part received, and once at end. +The ``begin`` and ``end`` calls are to allow the part handler to do initialization or teardown. + +Begins with: ``#part-handler`` or ``Content-Type: text/part-handler`` when using a MIME archive. + +Example +~~~~~~~ + +.. literalinclude:: ../../examples/part-handler.txt + :language: python + :linenos: + +Also this `blog`_ post offers another example for more advanced usage. + +.. [#] See your cloud provider for applicable user-data size limitations... +.. _blog: http://foss-boss.blogspot.com/2011/01/advanced-cloud-init-custom-handlers.html diff --git a/doc/rtd/topics/modules.rst b/doc/rtd/topics/modules.rst new file mode 100644 index 00000000..d4dd55df --- /dev/null +++ b/doc/rtd/topics/modules.rst @@ -0,0 +1,3 @@ +========= +Modules +========= diff --git a/doc/rtd/topics/moreinfo.rst b/doc/rtd/topics/moreinfo.rst new file mode 100644 index 00000000..2e436c3c --- /dev/null +++ b/doc/rtd/topics/moreinfo.rst @@ -0,0 +1,12 @@ +========= +More information +========= + +Useful external references +---------------- + +- `The beauty of cloudinit`_ +- `Introduction to cloud-init`_ (video) + +.. _Introduction to cloud-init: http://www.youtube.com/watch?v=-zL3BdbKyGY +.. _The beauty of cloudinit: http://brandon.fuller.name/archives/2011/05/02/06.40.57/ diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 92540b0c..4258a29d 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -2,6 +2,9 @@ import os import sys import unittest +from contextlib import contextmanager + +from mocker import Mocker from mocker import MockerTestCase from cloudinit import helpers as ch @@ -31,6 +34,17 @@ else: pass +@contextmanager +def mocker(verify_calls=True): + m = Mocker() + try: + yield m + finally: + m.restore() + if verify_calls: + m.verify() + + # Makes the old path start # with new base instead of whatever # it previously had diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index aa5b98ed..930086db 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -11,6 +11,7 @@ from cloudinit import settings from cloudinit.sources import DataSourceConfigDrive as ds from cloudinit import util +from tests.unittests import helpers as unit_helpers PUBKEY = u'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460\n' EC2_META = { @@ -89,23 +90,22 @@ class TestConfigDriveDataSource(MockerTestCase): 'swap': '/dev/vda3', } for name, dev_name in name_tests.items(): - my_mock = mocker.Mocker() - find_mock = my_mock.replace(util.find_devs_with, - spec=False, passthrough=False) - provided_name = dev_name[len('/dev/'):] - provided_name = "s" + provided_name[1:] - find_mock(mocker.ARGS) - my_mock.result([provided_name]) - exists_mock = my_mock.replace(os.path.exists, - spec=False, passthrough=False) - exists_mock(mocker.ARGS) - my_mock.result(False) - exists_mock(mocker.ARGS) - my_mock.result(True) - my_mock.replay() - device = cfg_ds.device_name_to_device(name) - my_mock.restore() - self.assertEquals(dev_name, device) + with unit_helpers.mocker() as my_mock: + find_mock = my_mock.replace(util.find_devs_with, + spec=False, passthrough=False) + provided_name = dev_name[len('/dev/'):] + provided_name = "s" + provided_name[1:] + find_mock(mocker.ARGS) + my_mock.result([provided_name]) + exists_mock = my_mock.replace(os.path.exists, + spec=False, passthrough=False) + exists_mock(mocker.ARGS) + my_mock.result(False) + exists_mock(mocker.ARGS) + my_mock.result(True) + my_mock.replay() + device = cfg_ds.device_name_to_device(name) + self.assertEquals(dev_name, device) def test_dev_os_map(self): populate_dir(self.tmp, CFG_DRIVE_FILES_V2) @@ -122,19 +122,18 @@ class TestConfigDriveDataSource(MockerTestCase): 'swap': '/dev/vda3', } for name, dev_name in name_tests.items(): - my_mock = mocker.Mocker() - find_mock = my_mock.replace(util.find_devs_with, - spec=False, passthrough=False) - find_mock(mocker.ARGS) - my_mock.result([dev_name]) - exists_mock = my_mock.replace(os.path.exists, - spec=False, passthrough=False) - exists_mock(mocker.ARGS) - my_mock.result(True) - my_mock.replay() - device = cfg_ds.device_name_to_device(name) - my_mock.restore() - self.assertEquals(dev_name, device) + with unit_helpers.mocker() as my_mock: + find_mock = my_mock.replace(util.find_devs_with, + spec=False, passthrough=False) + find_mock(mocker.ARGS) + my_mock.result([dev_name]) + exists_mock = my_mock.replace(os.path.exists, + spec=False, passthrough=False) + exists_mock(mocker.ARGS) + my_mock.result(True) + my_mock.replay() + device = cfg_ds.device_name_to_device(name) + self.assertEquals(dev_name, device) def test_dev_ec2_remap(self): populate_dir(self.tmp, CFG_DRIVE_FILES_V2) @@ -156,17 +155,16 @@ class TestConfigDriveDataSource(MockerTestCase): 'root2k': None, } for name, dev_name in name_tests.items(): - my_mock = mocker.Mocker() - exists_mock = my_mock.replace(os.path.exists, - spec=False, passthrough=False) - exists_mock(mocker.ARGS) - my_mock.result(False) - exists_mock(mocker.ARGS) - my_mock.result(True) - my_mock.replay() - device = cfg_ds.device_name_to_device(name) - self.assertEquals(dev_name, device) - my_mock.restore() + with unit_helpers.mocker(verify_calls=False) as my_mock: + exists_mock = my_mock.replace(os.path.exists, + spec=False, passthrough=False) + exists_mock(mocker.ARGS) + my_mock.result(False) + exists_mock(mocker.ARGS) + my_mock.result(True) + my_mock.replay() + device = cfg_ds.device_name_to_device(name) + self.assertEquals(dev_name, device) def test_dev_ec2_map(self): populate_dir(self.tmp, CFG_DRIVE_FILES_V2) @@ -259,19 +257,25 @@ class TestConfigDriveDataSource(MockerTestCase): ds.read_config_drive_dir, my_d) def test_find_candidates(self): - devs_with_answers = { - "TYPE=vfat": [], - "TYPE=iso9660": ["/dev/vdb"], - "LABEL=config-2": ["/dev/vdb"], - } + devs_with_answers = {} def my_devs_with(criteria): return devs_with_answers[criteria] + def my_is_partition(dev): + return dev[-1] in "0123456789" and not dev.startswith("sr") + try: orig_find_devs_with = util.find_devs_with util.find_devs_with = my_devs_with + orig_is_partition = util.is_partition + util.is_partition = my_is_partition + + devs_with_answers = {"TYPE=vfat": [], + "TYPE=iso9660": ["/dev/vdb"], + "LABEL=config-2": ["/dev/vdb"], + } self.assertEqual(["/dev/vdb"], ds.find_candidate_devs()) # add a vfat item @@ -287,6 +291,7 @@ class TestConfigDriveDataSource(MockerTestCase): finally: util.find_devs_with = orig_find_devs_with + util.is_partition = orig_is_partition def test_pubkeys_v2(self): """Verify that public-keys work in config-drive-v2.""" diff --git a/tools/make-mime.py b/tools/make-mime.py new file mode 100755 index 00000000..72b29fb9 --- /dev/null +++ b/tools/make-mime.py @@ -0,0 +1,60 @@ +#!/usr/bin/python + +import argparse +import sys + +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText + +KNOWN_CONTENT_TYPES = [ + 'text/x-include-once-url', + 'text/x-include-url', + 'text/cloud-config-archive', + 'text/upstart-job', + 'text/cloud-config', + 'text/part-handler', + 'text/x-shellscript', + 'text/cloud-boothook', +] + + +def file_content_type(text): + try: + filename, content_type = text.split(":", 1) + return (open(filename, 'r'), filename, content_type.strip()) + except: + raise argparse.ArgumentError("Invalid value for %r" % (text)) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("-a", "--attach", + dest="files", + type=file_content_type, + action='append', + default=[], + required=True, + metavar="<file>:<content-type>", + help="attach the given file in the specified " + "content type") + args = parser.parse_args() + sub_messages = [] + for i, (fh, filename, format_type) in enumerate(args.files): + contents = fh.read() + sub_message = MIMEText(contents, format_type, sys.getdefaultencoding()) + sub_message.add_header('Content-Disposition', + 'attachment; filename="%s"' % (filename)) + content_type = sub_message.get_content_type().lower() + if content_type not in KNOWN_CONTENT_TYPES: + sys.stderr.write(("WARNING: content type %r for attachment %s " + "may be incorrect!\n") % (content_type, i + 1)) + sub_messages.append(sub_message) + combined_message = MIMEMultipart() + for msg in sub_messages: + combined_message.attach(msg) + print(combined_message) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) |
