From 41d46bfb85929c79dabcec3cf21c8d71401fd2b8 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 28 Sep 2016 13:20:55 -0700 Subject: cloud.cfg: move to a template. setup.py changes along the way. Here we move the config/cloud.cfg to be rendered as a template. That allows us to maintain deltas between distros in one place. Currently we use 'variant' variable to make decisions. A tools/render-cloudcfg is provided to render the file. There were changes to setup.py, MANIFEST.in to allow us to put all files into a virtual env installation and to render the cloud-config file in 'install' or 'bdist' targets. We have also included some config changes that were found in the redhat distro spec. * include some config changes from the redhat distro spec. The rendered cloud.cfg has some differences. Ubuntu: white space and comment changes only. Freebsd: - whitespace changes and comment changes - datasource_list definition moved to be closer to 'datasource'. - enable modules: migrator, write_files - move package-update-upgrade-install to final. The initial work was done by Josh Harlow. --- config/cloud.cfg | 117 ---------------------------- config/cloud.cfg-freebsd | 88 --------------------- config/cloud.cfg.tmpl | 194 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 194 insertions(+), 205 deletions(-) delete mode 100644 config/cloud.cfg delete mode 100644 config/cloud.cfg-freebsd create mode 100644 config/cloud.cfg.tmpl (limited to 'config') diff --git a/config/cloud.cfg b/config/cloud.cfg deleted file mode 100644 index 1b93e7f9..00000000 --- a/config/cloud.cfg +++ /dev/null @@ -1,117 +0,0 @@ -# The top level settings are used as module -# and system configuration. - -# A set of users which may be applied and/or used by various modules -# when a 'default' entry is found it will reference the 'default_user' -# from the distro configuration specified below -users: - - default - -# If this is set, 'root' will not be able to ssh in and they -# will get a message to login instead as the above $user (ubuntu) -disable_root: true - -# This will cause the set+update hostname module to not operate (if true) -preserve_hostname: false - -# Example datasource config -# datasource: -# Ec2: -# metadata_urls: [ 'blah.com' ] -# timeout: 5 # (defaults to 50 seconds) -# max_wait: 10 # (defaults to 120 seconds) - -# The modules that run in the 'init' stage -cloud_init_modules: - - migrator - - ubuntu-init-switch - - seed_random - - bootcmd - - write-files - - growpart - - resizefs - - disk_setup - - mounts - - set_hostname - - update_hostname - - update_etc_hosts - - ca-certs - - rsyslog - - users-groups - - ssh - -# The modules that run in the 'config' stage -cloud_config_modules: -# Emit the cloud config ready event -# this can be used by upstart jobs for 'start on cloud-config'. - - emit_upstart - - snap_config - - ssh-import-id - - locale - - set-passwords - - grub-dpkg - - apt-pipelining - - apt-configure - - ntp - - timezone - - disable-ec2-metadata - - runcmd - - byobu - -# The modules that run in the 'final' stage -cloud_final_modules: - - snappy - - package-update-upgrade-install - - fan - - landscape - - lxd - - puppet - - chef - - salt-minion - - mcollective - - rightscale_userdata - - scripts-vendor - - scripts-per-once - - scripts-per-boot - - scripts-per-instance - - scripts-user - - ssh-authkey-fingerprints - - keys-to-console - - phone-home - - final-message - - power-state-change - -# System and/or distro specific settings -# (not accessible to handlers/transforms) -system_info: - # This will affect which distro class gets used - distro: ubuntu - # Default user name + that default users groups (if added/used) - default_user: - name: ubuntu - lock_passwd: True - gecos: Ubuntu - groups: [adm, audio, cdrom, dialout, dip, floppy, lxd, netdev, plugdev, sudo, video] - sudo: ["ALL=(ALL) NOPASSWD:ALL"] - shell: /bin/bash - # Other config here will be given to the distro class and/or path classes - paths: - cloud_dir: /var/lib/cloud/ - templates_dir: /etc/cloud/templates/ - upstart_dir: /etc/init/ - package_mirrors: - - arches: [i386, amd64] - failsafe: - primary: http://archive.ubuntu.com/ubuntu - security: http://security.ubuntu.com/ubuntu - search: - primary: - - http://%(ec2_region)s.ec2.archive.ubuntu.com/ubuntu/ - - http://%(availability_zone)s.clouds.archive.ubuntu.com/ubuntu/ - - http://%(region)s.clouds.archive.ubuntu.com/ubuntu/ - security: [] - - arches: [armhf, armel, default] - failsafe: - primary: http://ports.ubuntu.com/ubuntu-ports - security: http://ports.ubuntu.com/ubuntu-ports - ssh_svcname: ssh diff --git a/config/cloud.cfg-freebsd b/config/cloud.cfg-freebsd deleted file mode 100644 index d666c397..00000000 --- a/config/cloud.cfg-freebsd +++ /dev/null @@ -1,88 +0,0 @@ -# The top level settings are used as module -# and system configuration. - -syslog_fix_perms: root:wheel - -# This should not be required, but leave it in place until the real cause of -# not beeing able to find -any- datasources is resolved. -datasource_list: ['ConfigDrive', 'Azure', 'OpenStack', 'Ec2'] - -# A set of users which may be applied and/or used by various modules -# when a 'default' entry is found it will reference the 'default_user' -# from the distro configuration specified below -users: - - default - -# If this is set, 'root' will not be able to ssh in and they -# will get a message to login instead as the above $user (ubuntu) -disable_root: false - -# This will cause the set+update hostname module to not operate (if true) -preserve_hostname: false - -# Example datasource config -# datasource: -# Ec2: -# metadata_urls: [ 'blah.com' ] -# timeout: 5 # (defaults to 50 seconds) -# max_wait: 10 # (defaults to 120 seconds) - -# The modules that run in the 'init' stage -cloud_init_modules: -# - migrator - - seed_random - - bootcmd -# - write-files - - growpart - - resizefs - - set_hostname - - update_hostname -# - update_etc_hosts -# - ca-certs -# - rsyslog - - users-groups - - ssh - -# The modules that run in the 'config' stage -cloud_config_modules: -# - disk_setup -# - mounts - - ssh-import-id - - locale - - set-passwords - - package-update-upgrade-install -# - landscape - - timezone -# - puppet -# - chef -# - salt-minion -# - mcollective - - disable-ec2-metadata - - runcmd -# - byobu - -# The modules that run in the 'final' stage -cloud_final_modules: - - rightscale_userdata - - scripts-vendor - - scripts-per-once - - scripts-per-boot - - scripts-per-instance - - scripts-user - - ssh-authkey-fingerprints - - keys-to-console - - phone-home - - final-message - - power-state-change - -# System and/or distro specific settings -# (not accessible to handlers/transforms) -system_info: - distro: freebsd - default_user: - name: freebsd - lock_passwd: True - gecos: FreeBSD - groups: [wheel] - sudo: ["ALL=(ALL) NOPASSWD:ALL"] - shell: /bin/tcsh diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl new file mode 100644 index 00000000..5af2a88f --- /dev/null +++ b/config/cloud.cfg.tmpl @@ -0,0 +1,194 @@ +## template:jinja +# The top level settings are used as module +# and system configuration. + +{% if variant in ["bsd"] %} +syslog_fix_perms: root:wheel +{% endif %} +# A set of users which may be applied and/or used by various modules +# when a 'default' entry is found it will reference the 'default_user' +# from the distro configuration specified below +users: + - default + +# If this is set, 'root' will not be able to ssh in and they +# will get a message to login instead as the default $user +{% if variant in ["bsd"] %} +disable_root: false +{% else %} +disable_root: true +{% endif %} + +{% if variant in ["centos", "fedora", "rhel"] %} +mount_default_fields: [~, ~, 'auto', 'defaults,nofail', '0', '2'] +resize_rootfs_tmp: /dev +ssh_deletekeys: 0 +ssh_genkeytypes: ~ +ssh_pwauth: 0 + +{% endif %} +# This will cause the set+update hostname module to not operate (if true) +preserve_hostname: false + +{% if variant in ["bsd"] %} +# This should not be required, but leave it in place until the real cause of +# not beeing able to find -any- datasources is resolved. +datasource_list: ['ConfigDrive', 'Azure', 'OpenStack', 'Ec2'] +{% endif %} +# Example datasource config +# datasource: +# Ec2: +# metadata_urls: [ 'blah.com' ] +# timeout: 5 # (defaults to 50 seconds) +# max_wait: 10 # (defaults to 120 seconds) + +# The modules that run in the 'init' stage +cloud_init_modules: + - migrator +{% if variant in ["ubuntu", "unknown", "debian"] %} + - ubuntu-init-switch +{% endif %} + - seed_random + - bootcmd + - write-files + - growpart + - resizefs +{% if variant not in ["bsd"] %} + - disk_setup + - mounts +{% endif %} + - set_hostname + - update_hostname +{% if variant not in ["bsd"] %} + - update_etc_hosts + - ca-certs + - rsyslog +{% endif %} + - users-groups + - ssh + +# The modules that run in the 'config' stage +cloud_config_modules: +{% if variant in ["ubuntu", "unknown", "debian"] %} +# Emit the cloud config ready event +# this can be used by upstart jobs for 'start on cloud-config'. + - emit_upstart + - snap_config +{% endif %} + - ssh-import-id + - locale + - set-passwords +{% if variant in ["rhel", "fedora"] %} + - spacewalk + - yum-add-repo +{% endif %} +{% if variant in ["ubuntu", "unknown", "debian"] %} + - grub-dpkg + - apt-pipelining + - apt-configure +{% endif %} +{% if variant not in ["bsd"] %} + - ntp +{% endif %} + - timezone + - disable-ec2-metadata + - runcmd +{% if variant in ["ubuntu", "unknown", "debian"] %} + - byobu +{% endif %} + +# The modules that run in the 'final' stage +cloud_final_modules: +{% if variant in ["ubuntu", "unknown", "debian"] %} + - snappy +{% endif %} + - package-update-upgrade-install +{% if variant in ["ubuntu", "unknown", "debian"] %} + - fan + - landscape + - lxd +{% endif %} +{% if variant not in ["bsd"] %} + - puppet + - chef + - salt-minion + - mcollective +{% endif %} + - rightscale_userdata + - scripts-vendor + - scripts-per-once + - scripts-per-boot + - scripts-per-instance + - scripts-user + - ssh-authkey-fingerprints + - keys-to-console + - phone-home + - final-message + - power-state-change + +# System and/or distro specific settings +# (not accessible to handlers/transforms) +system_info: + # This will affect which distro class gets used +{% if variant in ["centos", "debian", "fedora", "rhel", "ubuntu"] %} + distro: {{ variant }} +{% elif variant in ["bsd"] %} + distro: freebsd +{% else %} + # Unknown/fallback distro. + distro: ubuntu +{% endif %} +{% if variant in ["ubuntu", "unknown", "debian"] %} + # Default user name + that default users groups (if added/used) + default_user: + name: ubuntu + lock_passwd: True + gecos: Ubuntu + groups: [adm, audio, cdrom, dialout, dip, floppy, lxd, netdev, plugdev, sudo, video] + sudo: ["ALL=(ALL) NOPASSWD:ALL"] + shell: /bin/bash + # Other config here will be given to the distro class and/or path classes + paths: + cloud_dir: /var/lib/cloud/ + templates_dir: /etc/cloud/templates/ + upstart_dir: /etc/init/ + package_mirrors: + - arches: [i386, amd64] + failsafe: + primary: http://archive.ubuntu.com/ubuntu + security: http://security.ubuntu.com/ubuntu + search: + primary: + - http://%(ec2_region)s.ec2.archive.ubuntu.com/ubuntu/ + - http://%(availability_zone)s.clouds.archive.ubuntu.com/ubuntu/ + - http://%(region)s.clouds.archive.ubuntu.com/ubuntu/ + security: [] + - arches: [armhf, armel, default] + failsafe: + primary: http://ports.ubuntu.com/ubuntu-ports + security: http://ports.ubuntu.com/ubuntu-ports + ssh_svcname: ssh +{% elif variant in ["centos", "rhel", "fedora"] %} + # Default user name + that default users groups (if added/used) + default_user: + name: {{ variant }} + lock_passwd: True + gecos: {{ variant }} Cloud User + groups: [wheel, adm, systemd-journal] + sudo: ["ALL=(ALL) NOPASSWD:ALL"] + shell: /bin/bash + # Other config here will be given to the distro class and/or path classes + paths: + cloud_dir: /var/lib/cloud/ + templates_dir: /etc/cloud/templates/ + ssh_svcname: sshd +{% elif variant in ["bsd"] %} + # Default user name + that default users groups (if added/used) + default_user: + name: freebsd + lock_passwd: True + gecos: FreeBSD + groups: [wheel] + sudo: ["ALL=(ALL) NOPASSWD:ALL"] + shell: /bin/tcsh +{% endif %} -- cgit v1.2.3 From ecb408afa1104fe49ce6eb1dc5708be56abd5cb2 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 15 Jun 2017 10:03:45 -0400 Subject: FreeBSD: Make freebsd a variant, fix unittests and tools/build-on-freebsd. - Simplify the logic of 'variant' in util.system_info much of the data from https://github.com/hpcugent/easybuild/wiki/OS_flavor_name_version - fix get_resource_disk_on_freebsd when running on a system without an Azure resource disk. - fix tools/build-on-freebsd to replace oauth with oauthlib and add bash which is a dependency for tests. - update a fiew places that were checking for freebsd but not using the util.is_FreeBSD() --- cloudinit/config/cc_growpart.py | 2 +- cloudinit/config/cc_power_state_change.py | 2 +- cloudinit/sources/DataSourceAzure.py | 2 +- cloudinit/util.py | 46 ++++++++++-------------- config/cloud.cfg.tmpl | 20 +++++------ tests/unittests/test_handler/test_handler_ntp.py | 2 +- tests/unittests/test_util.py | 9 +++-- tools/build-on-freebsd | 6 ++-- 8 files changed, 40 insertions(+), 49 deletions(-) (limited to 'config') diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index d2bc6e6c..bafca9d8 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -214,7 +214,7 @@ def device_part_info(devpath): # FreeBSD doesn't know of sysfs so just get everything we need from # the device, like /dev/vtbd0p2. - if util.system_info()["platform"].startswith('FreeBSD'): + if util.is_FreeBSD(): m = re.search('^(/dev/.+)p([0-9])$', devpath) return (m.group(1), m.group(2)) diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py index c1c6fe7e..eba58b02 100644 --- a/cloudinit/config/cc_power_state_change.py +++ b/cloudinit/config/cc_power_state_change.py @@ -71,7 +71,7 @@ def givecmdline(pid): # Example output from procstat -c 1 # PID COMM ARGS # 1 init /bin/init -- - if util.system_info()["platform"].startswith('FreeBSD'): + if util.is_FreeBSD(): (output, _err) = util.subp(['procstat', '-c', str(pid)]) line = output.splitlines()[1] m = re.search('\d+ (\w|\.|-)+\s+(/\w.+)', line) diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 71e7c55c..4fe0d635 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -101,7 +101,7 @@ def get_dev_storvsc_sysctl(): sysctl_out, err = util.subp(['sysctl', 'dev.storvsc']) except util.ProcessExecutionError: LOG.debug("Fail to execute sysctl dev.storvsc") - return None + sysctl_out = "" return sysctl_out diff --git a/cloudinit/util.py b/cloudinit/util.py index ec68925e..c93b6d7e 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -573,7 +573,7 @@ def is_ipv4(instr): def is_FreeBSD(): - return system_info()['platform'].startswith('FreeBSD') + return system_info()['variant'] == "freebsd" def get_cfg_option_bool(yobj, key, default=False): @@ -598,37 +598,29 @@ def get_cfg_option_int(yobj, key, default=0): def system_info(): info = { 'platform': platform.platform(), + 'system': platform.system(), 'release': platform.release(), 'python': platform.python_version(), 'uname': platform.uname(), - 'dist': platform.linux_distribution(), # pylint: disable=W1505 + 'dist': platform.dist(), # pylint: disable=W1505 } - plat = info['platform'].lower() - # Try to get more info about what it actually is, in a format - # that we can easily use across linux and variants... - if plat.startswith('darwin'): - info['variant'] = 'darwin' - elif plat.endswith("bsd"): - info['variant'] = 'bsd' - elif plat.startswith('win'): - info['variant'] = 'windows' - elif 'linux' in plat: - # Try to get a single string out of these... - linux_dist, _version, _id = info['dist'] - linux_dist = linux_dist.lower() - if linux_dist in ('ubuntu', 'linuxmint', 'mint'): - info['variant'] = 'ubuntu' + system = info['system'].lower() + var = 'unknown' + if system == "linux": + linux_dist = info['dist'][0].lower() + if linux_dist in ('centos', 'fedora', 'debian'): + var = linux_dist + elif linux_dist in ('ubuntu', 'linuxmint', 'mint'): + var = 'ubuntu' + elif linux_dist == 'redhat': + var = 'rhel' else: - for prefix, variant in [('redhat', 'rhel'), - ('centos', 'centos'), - ('fedora', 'fedora'), - ('debian', 'debian')]: - if linux_dist.startswith(prefix): - info['variant'] = variant - if 'variant' not in info: - info['variant'] = 'linux' - if 'variant' not in info: - info['variant'] = 'unknown' + var = 'linux' + elif system in ('windows', 'darwin', "freebsd"): + var = system + + info['variant'] = var + return info diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl index 5af2a88f..f4b9069b 100644 --- a/config/cloud.cfg.tmpl +++ b/config/cloud.cfg.tmpl @@ -2,7 +2,7 @@ # The top level settings are used as module # and system configuration. -{% if variant in ["bsd"] %} +{% if variant in ["freebsd"] %} syslog_fix_perms: root:wheel {% endif %} # A set of users which may be applied and/or used by various modules @@ -13,7 +13,7 @@ users: # If this is set, 'root' will not be able to ssh in and they # will get a message to login instead as the default $user -{% if variant in ["bsd"] %} +{% if variant in ["freebsd"] %} disable_root: false {% else %} disable_root: true @@ -30,7 +30,7 @@ ssh_pwauth: 0 # This will cause the set+update hostname module to not operate (if true) preserve_hostname: false -{% if variant in ["bsd"] %} +{% if variant in ["freebsd"] %} # This should not be required, but leave it in place until the real cause of # not beeing able to find -any- datasources is resolved. datasource_list: ['ConfigDrive', 'Azure', 'OpenStack', 'Ec2'] @@ -53,13 +53,13 @@ cloud_init_modules: - write-files - growpart - resizefs -{% if variant not in ["bsd"] %} +{% if variant not in ["freebsd"] %} - disk_setup - mounts {% endif %} - set_hostname - update_hostname -{% if variant not in ["bsd"] %} +{% if variant not in ["freebsd"] %} - update_etc_hosts - ca-certs - rsyslog @@ -87,7 +87,7 @@ cloud_config_modules: - apt-pipelining - apt-configure {% endif %} -{% if variant not in ["bsd"] %} +{% if variant not in ["freebsd"] %} - ntp {% endif %} - timezone @@ -108,7 +108,7 @@ cloud_final_modules: - landscape - lxd {% endif %} -{% if variant not in ["bsd"] %} +{% if variant not in ["freebsd"] %} - puppet - chef - salt-minion @@ -130,10 +130,8 @@ cloud_final_modules: # (not accessible to handlers/transforms) system_info: # This will affect which distro class gets used -{% if variant in ["centos", "debian", "fedora", "rhel", "ubuntu"] %} +{% if variant in ["centos", "debian", "fedora", "rhel", "ubuntu", "freebsd"] %} distro: {{ variant }} -{% elif variant in ["bsd"] %} - distro: freebsd {% else %} # Unknown/fallback distro. distro: ubuntu @@ -182,7 +180,7 @@ system_info: cloud_dir: /var/lib/cloud/ templates_dir: /etc/cloud/templates/ ssh_svcname: sshd -{% elif variant in ["bsd"] %} +{% elif variant in ["freebsd"] %} # Default user name + that default users groups (if added/used) default_user: name: freebsd diff --git a/tests/unittests/test_handler/test_handler_ntp.py b/tests/unittests/test_handler/test_handler_ntp.py index c4299d94..7f278646 100644 --- a/tests/unittests/test_handler/test_handler_ntp.py +++ b/tests/unittests/test_handler/test_handler_ntp.py @@ -62,7 +62,7 @@ class TestNtp(FilesystemMockingTestCase): def test_ntp_rename_ntp_conf(self): """When NTP_CONF exists, rename_ntp moves it.""" ntpconf = self.tmp_path("ntp.conf", self.new_root) - os.mknod(ntpconf) + util.write_file(ntpconf, "") with mock.patch("cloudinit.config.cc_ntp.NTP_CONF", ntpconf): cc_ntp.rename_ntp_conf() self.assertFalse(os.path.exists(ntpconf)) diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 014aa6a3..a73fd26a 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -20,6 +20,9 @@ except ImportError: import mock +BASH = util.which('bash') + + class FakeSelinux(object): def __init__(self, match_what): @@ -544,17 +547,17 @@ class TestReadSeeded(helpers.TestCase): class TestSubp(helpers.TestCase): - stdin2err = ['bash', '-c', 'cat >&2'] + stdin2err = [BASH, '-c', 'cat >&2'] stdin2out = ['cat'] utf8_invalid = b'ab\xaadef' utf8_valid = b'start \xc3\xa9 end' utf8_valid_2 = b'd\xc3\xa9j\xc8\xa7' - printenv = ['bash', '-c', 'for n in "$@"; do echo "$n=${!n}"; done', '--'] + printenv = [BASH, '-c', 'for n in "$@"; do echo "$n=${!n}"; done', '--'] def printf_cmd(self, *args): # bash's printf supports \xaa. So does /usr/bin/printf # but by using bash, we remove dependency on another program. - return(['bash', '-c', 'printf "$@"', 'printf'] + list(args)) + return([BASH, '-c', 'printf "$@"', 'printf'] + list(args)) def test_subp_handles_utf8(self): # The given bytes contain utf-8 accented characters as seen in e.g. diff --git a/tools/build-on-freebsd b/tools/build-on-freebsd index ccc10b40..ff9153ad 100755 --- a/tools/build-on-freebsd +++ b/tools/build-on-freebsd @@ -8,6 +8,7 @@ fail() { echo "FAILED:" "$@" 1>&2; exit 1; } # Check dependencies: depschecked=/tmp/c-i.dependencieschecked pkgs=" + bash dmidecode e2fsprogs py27-Jinja2 @@ -16,7 +17,7 @@ pkgs=" py27-configobj py27-jsonpatch py27-jsonpointer - py27-oauth + py27-oauthlib py27-prettytable py27-requests py27-serial @@ -35,9 +36,6 @@ touch $depschecked python setup.py build python setup.py install -O1 --skip-build --prefix /usr/local/ --init-system sysvinit_freebsd -# Install the correct config file: -cp config/cloud.cfg-freebsd /etc/cloud/cloud.cfg - # Enable cloud-init in /etc/rc.conf: sed -i.bak -e "/cloudinit_enable=.*/d" /etc/rc.conf echo 'cloudinit_enable="YES"' >> /etc/rc.conf -- cgit v1.2.3