From 49b562c94838f4e2c0aa0df01a5247d06bc8a561 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 12 Apr 2018 15:51:45 -0600 Subject: tools: Fix make-tarball cli tool usage for development This tool is used to assist during the creation of ubuntu packages for release testing. Address the following on the command-line: * --help option now print usage * Add --orig-tarball which creates named output file cloud-init_.orig.tar.gz * drop unused --verbose option --- tools/make-tarball | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) (limited to 'tools') diff --git a/tools/make-tarball b/tools/make-tarball index 3197689f..8d540139 100755 --- a/tools/make-tarball +++ b/tools/make-tarball @@ -13,22 +13,28 @@ Usage: ${0##*/} [revision] create a tarball of revision (default HEAD) options: - -o | --output FILE write to file + -h | --help print usage + -o | --output FILE write to file + --orig-tarball Write file cloud-init_.orig.tar.gz + --long Use git describe --long for versioning EOF } short_opts="ho:v" -long_opts="help,output:,long,verbose" +long_opts="help,output:,orig-tarball,long" getopt_out=$(getopt --name "${0##*/}" \ --options "${short_opts}" --long "${long_opts}" -- "$@") && eval set -- "${getopt_out}" || { Usage 1>&2; exit 1; } long_opt="" +orig_opt="" while [ $# -ne 0 ]; do cur=$1; next=$2 case "$cur" in + -h|--help) Usage; exit 0;; -o|--output) output=$next; shift;; --long) long_opt="--long";; + --orig-tarball) orig_opt=".orig";; --) shift; break;; esac shift; @@ -39,7 +45,10 @@ version=$(git describe --abbrev=8 "--match=[0-9]*" ${long_opt} $rev) archive_base="cloud-init-$version" if [ -z "$output" ]; then - output="$archive_base.tar.gz" + if [ ! -z "$orig_opt" ]; then + archive_base="cloud-init_$version" + fi + output="$archive_base$orig_opt.tar.gz" fi # when building an archiving from HEAD, ensure that there aren't any -- cgit v1.2.3 From 6ef92c98c3d2b127b05d6708337efc8a81e00071 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 Apr 2018 16:24:24 -0500 Subject: IBMCloud: recognize provisioning environment during debug boots. When images are deployed from template in a production environment the artifacts of the provisioning stage (provisioningConfiguration.cfg) that cloud-init referenced are cleaned up. However, when provisioned in "debug" mode (internal to IBM) the artifacts are left. This changes the 'is_ibm_provisioning' implementations in both ds-identify and in the IBM datasource to identify the provisioning stage more correctly. The change is to consider provisioning only if the provisioing file existed and there was no log file or the log file was older than this boot. LP: #1767166 --- cloudinit/sources/DataSourceIBMCloud.py | 42 +++++++++----- cloudinit/tests/helpers.py | 13 ++++- tests/unittests/test_datasource/test_ibmcloud.py | 50 ++++++++++++++++ tests/unittests/test_ds_identify.py | 72 +++++++++++++++++++++--- tools/ds-identify | 21 ++++++- 5 files changed, 175 insertions(+), 23 deletions(-) (limited to 'tools') diff --git a/cloudinit/sources/DataSourceIBMCloud.py b/cloudinit/sources/DataSourceIBMCloud.py index cfa724bf..01106ec0 100644 --- a/cloudinit/sources/DataSourceIBMCloud.py +++ b/cloudinit/sources/DataSourceIBMCloud.py @@ -8,17 +8,11 @@ There are 2 different api exposed launch methods. * template: This is the legacy method of launching instances. When booting from an image template, the system boots first into a "provisioning" mode. There, host <-> guest mechanisms are utilized - to execute code in the guest and provision it. + to execute code in the guest and configure it. The configuration + includes configuring the system network and possibly installing + packages and other software stack. - Cloud-init will disable itself when it detects that it is in the - provisioning mode. It detects this by the presence of - a file '/root/provisioningConfiguration.cfg'. - - When provided with user-data, the "first boot" will contain a - ConfigDrive-like disk labeled with 'METADATA'. If there is no user-data - provided, then there is no data-source. - - Cloud-init never does any network configuration in this mode. + After the provisioning is finished, the system reboots. * os_code: Essentially "launch by OS Code" (Operating System Code). This is a more modern approach. There is no specific "provisioning" boot. @@ -200,8 +194,30 @@ def _is_xen(): return os.path.exists("/proc/xen") -def _is_ibm_provisioning(): - return os.path.exists("/root/provisioningConfiguration.cfg") +def _is_ibm_provisioning( + prov_cfg="/root/provisioningConfiguration.cfg", + inst_log="/root/swinstall.log", + boot_ref="/proc/1/environ"): + """Return boolean indicating if this boot is ibm provisioning boot.""" + if os.path.exists(prov_cfg): + msg = "config '%s' exists." % prov_cfg + result = True + if os.path.exists(inst_log): + if os.path.exists(boot_ref): + result = (os.stat(inst_log).st_mtime > + os.stat(boot_ref).st_mtime) + msg += (" log '%s' from %s boot." % + (inst_log, "current" if result else "previous")) + else: + msg += (" log '%s' existed, but no reference file '%s'." % + (inst_log, boot_ref)) + result = False + else: + msg += " log '%s' did not exist." % inst_log + else: + result, msg = (False, "config '%s' did not exist." % prov_cfg) + LOG.debug("ibm_provisioning=%s: %s", result, msg) + return result def get_ibm_platform(): @@ -251,7 +267,7 @@ def get_ibm_platform(): else: return (Platforms.TEMPLATE_LIVE_METADATA, metadata_path) elif _is_ibm_provisioning(): - return (Platforms.TEMPLATE_PROVISIONING_NODATA, None) + return (Platforms.TEMPLATE_PROVISIONING_NODATA, None) return not_found diff --git a/cloudinit/tests/helpers.py b/cloudinit/tests/helpers.py index 4999f1f6..117a9cfe 100644 --- a/cloudinit/tests/helpers.py +++ b/cloudinit/tests/helpers.py @@ -8,6 +8,7 @@ import os import shutil import sys import tempfile +import time import unittest import mock @@ -263,7 +264,8 @@ class FilesystemMockingTestCase(ResourceUsingTestCase): os.path: [('isfile', 1), ('exists', 1), ('islink', 1), ('isdir', 1), ('lexists', 1)], os: [('listdir', 1), ('mkdir', 1), - ('lstat', 1), ('symlink', 2)] + ('lstat', 1), ('symlink', 2), + ('stat', 1)] } if hasattr(os, 'scandir'): @@ -349,6 +351,15 @@ def populate_dir(path, files): return ret +def populate_dir_with_ts(path, data): + """data is {'file': ('contents', mtime)}. mtime relative to now.""" + populate_dir(path, dict((k, v[0]) for k, v in data.items())) + btime = time.time() + for fpath, (_contents, mtime) in data.items(): + ts = btime + mtime if mtime else btime + os.utime(os.path.sep.join((path, fpath)), (ts, ts)) + + def dir2dict(startdir, prefix=None): flist = {} if prefix is None: diff --git a/tests/unittests/test_datasource/test_ibmcloud.py b/tests/unittests/test_datasource/test_ibmcloud.py index 621cfe49..e639ae47 100644 --- a/tests/unittests/test_datasource/test_ibmcloud.py +++ b/tests/unittests/test_datasource/test_ibmcloud.py @@ -259,4 +259,54 @@ class TestReadMD(test_helpers.CiTestCase): ret['metadata']) +class TestIsIBMProvisioning(test_helpers.FilesystemMockingTestCase): + """Test the _is_ibm_provisioning method.""" + inst_log = "/root/swinstall.log" + prov_cfg = "/root/provisioningConfiguration.cfg" + boot_ref = "/proc/1/environ" + with_logs = True + + def _call_with_root(self, rootd): + self.reRoot(rootd) + return ibm._is_ibm_provisioning() + + def test_no_config(self): + """No provisioning config means not provisioning.""" + self.assertFalse(self._call_with_root(self.tmp_dir())) + + def test_config_only(self): + """A provisioning config without a log means provisioning.""" + rootd = self.tmp_dir() + test_helpers.populate_dir(rootd, {self.prov_cfg: "key=value"}) + self.assertTrue(self._call_with_root(rootd)) + + def test_config_with_old_log(self): + """A config with a log from previous boot is not provisioning.""" + rootd = self.tmp_dir() + data = {self.prov_cfg: ("key=value\nkey2=val2\n", -10), + self.inst_log: ("log data\n", -30), + self.boot_ref: ("PWD=/", 0)} + test_helpers.populate_dir_with_ts(rootd, data) + self.assertFalse(self._call_with_root(rootd=rootd)) + self.assertIn("from previous boot", self.logs.getvalue()) + + def test_config_with_new_log(self): + """A config with a log from this boot is provisioning.""" + rootd = self.tmp_dir() + data = {self.prov_cfg: ("key=value\nkey2=val2\n", -10), + self.inst_log: ("log data\n", 30), + self.boot_ref: ("PWD=/", 0)} + test_helpers.populate_dir_with_ts(rootd, data) + self.assertTrue(self._call_with_root(rootd=rootd)) + self.assertIn("from current boot", self.logs.getvalue()) + + def test_config_and_log_no_reference(self): + """If the config and log existed, but no reference, assume not.""" + rootd = self.tmp_dir() + test_helpers.populate_dir( + rootd, {self.prov_cfg: "key=value", self.inst_log: "log data\n"}) + self.assertFalse(self._call_with_root(rootd=rootd)) + self.assertIn("no reference file", self.logs.getvalue()) + + # vi: ts=4 expandtab diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py index 53643989..ad7fe41e 100644 --- a/tests/unittests/test_ds_identify.py +++ b/tests/unittests/test_ds_identify.py @@ -1,5 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. +from collections import namedtuple import copy import os from uuid import uuid4 @@ -7,7 +8,7 @@ from uuid import uuid4 from cloudinit import safeyaml from cloudinit import util from cloudinit.tests.helpers import ( - CiTestCase, dir2dict, populate_dir) + CiTestCase, dir2dict, populate_dir, populate_dir_with_ts) from cloudinit.sources import DataSourceIBMCloud as dsibm @@ -66,7 +67,6 @@ P_SYS_VENDOR = "sys/class/dmi/id/sys_vendor" P_SEED_DIR = "var/lib/cloud/seed" P_DSID_CFG = "etc/cloud/ds-identify.cfg" -IBM_PROVISIONING_CHECK_PATH = "/root/provisioningConfiguration.cfg" IBM_CONFIG_UUID = "9796-932E" MOCK_VIRT_IS_KVM = {'name': 'detect_virt', 'RET': 'kvm', 'ret': 0} @@ -74,11 +74,17 @@ MOCK_VIRT_IS_VMWARE = {'name': 'detect_virt', 'RET': 'vmware', 'ret': 0} MOCK_VIRT_IS_XEN = {'name': 'detect_virt', 'RET': 'xen', 'ret': 0} MOCK_UNAME_IS_PPC64 = {'name': 'uname', 'out': UNAME_PPC64EL, 'ret': 0} +shell_true = 0 +shell_false = 1 -class TestDsIdentify(CiTestCase): +CallReturn = namedtuple('CallReturn', + ['rc', 'stdout', 'stderr', 'cfg', 'files']) + + +class DsIdentifyBase(CiTestCase): dsid_path = os.path.realpath('tools/ds-identify') - def call(self, rootd=None, mocks=None, args=None, files=None, + def call(self, rootd=None, mocks=None, func="main", args=None, files=None, policy_dmi=DI_DEFAULT_POLICY, policy_no_dmi=DI_DEFAULT_POLICY_NO_DMI, ec2_strict_id=DI_EC2_STRICT_ID_DEFAULT): @@ -135,7 +141,7 @@ class TestDsIdentify(CiTestCase): mocklines.append(write_mock(d)) endlines = [ - 'main %s' % ' '.join(['"%s"' % s for s in args]) + func + ' ' + ' '.join(['"%s"' % s for s in args]) ] with open(wrap, "w") as fp: @@ -159,7 +165,7 @@ class TestDsIdentify(CiTestCase): cfg = {"_INVALID_YAML": contents, "_EXCEPTION": str(e)} - return rc, out, err, cfg, dir2dict(rootd) + return CallReturn(rc, out, err, cfg, dir2dict(rootd)) def _call_via_dict(self, data, rootd=None, **kwargs): # return output of self.call with a dict input like VALID_CFG[item] @@ -190,6 +196,8 @@ class TestDsIdentify(CiTestCase): _print_run_output(rc, out, err, cfg, files) return rc, out, err, cfg, files + +class TestDsIdentify(DsIdentifyBase): def test_wb_print_variables(self): """_print_info reports an array of discovered variables to stderr.""" data = VALID_CFG['Azure-dmi-detection'] @@ -250,7 +258,10 @@ class TestDsIdentify(CiTestCase): Template provisioning with user-data has METADATA disk, datasource should return not found.""" data = copy.deepcopy(VALID_CFG['IBMCloud-metadata']) - data['files'] = {IBM_PROVISIONING_CHECK_PATH: 'xxx'} + # change the 'is_ibm_provisioning' mock to return 1 (false) + isprov_m = [m for m in data['mocks'] + if m["name"] == "is_ibm_provisioning"][0] + isprov_m['ret'] = shell_true return self._check_via_dict(data, RC_NOT_FOUND) def test_ibmcloud_template_userdata(self): @@ -265,7 +276,8 @@ class TestDsIdentify(CiTestCase): no disks attached. Datasource should return not found.""" data = copy.deepcopy(VALID_CFG['IBMCloud-nodisks']) - data['files'] = {IBM_PROVISIONING_CHECK_PATH: 'xxx'} + data['mocks'].append( + {'name': 'is_ibm_provisioning', 'ret': shell_true}) return self._check_via_dict(data, RC_NOT_FOUND) def test_ibmcloud_template_no_userdata(self): @@ -446,6 +458,47 @@ class TestDsIdentify(CiTestCase): self._test_ds_found('Hetzner') +class TestIsIBMProvisioning(DsIdentifyBase): + """Test the is_ibm_provisioning method in ds-identify.""" + + inst_log = "/root/swinstall.log" + prov_cfg = "/root/provisioningConfiguration.cfg" + boot_ref = "/proc/1/environ" + funcname = "is_ibm_provisioning" + + def test_no_config(self): + """No provisioning config means not provisioning.""" + ret = self.call(files={}, func=self.funcname) + self.assertEqual(shell_false, ret.rc) + + def test_config_only(self): + """A provisioning config without a log means provisioning.""" + ret = self.call(files={self.prov_cfg: "key=value"}, func=self.funcname) + self.assertEqual(shell_true, ret.rc) + + def test_config_with_old_log(self): + """A config with a log from previous boot is not provisioning.""" + rootd = self.tmp_dir() + data = {self.prov_cfg: ("key=value\nkey2=val2\n", -10), + self.inst_log: ("log data\n", -30), + self.boot_ref: ("PWD=/", 0)} + populate_dir_with_ts(rootd, data) + ret = self.call(rootd=rootd, func=self.funcname) + self.assertEqual(shell_false, ret.rc) + self.assertIn("from previous boot", ret.stderr) + + def test_config_with_new_log(self): + """A config with a log from this boot is provisioning.""" + rootd = self.tmp_dir() + data = {self.prov_cfg: ("key=value\nkey2=val2\n", -10), + self.inst_log: ("log data\n", 30), + self.boot_ref: ("PWD=/", 0)} + populate_dir_with_ts(rootd, data) + ret = self.call(rootd=rootd, func=self.funcname) + self.assertEqual(shell_true, ret.rc) + self.assertIn("from current boot", ret.stderr) + + def blkid_out(disks=None): """Convert a list of disk dictionaries into blkid content.""" if disks is None: @@ -639,6 +692,7 @@ VALID_CFG = { 'ds': 'IBMCloud', 'mocks': [ MOCK_VIRT_IS_XEN, + {'name': 'is_ibm_provisioning', 'ret': shell_false}, {'name': 'blkid', 'ret': 0, 'out': blkid_out( [{'DEVNAME': 'xvda1', 'TYPE': 'vfat', 'PARTUUID': uuid4()}, @@ -652,6 +706,7 @@ VALID_CFG = { 'ds': 'IBMCloud', 'mocks': [ MOCK_VIRT_IS_XEN, + {'name': 'is_ibm_provisioning', 'ret': shell_false}, {'name': 'blkid', 'ret': 0, 'out': blkid_out( [{'DEVNAME': 'xvda1', 'TYPE': 'ext3', 'PARTUUID': uuid4(), @@ -669,6 +724,7 @@ VALID_CFG = { 'ds': 'IBMCloud', 'mocks': [ MOCK_VIRT_IS_XEN, + {'name': 'is_ibm_provisioning', 'ret': shell_false}, {'name': 'blkid', 'ret': 0, 'out': blkid_out( [{'DEVNAME': 'xvda1', 'TYPE': 'vfat', 'PARTUUID': uuid4()}, diff --git a/tools/ds-identify b/tools/ds-identify index 9a2db5c4..7fff5d1e 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -125,6 +125,7 @@ DI_ON_NOTFOUND="" DI_EC2_STRICT_ID_DEFAULT="true" _IS_IBM_CLOUD="" +_IS_IBM_PROVISIONING="" error() { set -- "ERROR:" "$@"; @@ -1006,7 +1007,25 @@ dscheck_Hetzner() { } is_ibm_provisioning() { - [ -f "${PATH_ROOT}/root/provisioningConfiguration.cfg" ] + local pcfg="${PATH_ROOT}/root/provisioningConfiguration.cfg" + local logf="${PATH_ROOT}/root/swinstall.log" + local is_prov=false msg="config '$pcfg' did not exist." + if [ -f "$pcfg" ]; then + msg="config '$pcfg' exists." + is_prov=true + if [ -f "$logf" ]; then + if [ "$logf" -nt "$PATH_PROC_1_ENVIRON" ]; then + msg="$msg log '$logf' from current boot." + else + is_prov=false + msg="$msg log '$logf' from previous boot." + fi + else + msg="$msg log '$logf' did not exist." + fi + fi + debug 2 "ibm_provisioning=$is_prov: $msg" + [ "$is_prov" = "true" ] } is_ibm_cloud() { -- cgit v1.2.3 From 11172924a48a47a7231d19d9cefe628dfddda8bf Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 30 Apr 2018 13:21:51 -0600 Subject: IBMCloud: Disable config-drive and nocloud only if IBMCloud is enabled. Ubuntu images on IBMCloud for 16.04 have some seed data in /var/lib/cloud/data/seed/nocloud-net. In order to have systems with IBMCloud enabled, we modified ds-identify detection to skip that seed if the system was on IBMCloud. That change did not consider the fact that IBMCloud might not be in the datasource list. There was similar logic in the ConfigDrive datasource in ds-identify and the datasource itself. Config drive is now updated to only check and avoid IBMCloud if IBMCloud is enabled. The check in ds-identify for nocloud was dropped. If a user provides a nocloud seed on IBMCloud, then that can be used. This means that systems running Xenial will continue to get their old datasources. LP: #1766401 --- cloudinit/sources/DataSourceConfigDrive.py | 11 +++-- tests/unittests/test_ds_identify.py | 77 +++++++++++++++++++++++++++--- tools/ds-identify | 17 +++++-- 3 files changed, 91 insertions(+), 14 deletions(-) (limited to 'tools') diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index c7b5fe5f..121cf215 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -69,7 +69,8 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): util.logexc(LOG, "Failed reading config drive from %s", sdir) if not found: - for dev in find_candidate_devs(): + dslist = self.sys_cfg.get('datasource_list') + for dev in find_candidate_devs(dslist=dslist): try: # Set mtype if freebsd and turn off sync if dev.startswith("/dev/cd"): @@ -211,7 +212,7 @@ def write_injected_files(files): util.logexc(LOG, "Failed writing file: %s", filename) -def find_candidate_devs(probe_optical=True): +def find_candidate_devs(probe_optical=True, dslist=None): """Return a list of devices that may contain the config drive. The returned list is sorted by search order where the first item has @@ -227,6 +228,9 @@ def find_candidate_devs(probe_optical=True): * either vfat or iso9660 formated * labeled with 'config-2' or 'CONFIG-2' """ + if dslist is None: + dslist = [] + # query optical drive to get it in blkid cache for 2.6 kernels if probe_optical: for device in OPTICAL_DEVICES: @@ -257,7 +261,8 @@ def find_candidate_devs(probe_optical=True): devices = [d for d in candidates if d in by_label or not util.is_partition(d)] - if devices: + LOG.debug("devices=%s dslist=%s", devices, dslist) + if devices and "IBMCloud" in dslist: # IBMCloud uses config-2 label, but limited to a single UUID. ibm_platform, ibm_path = get_ibm_platform() if ibm_path in devices: diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py index ad7fe41e..4d8a4360 100644 --- a/tests/unittests/test_ds_identify.py +++ b/tests/unittests/test_ds_identify.py @@ -184,17 +184,18 @@ class DsIdentifyBase(CiTestCase): data, RC_FOUND, dslist=[data.get('ds'), DS_NONE]) def _check_via_dict(self, data, rc, dslist=None, **kwargs): - found_rc, out, err, cfg, files = self._call_via_dict(data, **kwargs) + ret = self._call_via_dict(data, **kwargs) good = False try: - self.assertEqual(rc, found_rc) + self.assertEqual(rc, ret.rc) if dslist is not None: - self.assertEqual(dslist, cfg['datasource_list']) + self.assertEqual(dslist, ret.cfg['datasource_list']) good = True finally: if not good: - _print_run_output(rc, out, err, cfg, files) - return rc, out, err, cfg, files + _print_run_output(ret.rc, ret.stdout, ret.stderr, ret.cfg, + ret.files) + return ret class TestDsIdentify(DsIdentifyBase): @@ -245,13 +246,40 @@ class TestDsIdentify(DsIdentifyBase): def test_config_drive(self): """ConfigDrive datasource has a disk with LABEL=config-2.""" self._test_ds_found('ConfigDrive') - return def test_config_drive_upper(self): """ConfigDrive datasource has a disk with LABEL=CONFIG-2.""" self._test_ds_found('ConfigDriveUpper') return + def test_config_drive_seed(self): + """Config Drive seed directory.""" + self._test_ds_found('ConfigDrive-seed') + + def test_config_drive_interacts_with_ibmcloud_config_disk(self): + """Verify ConfigDrive interaction with IBMCloud. + + If ConfigDrive is enabled and not IBMCloud, then ConfigDrive + should claim the ibmcloud 'config-2' disk. + If IBMCloud is enabled, then ConfigDrive should skip.""" + data = copy.deepcopy(VALID_CFG['IBMCloud-config-2']) + files = data.get('files', {}) + if not files: + data['files'] = files + cfgpath = 'etc/cloud/cloud.cfg.d/99_networklayer_common.cfg' + + # with list including IBMCloud, config drive should be not found. + files[cfgpath] = 'datasource_list: [ ConfigDrive, IBMCloud ]\n' + ret = self._check_via_dict(data, shell_true) + self.assertEqual( + ret.cfg.get('datasource_list'), ['IBMCloud', 'None']) + + # But if IBMCloud is not enabled, config drive should claim this. + files[cfgpath] = 'datasource_list: [ ConfigDrive, NoCloud ]\n' + ret = self._check_via_dict(data, shell_true) + self.assertEqual( + ret.cfg.get('datasource_list'), ['ConfigDrive', 'None']) + def test_ibmcloud_template_userdata_in_provisioning(self): """Template provisioned with user-data during provisioning stage. @@ -307,6 +335,37 @@ class TestDsIdentify(DsIdentifyBase): self._check_via_dict( data, rc=RC_FOUND, dslist=['ConfigDrive', DS_NONE]) + def test_ibmcloud_with_nocloud_seed(self): + """NoCloud seed should be preferred over IBMCloud. + + A nocloud seed should be preferred over IBMCloud even if enabled. + Ubuntu 16.04 images have /seed/nocloud-net. LP: #1766401.""" + data = copy.deepcopy(VALID_CFG['IBMCloud-config-2']) + files = data.get('files', {}) + if not files: + data['files'] = files + files.update(VALID_CFG['NoCloud-seed']['files']) + ret = self._check_via_dict(data, shell_true) + self.assertEqual( + ['NoCloud', 'IBMCloud', 'None'], + ret.cfg.get('datasource_list')) + + def test_ibmcloud_with_configdrive_seed(self): + """ConfigDrive seed should be preferred over IBMCloud. + + A ConfigDrive seed should be preferred over IBMCloud even if enabled. + Ubuntu 16.04 images have a fstab entry that mounts the + METADATA disk into /seed/config_drive. LP: ##1766401.""" + data = copy.deepcopy(VALID_CFG['IBMCloud-config-2']) + files = data.get('files', {}) + if not files: + data['files'] = files + files.update(VALID_CFG['ConfigDrive-seed']['files']) + ret = self._check_via_dict(data, shell_true) + self.assertEqual( + ['ConfigDrive', 'IBMCloud', 'None'], + ret.cfg.get('datasource_list')) + def test_policy_disabled(self): """A Builtin policy of 'disabled' should return not found. @@ -684,6 +743,12 @@ VALID_CFG = { }, ], }, + 'ConfigDrive-seed': { + 'ds': 'ConfigDrive', + 'files': { + os.path.join(P_SEED_DIR, 'config_drive', 'openstack', + 'latest', 'meta_data.json'): 'md\n'}, + }, 'Hetzner': { 'ds': 'Hetzner', 'files': {P_SYS_VENDOR: 'Hetzner\n'}, diff --git a/tools/ds-identify b/tools/ds-identify index 7fff5d1e..9f0d96f7 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -601,7 +601,6 @@ dscheck_NoCloud() { *\ ds=nocloud*) return ${DS_FOUND};; esac - is_ibm_cloud && return ${DS_NOT_FOUND} for d in nocloud nocloud-net; do check_seed_dir "$d" meta-data user-data && return ${DS_FOUND} check_writable_seed_dir "$d" meta-data user-data && return ${DS_FOUND} @@ -612,11 +611,12 @@ dscheck_NoCloud() { return ${DS_NOT_FOUND} } +is_ds_enabled() { + local name="$1" pad=" ${DI_DSLIST} " + [ "${pad#* $name }" != "${pad}" ] +} + check_configdrive_v2() { - is_ibm_cloud && return ${DS_NOT_FOUND} - if has_fs_with_label CONFIG-2 config-2; then - return ${DS_FOUND} - fi # look in /config-drive /seed/config_drive for a directory # openstack/YYYY-MM-DD format with a file meta_data.json local d="" @@ -631,6 +631,13 @@ check_configdrive_v2() { debug 1 "config drive seeded directory had only 'latest'" return ${DS_FOUND} fi + + is_ds_enabled "IBMCloud" + debug 1 "is_ds_enabled returned $?: $DI_DSLIST" + is_ds_enabled "IBMCloud" && is_ibm_cloud && return ${DS_NOT_FOUND} + if has_fs_with_label CONFIG-2 config-2; then + return ${DS_FOUND} + fi return ${DS_NOT_FOUND} } -- cgit v1.2.3 From 4c1af5c7eb8db67f51f35130e13157a735256d2b Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 8 May 2018 13:59:33 -0400 Subject: ds-identify: make shellcheck 0.4.6 happy with ds-identify. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This fixes warnings reported by shellcheck at 0.4.6. The complaints that we are ignoring globally (top of the file) are:  2015: Note that A && B || C is not if-then-else. C may run if A is true.  2039: In POSIX sh, 'local' is undefined.  2162: read without -r will mangle backslashes.  2166: Prefer [ p ] && [ q ] as [ p -a q ] is not well defined. Most of the complaints were just noise, but a few unused variables were reported and fixed. Related shellcheck issues opened: - https://github.com/koalaman/shellcheck/issues/1191 - https://github.com/koalaman/shellcheck/issues/1192 - https://github.com/koalaman/shellcheck/issues/1193 - https://github.com/koalaman/shellcheck/issues/1194 --- tools/ds-identify | 50 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 18 deletions(-) (limited to 'tools') diff --git a/tools/ds-identify b/tools/ds-identify index 9f0d96f7..a062e4d7 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -1,4 +1,5 @@ #!/bin/sh +# shellcheck disable=2015,2039,2162,2166 # # ds-identify is configured via /etc/cloud/ds-identify.cfg # or on the kernel command line. It takes primarily 2 inputs: @@ -125,7 +126,6 @@ DI_ON_NOTFOUND="" DI_EC2_STRICT_ID_DEFAULT="true" _IS_IBM_CLOUD="" -_IS_IBM_PROVISIONING="" error() { set -- "ERROR:" "$@"; @@ -211,7 +211,9 @@ read_fs_info() { # 'set --' will collapse multiple consecutive entries in IFS for # whitespace characters (\n, tab, " ") so we cannot rely on getting # empty lines in "$@" below. - IFS="$CR"; set -- $out; IFS="$oifs" + + # shellcheck disable=2086 + { IFS="$CR"; set -- $out; IFS="$oifs"; } for line in "$@"; do case "${line}" in @@ -311,6 +313,7 @@ read_dmi_product_serial() { DI_DMI_PRODUCT_SERIAL="$_RET" } +# shellcheck disable=2034 read_uname_info() { # run uname, and parse output. # uname is tricky to parse as it outputs always in a given order @@ -330,6 +333,7 @@ read_uname_info() { return $ret } fi + # shellcheck disable=2086 set -- $out DI_UNAME_KERNEL_NAME="$1" DI_UNAME_NODENAME="$2" @@ -357,7 +361,8 @@ parse_yaml_array() { # the fix was to quote the open bracket (val=${val#"["}) (LP: #1689648) val=${val#"["} val=${val%"]"} - IFS=","; set -- $val; IFS="$oifs" + # shellcheck disable=2086 + { IFS=","; set -- $val; IFS="$oifs"; } for tok in "$@"; do trim "$tok" unquote "$_RET" @@ -393,7 +398,7 @@ read_datasource_list() { fi if [ -z "$dslist" ]; then dslist=${DI_DSLIST_DEFAULT} - debug 1 "no datasource_list found, using default:" $dslist + debug 1 "no datasource_list found, using default: $dslist" fi DI_DSLIST=$dslist return 0 @@ -404,7 +409,8 @@ read_pid1_product_name() { cached "${DI_PID_1_PRODUCT_NAME}" && return [ -r "${PATH_PROC_1_ENVIRON}" ] || return out=$(tr '\0' '\n' <"${PATH_PROC_1_ENVIRON}") - IFS="$CR"; set -- $out; IFS="$oifs" + # shellcheck disable=2086 + { IFS="$CR"; set -- $out; IFS="$oifs"; } for tok in "$@"; do key=${tok%%=*} [ "$key" != "$tok" ] || continue @@ -471,6 +477,7 @@ nocase_equal() { [ "$1" = "$2" ] && return 0 local delim="-delim-" + # shellcheck disable=2018,2019 out=$(echo "$1${delim}$2" | tr A-Z a-z) [ "${out#*${delim}}" = "${out%${delim}*}" ] } @@ -547,11 +554,13 @@ check_config() { else files="$*" fi - set +f; set -- $files; set -f; + # shellcheck disable=2086 + { set +f; set -- $files; set -f; } if [ "$1" = "$files" -a ! -f "$1" ]; then return 1 fi local fname="" line="" ret="" found=0 found_fn="" + # shellcheck disable=2094 for fname in "$@"; do [ -f "$fname" ] || continue while read line; do @@ -794,7 +803,7 @@ ec2_read_strict_setting() { # 3. look for the key 'strict_id' (datasource/Ec2/strict_id) # only in cloud.cfg or cloud.cfg.d/EC2.cfg (case insensitive) local cfg="${PATH_ETC_CI_CFG}" cfg_d="${PATH_ETC_CI_CFG_D}" - if check_config strict_id $cfg "$cfg_d/*[Ee][Cc]2*.cfg"; then + if check_config strict_id "$cfg" "$cfg_d/*[Ee][Cc]2*.cfg"; then debug 2 "${_RET_fname} set strict_id to $_RET" return 0 fi @@ -1001,7 +1010,7 @@ dscheck_Scaleway() { *\ scaleway\ *) return ${DS_FOUND};; esac - if [ -f ${PATH_ROOT}/var/run/scaleway ]; then + if [ -f "${PATH_ROOT}/var/run/scaleway" ]; then return ${DS_FOUND} fi @@ -1156,6 +1165,7 @@ found() { } trim() { + # shellcheck disable=2048,2086 set -- $* _RET="$*" } @@ -1176,7 +1186,7 @@ _read_config() { # if no parameters are set, modifies _rc scoped environment vars. # if keyname is provided, then returns found value of that key. local keyname="${1:-_unset}" - local line="" hash="#" ckey="" key="" val="" + local line="" hash="#" key="" val="" while read line; do line=${line%%${hash}*} key="${line%%:*}" @@ -1254,7 +1264,8 @@ parse_policy() { local mode="" report="" found="" maybe="" notfound="" local oifs="$IFS" tok="" val="" - IFS=","; set -- $policy; IFS="$oifs" + # shellcheck disable=2086 + { IFS=","; set -- $policy; IFS="$oifs"; } for tok in "$@"; do val=${tok#*=} case "$tok" in @@ -1321,15 +1332,15 @@ manual_clean_and_existing() { } read_uptime() { - local up idle + local up _ _RET="${UNAVAILABLE}" - [ -f "$PATH_PROC_UPTIME" ] && - read up idle < "$PATH_PROC_UPTIME" && _RET="$up" + [ -f "$PATH_PROC_UPTIME" ] && read up _ < "$PATH_PROC_UPTIME" && + _RET="$up" return } _main() { - local dscheck="" ret_dis=1 ret_en=0 + local dscheck_fn="" ret_dis=1 ret_en=0 read_uptime debug 1 "[up ${_RET}s]" "ds-identify $*" @@ -1364,8 +1375,9 @@ _main() { return fi - # if there is only a single entry in $DI_DSLIST + # shellcheck disable=2086 set -- $DI_DSLIST + # if there is only a single entry in $DI_DSLIST if [ $# -eq 1 ] || [ $# -eq 2 -a "$2" = "None" ] ; then debug 1 "single entry in datasource_list ($DI_DSLIST) use that." found "$@" @@ -1398,6 +1410,7 @@ _main() { done debug 2 "found=${found# } maybe=${maybe# }" + # shellcheck disable=2086 set -- $found if [ $# -ne 0 ]; then if [ $# -eq 1 ]; then @@ -1413,6 +1426,7 @@ _main() { return fi + # shellcheck disable=2086 set -- $maybe if [ $# -ne 0 -a "${DI_ON_MAYBE}" != "none" ]; then debug 1 "$# datasources returned maybe: $*" @@ -1441,7 +1455,7 @@ _main() { *) error "Unexpected result";; esac debug 1 "$msg" - return $ret + return "$ret" } main() { @@ -1452,7 +1466,7 @@ main() { if read ret < "$PATH_RUN_DI_RESULT"; then if [ "$ret" = "0" ] || [ "$ret" = "1" ]; then debug 2 "used cached result $ret. pass --force to re-run." - return $ret; + return "$ret"; fi debug 1 "previous run returned unexpected '$ret'. Re-running." else @@ -1464,7 +1478,7 @@ main() { echo "$ret" > "$PATH_RUN_DI_RESULT" read_uptime debug 1 "[up ${_RET}s]" "returning $ret" - return $ret + return "$ret" } noop() { -- cgit v1.2.3 From bde30070ec5f20aeb4d48cee8cf6c49b900ee311 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 9 May 2018 20:08:53 -0600 Subject: ds-identify: Remove dupe call to is_ds_enabled, improve debug message. We had two calls to is_ds_enabled, and the debug message looked something like this: is_ds_enabled returned 1: ConfigDrive NoCloud Now instead we have just one call, and the debug message like: is_ds_enabled(IBMCloud) = true --- tools/ds-identify | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'tools') diff --git a/tools/ds-identify b/tools/ds-identify index a062e4d7..435a5bcb 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -641,9 +641,11 @@ check_configdrive_v2() { return ${DS_FOUND} fi - is_ds_enabled "IBMCloud" - debug 1 "is_ds_enabled returned $?: $DI_DSLIST" - is_ds_enabled "IBMCloud" && is_ibm_cloud && return ${DS_NOT_FOUND} + local ibm_enabled=false + is_ds_enabled "IBMCloud" && ibm_enabled=true + debug 1 "is_ds_enabled(IBMCloud) = $ibm_enabled." + [ "$ibm_enabled" = "true" ] && is_ibm_cloud && return ${DS_NOT_FOUND} + if has_fs_with_label CONFIG-2 config-2; then return ${DS_FOUND} fi -- cgit v1.2.3 From 0d7ee5592621d09699d079945ffd6febf16669b2 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 15 May 2018 13:38:20 -0400 Subject: ds-identify: recognize container-other as a container, test SmartOS. In playing with a SmartOS container I found that ds-identify did not identify the container there as a container. Systemd-detect-virt identifies it as 'container-other'. Also here are tests for ds-identify for the SmartOS platform identification, and some indentation fixes in ds-identify. --- cloudinit/sources/DataSourceSmartOS.py | 4 +-- tests/unittests/test_ds_identify.py | 51 ++++++++++++++++++++++++++++++++-- tools/ds-identify | 12 ++++---- 3 files changed, 57 insertions(+), 10 deletions(-) (limited to 'tools') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 4ea00eb1..fcb46b14 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -745,7 +745,7 @@ def get_smartos_environ(uname_version=None, product_name=None): # report 'BrandZ virtual linux' as the kernel version if uname_version is None: uname_version = uname[3] - if uname_version.lower() == 'brandz virtual linux': + if uname_version == 'BrandZ virtual linux': return SMARTOS_ENV_LX_BRAND if product_name is None: @@ -753,7 +753,7 @@ def get_smartos_environ(uname_version=None, product_name=None): else: system_type = product_name - if system_type and 'smartdc' in system_type.lower(): + if system_type and system_type.startswith('SmartDC'): return SMARTOS_ENV_KVM return None diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py index 4d8a4360..7f12be59 100644 --- a/tests/unittests/test_ds_identify.py +++ b/tests/unittests/test_ds_identify.py @@ -10,7 +10,8 @@ from cloudinit import util from cloudinit.tests.helpers import ( CiTestCase, dir2dict, populate_dir, populate_dir_with_ts) -from cloudinit.sources import DataSourceIBMCloud as dsibm +from cloudinit.sources import DataSourceIBMCloud as ds_ibm +from cloudinit.sources import DataSourceSmartOS as ds_smartos UNAME_MYSYS = ("Linux bart 4.4.0-62-generic #83-Ubuntu " "SMP Wed Jan 18 14:10:15 UTC 2017 x86_64 GNU/Linux") @@ -69,8 +70,12 @@ P_DSID_CFG = "etc/cloud/ds-identify.cfg" IBM_CONFIG_UUID = "9796-932E" +MOCK_VIRT_IS_CONTAINER_OTHER = {'name': 'detect_virt', + 'RET': 'container-other', 'ret': 0} MOCK_VIRT_IS_KVM = {'name': 'detect_virt', 'RET': 'kvm', 'ret': 0} MOCK_VIRT_IS_VMWARE = {'name': 'detect_virt', 'RET': 'vmware', 'ret': 0} +# currenty' SmartOS hypervisor "bhyve" is unknown by systemd-detect-virt. +MOCK_VIRT_IS_VM_OTHER = {'name': 'detect_virt', 'RET': 'vm-other', 'ret': 0} MOCK_VIRT_IS_XEN = {'name': 'detect_virt', 'RET': 'xen', 'ret': 0} MOCK_UNAME_IS_PPC64 = {'name': 'uname', 'out': UNAME_PPC64EL, 'ret': 0} @@ -330,7 +335,7 @@ class TestDsIdentify(DsIdentifyBase): break if not offset: raise ValueError("Expected to find 'blkid' mock, but did not.") - data['mocks'][offset]['out'] = d['out'].replace(dsibm.IBM_CONFIG_UUID, + data['mocks'][offset]['out'] = d['out'].replace(ds_ibm.IBM_CONFIG_UUID, "DEAD-BEEF") self._check_via_dict( data, rc=RC_FOUND, dslist=['ConfigDrive', DS_NONE]) @@ -516,6 +521,20 @@ class TestDsIdentify(DsIdentifyBase): """Hetzner cloud is identified in sys_vendor.""" self._test_ds_found('Hetzner') + def test_smartos_bhyve(self): + """SmartOS cloud identified by SmartDC in dmi.""" + self._test_ds_found('SmartOS-bhyve') + + def test_smartos_lxbrand(self): + """SmartOS cloud identified on lxbrand container.""" + self._test_ds_found('SmartOS-lxbrand') + + def test_smartos_lxbrand_requires_socket(self): + """SmartOS cloud should not be identified if no socket file.""" + mycfg = copy.deepcopy(VALID_CFG['SmartOS-lxbrand']) + del mycfg['files'][ds_smartos.METADATA_SOCKFILE] + self._check_via_dict(mycfg, rc=RC_NOT_FOUND, policy_dmi="disabled") + class TestIsIBMProvisioning(DsIdentifyBase): """Test the is_ibm_provisioning method in ds-identify.""" @@ -777,7 +796,7 @@ VALID_CFG = { [{'DEVNAME': 'xvda1', 'TYPE': 'ext3', 'PARTUUID': uuid4(), 'UUID': uuid4(), 'LABEL': 'cloudimg-bootfs'}, {'DEVNAME': 'xvdb', 'TYPE': 'vfat', 'LABEL': 'config-2', - 'UUID': dsibm.IBM_CONFIG_UUID}, + 'UUID': ds_ibm.IBM_CONFIG_UUID}, {'DEVNAME': 'xvda2', 'TYPE': 'ext4', 'LABEL': 'cloudimg-rootfs', 'PARTUUID': uuid4(), 'UUID': uuid4()}, @@ -798,6 +817,32 @@ VALID_CFG = { }, ], }, + 'SmartOS-bhyve': { + 'ds': 'SmartOS', + 'mocks': [ + MOCK_VIRT_IS_VM_OTHER, + {'name': 'blkid', 'ret': 0, + 'out': blkid_out( + [{'DEVNAME': 'vda1', 'TYPE': 'ext4', + 'PARTUUID': '49ec635a-01'}, + {'DEVNAME': 'vda2', 'TYPE': 'swap', + 'LABEL': 'cloudimg-swap', 'PARTUUID': '49ec635a-02'}]), + }, + ], + 'files': {P_PRODUCT_NAME: 'SmartDC HVM\n'}, + }, + 'SmartOS-lxbrand': { + 'ds': 'SmartOS', + 'mocks': [ + MOCK_VIRT_IS_CONTAINER_OTHER, + {'name': 'uname', 'ret': 0, + 'out': ("Linux d43da87a-daca-60e8-e6d4-d2ed372662a3 4.3.0 " + "BrandZ virtual linux x86_64 GNU/Linux")}, + {'name': 'blkid', 'ret': 2, 'out': ''}, + ], + 'files': {ds_smartos.METADATA_SOCKFILE: 'would be a socket\n'}, + } + } # vi: ts=4 expandtab diff --git a/tools/ds-identify b/tools/ds-identify index 435a5bcb..dc7ac3a7 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -261,7 +261,7 @@ read_virt() { is_container() { case "${DI_VIRT}" in - lxc|lxc-libvirt|systemd-nspawn|docker|rkt) return 0;; + container-other|lxc|lxc-libvirt|systemd-nspawn|docker|rkt) return 0;; *) return 1;; esac } @@ -990,12 +990,14 @@ dscheck_SmartOS() { # joyent cloud has two virt types: kvm and container # on kvm, product name on joyent public cloud shows 'SmartDC HVM' # on the container platform, uname's version has: BrandZ virtual linux + # for container, we also verify that the socketfile exists to protect + # against embedded containers (lxd running on brandz) local smartdc_kver="BrandZ virtual linux" + local metadata_sockfile="${PATH_ROOT}/native/.zonecontrol/metadata.sock" dmi_product_name_matches "SmartDC*" && return $DS_FOUND - if [ "${DI_UNAME_KERNEL_VERSION}" = "${smartdc_kver}" ] && - [ "${DI_VIRT}" = "container-other" ]; then - return ${DS_FOUND} - fi + [ "${DI_UNAME_KERNEL_VERSION}" = "${smartdc_kver}" ] && + [ -e "${metadata_sockfile}" ] && + return ${DS_FOUND} return ${DS_NOT_FOUND} } -- cgit v1.2.3 From b4ae0e1fb8a48a83ea325cf032eb1acb196ee31c Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 22 May 2018 10:07:59 -0400 Subject: ds-identify: ensure that we have certain tokens in PATH. SuSE builds were not getting a PATH set in generator's environment. This may seem like mis-configuration on the system, but caused ds-identify to fail to find blkid (or any other program). The change here just ensures that we get /sbin /usr/sbin /bin /usr/bin into the PATH when main is run. LP: #1771382 --- tests/unittests/test_ds_identify.py | 23 ++++++++++++++++++++++- tools/ds-identify | 11 +++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) (limited to 'tools') diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py index 7f12be59..64d9f9f8 100644 --- a/tests/unittests/test_ds_identify.py +++ b/tests/unittests/test_ds_identify.py @@ -175,7 +175,9 @@ class DsIdentifyBase(CiTestCase): def _call_via_dict(self, data, rootd=None, **kwargs): # return output of self.call with a dict input like VALID_CFG[item] xwargs = {'rootd': rootd} - for k in ('mocks', 'args', 'policy_dmi', 'policy_no_dmi', 'files'): + passthrough = ('mocks', 'func', 'args', 'policy_dmi', + 'policy_no_dmi', 'files') + for k in passthrough: if k in data: xwargs[k] = data[k] if k in kwargs: @@ -535,6 +537,25 @@ class TestDsIdentify(DsIdentifyBase): del mycfg['files'][ds_smartos.METADATA_SOCKFILE] self._check_via_dict(mycfg, rc=RC_NOT_FOUND, policy_dmi="disabled") + def test_path_env_gets_set_from_main(self): + """PATH environment should always have some tokens when main is run. + + We explicitly call main as we want to ensure it updates PATH.""" + cust = copy.deepcopy(VALID_CFG['NoCloud']) + rootd = self.tmp_dir() + mpp = 'main-printpath' + pre = "MYPATH=" + cust['files'][mpp] = ( + 'PATH="/mycust/path"; main; r=$?; echo ' + pre + '$PATH; exit $r;') + ret = self._check_via_dict( + cust, RC_FOUND, + func=".", args=[os.path.join(rootd, mpp)], rootd=rootd) + line = [l for l in ret.stdout.splitlines() if l.startswith(pre)][0] + toks = line.replace(pre, "").split(":") + expected = ["/sbin", "/bin", "/usr/sbin", "/usr/bin", "/mycust/path"] + self.assertEqual(expected, [p for p in expected if p in toks], + "path did not have expected tokens") + class TestIsIBMProvisioning(DsIdentifyBase): """Test the is_ibm_provisioning method in ds-identify.""" diff --git a/tools/ds-identify b/tools/ds-identify index dc7ac3a7..ce0477a5 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -187,6 +187,16 @@ block_dev_with_label() { return 0 } +ensure_sane_path() { + local t + for t in /sbin /usr/sbin /bin /usr/bin; do + case ":$PATH:" in + *:$t:*|*:$t/:*) continue;; + esac + PATH="${PATH:+${PATH}:}$t" + done +} + read_fs_info() { cached "${DI_BLKID_OUTPUT}" && return 0 # do not rely on links in /dev/disk which might not be present yet. @@ -1464,6 +1474,7 @@ _main() { main() { local ret="" + ensure_sane_path [ -d "$PATH_RUN_CI" ] || mkdir -p "$PATH_RUN_CI" if [ "${1:+$1}" != "--force" ] && [ -f "$PATH_RUN_CI_CFG" ] && [ -f "$PATH_RUN_DI_RESULT" ]; then -- cgit v1.2.3 From a821cde1722b18ec54279a5fbfb5da96032ef95e Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 23 May 2018 12:31:48 -0400 Subject: tools/run-container: replace tools/run-centos with more generic. tools/run-container is like tools/run-centos, but currently supports the following images from lxc-images opensuse/42.3 centos/6 centos/7 ubuntu/16.04 debian/10 debian/sid Also here is to make installation via zypper in tools/read-dependencies not prompt user. --- tools/read-dependencies | 8 +- tools/run-centos | 340 +++------------------------- tools/run-container | 590 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 626 insertions(+), 312 deletions(-) create mode 100755 tools/run-container (limited to 'tools') diff --git a/tools/read-dependencies b/tools/read-dependencies index 421f470a..b4656e69 100755 --- a/tools/read-dependencies +++ b/tools/read-dependencies @@ -51,6 +51,10 @@ MAYBE_RELIABLE_YUM_INSTALL = [ """, 'reliable-yum-install'] +ZYPPER_INSTALL = [ + 'zypper', '--non-interactive', '--gpg-auto-import-keys', 'install', + '--auto-agree-with-licenses'] + DRY_DISTRO_INSTALL_PKG_CMD = { 'centos': ['yum', 'install', '--assumeyes'], 'redhat': ['yum', 'install', '--assumeyes'], @@ -61,8 +65,8 @@ DISTRO_INSTALL_PKG_CMD = { 'redhat': MAYBE_RELIABLE_YUM_INSTALL, 'debian': ['apt', 'install', '-y'], 'ubuntu': ['apt', 'install', '-y'], - 'opensuse': ['zypper', 'install'], - 'suse': ['zypper', 'install'] + 'opensuse': ZYPPER_INSTALL, + 'suse': ZYPPER_INSTALL, } diff --git a/tools/run-centos b/tools/run-centos index cb241ee5..4506b20d 100755 --- a/tools/run-centos +++ b/tools/run-centos @@ -1,18 +1,17 @@ #!/bin/bash # This file is part of cloud-init. See LICENSE file for license information. -set -u - -VERBOSITY=0 -TEMP_D="" -KEEP=false -CONTAINER="" - -error() { echo "$@" 1>&2; } -fail() { [ $# -eq 0 ] || error "$@"; exit 1; } -errorrc() { local r=$?; error "$@" "ret=$r"; return $r; } +deprecated() { +cat <&2; [ $# -eq 0 ] || error "$@"; return 1; } -cleanup() { - if [ -n "$CONTAINER" -a "$KEEP" = "false" ]; then - delete_container "$CONTAINER" - fi - [ -z "${TEMP_D}" -o ! -d "${TEMP_D}" ] || rm -Rf "${TEMP_D}" -} - -debug() { - local level=${1}; shift; - [ "${level}" -gt "${VERBOSITY}" ] && return - error "${@}" -} - - -inside_as() { - # inside_as(container_name, user, cmd[, args]) - # executes cmd with args inside container as user in users home dir. - local name="$1" user="$2" - shift 2 - if [ "$user" = "root" ]; then - inside "$name" "$@" - return - fi - local stuffed="" b64="" - stuffed=$(getopt --shell sh --options "" -- -- "$@") - stuffed=${stuffed# -- } - b64=$(printf "%s\n" "$stuffed" | base64 --wrap=0) - inside "$name" su "$user" -c \ - 'cd; eval set -- "$(echo '$b64' | base64 --decode)" && exec "$@"' -} - -inside_as_cd() { - local name="$1" user="$2" dir="$3" - shift 3 - inside_as "$name" "$user" sh -c 'cd "$0" && exec "$@"' "$dir" "$@" -} - -inside() { - local name="$1" - shift - lxc exec "$name" -- "$@" -} - -inject_cloud_init(){ - # take current cloud-init git dir and put it inside $name at - # ~$user/cloud-init. - local name="$1" user="$2" dirty="$3" - local changes="" top_d="" dname="cloud-init" pstat="" - local gitdir="" commitish="" - gitdir=$(git rev-parse --git-dir) || { - errorrc "Failed to get git dir in $PWD"; - return - } - local t=${gitdir%/*} - case "$t" in - */worktrees) - if [ -f "${t%worktrees}/config" ]; then - gitdir="${t%worktrees}" - fi - esac - - # attempt to get branch name. - commitish=$(git rev-parse --abbrev-ref HEAD) || { - errorrc "Failed git rev-parse --abbrev-ref HEAD" - return - } - if [ "$commitish" = "HEAD" ]; then - # detached head - commitish=$(git rev-parse HEAD) || { - errorrc "failed git rev-parse HEAD" - return - } - fi - - local local_changes=false - if ! git diff --quiet "$commitish"; then - # there are local changes not committed. - local_changes=true - if [ "$dirty" = "false" ]; then - error "WARNING: You had uncommitted changes. Those changes will " - error "be put into 'local-changes.diff' inside the container. " - error "To test these changes you must pass --dirty." - fi - fi - - debug 1 "collecting ${gitdir} ($dname) into user $user in $name." - tar -C "${gitdir}" -cpf - . | - inside_as "$name" "$user" sh -ec ' - dname=$1 - commitish=$2 - rm -Rf "$dname" - mkdir -p $dname/.git - cd $dname/.git - tar -xpf - - cd .. - git config core.bare false - out=$(git checkout $commitish 2>&1) || - { echo "failed git checkout $commitish: $out" 1>&2; exit 1; } - out=$(git checkout . 2>&1) || - { echo "failed git checkout .: $out" 1>&2; exit 1; } - ' extract "$dname" "$commitish" - [ "${PIPESTATUS[*]}" = "0 0" ] || { - error "Failed to push tarball of '$gitdir' into $name" \ - " for user $user (dname=$dname)" - return 1 - } - echo "local_changes=$local_changes dirty=$dirty" - if [ "$local_changes" = "true" ]; then - git diff "$commitish" | - inside_as "$name" "$user" sh -exc ' - cd "$1" - if [ "$2" = "true" ]; then - git apply - else - cat > local-changes.diff - fi - ' insert_changes "$dname" "$dirty" - [ "${PIPESTATUS[*]}" = "0 0" ] || { - error "Failed to apply local changes." - return 1 - } - fi - - return 0 -} - -prep() { - # we need some very basic things not present in the container. - # - git - # - tar (CentOS 6 lxc container does not have it) - # - python-argparse (or python3) - local needed="" pair="" pkg="" cmd="" needed="" - for pair in tar:tar git:git; do - pkg=${pair#*:} - cmd=${pair%%:*} - command -v $cmd >/dev/null 2>&1 || needed="${needed} $pkg" - done - if ! command -v python3; then - python -c "import argparse" >/dev/null 2>&1 || - needed="${needed} python-argparse" - fi - needed=${needed# } - if [ -z "$needed" ]; then - error "No prep packages needed" - return 0 +main() { + if [ "$1" = "-h" -o "$1" == "--help" ]; then + Usage 1>&2; + exit 0; fi - error "Installing prep packages: ${needed}" - set -- $needed - local n max r - n=0; max=10; - bcmd="yum install --downloadonly --assumeyes --setopt=keepcache=1" - while n=$(($n+1)); do - error ":: running $bcmd $* [$n/$max]" - $bcmd "$@" - r=$? - [ $r -eq 0 ] && break - [ $n -ge $max ] && { error "gave up on $bcmd"; exit $r; } - nap=$(($n*5)) - error ":: failed [$r] ($n/$max). sleeping $nap." - sleep $nap - done - error ":: running yum install --cacheonly --assumeyes $*" - yum install --cacheonly --assumeyes "$@" -} - -start_container() { - local src="$1" name="$2" - debug 1 "starting container $name from '$src'" - lxc launch "$src" "$name" || { - errorrc "Failed to start container '$name' from '$src'"; + local pt="" mydir=$(dirname "$0") + local run_container="$mydir/run-container" + if [ ! -x "$run_container" ]; then + bad_Usage "Could not find run-container." return - } - CONTAINER=$name - - local out="" ret="" - debug 1 "waiting for networking" - out=$(inside "$name" sh -c ' - i=0 - while [ $i -lt 60 ]; do - getent hosts mirrorlist.centos.org && exit 0 - sleep 2 - done' 2>&1) - ret=$? - if [ $ret -ne 0 ]; then - error "Waiting for network in container '$name' failed. [$ret]" - error "$out" - return $ret - fi - - if [ ! -z "${http_proxy-}" ]; then - debug 1 "configuring proxy ${http_proxy}" - inside "$name" sh -c "echo proxy=$http_proxy >> /etc/yum.conf" - inside "$name" sed -i s/enabled=1/enabled=0/ /etc/yum/pluginconf.d/fastestmirror.conf fi -} - -delete_container() { - debug 1 "removing container $1 [--keep to keep]" - lxc delete --force "$1" -} - -main() { - local short_opts="ahkrsuv" - local long_opts="artifact,dirty,help,keep,rpm,srpm,unittest,verbose" - local getopt_out="" - getopt_out=$(getopt --name "${0##*/}" \ - --options "${short_opts}" --long "${long_opts}" -- "$@") && - eval set -- "${getopt_out}" || - { bad_Usage; return; } - - local cur="" next="" - local artifact="" keep="" rpm="" srpm="" unittest="" version="" - local dirty=false - + + pt=( "$run_container" ) while [ $# -ne 0 ]; do cur="${1:-}"; next="${2:-}"; case "$cur" in - -a|--artifact) artifact=1;; - --dirty) dirty=true;; - -h|--help) Usage ; exit 0;; - -k|--keep) KEEP=true;; - -r|--rpm) rpm=1;; - -s|--srpm) srpm=1;; - -u|--unittest) unittest=1;; - -v|--verbose) VERBOSITY=$((${VERBOSITY}+1));; - --) shift; break;; + -r|--rpm) cur="--package";; + -s|--srpm) cur="--source-package";; + -a|--artifact) cur="--artifacts=.";; + 6|7) cur="centos/$cur";; esac + pt[${#pt[@]}]="$cur" shift; done - - [ $# -eq 1 ] || { bad_Usage "ERROR: Must provide version!"; return; } - version="$1" - case "$version" in - 6|7) :;; - *) error "Expected version of 6 or 7, not '$version'"; return;; - esac - - TEMP_D=$(mktemp -d "${TMPDIR:-/tmp}/${0##*/}.XXXXXX") || - fail "failed to make tempdir" - trap cleanup EXIT - - # program starts here - local uuid="" name="" user="ci-test" cdir="" - cdir="/home/$user/cloud-init" - uuid=$(uuidgen -t) || { error "no uuidgen"; return 1; } - name="cloud-init-centos-${uuid%%-*}" - - start_container "images:centos/$version" "$name" - - # prep the container (install very basic dependencies) - inside "$name" bash -s prep <"$0" || - { errorrc "Failed to prep container $name"; return; } - - # add the user - inside "$name" useradd "$user" - - debug 1 "inserting cloud-init" - inject_cloud_init "$name" "$user" "$dirty" || { - errorrc "FAIL: injecting cloud-init into $name failed." - return - } - - inside_as_cd "$name" root "$cdir" \ - ./tools/read-dependencies --distro=centos --test-distro || { - errorrc "FAIL: failed to install dependencies with read-dependencies" - return - } - - local errors=0 - inside_as_cd "$name" "$user" "$cdir" \ - sh -ec "git status" || - { errorrc "git checkout failed."; errors=$(($errors+1)); } - - if [ -n "$unittest" ]; then - debug 1 "running unit tests." - inside_as_cd "$name" "$user" "$cdir" \ - nosetests tests/unittests cloudinit || - { errorrc "nosetests failed."; errors=$(($errors+1)); } - fi - - if [ -n "$srpm" ]; then - debug 1 "building srpm." - inside_as_cd "$name" "$user" "$cdir" ./packages/brpm --srpm || - { errorrc "brpm --srpm."; errors=$(($errors+1)); } - fi - - if [ -n "$rpm" ]; then - debug 1 "building rpm." - inside_as_cd "$name" "$user" "$cdir" ./packages/brpm || - { errorrc "brpm failed."; errors=$(($errors+1)); } - fi - - if [ -n "$artifact" ]; then - for built_rpm in $(inside "$name" sh -c "echo $cdir/*.rpm"); do - lxc file pull "$name/$built_rpm" . - done - fi - - if [ "$errors" != "0" ]; then - error "there were $errors errors." - return 1 - fi - return 0 + deprecated + exec "${pt[@]}" } -if [ "${1:-}" = "prep" ]; then - shift - prep "$@" -else - main "$@" -fi +main "$@" + # vi: ts=4 expandtab diff --git a/tools/run-container b/tools/run-container new file mode 100755 index 00000000..499e85b0 --- /dev/null +++ b/tools/run-container @@ -0,0 +1,590 @@ +#!/bin/bash +# This file is part of cloud-init. See LICENSE file for license information. +# +# shellcheck disable=2015,2016,2039,2162,2166 + +set -u + +VERBOSITY=0 +KEEP=false +CONTAINER="" +DEFAULT_WAIT_MAX=30 + +error() { echo "$@" 1>&2; } +fail() { [ $# -eq 0 ] || error "$@"; exit 1; } +errorrc() { local r=$?; error "$@" "ret=$r"; return $r; } + +Usage() { + cat <&2; [ $# -eq 0 ] || error "$@"; return 1; } +cleanup() { + if [ -n "$CONTAINER" ]; then + if [ "$KEEP" = "true" ]; then + error "not deleting container '$CONTAINER' due to --keep" + else + delete_container "$CONTAINER" + fi + fi +} + +debug() { + local level=${1}; shift; + [ "${level}" -gt "${VERBOSITY}" ] && return + error "${@}" +} + + +inside_as() { + # inside_as(container_name, user, cmd[, args]) + # executes cmd with args inside container as user in users home dir. + local name="$1" user="$2" + shift 2 + if [ "$user" = "root" ]; then + inside "$name" "$@" + return + fi + local stuffed="" b64="" + stuffed=$(getopt --shell sh --options "" -- -- "$@") + stuffed=${stuffed# -- } + b64=$(printf "%s\n" "$stuffed" | base64 --wrap=0) + inside "$name" su "$user" -c \ + 'cd; eval set -- "$(echo '"$b64"' | base64 --decode)" && exec "$@"'; +} + +inside_as_cd() { + local name="$1" user="$2" dir="$3" + shift 3 + inside_as "$name" "$user" sh -c 'cd "$0" && exec "$@"' "$dir" "$@" +} + +inside() { + local name="$1" + shift + lxc exec "$name" -- "$@" +} + +inject_cloud_init(){ + # take current cloud-init git dir and put it inside $name at + # ~$user/cloud-init. + local name="$1" user="$2" dirty="$3" + local dname="cloud-init" gitdir="" commitish="" + gitdir=$(git rev-parse --git-dir) || { + errorrc "Failed to get git dir in $PWD"; + return + } + local t=${gitdir%/*} + case "$t" in + */worktrees) + if [ -f "${t%worktrees}/config" ]; then + gitdir="${t%worktrees}" + fi + esac + + # attempt to get branch name. + commitish=$(git rev-parse --abbrev-ref HEAD) || { + errorrc "Failed git rev-parse --abbrev-ref HEAD" + return + } + if [ "$commitish" = "HEAD" ]; then + # detached head + commitish=$(git rev-parse HEAD) || { + errorrc "failed git rev-parse HEAD" + return + } + fi + + local local_changes=false + if ! git diff --quiet "$commitish"; then + # there are local changes not committed. + local_changes=true + if [ "$dirty" = "false" ]; then + error "WARNING: You had uncommitted changes. Those changes will " + error "be put into 'local-changes.diff' inside the container. " + error "To test these changes you must pass --dirty." + fi + fi + + debug 1 "collecting ${gitdir} ($dname) into user $user in $name." + tar -C "${gitdir}" -cpf - . | + inside_as "$name" "$user" sh -ec ' + dname=$1 + commitish=$2 + rm -Rf "$dname" + mkdir -p $dname/.git + cd $dname/.git + tar -xpf - + cd .. + git config core.bare false + out=$(git checkout $commitish 2>&1) || + { echo "failed git checkout $commitish: $out" 1>&2; exit 1; } + out=$(git checkout . 2>&1) || + { echo "failed git checkout .: $out" 1>&2; exit 1; } + ' extract "$dname" "$commitish" + [ "${PIPESTATUS[*]}" = "0 0" ] || { + error "Failed to push tarball of '$gitdir' into $name" \ + " for user $user (dname=$dname)" + return 1 + } + + echo "local_changes=$local_changes dirty=$dirty" + if [ "$local_changes" = "true" ]; then + git diff "$commitish" | + inside_as "$name" "$user" sh -exc ' + cd "$1" + if [ "$2" = "true" ]; then + git apply + else + cat > local-changes.diff + fi + ' insert_changes "$dname" "$dirty" + [ "${PIPESTATUS[*]}" = "0 0" ] || { + error "Failed to apply local changes." + return 1 + } + fi + + return 0 +} + +get_os_info_in() { + # prep the container (install very basic dependencies) + [ -n "${OS_VERSION:-}" -a -n "${OS_NAME:-}" ] && return 0 + data=$(run_self_inside "$name" os_info) || + { errorrc "Failed to get os-info in container $name"; return; } + eval "$data" && [ -n "${OS_VERSION:-}" -a -n "${OS_NAME:-}" ] || return + debug 1 "determined $name is $OS_NAME/$OS_VERSION" +} + +os_info() { + get_os_info || return + echo "OS_NAME=$OS_NAME" + echo "OS_VERSION=$OS_VERSION" +} + +get_os_info() { + # run inside container, set OS_NAME, OS_VERSION + # example OS_NAME are centos, debian, opensuse + [ -n "${OS_NAME:-}" -a -n "${OS_VERSION:-}" ] && return 0 + if [ -f /etc/os-release ]; then + OS_NAME=$(sh -c '. /etc/os-release; echo $ID') + OS_VERSION=$(sh -c '. /etc/os-release; echo $VERSION_ID') + if [ -z "$OS_VERSION" ]; then + local pname="" + pname=$(sh -c '. /etc/os-release; echo $PRETTY_NAME') + case "$pname" in + *buster*) OS_VERSION=10;; + *sid*) OS_VERSION="sid";; + esac + fi + elif [ -f /etc/centos-release ]; then + local line="" + read line < /etc/centos-release + case "$line" in + CentOS\ *\ 6.*) OS_VERSION="6"; OS_NAME="centos";; + esac + fi + [ -n "${OS_NAME:-}" -a -n "${OS_VERSION:-}" ] || + { error "Unable to determine OS_NAME/OS_VERSION"; return 1; } +} + +yum_install() { + local n=0 max=10 ret + bcmd="yum install --downloadonly --assumeyes --setopt=keepcache=1" + while n=$((n+1)); do + error ":: running $bcmd $* [$n/$max]" + $bcmd "$@" + ret=$? + [ $ret -eq 0 ] && break + [ $n -ge $max ] && { error "gave up on $bcmd"; exit $ret; } + nap=$((n*5)) + error ":: failed [$ret] ($n/$max). sleeping $nap." + sleep $nap + done + error ":: running yum install --cacheonly --assumeyes $*" + yum install --cacheonly --assumeyes "$@" +} + +zypper_install() { + local pkgs="$*" + set -- zypper --non-interactive --gpg-auto-import-keys install \ + --auto-agree-with-licenses "$@" + debug 1 ":: installing $pkgs with zypper: $*" + "$@" +} + +apt_install() { + apt-get update -q && apt-get install --no-install-recommends "$@" +} + +install_packages() { + get_os_info || return + case "$OS_NAME" in + centos) yum_install "$@";; + opensuse) zypper_install "$@";; + debian|ubuntu) apt_install "$@";; + *) error "Do not know how to install packages on ${OS_NAME}"; + return 1;; + esac +} + +prep() { + # we need some very basic things not present in the container. + # - git + # - tar (CentOS 6 lxc container does not have it) + # - python-argparse (or python3) + local needed="" pair="" pkg="" cmd="" needed="" + local pairs="tar:tar git:git" + local pyexe="$1" + get_os_info + local py2pkg="python2" py3pkg="python3" + case "$OS_NAME" in + opensuse) + py2pkg="python-base" + py3pkg="python3-base";; + esac + + case "$pyexe" in + python2) pairs="$pairs python2:$py2pkg";; + python3) pairs="$pairs python3:$py3pkg";; + esac + + for pair in $pairs; do + pkg=${pair#*:} + cmd=${pair%%:*} + command -v "$cmd" >/dev/null 2>&1 || needed="${needed} $pkg" + done + if [ "$OS_NAME" = "centos" -a "$pyexe" = "python2" ]; then + python -c "import argparse" >/dev/null 2>&1 || + needed="${needed} python-argparse" + fi + needed=${needed# } + if [ -z "$needed" ]; then + error "No prep packages needed" + return 0 + fi + error "Installing prep packages: ${needed}" + # shellcheck disable=SC2086 + set -- $needed + install_packages "$@" +} + +nose() { + local pyexe="$1" cmd="" + shift + get_os_info + if [ "$OS_NAME/$OS_VERSION" = "centos/6" ]; then + cmd="nosetests" + else + cmd="$pyexe -m nose" + fi + ${cmd} "$@" +} + +is_done_cloudinit() { + [ -e "/run/cloud-init/result.json" ] + _RET="" +} + +is_done_systemd() { + local s="" num="$1" + s=$(systemctl is-system-running 2>&1); + _RET="$? $s" + case "$s" in + initializing|starting) return 1;; + *[Ff]ailed*connect*bus*) + # warn if not the first run. + [ "$num" -lt 5 ] || + error "Failed to connect to systemd bus [${_RET%% *}]"; + return 1;; + esac + return 0 +} + +is_done_other() { + local out="" + out=$(getent hosts ubuntu.com 2>&1) + return +} + +wait_inside() { + local name="$1" max="${2:-${DEFAULT_WAIT_MAX}}" debug=${3:-0} + local i=0 check="is_done_other"; + if [ -e /run/systemd ]; then + check=is_done_systemd + elif [ -x /usr/bin/cloud-init ]; then + check=is_done_cloudinit + fi + [ "$debug" != "0" ] && debug 1 "check=$check" + while ! $check $i && i=$((i+1)); do + [ "$i" -ge "$max" ] && exit 1 + [ "$debug" = "0" ] || echo -n . + sleep 1 + done + if [ "$debug" != "0" ]; then + read up _ > /etc/yum.conf" + inside "$name" sed -i s/enabled=1/enabled=0/ \ + /etc/yum/pluginconf.d/fastestmirror.conf + else + debug 1 "do not know how to configure proxy on $OS_NAME" + fi + fi +} + +start_container() { + local src="$1" name="$2" + debug 1 "starting container $name from '$src'" + lxc launch "$src" "$name" || { + errorrc "Failed to start container '$name' from '$src'"; + return + } + CONTAINER=$name + wait_for_boot "$name" +} + +delete_container() { + debug 1 "removing container $1 [--keep to keep]" + lxc delete --force "$1" +} + +run_self_inside() { + # run_self_inside(container, args) + local name="$1" + shift + inside "$name" bash -s "$@" <"$0" +} + +run_self_inside_as_cd() { + local name="$1" user="$2" dir="$3" + shift 3 + inside_as_cd "$name" "$user" "$dir" bash -s "$@" <"$0" +} + +main() { + local short_opts="a:hknpsuv" + local long_opts="artifacts:,dirty,help,keep,name:,pyexe:,package,source-package,unittest,verbose" + local getopt_out="" + getopt_out=$(getopt --name "${0##*/}" \ + --options "${short_opts}" --long "${long_opts}" -- "$@") && + eval set -- "${getopt_out}" || + { bad_Usage; return; } + + local cur="" next="" + local package="" source_package="" unittest="" name="" + local dirty=false pyexe="auto" artifact_d="." + + while [ $# -ne 0 ]; do + cur="${1:-}"; next="${2:-}"; + case "$cur" in + -a|--artifacts) artifact_d="$next";; + --dirty) dirty=true;; + -h|--help) Usage ; exit 0;; + -k|--keep) KEEP=true;; + -n|--name) name="$next"; shift;; + --pyexe) pyexe=$next; shift;; + -p|--package) package=1;; + -s|--source-package) source_package=1;; + -u|--unittest) unittest=1;; + -v|--verbose) VERBOSITY=$((VERBOSITY+1));; + --) shift; break;; + esac + shift; + done + + [ $# -eq 1 ] || { bad_Usage "Expected 1 arg, got $# ($*)"; return; } + local img_ref_in="$1" + case "${img_ref_in}" in + *:*) img_ref="${img_ref_in}";; + *) img_ref="images:${img_ref_in}";; + esac + + # program starts here + local out="" user="ci-test" cdir="" home="" + home="/home/$user" + cdir="$home/cloud-init" + if [ -z "$name" ]; then + if out=$(petname 2>&1); then + name="ci-${out}" + elif out=$(uuidgen -t 2>&1); then + name="ci-${out%%-*}" + else + error "Must provide name or have petname or uuidgen" + return 1 + fi + fi + + trap cleanup EXIT + + start_container "$img_ref" "$name" || + { errorrc "Failed to start container for $img_ref"; return; } + + get_os_info_in "$name" || + { errorrc "failed to get os_info in $name"; return; } + + if [ "$pyexe" = "auto" ]; then + case "$OS_NAME/$OS_VERSION" in + centos/*|opensuse/*) pyexe=python2;; + *) pyexe=python3;; + esac + debug 1 "set pyexe=$pyexe for $OS_NAME/$OS_VERSION" + fi + + # prep the container (install very basic dependencies) + run_self_inside "$name" prep "$pyexe" || + { errorrc "Failed to prep container $name"; return; } + + # add the user + inside "$name" useradd "$user" --create-home "--home-dir=$home" || + { errorrc "Failed to add user '$user' in '$name'"; return 1; } + + debug 1 "inserting cloud-init" + inject_cloud_init "$name" "$user" "$dirty" || { + errorrc "FAIL: injecting cloud-init into $name failed." + return + } + + inside_as_cd "$name" root "$cdir" \ + $pyexe ./tools/read-dependencies "--distro=${OS_NAME}" \ + --test-distro || { + errorrc "FAIL: failed to install dependencies with read-dependencies" + return + } + + local errors=( ) + inside_as_cd "$name" "$user" "$cdir" git status || { + errorrc "git checkout failed." + errors[${#errors[@]}]="git checkout"; + } + + if [ -n "$unittest" ]; then + debug 1 "running unit tests." + run_self_inside_as_cd "$name" "$user" "$cdir" nose "$pyexe" \ + tests/unittests cloudinit/ || { + errorrc "nosetests failed."; + errors[${#errors[@]}]="nosetests" + } + fi + + local build_pkg="" build_srcpkg="" pkg_ext="" distflag="" + case "$OS_NAME" in + centos) distflag="--distro=redhat";; + opensuse) distflag="--distro=suse";; + esac + + case "$OS_NAME" in + debian|ubuntu) + build_pkg="./packages/bddeb -d" + build_srcpkg="./packages/bddeb -S -d" + pkg_ext=".deb";; + centos|opensuse) + build_pkg="./packages/brpm $distflag" + build_srcpkg="./packages/brpm $distflag --srpm" + pkg_ext=".rpm";; + esac + if [ -n "$source_package" ]; then + [ -n "$build_pkg" ] || { + error "Unknown package command for $OS_NAME" + return 1 + } + debug 1 "building source package with $build_srcpkg." + # shellcheck disable=SC2086 + inside_as_cd "$name" "$user" "$cdir" $pyexe $build_srcpkg || { + errorrc "failed: $build_srcpkg"; + errors[${#errors[@]}]="source package" + } + fi + + if [ -n "$package" ]; then + [ -n "$build_srcpkg" ] || { + error "Unknown build source command for $OS_NAME" + return 1 + } + debug 1 "building binary package with $build_pkg." + inside_as_cd "$name" "$user" "$cdir" $pyexe $build_pkg || { + errorrc "failed: $build_pkg"; + errors[${#errors[@]}]="binary package" + } + fi + + if [ -n "$artifact_d" ]; then + local art="" + artifact_d="${artifact_d%/}/" + [ -d "${artifact_d}" ] || mkdir -p "$artifact_d" || { + errorrc "failed to create artifact dir '$artifact_d'" + return + } + + for art in $(inside "$name" sh -c "echo $cdir/*${pkg_ext}"); do + lxc file pull "$name/$art" "$artifact_d" || { + errorrc "Failed to pull '$name/$art' to ${artifact_d}" + errors[${#errors[@]}]="artifact copy: $art" + } + debug 1 "wrote ${artifact_d}${art##*/}" + done + fi + + if [ "${#errors[@]}" != "0" ]; then + local e="" + error "there were ${#errors[@]} errors." + for e in "${errors[@]}"; do + error " $e" + done + return 1 + fi + return 0 +} + +case "${1:-}" in + prep|os_info|wait_inside|nose) _n=$1; shift; "$_n" "$@";; + *) main "$@";; +esac + +# vi: ts=4 expandtab -- cgit v1.2.3