From a5dc0f425facf404344fb7baaf2b9136df143ecf Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 6 Dec 2017 17:26:52 -0700 Subject: OVF: improve ds-identify to support finding OVF iso transport. Previously the OVF transport would not be identified except for when config files set 'ovf_vmware_guest_customization'. It would also return DS_MAYBE almost always. The change here is to add support to ds-identify for storing the iso9660 filesystems that it finds (ISO9660_DEVS). Then the OVF check will check that the iso9660 filesystem has ovf-env.xml on it. The least wonderful part of this is that the check is done by 'grep' for case insensitive ovf-env.xml. Future improvement would be to identify VMware's OVF by label or UUID so we could avoid the grep. LP: #1731868 --- tools/ds-identify | 112 +++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 77 insertions(+), 35 deletions(-) (limited to 'tools/ds-identify') diff --git a/tools/ds-identify b/tools/ds-identify index ee5e05a4..4c59d7bc 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -83,6 +83,7 @@ _DI_LOGGED="" # set DI_MAIN='noop' in environment to source this file with no main called. DI_MAIN=${DI_MAIN:-main} +DI_BLKID_OUTPUT="" DI_DEFAULT_POLICY="search,found=all,maybe=all,notfound=${DI_DISABLED}" DI_DEFAULT_POLICY_NO_DMI="search,found=all,maybe=all,notfound=${DI_ENABLED}" DI_DMI_CHASSIS_ASSET_TAG="" @@ -91,6 +92,7 @@ DI_DMI_SYS_VENDOR="" DI_DMI_PRODUCT_SERIAL="" DI_DMI_PRODUCT_UUID="" DI_FS_LABELS="" +DI_ISO9660_DEVS="" DI_KERNEL_CMDLINE="" DI_VIRT="" DI_PID_1_PRODUCT_NAME="" @@ -181,32 +183,43 @@ block_dev_with_label() { return 0 } -read_fs_labels() { - cached "${DI_FS_LABELS}" && return 0 +read_fs_info() { + cached "${DI_BLKID_OUTPUT}" && return 0 # do not rely on links in /dev/disk which might not be present yet. # note that older blkid versions do not report DEVNAME in 'export' output. - local out="" ret=0 oifs="$IFS" line="" delim="," - local labels="" if is_container; then # blkid will in a container, or at least currently in lxd # not provide useful information. DI_FS_LABELS="$UNAVAILABLE:container" - else - out=$(blkid -c /dev/null -o export) || { - ret=$? - error "failed running [$ret]: blkid -c /dev/null -o export" - return $ret - } - IFS="$CR" - set -- $out - IFS="$oifs" - for line in "$@"; do - case "${line}" in - LABEL=*) labels="${labels}${line#LABEL=}${delim}";; - esac - done - DI_FS_LABELS="${labels%${delim}}" + DI_ISO9660_DEVS="$UNAVAILABLE:container" + return fi + local oifs="$IFS" line="" delim="," + local ret=0 out="" labels="" dev="" label="" ftype="" isodevs="" + out=$(blkid -c /dev/null -o export) || { + ret=$? + error "failed running [$ret]: blkid -c /dev/null -o export" + DI_FS_LABELS="$UNAVAILABLE:error" + DI_ISO9660_DEVS="$UNAVAILABLE:error" + return $ret + } + IFS="$CR" + set -- $out + IFS="$oifs" + for line in "$@" ""; do + case "${line}" in + DEVNAME=*) dev=${line#DEVNAME=};; + LABEL=*) label="${line#LABEL=}"; + labels="${labels}${line#LABEL=}${delim}";; + TYPE=*) ftype=${line#TYPE=};; + "") if [ "$ftype" = "iso9660" ]; then + isodevs="${isodevs} ${dev}=$label" + fi + ftype=""; devname=""; label=""; + esac + done + DI_FS_LABELS="${labels%${delim}}" + DI_ISO9660_DEVS="${isodevs# }" } cached() { @@ -214,10 +227,6 @@ cached() { } -has_cdrom() { - [ -e "${PATH_ROOT}/dev/cdrom" ] -} - detect_virt() { local virt="${UNAVAILABLE}" r="" out="" if [ -d /run/systemd ]; then @@ -621,14 +630,13 @@ ovf_vmware_guest_customization() { [ "${DI_VIRT}" = "vmware" ] || return 1 # we have to have the plugin to do vmware customization - local found="" pkg="" pre="/usr/lib" + local found="" pkg="" pre="${PATH_ROOT}/usr/lib" for pkg in vmware-tools open-vm-tools; do if [ -f "$pre/$pkg/plugins/vmsvc/libdeployPkgPlugin.so" ]; then found="$pkg"; break; fi done [ -n "$found" ] || return 1 - # vmware customization is disabled by default # (disable_vmware_customization=true). If it is set to false, then # user has requested customization. @@ -644,20 +652,55 @@ ovf_vmware_guest_customization() { return 1 } +is_cdrom_ovf() { + local dev="$1" label="$2" + # skip devices that don't look like cdrom paths. + case "$dev" in + /dev/sr[0-9]|/dev/hd[a-z]) :;; + *) debug 1 "skipping iso dev $d" + return 1;; + esac + + # fast path known 'OVF' labels + [ "$label" = "OVF-TRANSPORT" -o "$label" = "ovf-transport" ] && return 0 + + # explicitly skip known labels of other types. rd_rdfe is azure. + case "$label" in + config-2|rd_rdfe_stable*) return 1;; + esac + + local idstr="http://schemas.dmtf.org/ovf/environment/1" + grep --quiet --ignore-case "$idstr" "${PATH_ROOT}$dev" +} + dscheck_OVF() { - local p="" check_seed_dir ovf ovf-env.xml && return "${DS_FOUND}" + [ "${DI_VIRT}" = "none" ] && return ${DS_NOT_FOUND} + + # Azure provides ovf. Skip false positive by dis-allowing. + is_azure_chassis && return $DS_NOT_FOUND + + local isodevs="${DI_ISO9660_DEVS}" + case "$isodevs" in + ""|$UNAVAILABLE:*) return ${DS_NOT_FOUND};; + esac + + # DI_ISO9660_DEVS is =label, like /dev/sr0=OVF-TRANSPORT + for tok in $isodevs; do + is_cdrom_ovf "${tok%%=*}" "${tok#*=}" && return $DS_FOUND + done + if ovf_vmware_guest_customization; then return ${DS_FOUND} fi - has_cdrom || return ${DS_NOT_FOUND} + return ${DS_NOT_FOUND} +} - # FIXME: currently just return maybe if there is a cdrom - # ovf iso9660 transport does not specify an fs label. - # better would be to check if - return ${DS_MAYBE} +is_azure_chassis() { + local azure_chassis="7783-7084-3265-9085-8269-3286-77" + dmi_chassis_asset_tag_matches "${azure_chassis}" } dscheck_Azure() { @@ -667,8 +710,7 @@ dscheck_Azure() { # UUID="112D211272645f72" LABEL="rd_rdfe_stable.161212-1209" # TYPE="udf">/dev/sr0 # - local azure_chassis="7783-7084-3265-9085-8269-3286-77" - dmi_chassis_asset_tag_matches "${azure_chassis}" && return $DS_FOUND + is_azure_chassis && return $DS_FOUND check_seed_dir azure ovf-env.xml && return ${DS_FOUND} [ "${DI_VIRT}" = "microsoft" ] || return ${DS_NOT_FOUND} @@ -930,7 +972,7 @@ collect_info() { read_dmi_product_name read_dmi_product_serial read_dmi_product_uuid - read_fs_labels + read_fs_info } print_info() { @@ -942,7 +984,7 @@ _print_info() { local n="" v="" vars="" vars="DMI_PRODUCT_NAME DMI_SYS_VENDOR DMI_PRODUCT_SERIAL" vars="$vars DMI_PRODUCT_UUID PID_1_PRODUCT_NAME DMI_CHASSIS_ASSET_TAG" - vars="$vars FS_LABELS KERNEL_CMDLINE VIRT" + vars="$vars FS_LABELS ISO9660_DEVS KERNEL_CMDLINE VIRT" vars="$vars UNAME_KERNEL_NAME UNAME_KERNEL_RELEASE UNAME_KERNEL_VERSION" vars="$vars UNAME_MACHINE UNAME_NODENAME UNAME_OPERATING_SYSTEM" vars="$vars DSNAME DSLIST" -- cgit v1.2.3 From a30a3bb5baec4da1d8f91385849e9b5b826678bf Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 12 Dec 2017 11:41:26 -0700 Subject: ds-identify: failure in NoCloud due to unset variable usage. The previous OVF datasource change added a debug message that referenced an un-used variable. The failure path would be triggered if an image was booted with a iso9660 filesystem attached to a device that was not a cdrom. A unit test is added for the specific failure found. Additional safety to avoid 'cidata' labels is also added to the OVF checker. LP: #1737704 --- tests/unittests/test_ds_identify.py | 25 +++++++++++++++++++++++++ tools/ds-identify | 4 ++-- 2 files changed, 27 insertions(+), 2 deletions(-) (limited to 'tools/ds-identify') diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py index 3f1a6712..c9234edd 100644 --- a/tests/unittests/test_ds_identify.py +++ b/tests/unittests/test_ds_identify.py @@ -27,6 +27,14 @@ TYPE=ext4 PARTUUID=30c65c77-e07d-4039-b2fb-88b1fb5fa1fc """ +# this is a Ubuntu 18.04 disk.img output (dual uefi and bios bootable) +BLKID_UEFI_UBUNTU = [ + {'DEVNAME': 'vda1', 'TYPE': 'ext4', 'PARTUUID': uuid4(), 'UUID': uuid4()}, + {'DEVNAME': 'vda14', 'PARTUUID': uuid4()}, + {'DEVNAME': 'vda15', 'TYPE': 'vfat', 'LABEL': 'UEFI', 'PARTUUID': uuid4(), + 'UUID': '5F55-129B'}] + + POLICY_FOUND_ONLY = "search,found=all,maybe=none,notfound=disabled" POLICY_FOUND_OR_MAYBE = "search,found=all,maybe=all,notfound=disabled" DI_DEFAULT_POLICY = "search,found=all,maybe=all,notfound=enabled" @@ -340,6 +348,10 @@ class TestDsIdentify(CiTestCase): self._check_via_dict( ovf_cdrom_by_label, rc=RC_FOUND, dslist=['OVF', DS_NONE]) + def test_default_nocloud_as_vdb_iso9660(self): + """NoCloud is found with iso9660 filesystem on non-cdrom disk.""" + self._test_ds_found('NoCloud') + def blkid_out(disks=None): """Convert a list of disk dictionaries into blkid content.""" @@ -422,6 +434,19 @@ VALID_CFG = { 'files': {P_PRODUCT_SERIAL: 'GoogleCloud-8f2e88f\n'}, 'mocks': [MOCK_VIRT_IS_KVM], }, + 'NoCloud': { + 'ds': 'NoCloud', + 'mocks': [ + MOCK_VIRT_IS_KVM, + {'name': 'blkid', 'ret': 0, + 'out': blkid_out( + BLKID_UEFI_UBUNTU + + [{'DEVNAME': 'vdb', 'TYPE': 'iso9660', 'LABEL': 'cidata'}])}, + ], + 'files': { + 'dev/vdb': 'pretend iso content for cidata\n', + } + }, 'OpenStack': { 'ds': 'OpenStack', 'files': {P_PRODUCT_NAME: 'OpenStack Nova\n'}, diff --git a/tools/ds-identify b/tools/ds-identify index 4c59d7bc..5893a761 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -657,7 +657,7 @@ is_cdrom_ovf() { # skip devices that don't look like cdrom paths. case "$dev" in /dev/sr[0-9]|/dev/hd[a-z]) :;; - *) debug 1 "skipping iso dev $d" + *) debug 1 "skipping iso dev $dev" return 1;; esac @@ -666,7 +666,7 @@ is_cdrom_ovf() { # explicitly skip known labels of other types. rd_rdfe is azure. case "$label" in - config-2|rd_rdfe_stable*) return 1;; + config-2|rd_rdfe_stable*|cidata) return 1;; esac local idstr="http://schemas.dmtf.org/ovf/environment/1" -- cgit v1.2.3 From eb70975eaf37cf9549949f72e7647addb81a52ac Mon Sep 17 00:00:00 2001 From: James Penick Date: Tue, 23 Jan 2018 14:22:54 -0700 Subject: Recognize uppercase vfat disk labels New mkfs.vfat and fatlabel tools included in the dosfsutils package no longer support creating vfat disks with lowercase labels. They silently default to an all uppercase label eg CONFIG-2 instead of config-2. This change makes cloud-init handle either upper or lower case. LP: #1598783 --- cloudinit/sources/DataSourceConfigDrive.py | 4 ++-- tests/unittests/test_datasource/test_configdrive.py | 6 ++++++ tests/unittests/test_ds_identify.py | 17 +++++++++++++++++ tools/ds-identify | 4 +++- 4 files changed, 28 insertions(+), 3 deletions(-) (limited to 'tools/ds-identify') diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 870b3688..b8db6267 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -25,7 +25,7 @@ DEFAULT_METADATA = { "instance-id": DEFAULT_IID, } FS_TYPES = ('vfat', 'iso9660') -LABEL_TYPES = ('config-2',) +LABEL_TYPES = ('config-2', 'CONFIG-2') POSSIBLE_MOUNTS = ('sr', 'cd') OPTICAL_DEVICES = tuple(('/dev/%s%s' % (z, i) for z in POSSIBLE_MOUNTS for i in range(0, 2))) @@ -224,7 +224,7 @@ def find_candidate_devs(probe_optical=True): config drive v2: Disk should be: * either vfat or iso9660 formated - * labeled with 'config-2' + * labeled with 'config-2' or 'CONFIG-2' """ # query optical drive to get it in blkid cache for 2.6 kernels if probe_optical: diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index 6ef5a35c..68400f22 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -458,6 +458,12 @@ class TestConfigDriveDataSource(CiTestCase): self.assertEqual(["/dev/vdb3"], ds.find_candidate_devs()) + # Verify that uppercase labels are also found. + devs_with_answers = {"TYPE=vfat": [], + "TYPE=iso9660": ["/dev/vdb"], + "LABEL=CONFIG-2": ["/dev/vdb"]} + self.assertEqual(["/dev/vdb"], ds.find_candidate_devs()) + finally: util.find_devs_with = orig_find_devs_with util.is_partition = orig_is_partition diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py index c9234edd..ad6c5cf4 100644 --- a/tests/unittests/test_ds_identify.py +++ b/tests/unittests/test_ds_identify.py @@ -232,6 +232,11 @@ class TestDsIdentify(CiTestCase): self._test_ds_found('ConfigDrive') return + def test_config_drive_upper(self): + """ConfigDrive datasource has a disk with LABEL=CONFIG-2.""" + self._test_ds_found('ConfigDriveUpper') + return + def test_policy_disabled(self): """A Builtin policy of 'disabled' should return not found. @@ -503,6 +508,18 @@ VALID_CFG = { }, ], }, + 'ConfigDriveUpper': { + 'ds': 'ConfigDrive', + 'mocks': [ + {'name': 'blkid', 'ret': 0, + 'out': blkid_out( + [{'DEVNAME': 'vda1', 'TYPE': 'vfat', 'PARTUUID': uuid4()}, + {'DEVNAME': 'vda2', 'TYPE': 'ext4', + 'LABEL': 'cloudimg-rootfs', 'PARTUUID': uuid4()}, + {'DEVNAME': 'vdb', 'TYPE': 'vfat', 'LABEL': 'CONFIG-2'}]) + }, + ], + }, } # vi: ts=4 expandtab diff --git a/tools/ds-identify b/tools/ds-identify index 5893a761..374c3ad1 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -579,6 +579,8 @@ dscheck_NoCloud() { check_configdrive_v2() { if has_fs_with_label "config-2"; then return ${DS_FOUND} + elif has_fs_with_label "CONFIG-2"; then + return ${DS_FOUND} fi # look in /config-drive /seed/config_drive for a directory # openstack/YYYY-MM-DD format with a file meta_data.json @@ -666,7 +668,7 @@ is_cdrom_ovf() { # explicitly skip known labels of other types. rd_rdfe is azure. case "$label" in - config-2|rd_rdfe_stable*|cidata) return 1;; + config-2|CONFIG-2|rd_rdfe_stable*|cidata) return 1;; esac local idstr="http://schemas.dmtf.org/ovf/environment/1" -- cgit v1.2.3 From 5e5dc9731f39e8b1df767fbaf850fbbd31355a1a Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 25 Jan 2018 12:51:03 -0500 Subject: OVF: Extend well-known labels to include OVFENV. Fujitsu Cloud Service attaches a ovf iso transport with a label 'OVFENV'. This seems to be a reasonable value as a label. While the for bug 1731868 would likely fix cloud-init on fujitsu cloud, this change will find it faster. LP: #1698669 --- tests/unittests/test_ds_identify.py | 8 +++++--- tools/ds-identify | 4 +++- 2 files changed, 8 insertions(+), 4 deletions(-) (limited to 'tools/ds-identify') diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py index ad6c5cf4..31cc6223 100644 --- a/tests/unittests/test_ds_identify.py +++ b/tests/unittests/test_ds_identify.py @@ -338,7 +338,7 @@ class TestDsIdentify(CiTestCase): self._test_ds_found('OVF-vmware-customization') def test_ovf_on_vmware_iso_found_by_cdrom_with_matching_fs_label(self): - """OVF is identified when iso9660 cdrom label has ovf-transport.""" + """OVF is identified by well-known iso9660 labels.""" ovf_cdrom_by_label = copy.deepcopy(VALID_CFG['OVF']) # Unset matching cdrom ovf schema content ovf_cdrom_by_label['files']['dev/sr0'] = 'No content match' @@ -346,10 +346,12 @@ class TestDsIdentify(CiTestCase): ovf_cdrom_by_label, rc=RC_NOT_FOUND, policy_dmi="disabled") # Add recognized labels - for valid_fs_label in ['ovf-transport', 'OVF-TRANSPORT']: + valid_ovf_labels = ['ovf-transport', 'OVF-TRANSPORT', + "OVFENV", "ovfenv"] + for valid_ovf_label in valid_ovf_labels: ovf_cdrom_by_label['mocks'][0]['out'] = blkid_out([ {'DEVNAME': 'sr0', 'TYPE': 'iso9660', - 'LABEL': valid_fs_label}]) + 'LABEL': valid_ovf_label}]) self._check_via_dict( ovf_cdrom_by_label, rc=RC_FOUND, dslist=['OVF', DS_NONE]) diff --git a/tools/ds-identify b/tools/ds-identify index 374c3ad1..cd268242 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -664,7 +664,9 @@ is_cdrom_ovf() { esac # fast path known 'OVF' labels - [ "$label" = "OVF-TRANSPORT" -o "$label" = "ovf-transport" ] && return 0 + case "$label" in + OVF-TRANSPORT|ovf-transport|OVFENV|ovfenv) return 0;; + esac # explicitly skip known labels of other types. rd_rdfe is azure. case "$label" in -- cgit v1.2.3