From 0a71d5a870b416f2c86c8bc196004bb3fc0768a0 Mon Sep 17 00:00:00 2001 From: Hongjiang Zhang Date: Fri, 13 Jan 2017 15:08:22 +0800 Subject: FreeBSD: improvements and fixes for use on Azure This patch targets to make FreeBSD 10.3 or 11 work on Azure. The modifications abide by the rule of: * making as less modification as possible * delegate to the distro or datasource where possible. The main modifications are: 1. network configuration improvements, and movement into distro path. 2. Fix setting of password. Password setting through "pw" can only work through pipe. 3. Add 'root:wheel' to syslog_fix_perms field. 4. Support resizing default file system (ufs) 5. copy cloud.cfg for freebsd to /etc/cloud/cloud.cfg rather than /usr/local/etc/cloud/cloud.cfg. 6. Azure specific changes: a. When reading the azure endpoint, search in a different path and read a different option name (option-245 vs. unknown-245). so, the lease file path should be generated according to platform. b. adjust the handling of ephemeral mounts for ufs filesystem and for finding the ephemeral device. c. fix mounting of cdrom LP: #1636345 --- cloudinit/util.py | 52 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) (limited to 'cloudinit/util.py') diff --git a/cloudinit/util.py b/cloudinit/util.py index 22af99dd..27a98330 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -565,6 +565,10 @@ def is_ipv4(instr): return len(toks) == 4 +def is_FreeBSD(): + return system_info()['platform'].startswith('FreeBSD') + + def get_cfg_option_bool(yobj, key, default=False): if key not in yobj: return default @@ -2055,11 +2059,56 @@ def parse_mtab(path): return None +def find_freebsd_part(label_part): + if label_part.startswith("/dev/label/"): + target_label = label_part[5:] + (label_part, err) = subp(['glabel', 'status', '-s']) + for labels in label_part.split("\n"): + items = labels.split() + if len(items) > 0 and items[0].startswith(target_label): + label_part = items[2] + break + label_part = str(label_part) + return label_part + + +def get_path_dev_freebsd(path, mnt_list): + path_found = None + for line in mnt_list.split("\n"): + items = line.split() + if (len(items) > 2 and os.path.exists(items[1] + path)): + path_found = line + break + return path_found + + +def get_mount_info_freebsd(path, log=LOG): + (result, err) = subp(['mount', '-p', path], rcs=[0, 1]) + if len(err): + # find a path if the input is not a mounting point + (mnt_list, err) = subp(['mount', '-p']) + path_found = get_path_dev_freebsd(path, mnt_list) + if (path_found is None): + return None + result = path_found + ret = result.split() + label_part = find_freebsd_part(ret[0]) + return "/dev/" + label_part, ret[2], ret[1] + + def parse_mount(path): (mountoutput, _err) = subp("mount") mount_locs = mountoutput.splitlines() for line in mount_locs: m = re.search(r'^(/dev/[\S]+) on (/.*) \((.+), .+, (.+)\)$', line) + if not m: + continue + # check whether the dev refers to a label on FreeBSD + # for example, if dev is '/dev/label/rootfs', we should + # continue finding the real device like '/dev/da0'. + devm = re.search('^(/dev/.+)p([0-9])$', m.group(1)) + if (not devm and is_FreeBSD()): + return get_mount_info_freebsd(path) devpth = m.group(1) mount_point = m.group(2) fs_type = m.group(3) @@ -2336,7 +2385,8 @@ def read_dmi_data(key): uname_arch = os.uname()[4] if not (uname_arch == "x86_64" or (uname_arch.startswith("i") and uname_arch[2:] == "86") or - uname_arch == 'aarch64'): + uname_arch == 'aarch64' or + uname_arch == 'amd64'): LOG.debug("dmidata is not supported on %s", uname_arch) return None -- cgit v1.2.3 From 4bcc947301bedc5ebf430cfaf6e4597bfb174aa7 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 16 May 2017 13:43:55 -0400 Subject: Improve detection of snappy to include os-release and kernel cmdline. Recent core snap images (edge channel revision 1886) do not contain the previously known files used to detect that a system is ubuntu core. The changes here are to look in 2 additional locations to determine if a system is snappy. LP: #1689944 --- cloudinit/net/cmdline.py | 31 +------------------- cloudinit/util.py | 37 ++++++++++++++++++++++++ tests/unittests/helpers.py | 7 ++--- tests/unittests/test_util.py | 69 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 110 insertions(+), 34 deletions(-) (limited to 'cloudinit/util.py') diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py index 61e23697..38b27a52 100755 --- a/cloudinit/net/cmdline.py +++ b/cloudinit/net/cmdline.py @@ -9,41 +9,12 @@ import base64 import glob import gzip import io -import shlex -import sys - -import six from . import get_devicelist from . import read_sys_net_safe from cloudinit import util -PY26 = sys.version_info[0:2] == (2, 6) - - -def _shlex_split(blob): - if PY26 and isinstance(blob, six.text_type): - # Older versions don't support unicode input - blob = blob.encode("utf8") - return shlex.split(blob) - - -def _load_shell_content(content, add_empty=False, empty_val=None): - """Given shell like syntax (key=value\nkey2=value2\n) in content - return the data in dictionary form. If 'add_empty' is True - then add entries in to the returned dictionary for 'VAR=' - variables. Set their value to empty_val.""" - data = {} - for line in _shlex_split(content): - key, value = line.split("=", 1) - if not value: - value = empty_val - if add_empty or value: - data[key] = value - - return data - def _klibc_to_config_entry(content, mac_addrs=None): """Convert a klibc written shell content file to a 'config' entry @@ -63,7 +34,7 @@ def _klibc_to_config_entry(content, mac_addrs=None): if mac_addrs is None: mac_addrs = {} - data = _load_shell_content(content) + data = util.load_shell_content(content) try: name = data['DEVICE'] if 'DEVICE' in data else data['DEVICE6'] except KeyError: diff --git a/cloudinit/util.py b/cloudinit/util.py index 27a98330..67ff7ba3 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -24,6 +24,7 @@ import platform import pwd import random import re +import shlex import shutil import socket import stat @@ -75,6 +76,7 @@ CONTAINER_TESTS = (['systemd-detect-virt', '--quiet', '--container'], PROC_CMDLINE = None _LSB_RELEASE = {} +PY26 = sys.version_info[0:2] == (2, 6) def get_architecture(target=None): @@ -2424,6 +2426,18 @@ def system_is_snappy(): # channel.ini is configparser loadable. # snappy will move to using /etc/system-image/config.d/*.ini # this is certainly not a perfect test, but good enough for now. + orpath = "/etc/os-release" + try: + orinfo = load_shell_content(load_file(orpath, quiet=True)) + if orinfo.get('ID', '').lower() == "ubuntu-core": + return True + except ValueError as e: + LOG.warning("Unexpected error loading '%s': %s", orpath, e) + + cmdline = get_cmdline() + if 'snap_core=' in cmdline: + return True + content = load_file("/etc/system-image/channel.ini", quiet=True) if 'ubuntu-core' in content.lower(): return True @@ -2470,4 +2484,27 @@ def rootdev_from_cmdline(cmdline): return "/dev/" + found +def load_shell_content(content, add_empty=False, empty_val=None): + """Given shell like syntax (key=value\nkey2=value2\n) in content + return the data in dictionary form. If 'add_empty' is True + then add entries in to the returned dictionary for 'VAR=' + variables. Set their value to empty_val.""" + + def _shlex_split(blob): + if PY26 and isinstance(blob, six.text_type): + # Older versions don't support unicode input + blob = blob.encode("utf8") + return shlex.split(blob) + + data = {} + for line in _shlex_split(content): + key, value = line.split("=", 1) + if not value: + value = empty_val + if add_empty or value: + data[key] = value + + return data + + # vi: ts=4 expandtab diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index a711404c..d24f817d 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -106,7 +106,7 @@ class CiTestCase(TestCase): return os.path.normpath(os.path.abspath(os.path.join(dir, path))) -class ResourceUsingTestCase(TestCase): +class ResourceUsingTestCase(CiTestCase): def setUp(self): super(ResourceUsingTestCase, self).setUp() self.resource_path = None @@ -229,8 +229,7 @@ class FilesystemMockingTestCase(ResourceUsingTestCase): def reRoot(self, root=None): if root is None: - root = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, root) + root = self.tmp_dir() self.patchUtils(root) self.patchOS(root) return root @@ -256,7 +255,7 @@ def populate_dir(path, files): os.makedirs(path) ret = [] for (name, content) in files.items(): - p = os.path.join(path, name) + p = os.path.sep.join([path, name]) util.ensure_dir(os.path.dirname(p)) with open(p, "wb") as fp: if isinstance(content, six.binary_type): diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 189caca8..490760d1 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -712,4 +712,73 @@ class TestProcessExecutionError(helpers.TestCase): )).format(description=self.empty_description, empty_attr=self.empty_attr)) + +class TestSystemIsSnappy(helpers.FilesystemMockingTestCase): + def test_id_in_os_release_quoted(self): + """os-release containing ID="ubuntu-core" is snappy.""" + orcontent = '\n'.join(['ID="ubuntu-core"', '']) + root_d = self.tmp_dir() + helpers.populate_dir(root_d, {'etc/os-release': orcontent}) + self.reRoot(root_d) + self.assertTrue(util.system_is_snappy()) + + def test_id_in_os_release(self): + """os-release containing ID=ubuntu-core is snappy.""" + orcontent = '\n'.join(['ID=ubuntu-core', '']) + root_d = self.tmp_dir() + helpers.populate_dir(root_d, {'etc/os-release': orcontent}) + self.reRoot(root_d) + self.assertTrue(util.system_is_snappy()) + + @mock.patch('cloudinit.util.get_cmdline') + def test_bad_content_in_os_release_no_effect(self, m_cmdline): + """malformed os-release should not raise exception.""" + m_cmdline.return_value = 'root=/dev/sda' + orcontent = '\n'.join(['IDubuntu-core', '']) + root_d = self.tmp_dir() + helpers.populate_dir(root_d, {'etc/os-release': orcontent}) + self.reRoot() + self.assertFalse(util.system_is_snappy()) + + @mock.patch('cloudinit.util.get_cmdline') + def test_snap_core_in_cmdline_is_snappy(self, m_cmdline): + """The string snap_core= in kernel cmdline indicates snappy.""" + cmdline = ( + "BOOT_IMAGE=(loop)/kernel.img root=LABEL=writable " + "snap_core=core_x1.snap snap_kernel=pc-kernel_x1.snap ro " + "net.ifnames=0 init=/lib/systemd/systemd console=tty1 " + "console=ttyS0 panic=-1") + m_cmdline.return_value = cmdline + self.assertTrue(util.system_is_snappy()) + self.assertTrue(m_cmdline.call_count > 0) + + @mock.patch('cloudinit.util.get_cmdline') + def test_nothing_found_is_not_snappy(self, m_cmdline): + """If no positive identification, then not snappy.""" + m_cmdline.return_value = 'root=/dev/sda' + self.reRoot() + self.assertFalse(util.system_is_snappy()) + self.assertTrue(m_cmdline.call_count > 0) + + @mock.patch('cloudinit.util.get_cmdline') + def test_channel_ini_with_snappy_is_snappy(self, m_cmdline): + """A Channel.ini file with 'ubuntu-core' indicates snappy.""" + m_cmdline.return_value = 'root=/dev/sda' + root_d = self.tmp_dir() + content = '\n'.join(["[Foo]", "source = 'ubuntu-core'", ""]) + helpers.populate_dir( + root_d, {'etc/system-image/channel.ini': content}) + self.reRoot(root_d) + self.assertTrue(util.system_is_snappy()) + + @mock.patch('cloudinit.util.get_cmdline') + def test_system_image_config_dir_is_snappy(self, m_cmdline): + """Existence of /etc/system-image/config.d indicates snappy.""" + m_cmdline.return_value = 'root=/dev/sda' + root_d = self.tmp_dir() + helpers.populate_dir( + root_d, {'etc/system-image/config.d/my.file': "_unused"}) + self.reRoot(root_d) + self.assertTrue(util.system_is_snappy()) + # vi: ts=4 expandtab -- cgit v1.2.3 From 9c33cb24b8a834aa8034db4c989725901a0814c6 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 22 May 2017 10:17:38 -0400 Subject: tox: move pylint target to 1.7.1 The motivation for this is to make tip-pylint target green. It does 2 things: a.) silence a warning that is generated in pylint 1.7.1, but not other versions of pylint. This bug in pylint is filed at https://github.com/PyCQA/pylint/issues/1444 b.) move tox -e pylint to use pylint 1.7.1 --- cloudinit/util.py | 5 +++-- tox.ini | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) (limited to 'cloudinit/util.py') diff --git a/cloudinit/util.py b/cloudinit/util.py index 67ff7ba3..135e4608 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -478,10 +478,11 @@ def decomp_gzip(data, quiet=True, decode=True): try: buf = six.BytesIO(encode_text(data)) with contextlib.closing(gzip.GzipFile(None, "rb", 1, buf)) as gh: + # E1101 is https://github.com/PyCQA/pylint/issues/1444 if decode: - return decode_binary(gh.read()) + return decode_binary(gh.read()) # pylint: disable=E1101 else: - return gh.read() + return gh.read() # pylint: disable=E1101 except Exception as e: if quiet: return data diff --git a/tox.ini b/tox.ini index 826f554e..fce07740 100644 --- a/tox.ini +++ b/tox.ini @@ -18,7 +18,7 @@ setenv = LC_ALL = en_US.utf-8 [testenv:pylint] -deps = pylint==1.6.5 +deps = pylint==1.7.1 commands = {envpython} -m pylint {posargs:cloudinit} [testenv:py3] -- cgit v1.2.3