From 0a4c7983613a134fa62b8ee0c11b558f9e405346 Mon Sep 17 00:00:00 2001 From: Alex Sirbu Date: Mon, 7 Mar 2016 09:13:17 +0000 Subject: Enable Bigstep data source in default configuration --- cloudinit/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudinit/settings.py b/cloudinit/settings.py index b61e5613..8c258ea1 100644 --- a/cloudinit/settings.py +++ b/cloudinit/settings.py @@ -42,6 +42,7 @@ CFG_BUILTIN = { 'CloudSigma', 'CloudStack', 'SmartOS', + 'Bigstep', # At the end to act as a 'catch' when none of the above work... 'None', ], -- cgit v1.2.3 From 9ec6c876b72ccfa2ae590505fe6dbf7c0c561520 Mon Sep 17 00:00:00 2001 From: Alex Sirbu Date: Mon, 7 Mar 2016 09:33:40 +0000 Subject: Returning false if file does not exist, instead of throwing error --- cloudinit/sources/DataSourceBigstep.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cloudinit/sources/DataSourceBigstep.py b/cloudinit/sources/DataSourceBigstep.py index c22ffdb6..2d66c609 100644 --- a/cloudinit/sources/DataSourceBigstep.py +++ b/cloudinit/sources/DataSourceBigstep.py @@ -5,6 +5,7 @@ # import json +import errno from cloudinit import log as logging from cloudinit import sources @@ -22,7 +23,13 @@ class DataSourceBigstep(sources.DataSource): self.userdata_raw = "" def get_data(self, apply_filter=False): - url = get_url_from_file() + try: + url = get_url_from_file() + except IOError as e: + if e.errno == errno.ENOENT: + return False + else: + raise response = url_helper.readurl(url) decoded = json.loads(response.contents) self.metadata = decoded["metadata"] -- cgit v1.2.3 From d23868d6d3e35a91c348b94ce8416f56514aaf15 Mon Sep 17 00:00:00 2001 From: Alex Sirbu Date: Mon, 7 Mar 2016 12:30:08 +0000 Subject: Implemented review concerning position of try and more information about the caught exception. --- cloudinit/sources/DataSourceBigstep.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/cloudinit/sources/DataSourceBigstep.py b/cloudinit/sources/DataSourceBigstep.py index 2d66c609..b5ee4129 100644 --- a/cloudinit/sources/DataSourceBigstep.py +++ b/cloudinit/sources/DataSourceBigstep.py @@ -23,13 +23,9 @@ class DataSourceBigstep(sources.DataSource): self.userdata_raw = "" def get_data(self, apply_filter=False): - try: - url = get_url_from_file() - except IOError as e: - if e.errno == errno.ENOENT: - return False - else: - raise + url = get_url_from_file() + if url is None: + return False response = url_helper.readurl(url) decoded = json.loads(response.contents) self.metadata = decoded["metadata"] @@ -39,7 +35,15 @@ class DataSourceBigstep(sources.DataSource): def get_url_from_file(): - content = util.load_file("/var/lib/cloud/data/seed/bigstep/url") + try: + content = util.load_file("/var/lib/cloud/data/seed/bigstep/url") + except IOError as e: + # If the file doesn't exist, then the server probably isn't a Bigstep + # instance; otherwise, another problem exists which needs investigation + if e.errno == errno.ENOENT: + return None + else: + raise return content # Used to match classes to dependencies -- cgit v1.2.3 From 6e31038b9cccbcb4a33693060b96fc4f71d86789 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 7 Mar 2016 21:31:25 -0500 Subject: No longer run pollinate by default in seed_random The user can still choose to run pollinate here to seed their random data. And in an environment with network datasource, that would be expected to work. However, we do not want to run it any more from cloud-init because a.) pollinate's own init system jobs should get it ran before ssh, which is the primary purpose of wanting cloud-init to run it. b.) with a local datasource, there is no network guarantee when init_modules run, so pollinate -q would often cause issues then. c.) cloud-init would run pollinate and log the failure causing many cloud-init specific failures that it could do nothing about. LP: #1554152 --- ChangeLog | 1 + cloudinit/config/cc_seed_random.py | 2 +- tests/unittests/test_handler/test_handler_seed_random.py | 14 ++++++++------ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/ChangeLog b/ChangeLog index a80a5d5f..6da276b5 100644 --- a/ChangeLog +++ b/ChangeLog @@ -85,6 +85,7 @@ unless it is already a file (LP: #1543025). - Enable password changing via a hashed string [Alex Sirbu] - Added BigStep datasource [Alex Sirbu] + - No longer run pollinate in seed_random (LP: #1554152) 0.7.6: - open 0.7.6 diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py index 3288a853..1b011216 100644 --- a/cloudinit/config/cc_seed_random.py +++ b/cloudinit/config/cc_seed_random.py @@ -83,7 +83,7 @@ def handle(name, cfg, cloud, log, _args): len(seed_data), seed_path) util.append_file(seed_path, seed_data) - command = mycfg.get('command', ['pollinate', '-q']) + command = mycfg.get('command', None) req = mycfg.get('command_required', False) try: env = os.environ.copy() diff --git a/tests/unittests/test_handler/test_handler_seed_random.py b/tests/unittests/test_handler/test_handler_seed_random.py index 34d11f21..98bc9b81 100644 --- a/tests/unittests/test_handler/test_handler_seed_random.py +++ b/tests/unittests/test_handler/test_handler_seed_random.py @@ -170,28 +170,30 @@ class TestRandomSeed(t_help.TestCase): contents = util.load_file(self._seed_file) self.assertEquals('tiny-tim-was-here-so-was-josh', contents) - def test_seed_command_not_provided_pollinate_available(self): + def test_seed_command_provided_and_available(self): c = self._get_cloud('ubuntu', {}) self.whichdata = {'pollinate': '/usr/bin/pollinate'} - cc_seed_random.handle('test', {}, c, LOG, []) + cfg = {'random_seed': {'command': ['pollinate', '-q']}} + cc_seed_random.handle('test', cfg, c, LOG, []) subp_args = [f['args'] for f in self.subp_called] self.assertIn(['pollinate', '-q'], subp_args) - def test_seed_command_not_provided_pollinate_not_available(self): + def test_seed_command_not_provided(self): c = self._get_cloud('ubuntu', {}) self.whichdata = {} cc_seed_random.handle('test', {}, c, LOG, []) # subp should not have been called as which would say not available - self.assertEquals(self.subp_called, list()) + self.assertFalse(self.subp_called) def test_unavailable_seed_command_and_required_raises_error(self): c = self._get_cloud('ubuntu', {}) self.whichdata = {} + cfg = {'random_seed': {'command': ['THIS_NO_COMMAND'], + 'command_required': True}} self.assertRaises(ValueError, cc_seed_random.handle, - 'test', {'random_seed': {'command_required': True}}, - c, LOG, []) + 'test', cfg, c, LOG, []) def test_seed_command_and_required(self): c = self._get_cloud('ubuntu', {}) -- cgit v1.2.3 From f39e9b337778a0348ab08161d19c116408de5312 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 8 Mar 2016 11:20:41 -0500 Subject: add doc --- doc/examples/cloud-config-seed-random.txt | 32 +++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 doc/examples/cloud-config-seed-random.txt diff --git a/doc/examples/cloud-config-seed-random.txt b/doc/examples/cloud-config-seed-random.txt new file mode 100644 index 00000000..08f69a9f --- /dev/null +++ b/doc/examples/cloud-config-seed-random.txt @@ -0,0 +1,32 @@ +#cloud-config +# +# random_seed is a dictionary. +# +# The config module will write seed data from the datasource +# to 'file' described below. +# +# Entries in this dictionary are: +# file: the file to write random data to (default is /dev/urandom) +# data: this data will be written to 'file' before data from +# the datasource +# encoding: this will be used to decode 'data' provided. +# allowed values are 'encoding', 'raw', 'base64', 'b64' +# 'gzip', or 'gz'. Default is 'raw' +# +# command: execute this command to seed random. +# the command will have RANDOM_SEED_FILE in its environment +# set to the value of 'file' above. +# command_required: default False +# if true, and 'command' is not available to be run +# then exception is raised and cloud-init will record failure. +# Otherwise, only debug error is mentioned. +# +# Note: command could be ['pollinate', +# '--server=http://local.pollinate.server'] +# which would have pollinate populate /dev/urandom from provided server +seed_random: + file: '/dev/urandom' + data: 'my random string' + encoding: 'raw' + command: ['sh', '-c', 'dd if=/dev/urandom of=$RANDOM_SEED_FILE'] + command_required: True -- cgit v1.2.3 From 075fdbfcc567de07fcf54566872d812c8573efb9 Mon Sep 17 00:00:00 2001 From: Robert Jennings Date: Wed, 9 Mar 2016 16:34:11 -0600 Subject: Add default ubuntu user to lxd group (LP: #1539317) LP: #1539317 --- config/cloud.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/cloud.cfg b/config/cloud.cfg index 795df19f..a6afcc83 100644 --- a/config/cloud.cfg +++ b/config/cloud.cfg @@ -89,7 +89,7 @@ system_info: name: ubuntu lock_passwd: True gecos: Ubuntu - groups: [adm, audio, cdrom, dialout, dip, floppy, netdev, plugdev, sudo, video] + groups: [adm, audio, cdrom, dialout, dip, floppy, lxd, netdev, plugdev, sudo, video] sudo: ["ALL=(ALL) NOPASSWD:ALL"] shell: /bin/bash # Other config here will be given to the distro class and/or path classes -- cgit v1.2.3 From 2eb86ca13cb658b51440bb28cab47205b641d0ee Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 10 Mar 2016 10:32:08 -0500 Subject: initial systemd service suggestions provided by pitti This should mean that cloud-init-local is running earlier now (DefaultDependencies=no). And also blocking networking coming up (Before=network-pre.target) cloud-init.service should now actually block network-online.target from being made (meaning it will run before anything that expects that) but after networking.service, which is what actually does the bringup on ifupdown / ubuntu. --- systemd/cloud-init-local.service | 4 ++++ systemd/cloud-init.service | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/systemd/cloud-init-local.service b/systemd/cloud-init-local.service index 73aa46f6..475a2e11 100644 --- a/systemd/cloud-init-local.service +++ b/systemd/cloud-init-local.service @@ -1,7 +1,11 @@ [Unit] Description=Initial cloud-init job (pre-networking) +DefaultDependencies=no Wants=local-fs.target After=local-fs.target +Conflicts=shutdown.target +Before=network-pre.target +Before=shutdown.target [Service] Type=oneshot diff --git a/systemd/cloud-init.service b/systemd/cloud-init.service index 1f656f7f..6fb655e6 100644 --- a/systemd/cloud-init.service +++ b/systemd/cloud-init.service @@ -1,8 +1,8 @@ [Unit] Description=Initial cloud-init job (metadata service crawler) -After=local-fs.target network-online.target cloud-init-local.service -Before=sshd.service sshd-keygen.service systemd-user-sessions.service -Requires=network-online.target +After=cloud-init-local.service networking.service +Before=network-online.target sshd.service sshd-keygen.service systemd-user-sessions.service +Requires=networking.service Wants=local-fs.target cloud-init-local.service sshd.service sshd-keygen.service [Service] -- cgit v1.2.3 From 2f29e70d99906dd7bff0c64109a77f609577f063 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 10 Mar 2016 10:35:17 -0500 Subject: generator: support reading cmdline of pid 1 in a container This might need cleaning up in the future as I believe in some containers /proc/cmdline is provided, and in that case it would be preferred to pid 1's command line. --- systemd/cloud-init-generator | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/systemd/cloud-init-generator b/systemd/cloud-init-generator index 9d1e22f0..b7a2f17a 100755 --- a/systemd/cloud-init-generator +++ b/systemd/cloud-init-generator @@ -32,15 +32,16 @@ etc_file() { } read_proc_cmdline() { - if [ "$CONTAINER" = "lxc" ]; then - _RET_MSG="ignored: \$container=$CONTAINER" - _RET="" - return 0 - fi - - if systemd-detect-virt --container --quiet; then - _RET_MSG="ignored: detect-virt is container" + # return /proc/cmdline for non-container, and /proc/1/cmdline for container + local ctname="systemd" + if [ -n "$CONTAINER" ] && ctname=$CONTAINER || systemd-detect-virt --container --quiet; then + local + if _RET=$(tr '\0' ' ' < /proc/1/cmdline) >/dev/null 2>&1; then + _RET_MSG="container[$ctname]: pid 1 cmdline" + return + fi _RET="" + _RET_MSG="container[$ctname]: pid 1 cmdline not available" return 0 fi -- cgit v1.2.3 From b1d9c92f5bc6f711e104ceb981ded658249c3255 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 10 Mar 2016 10:41:48 -0500 Subject: generator: use /run/cloud-init instead of /run --- systemd/cloud-init-generator | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/systemd/cloud-init-generator b/systemd/cloud-init-generator index b7a2f17a..8156db58 100755 --- a/systemd/cloud-init-generator +++ b/systemd/cloud-init-generator @@ -3,7 +3,7 @@ set -f LOG="" DEBUG_LEVEL=1 -LOG_D="/run" +LOG_D="/run/cloud-init" ENABLE="enabled" DISABLE="disabled" CLOUD_SYSTEM_TARGET="/lib/systemd/system/cloud-init.target" @@ -17,7 +17,8 @@ debug() { [ "$lvl" -gt "$DEBUG_LEVEL" ] && return if [ -z "$LOG" ]; then local log="$LOG_D/${0##*/}.log" - { : > "$log"; } >/dev/null 2>&1 && LOG="$log" || + { [ -d "$LOG_D" ] || mkdir -p "$LOG_D"; } && + { : > "$log"; } >/dev/null 2>&1 && LOG="$log" || LOG="/dev/kmsg" fi echo "$@" >> "$LOG" -- cgit v1.2.3 From b839ad32b9bf4541583ecbe68a0bd5dd9f12345a Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 10 Mar 2016 12:32:46 -0500 Subject: dmi data: fix failure of reading dmi data for unset dmi values it is not uncommon to find dmi data in /sys full of 'ff'. utf-8 decoding of those would fail, causing warning and stacktrace. Return '.' instead of \xff. This maps to what dmidecode would return $ dmidecode --string system-product-name ................................. --- ChangeLog | 1 + cloudinit/util.py | 13 ++++++++++--- tests/unittests/test_util.py | 9 +++++++++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index da1ca9ee..ebaacf6a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -88,6 +88,7 @@ - No longer run pollinate in seed_random (LP: #1554152) - groups: add defalt user to 'lxd' group. Create groups listed for a user if they do not exist. (LP: #1539317) + - dmi data: fix failure of reading dmi data for unset dmi values 0.7.6: - open 0.7.6 diff --git a/cloudinit/util.py b/cloudinit/util.py index e7407ea4..1d50edc9 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2140,13 +2140,20 @@ def _read_dmi_syspath(key): LOG.debug("did not find %s", dmi_key_path) return None - key_data = load_file(dmi_key_path) + key_data = load_file(dmi_key_path, decode=False) if not key_data: LOG.debug("%s did not return any data", dmi_key_path) return None - LOG.debug("dmi data %s returned %s", dmi_key_path, key_data) - return key_data.strip() + # in the event that this is all \xff and a carriage return + # then return '.' in its place. + if key_data == b'\xff' * (len(key_data) - 1) + b'\n': + key_data = b'.' * (len(key_data) - 1) + b'\n' + + str_data = key_data.decode('utf8').strip() + + LOG.debug("dmi data %s returned %s", dmi_key_path, str_data) + return str_data except Exception: logexc(LOG, "failed read of %s", dmi_key_path) diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 95990165..542e4075 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -385,6 +385,15 @@ class TestReadDMIData(helpers.FilesystemMockingTestCase): self.patch_mapping({}) self.assertEqual(None, util.read_dmi_data('expect-fail')) + def test_dots_returned_instead_of_foxfox(self): + my_len = 32 + dmi_value = b'\xff' * my_len + b'\n' + expected = '.' * my_len + dmi_key = 'system-product-name' + sysfs_key = 'product_name' + self._create_sysfs_file(sysfs_key, dmi_value) + self.assertEqual(expected, util.read_dmi_data(dmi_key)) + class TestMultiLog(helpers.FilesystemMockingTestCase): -- cgit v1.2.3 From be38478cd8e11b0e29c70bb881a676628e9f74d5 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 10 Mar 2016 12:47:55 -0500 Subject: improve comment --- cloudinit/util.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cloudinit/util.py b/cloudinit/util.py index 1d50edc9..1a517c79 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2145,13 +2145,12 @@ def _read_dmi_syspath(key): LOG.debug("%s did not return any data", dmi_key_path) return None - # in the event that this is all \xff and a carriage return - # then return '.' in its place. + # uninitialized dmi values show as all \xff and /sys appends a '\n'. + # in that event, return a string of '.' in the same length. if key_data == b'\xff' * (len(key_data) - 1) + b'\n': key_data = b'.' * (len(key_data) - 1) + b'\n' str_data = key_data.decode('utf8').strip() - LOG.debug("dmi data %s returned %s", dmi_key_path, str_data) return str_data -- cgit v1.2.3 From 03f80fa62eb85270a7a96850c5e689a1c4bc0049 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 14 Mar 2016 09:21:02 -0400 Subject: change return value for dmi data of all \xff to be "" Previously we returned a string of "." the same length as the dmi field. That seems confusing to the user as "." would seem like a valid response when in fact this value should not be considered valid. So now, in this case, return empty string. --- cloudinit/util.py | 7 +++++-- tests/unittests/test_util.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cloudinit/util.py b/cloudinit/util.py index 1a517c79..caae17ce 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2148,7 +2148,7 @@ def _read_dmi_syspath(key): # uninitialized dmi values show as all \xff and /sys appends a '\n'. # in that event, return a string of '.' in the same length. if key_data == b'\xff' * (len(key_data) - 1) + b'\n': - key_data = b'.' * (len(key_data) - 1) + b'\n' + key_data = b"" str_data = key_data.decode('utf8').strip() LOG.debug("dmi data %s returned %s", dmi_key_path, str_data) @@ -2193,7 +2193,10 @@ def read_dmi_data(key): dmidecode_path = which('dmidecode') if dmidecode_path: - return _call_dmidecode(key, dmidecode_path) + ret = _call_dmidecode(key, dmidecode_path) + if ret is not None and ret.replace(".", "") == "": + return "" + return ret LOG.warn("did not find either path %s or dmidecode command", DMI_SYS_PATH) diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 542e4075..bdee9719 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -388,7 +388,7 @@ class TestReadDMIData(helpers.FilesystemMockingTestCase): def test_dots_returned_instead_of_foxfox(self): my_len = 32 dmi_value = b'\xff' * my_len + b'\n' - expected = '.' * my_len + expected = "" dmi_key = 'system-product-name' sysfs_key = 'product_name' self._create_sysfs_file(sysfs_key, dmi_value) -- cgit v1.2.3 From f8fe3182ac5e6b7b3b4a81e034e87bfd1327f82b Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 14 Mar 2016 09:25:50 -0400 Subject: change where we handle the translation --- cloudinit/util.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cloudinit/util.py b/cloudinit/util.py index caae17ce..f84f120e 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2168,6 +2168,8 @@ def _call_dmidecode(key, dmidecode_path): cmd = [dmidecode_path, "--string", key] (result, _err) = subp(cmd) LOG.debug("dmidecode returned '%s' for '%s'", result, key) + if result.replace(".", "") == "": + return "" return result except (IOError, OSError) as _err: LOG.debug('failed dmidecode cmd: %s\n%s', cmd, _err.message) @@ -2193,10 +2195,7 @@ def read_dmi_data(key): dmidecode_path = which('dmidecode') if dmidecode_path: - ret = _call_dmidecode(key, dmidecode_path) - if ret is not None and ret.replace(".", "") == "": - return "" - return ret + return _call_dmidecode(key, dmidecode_path) LOG.warn("did not find either path %s or dmidecode command", DMI_SYS_PATH) -- cgit v1.2.3 From 001057f01e698c3ca0c078d9535f05fdebec2d80 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 14 Mar 2016 09:34:46 -0400 Subject: strip return of dmidecode and do so before checking for all "." --- cloudinit/util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudinit/util.py b/cloudinit/util.py index f84f120e..f9e37a79 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2168,6 +2168,7 @@ def _call_dmidecode(key, dmidecode_path): cmd = [dmidecode_path, "--string", key] (result, _err) = subp(cmd) LOG.debug("dmidecode returned '%s' for '%s'", result, key) + result = result.strip() if result.replace(".", "") == "": return "" return result -- cgit v1.2.3