T2117: Cloud-init updated to 20.3

Merged with 20.3 tag from the upstream Cloud-init repository

4 files changed, 65 insertions, 37 deletions

diff --git a/cloudinit/analyze/dump.py b/cloudinit/analyze/dump.py
index 1f3060d0..62ad51fe 100644
--- a/cloudinit/analyze/dump.py
+++ b/cloudinit/analyze/dump.py
@@ -4,6 +4,7 @@ import calendar
 from datetime import datetime
 import sys
 
+from cloudinit import subp
 from cloudinit import util
 
 stage_to_description = {
@@ -51,7 +52,7 @@ def parse_timestamp(timestampstr):
 
 
 def parse_timestamp_from_date(timestampstr):
-    out, _ = util.subp(['date', '+%s.%3N', '-d', timestampstr])
+    out, _ = subp.subp(['date', '+%s.%3N', '-d', timestampstr])
     timestamp = out.strip()
     return float(timestamp)
 
@@ -74,8 +75,12 @@ def parse_ci_logline(line):
     #
     # 2017-05-22 18:02:01,088 - util.py[DEBUG]: Cloud-init v. 0.7.9 running \
     #         'init-local' at Mon, 22 May 2017 18:02:01 +0000. Up 2.0 seconds.
+    #
+    # Apr 30 19:39:11 cloud-init[2673]: handlers.py[DEBUG]: start: \
+    #          init-local/check-cache: attempting to read from cache [check]
 
-    separators = [' - ', ' [CLOUDINIT] ']
+    amazon_linux_2_sep = ' cloud-init['
+    separators = [' - ', ' [CLOUDINIT] ', amazon_linux_2_sep]
     found = False
     for sep in separators:
         if sep in line:
@@ -98,7 +103,14 @@ def parse_ci_logline(line):
             hostname = extra.split()[-1]
     else:
         hostname = timehost.split()[-1]
-        timestampstr = timehost.split(hostname)[0].strip()
+        if sep == amazon_linux_2_sep:
+            # This is an Amazon Linux style line, with no hostname and a PID.
+            # Use the whole of timehost as timestampstr, and strip off the PID
+            # from the start of eventstr.
+            timestampstr = timehost.strip()
+            eventstr = eventstr.split(maxsplit=1)[1]
+        else:
+            timestampstr = timehost.split(hostname)[0].strip()
     if 'Cloud-init v.' in eventstr:
         event_type = 'start'
         if 'running' in eventstr:
diff --git a/cloudinit/analyze/show.py b/cloudinit/analyze/show.py
index fb152b1d..01a4d3e5 100644
--- a/cloudinit/analyze/show.py
+++ b/cloudinit/analyze/show.py
@@ -11,31 +11,29 @@ import os
 import time
 import sys
 
+from cloudinit import subp
 from cloudinit import util
 from cloudinit.distros import uses_systemd
 
-#  An event:
-'''
-{
-        "description": "executing late commands",
-        "event_type": "start",
-        "level": "INFO",
-        "name": "cmd-install/stage-late"
-        "origin": "cloudinit",
-        "timestamp": 1461164249.1590767,
-},
-
-    {
-        "description": "executing late commands",
-        "event_type": "finish",
-        "level": "INFO",
-        "name": "cmd-install/stage-late",
-        "origin": "cloudinit",
-        "result": "SUCCESS",
-        "timestamp": 1461164249.1590767
-    }
-
-'''
+# Example events:
+#     {
+#             "description": "executing late commands",
+#             "event_type": "start",
+#             "level": "INFO",
+#             "name": "cmd-install/stage-late"
+#             "origin": "cloudinit",
+#             "timestamp": 1461164249.1590767,
+#     }
+#     {
+#         "description": "executing late commands",
+#         "event_type": "finish",
+#         "level": "INFO",
+#         "name": "cmd-install/stage-late",
+#         "origin": "cloudinit",
+#         "result": "SUCCESS",
+#         "timestamp": 1461164249.1590767
+#     }
+
 format_key = {
     '%d': 'delta',
     '%D': 'description',
@@ -155,7 +153,7 @@ class SystemctlReader(object):
         :return: whether the subp call failed or not
         '''
         try:
-            value, err = util.subp(self.args, capture=True)
+            value, err = subp.subp(self.args, capture=True)
             if err:
                 return err
             self.epoch = value
@@ -215,7 +213,7 @@ def gather_timestamps_using_dmesg():
     with gather_timestamps_using_systemd
     '''
     try:
-        data, _ = util.subp(['dmesg'], capture=True)
+        data, _ = subp.subp(['dmesg'], capture=True)
         split_entries = data[0].splitlines()
         for i in split_entries:
             if i.decode('UTF-8').find('user') != -1:
@@ -269,7 +267,7 @@ def gather_timestamps_using_systemd():
                 except OSError as err:
                     raise RuntimeError('Could not determine container boot '
                                        'time from /proc/1/cmdline. ({})'
-                                       .format(err))
+                                       .format(err)) from err
                 status = CONTAINER_CODE
             else:
                 status = FAIL_CODE
diff --git a/cloudinit/analyze/tests/test_boot.py b/cloudinit/analyze/tests/test_boot.py
index f4001c14..f69423c3 100644
--- a/cloudinit/analyze/tests/test_boot.py
+++ b/cloudinit/analyze/tests/test_boot.py
@@ -25,7 +25,7 @@ class TestDistroChecker(CiTestCase):
                                       m_get_linux_distro, m_is_FreeBSD):
         self.assertEqual(err_code, dist_check_timestamp())
 
-    @mock.patch('cloudinit.util.subp', return_value=(0, 1))
+    @mock.patch('cloudinit.subp.subp', return_value=(0, 1))
     def test_subp_fails(self, m_subp):
         self.assertEqual(err_code, dist_check_timestamp())
 
@@ -42,7 +42,7 @@ class TestSystemCtlReader(CiTestCase):
         with self.assertRaises(RuntimeError):
             reader.parse_epoch_as_float()
 
-    @mock.patch('cloudinit.util.subp', return_value=('U=1000000', None))
+    @mock.patch('cloudinit.subp.subp', return_value=('U=1000000', None))
     def test_systemctl_works_correctly_threshold(self, m_subp):
         reader = SystemctlReader('dummyProperty', 'dummyParameter')
         self.assertEqual(1.0, reader.parse_epoch_as_float())
@@ -50,12 +50,12 @@ class TestSystemCtlReader(CiTestCase):
         self.assertTrue(thresh < 1e-6)
         self.assertTrue(thresh > (-1 * 1e-6))
 
-    @mock.patch('cloudinit.util.subp', return_value=('U=0', None))
+    @mock.patch('cloudinit.subp.subp', return_value=('U=0', None))
     def test_systemctl_succeed_zero(self, m_subp):
         reader = SystemctlReader('dummyProperty', 'dummyParameter')
         self.assertEqual(0.0, reader.parse_epoch_as_float())
 
-    @mock.patch('cloudinit.util.subp', return_value=('U=1', None))
+    @mock.patch('cloudinit.subp.subp', return_value=('U=1', None))
     def test_systemctl_succeed_distinct(self, m_subp):
         reader = SystemctlReader('dummyProperty', 'dummyParameter')
         val1 = reader.parse_epoch_as_float()
@@ -64,13 +64,13 @@ class TestSystemCtlReader(CiTestCase):
         val2 = reader2.parse_epoch_as_float()
         self.assertNotEqual(val1, val2)
 
-    @mock.patch('cloudinit.util.subp', return_value=('100', None))
+    @mock.patch('cloudinit.subp.subp', return_value=('100', None))
     def test_systemctl_epoch_not_splittable(self, m_subp):
         reader = SystemctlReader('dummyProperty', 'dummyParameter')
         with self.assertRaises(IndexError):
             reader.parse_epoch_as_float()
 
-    @mock.patch('cloudinit.util.subp', return_value=('U=foobar', None))
+    @mock.patch('cloudinit.subp.subp', return_value=('U=foobar', None))
     def test_systemctl_cannot_convert_epoch_to_float(self, m_subp):
         reader = SystemctlReader('dummyProperty', 'dummyParameter')
         with self.assertRaises(ValueError):
@@ -130,7 +130,7 @@ class TestAnalyzeBoot(CiTestCase):
         self.assertEqual(err_string, data)
 
     @mock.patch("cloudinit.util.is_container", return_value=True)
-    @mock.patch('cloudinit.util.subp', return_value=('U=1000000', None))
+    @mock.patch('cloudinit.subp.subp', return_value=('U=1000000', None))
     def test_container_no_ci_log_line(self, m_is_container, m_subp):
         path = os.path.dirname(os.path.abspath(__file__))
         log_path = path + '/boot-test.log'
@@ -148,7 +148,7 @@ class TestAnalyzeBoot(CiTestCase):
         self.assertEqual(FAIL_CODE, finish_code)
 
     @mock.patch("cloudinit.util.is_container", return_value=True)
-    @mock.patch('cloudinit.util.subp', return_value=('U=1000000', None))
+    @mock.patch('cloudinit.subp.subp', return_value=('U=1000000', None))
     @mock.patch('cloudinit.analyze.__main__._get_events', return_value=[{
         'name': 'init-local', 'description': 'starting search', 'timestamp':
         100000}])
diff --git a/cloudinit/analyze/tests/test_dump.py b/cloudinit/analyze/tests/test_dump.py
index db2a667b..dac1efb6 100644
--- a/cloudinit/analyze/tests/test_dump.py
+++ b/cloudinit/analyze/tests/test_dump.py
@@ -5,7 +5,8 @@ from textwrap import dedent
 
 from cloudinit.analyze.dump import (
     dump_events, parse_ci_logline, parse_timestamp)
-from cloudinit.util import which, write_file
+from cloudinit.util import write_file
+from cloudinit.subp import which
 from cloudinit.tests.helpers import CiTestCase, mock, skipIf
 
 
@@ -119,6 +120,23 @@ class TestParseCILogLine(CiTestCase):
         m_parse_from_date.assert_has_calls(
             [mock.call("2016-08-30 21:53:25.972325+00:00")])
 
+    def test_parse_logline_returns_event_for_amazon_linux_2_line(self):
+        line = (
+            "Apr 30 19:39:11 cloud-init[2673]: handlers.py[DEBUG]: start:"
+            " init-local/check-cache: attempting to read from cache [check]")
+        # Generate the expected value using `datetime`, so that TZ
+        # determination is consistent with the code under test.
+        timestamp_dt = datetime.strptime(
+            "Apr 30 19:39:11", "%b %d %H:%M:%S"
+        ).replace(year=datetime.now().year)
+        expected = {
+            'description': 'attempting to read from cache [check]',
+            'event_type': 'start',
+            'name': 'init-local/check-cache',
+            'origin': 'cloudinit',
+            'timestamp': timestamp_dt.timestamp()}
+        self.assertEqual(expected, parse_ci_logline(line))
+
 
 SAMPLE_LOGS = dedent("""\
 Nov 03 06:51:06.074410 x2 cloud-init[106]: [CLOUDINIT] util.py[DEBUG]:\