smoketest: T7948: always verify FRRs mgmtd PID is unchanged

Upgrades to FRR 10.4 have shown that there are cases where FRRs mgmtd will
SIGSEGV - to cover for this case and fail the smoketests we will monitor the
PID of the mgmtd process.

4 files changed, 27 insertions, 6 deletions

diff --git a/smoketest/scripts/cli/base_vyostest_shim.py b/smoketest/scripts/cli/base_vyostest_shim.py
index 6415a0e3c..32d878a56 100644
--- a/smoketest/scripts/cli/base_vyostest_shim.py
+++ b/smoketest/scripts/cli/base_vyostest_shim.py
@@ -23,11 +23,13 @@ import unittest
 from time import sleep
 from typing import Type
 
+from vyos import ConfigError
 from vyos.configsession import ConfigSession
 from vyos.configsession import ConfigSessionError
-from vyos import ConfigError
 from vyos.defaults import commit_lock
+from vyos.frrender import mgmt_daemon
 from vyos.utils.process import cmd
+from vyos.utils.process import process_named_running
 from vyos.utils.process import run
 
 save_config = '/tmp/vyos-smoketest-save'
@@ -42,11 +44,11 @@ save_config = '/tmp/vyos-smoketest-save'
 
 class VyOSUnitTestSHIM:
     class TestCase(unittest.TestCase):
-        # if enabled in derived class, print out each and every set/del command
-        # on the CLI. This is usefull to grap all the commands required to
-        # trigger the certain failure condition.
-        # Use "self.debug = True" in derived classes setUp() method
+        # If enabled, print out each and every set/del command on stdout.
+        # This is usefull to grap all the commands required to trigger the
+        # certain failure condition.
         debug = False
+        mgmt_daemon_pid = 0
 
         @staticmethod
         def debug_on():
@@ -69,6 +71,10 @@ class VyOSUnitTestSHIM:
             cls._session.save_config(save_config)
             cls.debug = cls.debug_on()
 
+            # Retrieve FRR mgmtd daemon PID - it is not allowed to crash, thus
+            # PID must remain the same
+            cls.mgmt_daemon_pid = process_named_running(mgmt_daemon)
+
         @classmethod
         def tearDownClass(cls):
             # discard any pending changes which might caused a messed up config
@@ -86,7 +92,8 @@ class VyOSUnitTestSHIM:
             pass
 
         def tearDown(self):
-            pass
+            # check process health and continuity
+            self.assertEqual(self.mgmt_daemon_pid, process_named_running(mgmt_daemon))
 
         def cli_set(self, path, value=None):
             if self.debug:
diff --git a/smoketest/scripts/cli/test_interfaces_ethernet.py b/smoketest/scripts/cli/test_interfaces_ethernet.py
index cdc3dad9e..6d0b7b772 100755
--- a/smoketest/scripts/cli/test_interfaces_ethernet.py
+++ b/smoketest/scripts/cli/test_interfaces_ethernet.py
@@ -28,11 +28,13 @@ from base_interfaces_test import BasicInterfaceTest
 from base_vyostest_shim import VyOSUnitTestSHIM
 
 from vyos.configsession import ConfigSessionError
+from vyos.frrender import mgmt_daemon
 from vyos.ifconfig import Section
 from vyos.utils.file import read_file
 from vyos.utils.network import is_intf_addr_assigned
 from vyos.utils.network import is_ipv6_link_local
 from vyos.utils.process import cmd
+from vyos.utils.process import process_named_running
 from vyos.utils.process import popen
 
 class EthernetInterfaceTest(BasicInterfaceTest.TestCase):
@@ -87,6 +89,9 @@ class EthernetInterfaceTest(BasicInterfaceTest.TestCase):
             ]
             self.assertListEqual(tmp, [])
 
+        # check process health and continuity
+        self.assertEqual(self.mgmt_daemon_pid, process_named_running(mgmt_daemon))
+
     def test_offloading_rps(self):
         # enable RPS on all available CPUs, RPS works with a CPU bitmask,
         # where each bit represents a CPU (core/thread). The formula below
diff --git a/smoketest/scripts/cli/test_interfaces_loopback.py b/smoketest/scripts/cli/test_interfaces_loopback.py
index 6cc0d4dbe..f3a3602d1 100755
--- a/smoketest/scripts/cli/test_interfaces_loopback.py
+++ b/smoketest/scripts/cli/test_interfaces_loopback.py
@@ -21,7 +21,9 @@ from base_interfaces_test import BasicInterfaceTest
 from base_interfaces_test import MSG_TESTCASE_UNSUPPORTED
 from base_vyostest_shim import VyOSUnitTestSHIM
 
+from vyos.frrender import mgmt_daemon
 from vyos.utils.network import is_intf_addr_assigned
+from vyos.utils.process import process_named_running
 
 loopbacks = ['127.0.0.1', '::1']
 
@@ -44,6 +46,9 @@ class LoopbackInterfaceTest(BasicInterfaceTest.TestCase):
         for intf in self._interfaces:
             self.assertIn(intf, interfaces())
 
+        # check process health and continuity
+        self.assertEqual(self.mgmt_daemon_pid, process_named_running(mgmt_daemon))
+
     def test_add_single_ip_address(self):
         super().test_add_single_ip_address()
         for addr in loopbacks:
diff --git a/smoketest/scripts/cli/test_interfaces_virtual-ethernet.py b/smoketest/scripts/cli/test_interfaces_virtual-ethernet.py
index c2149440b..0a581951d 100755
--- a/smoketest/scripts/cli/test_interfaces_virtual-ethernet.py
+++ b/smoketest/scripts/cli/test_interfaces_virtual-ethernet.py
@@ -20,6 +20,7 @@ from netifaces import interfaces # pylint: disable = no-name-in-module
 from base_interfaces_test import BasicInterfaceTest
 from base_vyostest_shim import VyOSUnitTestSHIM
 
+from vyos.frrender import mgmt_daemon
 from vyos.utils.process import process_named_running
 
 class VEthInterfaceTest(BasicInterfaceTest.TestCase):
@@ -47,6 +48,9 @@ class VEthInterfaceTest(BasicInterfaceTest.TestCase):
         for intf in self._interfaces:
             self.assertNotIn(intf, interfaces())
 
+        # check process health and continuity
+        self.assertEqual(self.mgmt_daemon_pid, process_named_running(mgmt_daemon))
+
     @classmethod
     def tearDownClass(cls):
         # No daemon started during tests should remain running