smoketest: T5783: check for any abnormal daemon termination

We need to ensure when stressing FRR with the smoketests that no unexpected
crash happens. We simply verify the PID of the individual FRR daemons.

(cherry picked from commit 080e117884196136cd63e5d312ff43fba15f7182)

12 files changed, 147 insertions, 58 deletions

diff --git a/smoketest/scripts/cli/test_protocols_bfd.py b/smoketest/scripts/cli/test_protocols_bfd.py
index 451565664..f209eae3a 100755
--- a/smoketest/scripts/cli/test_protocols_bfd.py
+++ b/smoketest/scripts/cli/test_protocols_bfd.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2021 VyOS maintainers and contributors
+# Copyright (C) 2021-2023 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -77,11 +77,23 @@ profiles = {
 }
 
 class TestProtocolsBFD(VyOSUnitTestSHIM.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        super(TestProtocolsBFD, cls).setUpClass()
+
+        # Retrieve FRR daemon PID - it is not allowed to crash, thus PID must remain the same
+        cls.daemon_pid = process_named_running(PROCESS_NAME)
+
+        # ensure we can also run this test on a live system - so lets clean
+        # out the current configuration :)
+        cls.cli_delete(cls, base_path)
+
     def tearDown(self):
         self.cli_delete(base_path)
         self.cli_commit()
-        # Check for running process
-        self.assertTrue(process_named_running(PROCESS_NAME))
+
+        # check process health and continuity
+        self.assertEqual(self.daemon_pid, process_named_running(PROCESS_NAME))
 
     def test_bfd_peer(self):
         self.cli_set(['vrf', 'name', vrf_name, 'table', '1000'])
diff --git a/smoketest/scripts/cli/test_protocols_bgp.py b/smoketest/scripts/cli/test_protocols_bgp.py
index 23e138ebe..71e2142f9 100755
--- a/smoketest/scripts/cli/test_protocols_bgp.py
+++ b/smoketest/scripts/cli/test_protocols_bgp.py
@@ -174,9 +174,16 @@ class TestProtocolsBGP(VyOSUnitTestSHIM.TestCase):
     def setUpClass(cls):
         super(TestProtocolsBGP, cls).setUpClass()
 
+        # Retrieve FRR daemon PID - it is not allowed to crash, thus PID must remain the same
+        cls.daemon_pid = process_named_running(PROCESS_NAME)
+
         # ensure we can also run this test on a live system - so lets clean
         # out the current configuration :)
         cls.cli_delete(cls, base_path)
+        cls.cli_delete(cls, ['policy', 'route-map'])
+        cls.cli_delete(cls, ['policy', 'prefix-list'])
+        cls.cli_delete(cls, ['policy', 'prefix-list6'])
+        cls.cli_delete(cls, ['vrf'])
 
         cls.cli_set(cls, ['policy', 'route-map', route_map_in, 'rule', '10', 'action', 'permit'])
         cls.cli_set(cls, ['policy', 'route-map', route_map_out, 'rule', '10', 'action', 'permit'])
@@ -192,18 +199,23 @@ class TestProtocolsBGP(VyOSUnitTestSHIM.TestCase):
 
     @classmethod
     def tearDownClass(cls):
-        cls.cli_delete(cls, ['policy'])
+        cls.cli_delete(cls, ['policy', 'route-map'])
+        cls.cli_delete(cls, ['policy', 'prefix-list'])
+        cls.cli_delete(cls, ['policy', 'prefix-list6'])
 
     def setUp(self):
         self.cli_set(base_path + ['system-as', ASN])
 
     def tearDown(self):
+        # cleanup any possible VRF mess
         self.cli_delete(['vrf'])
+        # always destrox the entire bgpd configuration to make the processes
+        # life as hard as possible
         self.cli_delete(base_path)
         self.cli_commit()
 
-        # Check for running process
-        self.assertTrue(process_named_running(PROCESS_NAME))
+        # check process health and continuity
+        self.assertEqual(self.daemon_pid, process_named_running(PROCESS_NAME))
 
     def create_bgp_instances_for_import_test(self):
         table = '1000'
diff --git a/smoketest/scripts/cli/test_protocols_igmp-proxy.py b/smoketest/scripts/cli/test_protocols_igmp-proxy.py
index a75003b12..df10442ea 100755
--- a/smoketest/scripts/cli/test_protocols_igmp-proxy.py
+++ b/smoketest/scripts/cli/test_protocols_igmp-proxy.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2019-2020 VyOS maintainers and contributors
+# Copyright (C) 2019-2023 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -29,14 +29,32 @@ upstream_if = 'eth1'
 downstream_if = 'eth2'
 
 class TestProtocolsIGMPProxy(VyOSUnitTestSHIM.TestCase):
-    def setUp(self):
-        self.cli_set(['interfaces', 'ethernet', upstream_if, 'address', '172.16.1.1/24'])
+    @classmethod
+    def setUpClass(cls):
+        # call base-classes classmethod
+        super(TestProtocolsIGMPProxy, cls).setUpClass()
+        # ensure we can also run this test on a live system - so lets clean
+        # out the current configuration :)
+        cls.cli_delete(cls, base_path)
+        cls.cli_set(cls, ['interfaces', 'ethernet', upstream_if, 'address', '172.16.1.1/24'])
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.cli_delete(cls, ['interfaces', 'ethernet', upstream_if, 'address'])
+
+        # call base-classes classmethod
+        super(TestProtocolsIGMPProxy, cls).tearDownClass()
 
     def tearDown(self):
-        self.cli_delete(['interfaces', 'ethernet', upstream_if, 'address'])
+        # Check for running process
+        self.assertTrue(process_named_running(PROCESS_NAME))
+
         self.cli_delete(base_path)
         self.cli_commit()
 
+        # Check for no longer running process
+        self.assertFalse(process_named_running(PROCESS_NAME))
+
     def test_igmpproxy(self):
         threshold = '20'
         altnet = '192.0.2.0/24'
@@ -74,8 +92,5 @@ class TestProtocolsIGMPProxy(VyOSUnitTestSHIM.TestCase):
         self.assertIn(f'whitelist {whitelist}', config)
         self.assertIn(f'phyint {downstream_if} downstream ratelimit 0 threshold 1', config)
 
-        # Check for running process
-        self.assertTrue(process_named_running(PROCESS_NAME))
-
 if __name__ == '__main__':
     unittest.main(verbosity=2)
diff --git a/smoketest/scripts/cli/test_protocols_isis.py b/smoketest/scripts/cli/test_protocols_isis.py
index 747fb5e80..c82836b72 100755
--- a/smoketest/scripts/cli/test_protocols_isis.py
+++ b/smoketest/scripts/cli/test_protocols_isis.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2021-2022 VyOS maintainers and contributors
+# Copyright (C) 2021-2023 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -31,20 +31,25 @@ class TestProtocolsISIS(VyOSUnitTestSHIM.TestCase):
     @classmethod
     def setUpClass(cls):
         cls._interfaces = Section.interfaces('ethernet')
-
         # call base-classes classmethod
         super(TestProtocolsISIS, cls).setUpClass()
-
+        # Retrieve FRR daemon PID - it is not allowed to crash, thus PID must remain the same
+        cls.daemon_pid = process_named_running(PROCESS_NAME)
         # ensure we can also run this test on a live system - so lets clean
         # out the current configuration :)
         cls.cli_delete(cls, base_path)
+        cls.cli_delete(cls, ['vrf'])
 
     def tearDown(self):
+        # cleanup any possible VRF mess
+        self.cli_delete(['vrf'])
+        # always destrox the entire isisd configuration to make the processes
+        # life as hard as possible
         self.cli_delete(base_path)
         self.cli_commit()
 
-        # Check for running process
-        self.assertTrue(process_named_running(PROCESS_NAME))
+        # check process health and continuity
+        self.assertEqual(self.daemon_pid, process_named_running(PROCESS_NAME))
 
     def isis_base_config(self):
         self.cli_set(base_path + ['net', net])
diff --git a/smoketest/scripts/cli/test_protocols_mpls.py b/smoketest/scripts/cli/test_protocols_mpls.py
index 06f21c6e1..0c1599f9b 100755
--- a/smoketest/scripts/cli/test_protocols_mpls.py
+++ b/smoketest/scripts/cli/test_protocols_mpls.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2021 VyOS maintainers and contributors
+# Copyright (C) 2021-2023 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -70,6 +70,9 @@ class TestProtocolsMPLS(VyOSUnitTestSHIM.TestCase):
     def setUpClass(cls):
         super(TestProtocolsMPLS, cls).setUpClass()
 
+        # Retrieve FRR daemon PID - it is not allowed to crash, thus PID must remain the same
+        cls.daemon_pid = process_named_running(PROCESS_NAME)
+
         # ensure we can also run this test on a live system - so lets clean
         # out the current configuration :)
         cls.cli_delete(cls, base_path)
@@ -77,8 +80,9 @@ class TestProtocolsMPLS(VyOSUnitTestSHIM.TestCase):
     def tearDown(self):
         self.cli_delete(base_path)
         self.cli_commit()
-        # Check for running process
-        self.assertTrue(process_named_running(PROCESS_NAME))
+
+        # check process health and continuity
+        self.assertEqual(self.daemon_pid, process_named_running(PROCESS_NAME))
 
     def test_mpls_basic(self):
         router_id = '1.2.3.4'
diff --git a/smoketest/scripts/cli/test_protocols_ospf.py b/smoketest/scripts/cli/test_protocols_ospf.py
index a6850db71..6bffc7c45 100755
--- a/smoketest/scripts/cli/test_protocols_ospf.py
+++ b/smoketest/scripts/cli/test_protocols_ospf.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2021-2022 VyOS maintainers and contributors
+# Copyright (C) 2021-2023 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -32,6 +32,9 @@ class TestProtocolsOSPF(VyOSUnitTestSHIM.TestCase):
     def setUpClass(cls):
         super(TestProtocolsOSPF, cls).setUpClass()
 
+        # Retrieve FRR daemon PID - it is not allowed to crash, thus PID must remain the same
+        cls.daemon_pid = process_named_running(PROCESS_NAME)
+
         cls.cli_set(cls, ['policy', 'route-map', route_map, 'rule', '10', 'action', 'permit'])
         cls.cli_set(cls, ['policy', 'route-map', route_map, 'rule', '20', 'action', 'permit'])
 
@@ -45,11 +48,12 @@ class TestProtocolsOSPF(VyOSUnitTestSHIM.TestCase):
         super(TestProtocolsOSPF, cls).tearDownClass()
 
     def tearDown(self):
-        # Check for running process
-        self.assertTrue(process_named_running(PROCESS_NAME))
         self.cli_delete(base_path)
         self.cli_commit()
 
+        # check process health and continuity
+        self.assertEqual(self.daemon_pid, process_named_running(PROCESS_NAME))
+
     def test_ospf_01_defaults(self):
         # commit changes
         self.cli_set(base_path)
diff --git a/smoketest/scripts/cli/test_protocols_ospfv3.py b/smoketest/scripts/cli/test_protocols_ospfv3.py
index 0d6c6c691..4ae7f05d9 100755
--- a/smoketest/scripts/cli/test_protocols_ospfv3.py
+++ b/smoketest/scripts/cli/test_protocols_ospfv3.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2021-2022 VyOS maintainers and contributors
+# Copyright (C) 2021-2023 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -35,6 +35,9 @@ class TestProtocolsOSPFv3(VyOSUnitTestSHIM.TestCase):
     def setUpClass(cls):
         super(TestProtocolsOSPFv3, cls).setUpClass()
 
+        # Retrieve FRR daemon PID - it is not allowed to crash, thus PID must remain the same
+        cls.daemon_pid = process_named_running(PROCESS_NAME)
+
         cls.cli_set(cls, ['policy', 'route-map', route_map, 'rule', '10', 'action', 'permit'])
         cls.cli_set(cls, ['policy', 'route-map', route_map, 'rule', '20', 'action', 'permit'])
 
@@ -48,11 +51,12 @@ class TestProtocolsOSPFv3(VyOSUnitTestSHIM.TestCase):
         super(TestProtocolsOSPFv3, cls).tearDownClass()
 
     def tearDown(self):
-        # Check for running process
-        self.assertTrue(process_named_running(PROCESS_NAME))
         self.cli_delete(base_path)
         self.cli_commit()
 
+        # check process health and continuity
+        self.assertEqual(self.daemon_pid, process_named_running(PROCESS_NAME))
+
     def test_ospfv3_01_basic(self):
         seq = '10'
         prefix = '2001:db8::/32'
diff --git a/smoketest/scripts/cli/test_protocols_pim6.py b/smoketest/scripts/cli/test_protocols_pim6.py
index e22a7c722..ba24edca2 100755
--- a/smoketest/scripts/cli/test_protocols_pim6.py
+++ b/smoketest/scripts/cli/test_protocols_pim6.py
@@ -25,15 +25,22 @@ PROCESS_NAME = 'pim6d'
 base_path = ['protocols', 'pim6']
 
 class TestProtocolsPIMv6(VyOSUnitTestSHIM.TestCase):
-    def tearDown(self):
-        # Check for running process
-        self.assertTrue(process_named_running(PROCESS_NAME))
+    @classmethod
+    def setUpClass(cls):
+        # call base-classes classmethod
+        super(TestProtocolsPIMv6, cls).setUpClass()
+        # Retrieve FRR daemon PID - it is not allowed to crash, thus PID must remain the same
+        cls.daemon_pid = process_named_running(PROCESS_NAME)
+        # ensure we can also run this test on a live system - so lets clean
+        # out the current configuration :)
+        cls.cli_delete(cls, base_path)
 
+    def tearDown(self):
         self.cli_delete(base_path)
         self.cli_commit()
 
-        # Check for running process
-        self.assertTrue(process_named_running(PROCESS_NAME))
+        # check process health and continuity
+        self.assertEqual(self.daemon_pid, process_named_running(PROCESS_NAME))
 
     def test_pim6_01_mld_simple(self):
         # commit changes
diff --git a/smoketest/scripts/cli/test_protocols_rip.py b/smoketest/scripts/cli/test_protocols_rip.py
index 925499fc8..bfc327fd4 100755
--- a/smoketest/scripts/cli/test_protocols_rip.py
+++ b/smoketest/scripts/cli/test_protocols_rip.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2021 VyOS maintainers and contributors
+# Copyright (C) 2021-2023 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -34,7 +34,8 @@ class TestProtocolsRIP(VyOSUnitTestSHIM.TestCase):
     @classmethod
     def setUpClass(cls):
         super(TestProtocolsRIP, cls).setUpClass()
-
+        # Retrieve FRR daemon PID - it is not allowed to crash, thus PID must remain the same
+        cls.daemon_pid = process_named_running(PROCESS_NAME)
         # ensure we can also run this test on a live system - so lets clean
         # out the current configuration :)
         cls.cli_delete(cls, base_path)
@@ -65,8 +66,8 @@ class TestProtocolsRIP(VyOSUnitTestSHIM.TestCase):
         self.cli_delete(base_path)
         self.cli_commit()
 
-        # Check for running process
-        self.assertTrue(process_named_running(PROCESS_NAME))
+        # check process health and continuity
+        self.assertEqual(self.daemon_pid, process_named_running(PROCESS_NAME))
 
     def test_rip_01_parameters(self):
         distance = '40'
diff --git a/smoketest/scripts/cli/test_protocols_ripng.py b/smoketest/scripts/cli/test_protocols_ripng.py
index 0a8ce7eef..0cfb065c6 100755
--- a/smoketest/scripts/cli/test_protocols_ripng.py
+++ b/smoketest/scripts/cli/test_protocols_ripng.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2021 VyOS maintainers and contributors
+# Copyright (C) 2021-2023 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -31,28 +31,43 @@ route_map = 'FooBar123'
 base_path = ['protocols', 'ripng']
 
 class TestProtocolsRIPng(VyOSUnitTestSHIM.TestCase):
-    def setUp(self):
-        self.cli_set(['policy', 'access-list6', acl_in, 'rule', '10', 'action', 'permit'])
-        self.cli_set(['policy', 'access-list6', acl_in, 'rule', '10', 'source', 'any'])
-        self.cli_set(['policy', 'access-list6', acl_out, 'rule', '20', 'action', 'deny'])
-        self.cli_set(['policy', 'access-list6', acl_out, 'rule', '20', 'source', 'any'])
-        self.cli_set(['policy', 'prefix-list6', prefix_list_in, 'rule', '100', 'action', 'permit'])
-        self.cli_set(['policy', 'prefix-list6', prefix_list_in, 'rule', '100', 'prefix', '2001:db8::/32'])
-        self.cli_set(['policy', 'prefix-list6', prefix_list_out, 'rule', '200', 'action', 'deny'])
-        self.cli_set(['policy', 'prefix-list6', prefix_list_out, 'rule', '200', 'prefix', '2001:db8::/32'])
-        self.cli_set(['policy', 'route-map', route_map, 'rule', '10', 'action', 'permit'])
+    @classmethod
+    def setUpClass(cls):
+        # call base-classes classmethod
+        super(TestProtocolsRIPng, cls).setUpClass()
+        # Retrieve FRR daemon PID - it is not allowed to crash, thus PID must remain the same
+        cls.daemon_pid = process_named_running(PROCESS_NAME)
+        # ensure we can also run this test on a live system - so lets clean
+        # out the current configuration :)
+        cls.cli_delete(cls, base_path)
+
+        cls.cli_set(cls, ['policy', 'access-list6', acl_in, 'rule', '10', 'action', 'permit'])
+        cls.cli_set(cls, ['policy', 'access-list6', acl_in, 'rule', '10', 'source', 'any'])
+        cls.cli_set(cls, ['policy', 'access-list6', acl_out, 'rule', '20', 'action', 'deny'])
+        cls.cli_set(cls, ['policy', 'access-list6', acl_out, 'rule', '20', 'source', 'any'])
+        cls.cli_set(cls, ['policy', 'prefix-list6', prefix_list_in, 'rule', '100', 'action', 'permit'])
+        cls.cli_set(cls, ['policy', 'prefix-list6', prefix_list_in, 'rule', '100', 'prefix', '2001:db8::/32'])
+        cls.cli_set(cls, ['policy', 'prefix-list6', prefix_list_out, 'rule', '200', 'action', 'deny'])
+        cls.cli_set(cls, ['policy', 'prefix-list6', prefix_list_out, 'rule', '200', 'prefix', '2001:db8::/32'])
+        cls.cli_set(cls, ['policy', 'route-map', route_map, 'rule', '10', 'action', 'permit'])
+
+    @classmethod
+    def tearDownClass(cls):
+        # call base-classes classmethod
+        super(TestProtocolsRIPng, cls).tearDownClass()
+
+        cls.cli_delete(cls, ['policy', 'access-list6', acl_in])
+        cls.cli_delete(cls, ['policy', 'access-list6', acl_out])
+        cls.cli_delete(cls, ['policy', 'prefix-list6', prefix_list_in])
+        cls.cli_delete(cls, ['policy', 'prefix-list6', prefix_list_out])
+        cls.cli_delete(cls, ['policy', 'route-map', route_map])
 
     def tearDown(self):
         self.cli_delete(base_path)
-        self.cli_delete(['policy', 'access-list6', acl_in])
-        self.cli_delete(['policy', 'access-list6', acl_out])
-        self.cli_delete(['policy', 'prefix-list6', prefix_list_in])
-        self.cli_delete(['policy', 'prefix-list6', prefix_list_out])
-        self.cli_delete(['policy', 'route-map', route_map])
         self.cli_commit()
 
-        # Check for running process
-        self.assertTrue(process_named_running(PROCESS_NAME))
+        # check process health and continuity
+        self.assertEqual(self.daemon_pid, process_named_running(PROCESS_NAME))
 
     def test_ripng_01_parameters(self):
         metric = '8'
diff --git a/smoketest/scripts/cli/test_protocols_rpki.py b/smoketest/scripts/cli/test_protocols_rpki.py
index f4aedcbc3..ab3f076ac 100755
--- a/smoketest/scripts/cli/test_protocols_rpki.py
+++ b/smoketest/scripts/cli/test_protocols_rpki.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2021 VyOS maintainers and contributors
+# Copyright (C) 2021-2023 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -31,6 +31,16 @@ rpki_ssh_key = '/config/auth/id_rsa_rpki'
 rpki_ssh_pub = f'{rpki_ssh_key}.pub'
 
 class TestProtocolsRPKI(VyOSUnitTestSHIM.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        # call base-classes classmethod
+        super(TestProtocolsRPKI, cls).setUpClass()
+        # Retrieve FRR daemon PID - it is not allowed to crash, thus PID must remain the same
+        cls.daemon_pid = process_named_running(PROCESS_NAME)
+        # ensure we can also run this test on a live system - so lets clean
+        # out the current configuration :)
+        cls.cli_delete(cls, base_path)
+
     def tearDown(self):
         self.cli_delete(base_path)
         self.cli_commit()
@@ -39,8 +49,8 @@ class TestProtocolsRPKI(VyOSUnitTestSHIM.TestCase):
         # frrconfig = self.getFRRconfig('rpki')
         # self.assertNotIn('rpki', frrconfig)
 
-        # Check for running process
-        self.assertTrue(process_named_running(PROCESS_NAME))
+        # check process health and continuity
+        self.assertEqual(self.daemon_pid, process_named_running(PROCESS_NAME))
 
     def test_rpki(self):
         polling = '7200'
diff --git a/smoketest/scripts/cli/test_vrf.py b/smoketest/scripts/cli/test_vrf.py
index 0f658f366..bb91eddea 100755
--- a/smoketest/scripts/cli/test_vrf.py
+++ b/smoketest/scripts/cli/test_vrf.py
@@ -489,4 +489,4 @@ class VRFTest(VyOSUnitTestSHIM.TestCase):
 
 
 if __name__ == '__main__':
-    unittest.main(verbosity=2, failfast=True)
+    unittest.main(verbosity=2)