Merge pull request #2544 from c-po/t5783-smoketests

smoketest: T5783: check for any abnormal daemon termination

11 files changed, 148 insertions, 59 deletions

diff --git a/smoketest/scripts/cli/test_protocols_bfd.py b/smoketest/scripts/cli/test_protocols_bfd.py
index 451565664..f209eae3a 100755
--- a/smoketest/scripts/cli/test_protocols_bfd.py
+++ b/smoketest/scripts/cli/test_protocols_bfd.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2021 VyOS maintainers and contributors
+# Copyright (C) 2021-2023 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -77,11 +77,23 @@ profiles = {
 }
 
 class TestProtocolsBFD(VyOSUnitTestSHIM.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        super(TestProtocolsBFD, cls).setUpClass()
+
+        # Retrieve FRR daemon PID - it is not allowed to crash, thus PID must remain the same
+        cls.daemon_pid = process_named_running(PROCESS_NAME)
+
+        # ensure we can also run this test on a live system - so lets clean
+        # out the current configuration :)
+        cls.cli_delete(cls, base_path)
+
     def tearDown(self):
         self.cli_delete(base_path)
         self.cli_commit()
-        # Check for running process
-        self.assertTrue(process_named_running(PROCESS_NAME))
+
+        # check process health and continuity
+        self.assertEqual(self.daemon_pid, process_named_running(PROCESS_NAME))
 
     def test_bfd_peer(self):
         self.cli_set(['vrf', 'name', vrf_name, 'table', '1000'])
diff --git a/smoketest/scripts/cli/test_protocols_bgp.py b/smoketest/scripts/cli/test_protocols_bgp.py
index 23e138ebe..71e2142f9 100755
--- a/smoketest/scripts/cli/test_protocols_bgp.py
+++ b/smoketest/scripts/cli/test_protocols_bgp.py
@@ -174,9 +174,16 @@ class TestProtocolsBGP(VyOSUnitTestSHIM.TestCase):
     def setUpClass(cls):
         super(TestProtocolsBGP, cls).setUpClass()
 
+        # Retrieve FRR daemon PID - it is not allowed to crash, thus PID must remain the same
+        cls.daemon_pid = process_named_running(PROCESS_NAME)
+
         # ensure we can also run this test on a live system - so lets clean
         # out the current configuration :)
         cls.cli_delete(cls, base_path)
+        cls.cli_delete(cls, ['policy', 'route-map'])
+        cls.cli_delete(cls, ['policy', 'prefix-list'])
+        cls.cli_delete(cls, ['policy', 'prefix-list6'])
+        cls.cli_delete(cls, ['vrf'])
 
         cls.cli_set(cls, ['policy', 'route-map', route_map_in, 'rule', '10', 'action', 'permit'])
         cls.cli_set(cls, ['policy', 'route-map', route_map_out, 'rule', '10', 'action', 'permit'])
@@ -192,18 +199,23 @@ class TestProtocolsBGP(VyOSUnitTestSHIM.TestCase):
 
     @classmethod
     def tearDownClass(cls):
-        cls.cli_delete(cls, ['policy'])
+        cls.cli_delete(cls, ['policy', 'route-map'])
+        cls.cli_delete(cls, ['policy', 'prefix-list'])
+        cls.cli_delete(cls, ['policy', 'prefix-list6'])
 
     def setUp(self):
         self.cli_set(base_path + ['system-as', ASN])
 
     def tearDown(self):
+        # cleanup any possible VRF mess
         self.cli_delete(['vrf'])
+        # always destrox the entire bgpd configuration to make the processes
+        # life as hard as possible
         self.cli_delete(base_path)
         self.cli_commit()
 
-        # Check for running process
-        self.assertTrue(process_named_running(PROCESS_NAME))
+        # check process health and continuity
+        self.assertEqual(self.daemon_pid, process_named_running(PROCESS_NAME))
 
     def create_bgp_instances_for_import_test(self):
         table = '1000'
diff --git a/smoketest/scripts/cli/test_protocols_igmp-proxy.py b/smoketest/scripts/cli/test_protocols_igmp-proxy.py
index a75003b12..df10442ea 100755
--- a/smoketest/scripts/cli/test_protocols_igmp-proxy.py
+++ b/smoketest/scripts/cli/test_protocols_igmp-proxy.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2019-2020 VyOS maintainers and contributors
+# Copyright (C) 2019-2023 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -29,14 +29,32 @@ upstream_if = 'eth1'
 downstream_if = 'eth2'
 
 class TestProtocolsIGMPProxy(VyOSUnitTestSHIM.TestCase):
-    def setUp(self):
-        self.cli_set(['interfaces', 'ethernet', upstream_if, 'address', '172.16.1.1/24'])
+    @classmethod
+    def setUpClass(cls):
+        # call base-classes classmethod
+        super(TestProtocolsIGMPProxy, cls).setUpClass()
+        # ensure we can also run this test on a live system - so lets clean
+        # out the current configuration :)
+        cls.cli_delete(cls, base_path)
+        cls.cli_set(cls, ['interfaces', 'ethernet', upstream_if, 'address', '172.16.1.1/24'])
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.cli_delete(cls, ['interfaces', 'ethernet', upstream_if, 'address'])
+
+        # call base-classes classmethod
+        super(TestProtocolsIGMPProxy, cls).tearDownClass()
 
     def tearDown(self):
-        self.cli_delete(['interfaces', 'ethernet', upstream_if, 'address'])
+        # Check for running process
+        self.assertTrue(process_named_running(PROCESS_NAME))
+
         self.cli_delete(base_path)
         self.cli_commit()
 
+        # Check for no longer running process
+        self.assertFalse(process_named_running(PROCESS_NAME))
+
     def test_igmpproxy(self):
         threshold = '20'
         altnet = '192.0.2.0/24'
@@ -74,8 +92,5 @@ class TestProtocolsIGMPProxy(VyOSUnitTestSHIM.TestCase):
         self.assertIn(f'whitelist {whitelist}', config)
         self.assertIn(f'phyint {downstream_if} downstream ratelimit 0 threshold 1', config)
 
-        # Check for running process
-        self.assertTrue(process_named_running(PROCESS_NAME))
-
 if __name__ == '__main__':
     unittest.main(verbosity=2)
diff --git a/smoketest/scripts/cli/test_protocols_isis.py b/smoketest/scripts/cli/test_protocols_isis.py
index 8b423dbea..aa5f2f38c 100755
--- a/smoketest/scripts/cli/test_protocols_isis.py
+++ b/smoketest/scripts/cli/test_protocols_isis.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2021-2022 VyOS maintainers and contributors
+# Copyright (C) 2021-2023 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -31,20 +31,25 @@ class TestProtocolsISIS(VyOSUnitTestSHIM.TestCase):
     @classmethod
     def setUpClass(cls):
         cls._interfaces = Section.interfaces('ethernet')
-
         # call base-classes classmethod
         super(TestProtocolsISIS, cls).setUpClass()
-
+        # Retrieve FRR daemon PID - it is not allowed to crash, thus PID must remain the same
+        cls.daemon_pid = process_named_running(PROCESS_NAME)
         # ensure we can also run this test on a live system - so lets clean
         # out the current configuration :)
         cls.cli_delete(cls, base_path)
+        cls.cli_delete(cls, ['vrf'])
 
     def tearDown(self):
+        # cleanup any possible VRF mess
+        self.cli_delete(['vrf'])
+        # always destrox the entire isisd configuration to make the processes
+        # life as hard as possible
         self.cli_delete(base_path)
         self.cli_commit()
 
-        # Check for running process
-        self.assertTrue(process_named_running(PROCESS_NAME))
+        # check process health and continuity
+        self.assertEqual(self.daemon_pid, process_named_running(PROCESS_NAME))
 
     def isis_base_config(self):
         self.cli_set(base_path + ['net', net])
@@ -333,7 +338,7 @@ class TestProtocolsISIS(VyOSUnitTestSHIM.TestCase):
         self.cli_set(base_path + ['interface', interface])
         self.cli_set(['policy', 'prefix-list', prefix_list, 'rule', '1', 'action', 'permit'])
         self.cli_set(['policy', 'prefix-list', prefix_list, 'rule', '1', 'prefix', prefix_list_address])
-        
+
         # Commit main ISIS changes
         self.cli_commit()
 
@@ -385,4 +390,4 @@ class TestProtocolsISIS(VyOSUnitTestSHIM.TestCase):
         self.cli_commit()
 
 if __name__ == '__main__':
-    unittest.main(verbosity=2)
\ No newline at end of file
+    unittest.main(verbosity=2)
diff --git a/smoketest/scripts/cli/test_protocols_mpls.py b/smoketest/scripts/cli/test_protocols_mpls.py
index 06f21c6e1..0c1599f9b 100755
--- a/smoketest/scripts/cli/test_protocols_mpls.py
+++ b/smoketest/scripts/cli/test_protocols_mpls.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2021 VyOS maintainers and contributors
+# Copyright (C) 2021-2023 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -70,6 +70,9 @@ class TestProtocolsMPLS(VyOSUnitTestSHIM.TestCase):
     def setUpClass(cls):
         super(TestProtocolsMPLS, cls).setUpClass()
 
+        # Retrieve FRR daemon PID - it is not allowed to crash, thus PID must remain the same
+        cls.daemon_pid = process_named_running(PROCESS_NAME)
+
         # ensure we can also run this test on a live system - so lets clean
         # out the current configuration :)
         cls.cli_delete(cls, base_path)
@@ -77,8 +80,9 @@ class TestProtocolsMPLS(VyOSUnitTestSHIM.TestCase):
     def tearDown(self):
         self.cli_delete(base_path)
         self.cli_commit()
-        # Check for running process
-        self.assertTrue(process_named_running(PROCESS_NAME))
+
+        # check process health and continuity
+        self.assertEqual(self.daemon_pid, process_named_running(PROCESS_NAME))
 
     def test_mpls_basic(self):
         router_id = '1.2.3.4'
diff --git a/smoketest/scripts/cli/test_protocols_ospf.py b/smoketest/scripts/cli/test_protocols_ospf.py
index a6850db71..6bffc7c45 100755
--- a/smoketest/scripts/cli/test_protocols_ospf.py
+++ b/smoketest/scripts/cli/test_protocols_ospf.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2021-2022 VyOS maintainers and contributors
+# Copyright (C) 2021-2023 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -32,6 +32,9 @@ class TestProtocolsOSPF(VyOSUnitTestSHIM.TestCase):
     def setUpClass(cls):
         super(TestProtocolsOSPF, cls).setUpClass()
 
+        # Retrieve FRR daemon PID - it is not allowed to crash, thus PID must remain the same
+        cls.daemon_pid = process_named_running(PROCESS_NAME)
+
         cls.cli_set(cls, ['policy', 'route-map', route_map, 'rule', '10', 'action', 'permit'])
         cls.cli_set(cls, ['policy', 'route-map', route_map, 'rule', '20', 'action', 'permit'])
 
@@ -45,11 +48,12 @@ class TestProtocolsOSPF(VyOSUnitTestSHIM.TestCase):
         super(TestProtocolsOSPF, cls).tearDownClass()
 
     def tearDown(self):
-        # Check for running process
-        self.assertTrue(process_named_running(PROCESS_NAME))
         self.cli_delete(base_path)
         self.cli_commit()
 
+        # check process health and continuity
+        self.assertEqual(self.daemon_pid, process_named_running(PROCESS_NAME))
+
     def test_ospf_01_defaults(self):
         # commit changes
         self.cli_set(base_path)
diff --git a/smoketest/scripts/cli/test_protocols_ospfv3.py b/smoketest/scripts/cli/test_protocols_ospfv3.py
index 0d6c6c691..4ae7f05d9 100755
--- a/smoketest/scripts/cli/test_protocols_ospfv3.py
+++ b/smoketest/scripts/cli/test_protocols_ospfv3.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2021-2022 VyOS maintainers and contributors
+# Copyright (C) 2021-2023 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -35,6 +35,9 @@ class TestProtocolsOSPFv3(VyOSUnitTestSHIM.TestCase):
     def setUpClass(cls):
         super(TestProtocolsOSPFv3, cls).setUpClass()
 
+        # Retrieve FRR daemon PID - it is not allowed to crash, thus PID must remain the same
+        cls.daemon_pid = process_named_running(PROCESS_NAME)
+
         cls.cli_set(cls, ['policy', 'route-map', route_map, 'rule', '10', 'action', 'permit'])
         cls.cli_set(cls, ['policy', 'route-map', route_map, 'rule', '20', 'action', 'permit'])
 
@@ -48,11 +51,12 @@ class TestProtocolsOSPFv3(VyOSUnitTestSHIM.TestCase):
         super(TestProtocolsOSPFv3, cls).tearDownClass()
 
     def tearDown(self):
-        # Check for running process
-        self.assertTrue(process_named_running(PROCESS_NAME))
         self.cli_delete(base_path)
         self.cli_commit()
 
+        # check process health and continuity
+        self.assertEqual(self.daemon_pid, process_named_running(PROCESS_NAME))
+
     def test_ospfv3_01_basic(self):
         seq = '10'
         prefix = '2001:db8::/32'
diff --git a/smoketest/scripts/cli/test_protocols_pim6.py b/smoketest/scripts/cli/test_protocols_pim6.py
index e22a7c722..ba24edca2 100755
--- a/smoketest/scripts/cli/test_protocols_pim6.py
+++ b/smoketest/scripts/cli/test_protocols_pim6.py
@@ -25,15 +25,22 @@ PROCESS_NAME = 'pim6d'
 base_path = ['protocols', 'pim6']
 
 class TestProtocolsPIMv6(VyOSUnitTestSHIM.TestCase):
-    def tearDown(self):
-        # Check for running process
-        self.assertTrue(process_named_running(PROCESS_NAME))
+    @classmethod
+    def setUpClass(cls):
+        # call base-classes classmethod
+        super(TestProtocolsPIMv6, cls).setUpClass()
+        # Retrieve FRR daemon PID - it is not allowed to crash, thus PID must remain the same
+        cls.daemon_pid = process_named_running(PROCESS_NAME)
+        # ensure we can also run this test on a live system - so lets clean
+        # out the current configuration :)
+        cls.cli_delete(cls, base_path)
 
+    def tearDown(self):
         self.cli_delete(base_path)
         self.cli_commit()
 
-        # Check for running process
-        self.assertTrue(process_named_running(PROCESS_NAME))
+        # check process health and continuity
+        self.assertEqual(self.daemon_pid, process_named_running(PROCESS_NAME))
 
     def test_pim6_01_mld_simple(self):
         # commit changes
diff --git a/smoketest/scripts/cli/test_protocols_rip.py b/smoketest/scripts/cli/test_protocols_rip.py
index 925499fc8..bfc327fd4 100755
--- a/smoketest/scripts/cli/test_protocols_rip.py
+++ b/smoketest/scripts/cli/test_protocols_rip.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2021 VyOS maintainers and contributors
+# Copyright (C) 2021-2023 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -34,7 +34,8 @@ class TestProtocolsRIP(VyOSUnitTestSHIM.TestCase):
     @classmethod
     def setUpClass(cls):
         super(TestProtocolsRIP, cls).setUpClass()
-
+        # Retrieve FRR daemon PID - it is not allowed to crash, thus PID must remain the same
+        cls.daemon_pid = process_named_running(PROCESS_NAME)
         # ensure we can also run this test on a live system - so lets clean
         # out the current configuration :)
         cls.cli_delete(cls, base_path)
@@ -65,8 +66,8 @@ class TestProtocolsRIP(VyOSUnitTestSHIM.TestCase):
         self.cli_delete(base_path)
         self.cli_commit()
 
-        # Check for running process
-        self.assertTrue(process_named_running(PROCESS_NAME))
+        # check process health and continuity
+        self.assertEqual(self.daemon_pid, process_named_running(PROCESS_NAME))
 
     def test_rip_01_parameters(self):
         distance = '40'
diff --git a/smoketest/scripts/cli/test_protocols_ripng.py b/smoketest/scripts/cli/test_protocols_ripng.py
index 0a8ce7eef..0cfb065c6 100755
--- a/smoketest/scripts/cli/test_protocols_ripng.py
+++ b/smoketest/scripts/cli/test_protocols_ripng.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2021 VyOS maintainers and contributors
+# Copyright (C) 2021-2023 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -31,28 +31,43 @@ route_map = 'FooBar123'
 base_path = ['protocols', 'ripng']
 
 class TestProtocolsRIPng(VyOSUnitTestSHIM.TestCase):
-    def setUp(self):
-        self.cli_set(['policy', 'access-list6', acl_in, 'rule', '10', 'action', 'permit'])
-        self.cli_set(['policy', 'access-list6', acl_in, 'rule', '10', 'source', 'any'])
-        self.cli_set(['policy', 'access-list6', acl_out, 'rule', '20', 'action', 'deny'])
-        self.cli_set(['policy', 'access-list6', acl_out, 'rule', '20', 'source', 'any'])
-        self.cli_set(['policy', 'prefix-list6', prefix_list_in, 'rule', '100', 'action', 'permit'])
-        self.cli_set(['policy', 'prefix-list6', prefix_list_in, 'rule', '100', 'prefix', '2001:db8::/32'])
-        self.cli_set(['policy', 'prefix-list6', prefix_list_out, 'rule', '200', 'action', 'deny'])
-        self.cli_set(['policy', 'prefix-list6', prefix_list_out, 'rule', '200', 'prefix', '2001:db8::/32'])
-        self.cli_set(['policy', 'route-map', route_map, 'rule', '10', 'action', 'permit'])
+    @classmethod
+    def setUpClass(cls):
+        # call base-classes classmethod
+        super(TestProtocolsRIPng, cls).setUpClass()
+        # Retrieve FRR daemon PID - it is not allowed to crash, thus PID must remain the same
+        cls.daemon_pid = process_named_running(PROCESS_NAME)
+        # ensure we can also run this test on a live system - so lets clean
+        # out the current configuration :)
+        cls.cli_delete(cls, base_path)
+
+        cls.cli_set(cls, ['policy', 'access-list6', acl_in, 'rule', '10', 'action', 'permit'])
+        cls.cli_set(cls, ['policy', 'access-list6', acl_in, 'rule', '10', 'source', 'any'])
+        cls.cli_set(cls, ['policy', 'access-list6', acl_out, 'rule', '20', 'action', 'deny'])
+        cls.cli_set(cls, ['policy', 'access-list6', acl_out, 'rule', '20', 'source', 'any'])
+        cls.cli_set(cls, ['policy', 'prefix-list6', prefix_list_in, 'rule', '100', 'action', 'permit'])
+        cls.cli_set(cls, ['policy', 'prefix-list6', prefix_list_in, 'rule', '100', 'prefix', '2001:db8::/32'])
+        cls.cli_set(cls, ['policy', 'prefix-list6', prefix_list_out, 'rule', '200', 'action', 'deny'])
+        cls.cli_set(cls, ['policy', 'prefix-list6', prefix_list_out, 'rule', '200', 'prefix', '2001:db8::/32'])
+        cls.cli_set(cls, ['policy', 'route-map', route_map, 'rule', '10', 'action', 'permit'])
+
+    @classmethod
+    def tearDownClass(cls):
+        # call base-classes classmethod
+        super(TestProtocolsRIPng, cls).tearDownClass()
+
+        cls.cli_delete(cls, ['policy', 'access-list6', acl_in])
+        cls.cli_delete(cls, ['policy', 'access-list6', acl_out])
+        cls.cli_delete(cls, ['policy', 'prefix-list6', prefix_list_in])
+        cls.cli_delete(cls, ['policy', 'prefix-list6', prefix_list_out])
+        cls.cli_delete(cls, ['policy', 'route-map', route_map])
 
     def tearDown(self):
         self.cli_delete(base_path)
-        self.cli_delete(['policy', 'access-list6', acl_in])
-        self.cli_delete(['policy', 'access-list6', acl_out])
-        self.cli_delete(['policy', 'prefix-list6', prefix_list_in])
-        self.cli_delete(['policy', 'prefix-list6', prefix_list_out])
-        self.cli_delete(['policy', 'route-map', route_map])
         self.cli_commit()
 
-        # Check for running process
-        self.assertTrue(process_named_running(PROCESS_NAME))
+        # check process health and continuity
+        self.assertEqual(self.daemon_pid, process_named_running(PROCESS_NAME))
 
     def test_ripng_01_parameters(self):
         metric = '8'
diff --git a/smoketest/scripts/cli/test_protocols_rpki.py b/smoketest/scripts/cli/test_protocols_rpki.py
index f4aedcbc3..ab3f076ac 100755
--- a/smoketest/scripts/cli/test_protocols_rpki.py
+++ b/smoketest/scripts/cli/test_protocols_rpki.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2021 VyOS maintainers and contributors
+# Copyright (C) 2021-2023 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -31,6 +31,16 @@ rpki_ssh_key = '/config/auth/id_rsa_rpki'
 rpki_ssh_pub = f'{rpki_ssh_key}.pub'
 
 class TestProtocolsRPKI(VyOSUnitTestSHIM.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        # call base-classes classmethod
+        super(TestProtocolsRPKI, cls).setUpClass()
+        # Retrieve FRR daemon PID - it is not allowed to crash, thus PID must remain the same
+        cls.daemon_pid = process_named_running(PROCESS_NAME)
+        # ensure we can also run this test on a live system - so lets clean
+        # out the current configuration :)
+        cls.cli_delete(cls, base_path)
+
     def tearDown(self):
         self.cli_delete(base_path)
         self.cli_commit()
@@ -39,8 +49,8 @@ class TestProtocolsRPKI(VyOSUnitTestSHIM.TestCase):
         # frrconfig = self.getFRRconfig('rpki')
         # self.assertNotIn('rpki', frrconfig)
 
-        # Check for running process
-        self.assertTrue(process_named_running(PROCESS_NAME))
+        # check process health and continuity
+        self.assertEqual(self.daemon_pid, process_named_running(PROCESS_NAME))
 
     def test_rpki(self):
         polling = '7200'