snmp: T8538: Persist engineBoots counter across reboots

Per RFC 3414 section 2.2 (Replay Protection), the `snmpEngineBoots`
counter must be stored in non-volatile storage and incremented on
every snmpd restart. VyOS was not persisting this value, causing
it to reset to 1 after every reboot.

SNMP managers cache the engineBoots value from previous sessions.
When VyOS resets the counter to 1 after reboot, managers reject
incoming SNMPv3 trap packets as "too old", producing errors such as:

```
  usm: Message too old.
  reboot count invalid
```

This change introduces `/config/snmp/engineboots.count` as a disk-backed
persist file and it uses to sync the counter into snmpd's conf
before the daemon starts.

1 files changed, 116 insertions, 0 deletions

diff --git a/smoketest/scripts/cli/test_service_snmp.py b/smoketest/scripts/cli/test_service_snmp.py
index 729dc2328..1f3a7a372 100755
--- a/smoketest/scripts/cli/test_service_snmp.py
+++ b/smoketest/scripts/cli/test_service_snmp.py
@@ -20,10 +20,12 @@ import unittest
 from base_vyostest_shim import VyOSUnitTestSHIM
 
 from vyos.configsession import ConfigSessionError
+from vyos.defaults import systemd_services
 from vyos.template import address_from_cidr
 from vyos.template import bracketize_ipv6
 from vyos.template import is_ipv4
 from vyos.template import is_ipv6
+from vyos.utils.process import cmd
 from vyos.utils.process import call
 from vyos.utils.process import DEVNULL
 from vyos.utils.file import read_file
@@ -33,6 +35,7 @@ from vyos.xml_ref import default_value
 
 PROCESS_NAME = 'snmpd'
 SNMPD_CONF = '/etc/snmp/snmpd.conf'
+SYSTEMD_SERVICE = systemd_services['snmpd']
 
 base_path = ['service', 'snmp']
 
@@ -49,6 +52,20 @@ def get_config_value(key):
     tmp = re.findall(r'\n?{}\s+(.*)'.format(key), tmp)
     return tmp[0]
 
+
+def get_engine_boots() -> int:
+    """Query engineBoots directly from the running snmpd via SNMP"""
+
+    engine_boots_oid = '1.3.6.1.6.3.10.2.1.2.0'
+    out = cmd(
+        f'snmpget -v3 -u {snmpv3_user} -l authPriv '
+        f'-a SHA -A {snmpv3_auth_pw} '
+        f'-x AES -X {snmpv3_priv_pw} '
+        f'127.0.0.1 {engine_boots_oid}'
+    )
+    # Output: SNMP-FRAMEWORK-MIB::snmpEngineBoots.0 = INTEGER: 3
+    return int(out.split()[-1]) if 'snmpEngineBoots' in out else 0
+
 class TestSNMPService(VyOSUnitTestSHIM.TestCase):
     @classmethod
     def setUpClass(cls):
@@ -292,6 +309,105 @@ class TestSNMPService(VyOSUnitTestSHIM.TestCase):
         self.assertEqual(get_config_value('extend default'), f'/config/user-data/{extensions["default"]}')
         self.assertEqual(get_config_value('extend external'), extensions["external"])
 
+    def test_snmp_engine_boots_increment(self):
+        # T8538: engineBoots must increment by 1 on every snmpd restart.
+
+        snmpd_file = '/var/lib/snmp/snmpd.conf'
+        persist_file = '/config/snmp/engineboots.count'
+
+        def _verify_engine_boots(value_before, value_after):
+            lib_snmpd_content = read_file(snmpd_file, sudo=True)
+            persist_count_content = read_file(persist_file)
+
+            with self.subTest(value_before=value_before, value_after=value_after):
+                self.assertGreater(
+                    value_after,
+                    value_before,
+                    'engineBoots must increase after snmpd restart',
+                )
+                self.assertIn(
+                    f'engineBoots {value_after}\n',
+                    lib_snmpd_content,
+                    f'{snmpd_file} does not contain `engineBoots {value_after}`',
+                )
+                self.assertEqual(
+                    str(value_after),
+                    persist_count_content,
+                    f'{persist_file} does not match the expected value `{value_after}`',
+                )
+
+        self.cli_set(base_path + ['v3', 'engineid', snmpv3_engine_id])
+        self.cli_set(base_path + ['v3', 'group', 'default', 'mode', 'ro'])
+        self.cli_set(base_path + ['v3', 'view', 'default', 'oid', '1'])
+        self.cli_set(base_path + ['v3', 'group', 'default', 'view', 'default'])
+
+        base_user_path = base_path + ['v3', 'user', snmpv3_user]
+        self.cli_set(base_user_path + ['auth', 'plaintext-password', snmpv3_auth_pw])
+        self.cli_set(base_user_path + ['auth', 'type', 'sha'])
+        self.cli_set(base_user_path + ['privacy', 'plaintext-password', snmpv3_priv_pw])
+        self.cli_set(base_user_path + ['privacy', 'type', 'aes'])
+        self.cli_set(base_user_path + ['group', 'default'])
+        self.cli_commit()
+
+        value_before = get_engine_boots()
+
+        # Simulates multiple commits (which stop/start snmpd) and checks
+        # the live OID value increases monotonically.
+        self.cli_set(base_path + ['v3', 'view', 'default', 'oid', '2'])
+        self.cli_commit()
+
+        value_after = get_engine_boots()
+        _verify_engine_boots(value_before, value_after)
+
+        value_before = get_engine_boots()
+
+        # Restart of the service also should trigger changing of engineBoots
+        call(f'sudo systemctl restart {SYSTEMD_SERVICE}')
+
+        value_after = get_engine_boots()
+        _verify_engine_boots(value_before, value_after)
+
+    def test_snmp_engine_boots_reset(self):
+        # T8538: engineBoots should be set to zero on every changing of engineID
+
+        self.cli_set(base_path + ['v3', 'engineid', snmpv3_engine_id])
+        self.cli_set(base_path + ['v3', 'group', 'default', 'mode', 'ro'])
+        self.cli_set(base_path + ['v3', 'view', 'default', 'oid', '1'])
+        self.cli_set(base_path + ['v3', 'group', 'default', 'view', 'default'])
+
+        base_user_path = base_path + ['v3', 'user', snmpv3_user]
+        self.cli_set(base_user_path + ['auth', 'plaintext-password', snmpv3_auth_pw])
+        self.cli_set(base_user_path + ['auth', 'type', 'sha'])
+        self.cli_set(base_user_path + ['privacy', 'plaintext-password', snmpv3_priv_pw])
+        self.cli_set(base_user_path + ['privacy', 'type', 'aes'])
+        self.cli_set(base_user_path + ['group', 'default'])
+        self.cli_commit()
+
+        # Restart of the service to trigger changing of engineBoots
+        call(f'sudo systemctl restart {SYSTEMD_SERVICE}')
+
+        value_before = get_engine_boots()
+        self.assertGreater(
+            value_before,
+            1,
+            f'engineBoots should be greater 1 after restart of {SYSTEMD_SERVICE}',
+        )
+
+        new_snmpv3_engine_id = '000000000000000000000004'
+        self.cli_set(base_path + ['v3', 'engineid', new_snmpv3_engine_id])
+        # Re-add passwords because they were hashed by old engine id
+        self.cli_set(base_user_path + ['auth', 'plaintext-password', snmpv3_auth_pw])
+        self.cli_set(base_user_path + ['privacy', 'plaintext-password', snmpv3_priv_pw])
+        self.cli_commit()
+
+        value_after = get_engine_boots()
+        self.assertEqual(
+            value_after,
+            1,
+            'engineBoots should be set to zero on every changing of engineID',
+        )
+
+
 
 if __name__ == '__main__':
     unittest.main(verbosity=2, failfast=VyOSUnitTestSHIM.TestCase.debug_on())