snmp: T8538: Persist engineBoots counter across reboots

Per RFC 3414 section 2.2 (Replay Protection), the `snmpEngineBoots`
counter must be stored in non-volatile storage and incremented on
every snmpd restart. VyOS was not persisting this value, causing
it to reset to 1 after every reboot.

SNMP managers cache the engineBoots value from previous sessions.
When VyOS resets the counter to 1 after reboot, managers reject
incoming SNMPv3 trap packets as "too old", producing errors such as:

```
  usm: Message too old.
  reboot count invalid
```

This change introduces `/config/snmp/engineboots.count` as a disk-backed
persist file and it uses to sync the counter into snmpd's conf
before the daemon starts.

5 files changed, 262 insertions, 0 deletions

diff --git a/data/templates/snmp/override.conf.j2 b/data/templates/snmp/override.conf.j2
index 42dc7a9d2..33b610fe3 100644
--- a/data/templates/snmp/override.conf.j2
+++ b/data/templates/snmp/override.conf.j2
@@ -8,5 +8,7 @@ Environment=
 Environment="MIBDIRS=/usr/share/snmp/mibs:/usr/share/snmp/mibs/iana:/usr/share/snmp/mibs/ietf:/usr/share/vyos/mibs"
 ExecStart=
 ExecStart={{ vrf_command }}/usr/sbin/snmpd -LS0-5d -Lf /dev/null -u Debian-snmp -g Debian-snmp -f -p /run/snmpd.pid
+# Sync engineBoot value between snmpd.conf and engineboots.count when user restarts the service manually
+ExecStartPost=!/usr/bin/python3 /usr/libexec/vyos/system/sync-snmp-engine-boots.py
 Restart=always
 RestartSec=10
diff --git a/data/templates/snmp/var.snmpd.conf.j2 b/data/templates/snmp/var.snmpd.conf.j2
index afab88abc..29abaa81c 100644
--- a/data/templates/snmp/var.snmpd.conf.j2
+++ b/data/templates/snmp/var.snmpd.conf.j2
@@ -14,3 +14,5 @@ createUser {{ vyos_user }} MD5 "{{ vyos_user_pass }}" DES
 oldEngineID 0x{{ v3.engineid }}
 {%     endif %}
 {% endif %}
+
+engineBoots {{ engine_boots }}
diff --git a/smoketest/scripts/cli/test_service_snmp.py b/smoketest/scripts/cli/test_service_snmp.py
index 729dc2328..1f3a7a372 100755
--- a/smoketest/scripts/cli/test_service_snmp.py
+++ b/smoketest/scripts/cli/test_service_snmp.py
@@ -20,10 +20,12 @@ import unittest
 from base_vyostest_shim import VyOSUnitTestSHIM
 
 from vyos.configsession import ConfigSessionError
+from vyos.defaults import systemd_services
 from vyos.template import address_from_cidr
 from vyos.template import bracketize_ipv6
 from vyos.template import is_ipv4
 from vyos.template import is_ipv6
+from vyos.utils.process import cmd
 from vyos.utils.process import call
 from vyos.utils.process import DEVNULL
 from vyos.utils.file import read_file
@@ -33,6 +35,7 @@ from vyos.xml_ref import default_value
 
 PROCESS_NAME = 'snmpd'
 SNMPD_CONF = '/etc/snmp/snmpd.conf'
+SYSTEMD_SERVICE = systemd_services['snmpd']
 
 base_path = ['service', 'snmp']
 
@@ -49,6 +52,20 @@ def get_config_value(key):
     tmp = re.findall(r'\n?{}\s+(.*)'.format(key), tmp)
     return tmp[0]
 
+
+def get_engine_boots() -> int:
+    """Query engineBoots directly from the running snmpd via SNMP"""
+
+    engine_boots_oid = '1.3.6.1.6.3.10.2.1.2.0'
+    out = cmd(
+        f'snmpget -v3 -u {snmpv3_user} -l authPriv '
+        f'-a SHA -A {snmpv3_auth_pw} '
+        f'-x AES -X {snmpv3_priv_pw} '
+        f'127.0.0.1 {engine_boots_oid}'
+    )
+    # Output: SNMP-FRAMEWORK-MIB::snmpEngineBoots.0 = INTEGER: 3
+    return int(out.split()[-1]) if 'snmpEngineBoots' in out else 0
+
 class TestSNMPService(VyOSUnitTestSHIM.TestCase):
     @classmethod
     def setUpClass(cls):
@@ -292,6 +309,105 @@ class TestSNMPService(VyOSUnitTestSHIM.TestCase):
         self.assertEqual(get_config_value('extend default'), f'/config/user-data/{extensions["default"]}')
         self.assertEqual(get_config_value('extend external'), extensions["external"])
 
+    def test_snmp_engine_boots_increment(self):
+        # T8538: engineBoots must increment by 1 on every snmpd restart.
+
+        snmpd_file = '/var/lib/snmp/snmpd.conf'
+        persist_file = '/config/snmp/engineboots.count'
+
+        def _verify_engine_boots(value_before, value_after):
+            lib_snmpd_content = read_file(snmpd_file, sudo=True)
+            persist_count_content = read_file(persist_file)
+
+            with self.subTest(value_before=value_before, value_after=value_after):
+                self.assertGreater(
+                    value_after,
+                    value_before,
+                    'engineBoots must increase after snmpd restart',
+                )
+                self.assertIn(
+                    f'engineBoots {value_after}\n',
+                    lib_snmpd_content,
+                    f'{snmpd_file} does not contain `engineBoots {value_after}`',
+                )
+                self.assertEqual(
+                    str(value_after),
+                    persist_count_content,
+                    f'{persist_file} does not match the expected value `{value_after}`',
+                )
+
+        self.cli_set(base_path + ['v3', 'engineid', snmpv3_engine_id])
+        self.cli_set(base_path + ['v3', 'group', 'default', 'mode', 'ro'])
+        self.cli_set(base_path + ['v3', 'view', 'default', 'oid', '1'])
+        self.cli_set(base_path + ['v3', 'group', 'default', 'view', 'default'])
+
+        base_user_path = base_path + ['v3', 'user', snmpv3_user]
+        self.cli_set(base_user_path + ['auth', 'plaintext-password', snmpv3_auth_pw])
+        self.cli_set(base_user_path + ['auth', 'type', 'sha'])
+        self.cli_set(base_user_path + ['privacy', 'plaintext-password', snmpv3_priv_pw])
+        self.cli_set(base_user_path + ['privacy', 'type', 'aes'])
+        self.cli_set(base_user_path + ['group', 'default'])
+        self.cli_commit()
+
+        value_before = get_engine_boots()
+
+        # Simulates multiple commits (which stop/start snmpd) and checks
+        # the live OID value increases monotonically.
+        self.cli_set(base_path + ['v3', 'view', 'default', 'oid', '2'])
+        self.cli_commit()
+
+        value_after = get_engine_boots()
+        _verify_engine_boots(value_before, value_after)
+
+        value_before = get_engine_boots()
+
+        # Restart of the service also should trigger changing of engineBoots
+        call(f'sudo systemctl restart {SYSTEMD_SERVICE}')
+
+        value_after = get_engine_boots()
+        _verify_engine_boots(value_before, value_after)
+
+    def test_snmp_engine_boots_reset(self):
+        # T8538: engineBoots should be set to zero on every changing of engineID
+
+        self.cli_set(base_path + ['v3', 'engineid', snmpv3_engine_id])
+        self.cli_set(base_path + ['v3', 'group', 'default', 'mode', 'ro'])
+        self.cli_set(base_path + ['v3', 'view', 'default', 'oid', '1'])
+        self.cli_set(base_path + ['v3', 'group', 'default', 'view', 'default'])
+
+        base_user_path = base_path + ['v3', 'user', snmpv3_user]
+        self.cli_set(base_user_path + ['auth', 'plaintext-password', snmpv3_auth_pw])
+        self.cli_set(base_user_path + ['auth', 'type', 'sha'])
+        self.cli_set(base_user_path + ['privacy', 'plaintext-password', snmpv3_priv_pw])
+        self.cli_set(base_user_path + ['privacy', 'type', 'aes'])
+        self.cli_set(base_user_path + ['group', 'default'])
+        self.cli_commit()
+
+        # Restart of the service to trigger changing of engineBoots
+        call(f'sudo systemctl restart {SYSTEMD_SERVICE}')
+
+        value_before = get_engine_boots()
+        self.assertGreater(
+            value_before,
+            1,
+            f'engineBoots should be greater 1 after restart of {SYSTEMD_SERVICE}',
+        )
+
+        new_snmpv3_engine_id = '000000000000000000000004'
+        self.cli_set(base_path + ['v3', 'engineid', new_snmpv3_engine_id])
+        # Re-add passwords because they were hashed by old engine id
+        self.cli_set(base_user_path + ['auth', 'plaintext-password', snmpv3_auth_pw])
+        self.cli_set(base_user_path + ['privacy', 'plaintext-password', snmpv3_priv_pw])
+        self.cli_commit()
+
+        value_after = get_engine_boots()
+        self.assertEqual(
+            value_after,
+            1,
+            'engineBoots should be set to zero on every changing of engineID',
+        )
+
+
 
 if __name__ == '__main__':
     unittest.main(verbosity=2, failfast=VyOSUnitTestSHIM.TestCase.debug_on())
diff --git a/src/conf_mode/service_snmp.py b/src/conf_mode/service_snmp.py
index b0b57a723..00993d269 100755
--- a/src/conf_mode/service_snmp.py
+++ b/src/conf_mode/service_snmp.py
@@ -15,12 +15,14 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
+import contextlib
 
 from sys import exit
 
 from vyos.base import Warning
 from vyos.config import Config
 from vyos.configdict import dict_merge
+from vyos.configdict import is_node_changed
 from vyos.configverify import verify_vrf
 from vyos.defaults import systemd_services
 from vyos.snmpv3_hashgen import plaintext_to_md5
@@ -33,6 +35,8 @@ from vyos.utils.dict import dict_search
 from vyos.utils.network import is_addr_assigned
 from vyos.utils.process import call
 from vyos.utils.permission import chmod_755
+from vyos.utils.file import read_file
+from vyos.utils.file import write_file
 from vyos.version import get_version_data
 from vyos import ConfigError
 from vyos import airbag
@@ -46,6 +50,34 @@ default_script_dir  = r'/config/user-data/'
 systemd_override    = r'/run/systemd/system/snmpd.service.d/override.conf'
 systemd_service     = systemd_services['snmpd']
 
+
+def _get_engine_boots_and_bump(reset=False):
+    """
+    Read, increment, persist, and return engineBoots counter.
+    Uses /config/snmp/engineboots.count as persistent storage
+    across reboots.
+
+    If the 'reset' flag is set, zero will be stored without reading the current state.
+    """
+    persist_count_file = '/config/snmp/engineboots.count'
+
+    # Ensure directory exists atomically
+    os.makedirs(os.path.dirname(persist_count_file), exist_ok=True)
+
+    count = 0
+
+    if not reset:
+        # Read current count, default to 0 on first run or corruption
+        raw = read_file(persist_count_file, defaultonfailure=str(count))
+        with contextlib.suppress(ValueError):
+            count = int(raw)
+
+    # Persist new value with increment immediately because snmpd will increase
+    # it automatically after restart the service
+    write_file(persist_count_file, str(count + 1))
+
+    return count
+
 def get_config(config=None):
     if config:
         conf = config
@@ -98,6 +130,26 @@ def get_config(config=None):
 
             snmp['script_extensions']['extension_name'][key]['script'] = script_path
 
+    # Per RFC 3414 section 2.3 we should reset the engineID to 0:
+    # > Note, that whenever the local value of snmpEngineID is
+    # > changed (e.g., through discovery) or when secure communications are
+    # > first established with an authoritative SNMP engine, the local values
+    # > of snmpEngineBoots and latestReceivedEngineTime should be set to
+    # > zero.
+    # It requires to track changing of this value and reset engineBoots.
+    if is_node_changed(conf, base + ['v3', 'engineid']):
+        effective_config = conf.get_config_dict(
+            base,
+            key_mangling=('-', '_'),
+            get_first_key=True,
+            no_tag_node_value_mangle=True,
+            effective=True,
+        )
+        current_engineid = dict_search('v3.engineid', snmp)
+        prev_engineid = dict_search('v3.engineid', effective_config)
+        if prev_engineid and current_engineid != prev_engineid:
+            snmp.update({'engineid_changed': {}})
+
     return snmp
 
 
@@ -210,6 +262,12 @@ def generate(snmp):
     if 'deleted' in snmp:
         return None
 
+    # RFC 3414 compliant:
+    #   - increments by 1 on every snmpd start
+    #   - reset to zero if engineID was changed
+    with_reset = 'engineid_changed' in snmp
+    snmp['engine_boots'] = _get_engine_boots_and_bump(reset=with_reset)
+
     if 'v3' in snmp:
         # SNMPv3 uses a hashed password. If CLI defines a plaintext password,
         # we will hash it in the background and replace the CLI node!
diff --git a/src/system/sync-snmp-engine-boots.py b/src/system/sync-snmp-engine-boots.py
new file mode 100644
index 000000000..4213325c4
--- /dev/null
+++ b/src/system/sync-snmp-engine-boots.py
@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) VyOS Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Called via systemd ExecStartPost= after snmpd starts.
+# Reads the live engineBoots value from /var/lib/snmp/snmpd.conf and
+# writes it to /config/snmp/engineboots.count only if the value differs.
+# Fixes T8538: ensures the persistent counter is always in sync with
+# what snmpd actually used, so the next restart increments correctly.
+
+import os
+import logging
+import contextlib
+
+import vyos.opmode
+
+from vyos.utils.file import read_file
+from vyos.utils.file import write_file
+
+SNMPD_CONF = '/var/lib/snmp/snmpd.conf'
+PERSIST_FILE = '/config/snmp/engineboots.count'
+
+# Configure logging
+logger = logging.getLogger(__name__)
+logger.addHandler(logging.StreamHandler())
+logger.setLevel(logging.DEBUG)
+
+
+def _read_snmpd_engine_boots() -> int | None:
+    """Return the engineBoots value from snmpd's persistent conf, or None."""
+
+    content = read_file(SNMPD_CONF, defaultonfailure='', sudo=True)
+    for line in content.splitlines():
+        if line.startswith('engineBoots'):
+            parts = line.split()
+            if len(parts) < 2:
+                continue
+            _, value, *_ = parts
+            with contextlib.suppress(ValueError):
+                return int(value)
+
+    return None
+
+
+def _read_persist_engine_boots() -> int | None:
+    """Return the currently saved engineBoots counter, or None."""
+
+    raw = read_file(PERSIST_FILE, defaultonfailure='')
+    with contextlib.suppress(ValueError):
+        return int(raw.strip())
+
+    return None
+
+
+if __name__ == '__main__':
+    snmpd_boots = _read_snmpd_engine_boots()
+    if snmpd_boots is None:
+        raise vyos.opmode.DataUnavailable(
+            f'Could not read engineBoots from {SNMPD_CONF}'
+        )
+
+    logger.debug(f'engineBoots from snmpd: {snmpd_boots}')
+
+    persist_boots = _read_persist_engine_boots()
+    logger.debug(f'engineBoots from persist file: {persist_boots}')
+
+    if persist_boots == snmpd_boots:
+        logger.debug('engineBoots already in sync, nothing to do')
+    else:
+        os.makedirs(os.path.dirname(PERSIST_FILE), exist_ok=True)
+        write_file(PERSIST_FILE, str(snmpd_boots))
+        logger.debug(f'engineBoots updated: {persist_boots} -> {snmpd_boots}')