Merge pull request #3587 from jestabro/config-default-system-version

migration: T6006: add system component version to config.boot.default by separating activation from migration

9 files changed, 362 insertions, 3 deletions

diff --git a/data/config.boot.default b/data/config.boot.default
new file mode 100644
index 000000000..93369d9b7
--- /dev/null
+++ b/data/config.boot.default
@@ -0,0 +1,53 @@
+interfaces {
+    loopback lo {
+    }
+}
+service {
+    ntp {
+        allow-client {
+            address "127.0.0.0/8"
+            address "169.254.0.0/16"
+            address "10.0.0.0/8"
+            address "172.16.0.0/12"
+            address "192.168.0.0/16"
+            address "::1/128"
+            address "fe80::/10"
+            address "fc00::/7"
+        }
+        server time1.vyos.net {
+        }
+        server time2.vyos.net {
+        }
+        server time3.vyos.net {
+        }
+    }
+}
+system {
+    config-management {
+        commit-revisions "100"
+    }
+    console {
+        device ttyS0 {
+            speed "115200"
+        }
+    }
+    host-name "vyos"
+    login {
+        user vyos {
+            authentication {
+                encrypted-password "$6$QxPS.uk6mfo$9QBSo8u1FkH16gMyAVhus6fU3LOzvLR9Z9.82m3tiHFAxTtIkhaZSWssSgzt4v4dGAL8rhVQxTg0oAG9/q11h/"
+                plaintext-password ""
+            }
+        }
+    }
+    syslog {
+        global {
+            facility all {
+                level "info"
+            }
+            facility local7 {
+                level "debug"
+            }
+        }
+    }
+}
diff --git a/debian/rules b/debian/rules
index d007089a4..9da40465f 100755
--- a/debian/rules
+++ b/debian/rules
@@ -11,6 +11,7 @@ VYOS_MIBS_DIR := usr/share/snmp/mibs
 VYOS_LOCALUI_DIR := srv/localui
 
 MIGRATION_SCRIPTS_DIR := opt/vyatta/etc/config-migrate/migrate
+ACTIVATION_SCRIPTS_DIR := usr/libexec/vyos/activate
 SYSTEM_SCRIPTS_DIR := usr/libexec/vyos/system
 SERVICES_DIR := usr/libexec/vyos/services
 
@@ -67,6 +68,10 @@ override_dh_auto_install:
 	mkdir -p $(DIR)/$(MIGRATION_SCRIPTS_DIR)
 	cp -r src/migration-scripts/* $(DIR)/$(MIGRATION_SCRIPTS_DIR)
 
+	# Install activation scripts
+	mkdir -p $(DIR)/$(ACTIVATION_SCRIPTS_DIR)
+	cp -r src/activation-scripts/* $(DIR)/$(ACTIVATION_SCRIPTS_DIR)
+
 	# Install system scripts
 	mkdir -p $(DIR)/$(SYSTEM_SCRIPTS_DIR)
 	cp -r src/system/* $(DIR)/$(SYSTEM_SCRIPTS_DIR)
diff --git a/debian/vyos-1x.install b/debian/vyos-1x.install
index 9e43669be..b3978d38a 100644
--- a/debian/vyos-1x.install
+++ b/debian/vyos-1x.install
@@ -28,6 +28,7 @@ usr/bin/vyos-config-to-commands
 usr/bin/vyos-config-to-json
 usr/bin/vyos-hostsd-client
 usr/lib
+usr/libexec/vyos/activate
 usr/libexec/vyos/completion
 usr/libexec/vyos/conf_mode
 usr/libexec/vyos/init
diff --git a/python/vyos/compose_config.py b/python/vyos/compose_config.py
new file mode 100644
index 000000000..efa28babe
--- /dev/null
+++ b/python/vyos/compose_config.py
@@ -0,0 +1,84 @@
+# Copyright 2024 VyOS maintainers and contributors <maintainers@vyos.io>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this library.  If not, see <http://www.gnu.org/licenses/>.
+
+"""This module allows iterating over function calls to modify an existing
+config.
+"""
+
+from pathlib import Path
+from typing import TypeAlias, Union, Callable
+
+from vyos.configtree import ConfigTree
+from vyos.configtree import deep_copy as ct_deep_copy
+from vyos.utils.system import load_as_module
+
+ConfigObj: TypeAlias = Union[str, ConfigTree]
+
+class ComposeConfigError(Exception):
+    """Raised when an error occurs modifying a config object.
+    """
+
+class ComposeConfig:
+    """Apply function to config tree: for iteration over functions or files.
+    """
+    def __init__(self, config_obj: ConfigObj, checkpoint_file=None):
+        if isinstance(config_obj, ConfigTree):
+            self.config_tree = config_obj
+        else:
+            self.config_tree = ConfigTree(config_obj)
+
+        self.checkpoint = self.config_tree
+        self.checkpoint_file = checkpoint_file
+
+    def apply_func(self, func: Callable):
+        """Apply the function to the config tree.
+        """
+        if not callable(func):
+            raise ComposeConfigError(f'{func.__name__} is not callable')
+
+        if self.checkpoint_file is not None:
+            self.checkpoint = ct_deep_copy(self.config_tree)
+
+        try:
+            func(self.config_tree)
+        except Exception as e:
+            self.config_tree = self.checkpoint
+            raise ComposeConfigError(e) from e
+
+    def apply_file(self, func_file: str, func_name: str):
+        """Apply named function from file.
+        """
+        try:
+            mod_name = Path(func_file).stem.replace('-', '_')
+            mod = load_as_module(mod_name, func_file)
+            func = getattr(mod, func_name)
+        except Exception as e:
+            raise ComposeConfigError(f'Error with {func_file}: {e}') from e
+
+        try:
+            self.apply_func(func)
+        except ComposeConfigError as e:
+            raise ComposeConfigError(f'Error in {func_file}: {e}') from e
+
+    def to_string(self, with_version=False) -> str:
+        """Return the rendered config tree.
+        """
+        return self.config_tree.to_string(no_version=not with_version)
+
+    def write(self, config_file: str, with_version=False):
+        """Write the config tree to a file.
+        """
+        config_str = self.to_string(with_version=with_version)
+        Path(config_file).write_text(config_str)
diff --git a/python/vyos/configtree.py b/python/vyos/configtree.py
index e4b282d72..afd6e030b 100644
--- a/python/vyos/configtree.py
+++ b/python/vyos/configtree.py
@@ -175,9 +175,11 @@ class ConfigTree(object):
     def get_version_string(self):
         return self.__version
 
-    def to_string(self, ordered_values=False):
+    def to_string(self, ordered_values=False, no_version=False):
         config_string = self.__to_string(self.__config, ordered_values).decode()
         config_string = unescape_backslash(config_string)
+        if no_version:
+            return config_string
         config_string = "{0}\n{1}".format(config_string, self.__version)
         return config_string
 
@@ -482,3 +484,9 @@ class DiffTree:
         add = self.add.to_commands()
         delete = self.delete.to_commands(op="delete")
         return delete + "\n" + add
+
+def deep_copy(config_tree: ConfigTree) -> ConfigTree:
+    """An inelegant, but reasonably fast, copy; replace with backend copy
+    """
+    D = DiffTree(None, config_tree)
+    return D.add
diff --git a/python/vyos/defaults.py b/python/vyos/defaults.py
index 64145a42e..e7cd69a8b 100644
--- a/python/vyos/defaults.py
+++ b/python/vyos/defaults.py
@@ -25,6 +25,7 @@ directories = {
   'services' : f'{base_dir}/services',
   'config' : '/opt/vyatta/etc/config',
   'migrate' : '/opt/vyatta/etc/config-migrate/migrate',
+  'activate' : f'{base_dir}/activate',
   'log' : '/var/log/vyatta',
   'templates' : '/usr/share/vyos/templates/',
   'certbot' : '/config/auth/letsencrypt',
@@ -46,3 +47,5 @@ cfg_vintage = 'vyos'
 commit_lock = '/opt/vyatta/config/.lock'
 
 component_version_json = os.path.join(directories['data'], 'component-versions.json')
+
+config_default = os.path.join(directories['data'], 'config.boot.default')
diff --git a/src/activation-scripts/20-ethernet_offload.py b/src/activation-scripts/20-ethernet_offload.py
new file mode 100755
index 000000000..33b0ea469
--- /dev/null
+++ b/src/activation-scripts/20-ethernet_offload.py
@@ -0,0 +1,103 @@
+# Copyright 2021-2024 VyOS maintainers and contributors <maintainers@vyos.io>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this library.  If not, see <http://www.gnu.org/licenses/>.
+
+# T3619: mirror Linux Kernel defaults for ethernet offloading options into VyOS
+#        CLI. See https://vyos.dev/T3619#102254 for all the details.
+# T3787: Remove deprecated UDP fragmentation offloading option
+# T6006: add to activation-scripts: migration-scripts/interfaces/20-to-21
+
+from vyos.ethtool import Ethtool
+from vyos.configtree import ConfigTree
+
+def activate(config: ConfigTree):
+    base = ['interfaces', 'ethernet']
+
+    if not config.exists(base):
+        return
+
+    for ifname in config.list_nodes(base):
+        eth = Ethtool(ifname)
+
+        # If GRO is enabled by the Kernel - we reflect this on the CLI. If GRO is
+        # enabled via CLI but not supported by the NIC - we remove it from the CLI
+        configured = config.exists(base + [ifname, 'offload', 'gro'])
+        enabled, fixed = eth.get_generic_receive_offload()
+        if configured and fixed:
+            config.delete(base + [ifname, 'offload', 'gro'])
+        elif enabled and not fixed:
+            config.set(base + [ifname, 'offload', 'gro'])
+
+        # If GSO is enabled by the Kernel - we reflect this on the CLI. If GSO is
+        # enabled via CLI but not supported by the NIC - we remove it from the CLI
+        configured = config.exists(base + [ifname, 'offload', 'gso'])
+        enabled, fixed = eth.get_generic_segmentation_offload()
+        if configured and fixed:
+            config.delete(base + [ifname, 'offload', 'gso'])
+        elif enabled and not fixed:
+            config.set(base + [ifname, 'offload', 'gso'])
+
+        # If LRO is enabled by the Kernel - we reflect this on the CLI. If LRO is
+        # enabled via CLI but not supported by the NIC - we remove it from the CLI
+        configured = config.exists(base + [ifname, 'offload', 'lro'])
+        enabled, fixed = eth.get_large_receive_offload()
+        if configured and fixed:
+            config.delete(base + [ifname, 'offload', 'lro'])
+        elif enabled and not fixed:
+            config.set(base + [ifname, 'offload', 'lro'])
+
+        # If SG is enabled by the Kernel - we reflect this on the CLI. If SG is
+        # enabled via CLI but not supported by the NIC - we remove it from the CLI
+        configured = config.exists(base + [ifname, 'offload', 'sg'])
+        enabled, fixed = eth.get_scatter_gather()
+        if configured and fixed:
+            config.delete(base + [ifname, 'offload', 'sg'])
+        elif enabled and not fixed:
+            config.set(base + [ifname, 'offload', 'sg'])
+
+        # If TSO is enabled by the Kernel - we reflect this on the CLI. If TSO is
+        # enabled via CLI but not supported by the NIC - we remove it from the CLI
+        configured = config.exists(base + [ifname, 'offload', 'tso'])
+        enabled, fixed = eth.get_tcp_segmentation_offload()
+        if configured and fixed:
+            config.delete(base + [ifname, 'offload', 'tso'])
+        elif enabled and not fixed:
+            config.set(base + [ifname, 'offload', 'tso'])
+
+        # Remove deprecated UDP fragmentation offloading option
+        if config.exists(base + [ifname, 'offload', 'ufo']):
+            config.delete(base + [ifname, 'offload', 'ufo'])
+
+        # Also while processing the interface configuration, not all adapters support
+        # changing the speed and duplex settings. If the desired speed and duplex
+        # values do not work for the NIC driver, we change them back to the default
+        # value of "auto" - which will be applied if the CLI node is deleted.
+        speed_path = base + [ifname, 'speed']
+        duplex_path = base + [ifname, 'duplex']
+        # speed and duplex must always be set at the same time if not set to "auto"
+        if config.exists(speed_path) and config.exists(duplex_path):
+            speed = config.return_value(speed_path)
+            duplex = config.return_value(duplex_path)
+            if speed != 'auto' and duplex != 'auto':
+                if not eth.check_speed_duplex(speed, duplex):
+                    config.delete(speed_path)
+                    config.delete(duplex_path)
+
+        # Also while processing the interface configuration, not all adapters support
+        # changing disabling flow-control - or change this setting. If disabling
+        # flow-control is not supported by the NIC, we remove the setting from CLI
+        flow_control_path = base + [ifname, 'disable-flow-control']
+        if config.exists(flow_control_path):
+            if not eth.check_flow_control():
+                config.delete(flow_control_path)
diff --git a/src/helpers/run-config-activation.py b/src/helpers/run-config-activation.py
new file mode 100755
index 000000000..58293702a
--- /dev/null
+++ b/src/helpers/run-config-activation.py
@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2024 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import re
+import logging
+from pathlib import Path
+from argparse import ArgumentParser
+
+from vyos.compose_config import ComposeConfig
+from vyos.compose_config import ComposeConfigError
+from vyos.defaults import directories
+
+parser = ArgumentParser()
+parser.add_argument('config_file', type=str,
+                    help="configuration file to modify with system-specific settings")
+parser.add_argument('--test-script', type=str,
+                    help="test effect of named script")
+
+args = parser.parse_args()
+
+checkpoint_file = '/run/vyos-activate-checkpoint'
+log_file = Path(directories['config']).joinpath('vyos-activate.log')
+
+logger = logging.getLogger(__name__)
+fh = logging.FileHandler(log_file)
+formatter = logging.Formatter('%(message)s')
+fh.setFormatter(formatter)
+logger.addHandler(fh)
+
+if 'vyos-activate-debug' in Path('/proc/cmdline').read_text():
+    print(f'\nactivate-debug enabled: file {checkpoint_file}_* on error')
+    debug = checkpoint_file
+    logger.setLevel(logging.DEBUG)
+else:
+    debug = None
+    logger.setLevel(logging.INFO)
+
+def sort_key(s: Path):
+    s = s.stem
+    pre, rem = re.match(r'(\d*)(?:-)?(.+)', s).groups()
+    return int(pre or 0), rem
+
+def file_ext(file_name: str) -> str:
+    """Return an identifier from file name for checkpoint file extension.
+    """
+    return Path(file_name).stem
+
+script_dir = Path(directories['activate'])
+
+if args.test_script:
+    script_list = [script_dir.joinpath(args.test_script)]
+else:
+    script_list = sorted(script_dir.glob('*.py'), key=sort_key)
+
+config_file = args.config_file
+config_str = Path(config_file).read_text()
+
+compose = ComposeConfig(config_str, checkpoint_file=debug)
+
+for file in script_list:
+    file = file.as_posix()
+    logger.info(f'calling {file}')
+    try:
+        compose.apply_file(file, func_name='activate')
+    except ComposeConfigError as e:
+        if debug:
+            compose.write(f'{compose.checkpoint_file}_{file_ext(file)}')
+        logger.error(f'config-activation error in {file}: {e}')
+
+compose.write(config_file, with_version=True)
diff --git a/src/init/vyos-router b/src/init/vyos-router
index 15e37df07..59004fdc1 100755
--- a/src/init/vyos-router
+++ b/src/init/vyos-router
@@ -22,6 +22,7 @@ declare progname=${0##*/}
 declare action=$1; shift
 
 declare -x BOOTFILE=$vyatta_sysconfdir/config/config.boot
+declare -x DEFAULT_BOOTFILE=$vyatta_sysconfdir/config.boot.default
 
 # If vyos-config= boot option is present, use that file instead
 for x in $(cat /proc/cmdline); do
@@ -129,9 +130,16 @@ unmount_encrypted_config() {
 
 # if necessary, provide initial config
 init_bootfile () {
+    # define and version default boot config if not present
+    if [ ! -r $DEFAULT_BOOTFILE ]; then
+        if [ -f $vyos_data_dir/config.boot.default ]; then
+            cp $vyos_data_dir/config.boot.default $DEFAULT_BOOTFILE
+            $vyos_libexec_dir/system-versions-foot.py >> $DEFAULT_BOOTFILE
+        fi
+    fi
     if [ ! -r $BOOTFILE ] ; then
-        if [ -f $vyatta_sysconfdir/config.boot.default ]; then
-            cp $vyatta_sysconfdir/config.boot.default $BOOTFILE
+        if [ -f $DEFAULT_BOOTFILE ]; then
+            cp $DEFAULT_BOOTFILE $BOOTFILE
         else
             $vyos_libexec_dir/system-versions-foot.py > $BOOTFILE
         fi
@@ -149,6 +157,15 @@ migrate_bootfile ()
     fi
 }
 
+# configure system-specific settings
+system_config ()
+{
+    if [ -x $vyos_libexec_dir/run-config-activation.py ]; then
+        log_progress_msg system
+        sg ${GROUP} -c "$vyos_libexec_dir/run-config-activation.py $BOOTFILE"
+    fi
+}
+
 # load the initial config
 load_bootfile ()
 {
@@ -493,6 +510,8 @@ start ()
 
     update_interface_config
 
+    disabled system_config || system_config
+
     for s in ${subinit[@]} ; do
     if ! disabled $s; then
         log_progress_msg $s