Merge pull request #4893 from c-po/configtests

smoketest: T8087: reorganize folderstructure for embedded configttests

55 files changed, 85 insertions, 10 deletions

diff --git a/debian/rules b/debian/rules
index f4540dacb..9649a6a36 100755
--- a/debian/rules
+++ b/debian/rules
@@ -149,12 +149,8 @@ override_dh_auto_install:
 	cp -r smoketest/scripts/* $(DIR)/$(VYOS_LIBEXEC_DIR)/tests/smoke
 
 	# Install smoke test configs
-	mkdir -p $(DIR)/$(VYOS_LIBEXEC_DIR)/tests/config/
-	cp -r smoketest/configs/* $(DIR)/$(VYOS_LIBEXEC_DIR)/tests/config
-
-	# Install smoke test config tests
-	mkdir -p $(DIR)/$(VYOS_LIBEXEC_DIR)/tests/config-tests/
-	cp -r smoketest/config-tests/* $(DIR)/$(VYOS_LIBEXEC_DIR)/tests/config-tests
+	mkdir -p $(DIR)/$(VYOS_LIBEXEC_DIR)/tests/configs/
+	cp -r smoketest/configs/* $(DIR)/$(VYOS_LIBEXEC_DIR)/tests/configs
 
 	# Install system programs
 	mkdir -p $(DIR)/$(VYOS_BIN_DIR)
diff --git a/debian/vyos-1x-smoketest.install b/debian/vyos-1x-smoketest.install
index 739cb189b..cadfb98dc 100644
--- a/debian/vyos-1x-smoketest.install
+++ b/debian/vyos-1x-smoketest.install
@@ -2,5 +2,4 @@ usr/bin/vyos-smoketest
 usr/bin/vyos-configtest
 usr/bin/vyos-configtest-pki
 usr/libexec/vyos/tests/smoke
-usr/libexec/vyos/tests/config
-usr/libexec/vyos/tests/config-tests
+usr/libexec/vyos/tests/configs
diff --git a/smoketest/README.md b/smoketest/README.md
new file mode 100644
index 000000000..5318fa55f
--- /dev/null
+++ b/smoketest/README.md
@@ -0,0 +1,80 @@
+# VyOS Smoketests
+
+A brief overview of the smoketesting platform
+
+## CLI set/delete and validation
+
+Smoketests execute predefined VyOS CLI commands and verify that the
+corresponding daemon is running and/or service configuration is correctly
+rendered.
+
+Smoketests are available for the system (e.g. Kernel) and the CLI. They can
+be run manually by simply executing the Python script on a live VyOS system.
+
+```
+vyos@vyos:~$ /usr/libexec/vyos/tests/smoke/cli/test_protocols_bgp.py
+test_bgp_01_simple (__main__.TestProtocolsBGP) ... ok
+test_bgp_02_neighbors (__main__.TestProtocolsBGP) ... ok
+...
+test_bgp_13_solo (__main__.TestProtocolsBGP) ... ok
+
+----------------------------------------------------------------------
+Ran 13 tests in 348.191s
+
+OK
+```
+
+It is possible to only execute a single testcase
+
+```
+vyos@vyos:~$ /usr/libexec/vyos/tests/smoke/cli/test_protocols_bgp.py -k test_bgp_02_neighbors
+test_bgp_02_neighbors (__main__.TestProtocolsBGP) ... ok
+
+----------------------------------------------------------------------
+Ran 1 test in 6.872s
+
+OK
+```
+
+## Configuration Migration
+
+The files in `smoketests/configs/` are real VyOS configurations taken from
+production or lab systems. They provide realistic inputs to test configuration
+migration and ensure that existing configs still load correctly after code
+changes.
+
+Example:
+
+After assembling a VyOS ISO image from vyos-build repository, a `make testc`
+loads the configuration files and its associated CLI commands that are asserted.
+After loading the configuration and executing any necessary migration scripts,
+we then check whether the resulting `show configuration commands` output
+contains all commands listed in `smoketests/configs/assert`. If all listed
+commands are present, the test passes.
+
+Each config file has exactly one matching expected-set file inside the
+`smoketests/configs/assert` folder.
+
+When modifying or adding a migration script, update the matching file in
+`smoketests/configs/assert` to reflect any new or changed CLI commands.
+
+**NOTE:** The migrated output does not need to match exactly. Missing lines in
+the assert file are not treated as failures. A test only fails when a listed
+line differs from the actual migrated output.
+
+```
+smoketests/
+  ├── configs/               # Input configuration files
+  │     ├── assert/          # Expected CLI commands after migration
+  │     │     ├── example1
+  │     │     ├── example2
+  │     │     └── ...
+  │     ├── no-load/         # Large configurations we do not automatically test
+  │     │     └── ...
+  │     ├── example1
+  │     ├── example2
+  │     └── ...
+  └── README.md              # Documentation for adding and running tests
+```
+
+
diff --git a/smoketest/bin/vyos-configtest b/smoketest/bin/vyos-configtest
index 23cbd4e54..6620c044f 100755
--- a/smoketest/bin/vyos-configtest
+++ b/smoketest/bin/vyos-configtest
@@ -23,8 +23,8 @@ import unittest
 from vyos.configsession import ConfigSession, ConfigSessionError
 from vyos import ConfigError
 
-config_dir = '/usr/libexec/vyos/tests/config'
-config_test_dir = '/usr/libexec/vyos/tests/config-tests'
+config_dir = '/usr/libexec/vyos/tests/configs'
+config_test_dir = f'{config_dir}/assert'
 save_config = '/tmp/vyos-configtest-save'
 
 class DynamicClassBase(unittest.TestCase):
diff --git a/smoketest/config-tests/basic-api-service b/smoketest/configs/assert/basic-api-service
index ca10cf4e9..ca10cf4e9 100644
--- a/smoketest/config-tests/basic-api-service
+++ b/smoketest/configs/assert/basic-api-service
diff --git a/smoketest/config-tests/basic-haproxy b/smoketest/configs/assert/basic-haproxy
index 7755fc4ea..7755fc4ea 100644
--- a/smoketest/config-tests/basic-haproxy
+++ b/smoketest/configs/assert/basic-haproxy
diff --git a/smoketest/config-tests/basic-ipv6 b/smoketest/configs/assert/basic-ipv6
index 232a42a07..232a42a07 100644
--- a/smoketest/config-tests/basic-ipv6
+++ b/smoketest/configs/assert/basic-ipv6
diff --git a/smoketest/config-tests/basic-syslog b/smoketest/configs/assert/basic-syslog
index a8acc18e9..a8acc18e9 100644
--- a/smoketest/config-tests/basic-syslog
+++ b/smoketest/configs/assert/basic-syslog
diff --git a/smoketest/config-tests/basic-vyos b/smoketest/configs/assert/basic-vyos
index 601051d8f..601051d8f 100644
--- a/smoketest/config-tests/basic-vyos
+++ b/smoketest/configs/assert/basic-vyos
diff --git a/smoketest/config-tests/basic-vyos-no-ntp b/smoketest/configs/assert/basic-vyos-no-ntp
index f00dea5d4..f00dea5d4 100644
--- a/smoketest/config-tests/basic-vyos-no-ntp
+++ b/smoketest/configs/assert/basic-vyos-no-ntp
diff --git a/smoketest/config-tests/bgp-azure-ipsec-gateway b/smoketest/configs/assert/bgp-azure-ipsec-gateway
index 823a2c36a..823a2c36a 100644
--- a/smoketest/config-tests/bgp-azure-ipsec-gateway
+++ b/smoketest/configs/assert/bgp-azure-ipsec-gateway
diff --git a/smoketest/config-tests/bgp-bfd-communities b/smoketest/configs/assert/bgp-bfd-communities
index 06e412c55..06e412c55 100644
--- a/smoketest/config-tests/bgp-bfd-communities
+++ b/smoketest/configs/assert/bgp-bfd-communities
diff --git a/smoketest/config-tests/bgp-big-as-cloud b/smoketest/configs/assert/bgp-big-as-cloud
index 072f5258a..072f5258a 100644
--- a/smoketest/config-tests/bgp-big-as-cloud
+++ b/smoketest/configs/assert/bgp-big-as-cloud
diff --git a/smoketest/config-tests/bgp-dmvpn-hub b/smoketest/configs/assert/bgp-dmvpn-hub
index f9ceba11c..f9ceba11c 100644
--- a/smoketest/config-tests/bgp-dmvpn-hub
+++ b/smoketest/configs/assert/bgp-dmvpn-hub
diff --git a/smoketest/config-tests/bgp-dmvpn-spoke b/smoketest/configs/assert/bgp-dmvpn-spoke
index a98275ba4..a98275ba4 100644
--- a/smoketest/config-tests/bgp-dmvpn-spoke
+++ b/smoketest/configs/assert/bgp-dmvpn-spoke
diff --git a/smoketest/config-tests/bgp-evpn-l2vpn-leaf b/smoketest/configs/assert/bgp-evpn-l2vpn-leaf
index 84cf0805e..84cf0805e 100644
--- a/smoketest/config-tests/bgp-evpn-l2vpn-leaf
+++ b/smoketest/configs/assert/bgp-evpn-l2vpn-leaf
diff --git a/smoketest/config-tests/bgp-evpn-l2vpn-spine b/smoketest/configs/assert/bgp-evpn-l2vpn-spine
index 5d0a70398..5d0a70398 100644
--- a/smoketest/config-tests/bgp-evpn-l2vpn-spine
+++ b/smoketest/configs/assert/bgp-evpn-l2vpn-spine
diff --git a/smoketest/config-tests/bgp-evpn-l3vpn-pe-router b/smoketest/configs/assert/bgp-evpn-l3vpn-pe-router
index 88309e424..88309e424 100644
--- a/smoketest/config-tests/bgp-evpn-l3vpn-pe-router
+++ b/smoketest/configs/assert/bgp-evpn-l3vpn-pe-router
diff --git a/smoketest/config-tests/bgp-medium-confederation b/smoketest/configs/assert/bgp-medium-confederation
index 71797fe93..71797fe93 100644
--- a/smoketest/config-tests/bgp-medium-confederation
+++ b/smoketest/configs/assert/bgp-medium-confederation
diff --git a/smoketest/config-tests/bgp-rpki b/smoketest/configs/assert/bgp-rpki
index 657d4abcc..657d4abcc 100644
--- a/smoketest/config-tests/bgp-rpki
+++ b/smoketest/configs/assert/bgp-rpki
diff --git a/smoketest/config-tests/bgp-small-internet-exchange b/smoketest/configs/assert/bgp-small-internet-exchange
index 2adb3fbb5..2adb3fbb5 100644
--- a/smoketest/config-tests/bgp-small-internet-exchange
+++ b/smoketest/configs/assert/bgp-small-internet-exchange
diff --git a/smoketest/config-tests/bgp-small-ipv4-unicast b/smoketest/configs/assert/bgp-small-ipv4-unicast
index f8820cb3c..f8820cb3c 100644
--- a/smoketest/config-tests/bgp-small-ipv4-unicast
+++ b/smoketest/configs/assert/bgp-small-ipv4-unicast
diff --git a/smoketest/config-tests/cluster-basic b/smoketest/configs/assert/cluster-basic
index 871b40bbb..871b40bbb 100644
--- a/smoketest/config-tests/cluster-basic
+++ b/smoketest/configs/assert/cluster-basic
diff --git a/smoketest/config-tests/conntrack-basic b/smoketest/configs/assert/conntrack-basic
index 8c375d244..8c375d244 100644
--- a/smoketest/config-tests/conntrack-basic
+++ b/smoketest/configs/assert/conntrack-basic
diff --git a/smoketest/config-tests/container-simple b/smoketest/configs/assert/container-simple
index fcc665100..fcc665100 100644
--- a/smoketest/config-tests/container-simple
+++ b/smoketest/configs/assert/container-simple
diff --git a/smoketest/config-tests/dialup-router-complex b/smoketest/configs/assert/dialup-router-complex
index 12edcfef2..12edcfef2 100644
--- a/smoketest/config-tests/dialup-router-complex
+++ b/smoketest/configs/assert/dialup-router-complex
diff --git a/smoketest/config-tests/dialup-router-medium-vpn b/smoketest/configs/assert/dialup-router-medium-vpn
index 87b35b18c..87b35b18c 100644
--- a/smoketest/config-tests/dialup-router-medium-vpn
+++ b/smoketest/configs/assert/dialup-router-medium-vpn
diff --git a/smoketest/config-tests/dialup-router-wireguard-ipv6 b/smoketest/configs/assert/dialup-router-wireguard-ipv6
index 269e9d722..269e9d722 100644
--- a/smoketest/config-tests/dialup-router-wireguard-ipv6
+++ b/smoketest/configs/assert/dialup-router-wireguard-ipv6
diff --git a/smoketest/config-tests/egp-igp-route-maps b/smoketest/configs/assert/egp-igp-route-maps
index 222325cd7..222325cd7 100644
--- a/smoketest/config-tests/egp-igp-route-maps
+++ b/smoketest/configs/assert/egp-igp-route-maps
diff --git a/smoketest/config-tests/firewall-bridged-global-options b/smoketest/configs/assert/firewall-bridged-global-options
index 1d960d6c1..1d960d6c1 100644
--- a/smoketest/config-tests/firewall-bridged-global-options
+++ b/smoketest/configs/assert/firewall-bridged-global-options
diff --git a/smoketest/config-tests/firewall-empty-nodes b/smoketest/configs/assert/firewall-empty-nodes
index c0fcd93be..c0fcd93be 100644
--- a/smoketest/config-tests/firewall-empty-nodes
+++ b/smoketest/configs/assert/firewall-empty-nodes
diff --git a/smoketest/config-tests/igmp-pim-small b/smoketest/configs/assert/igmp-pim-small
index 06051af41..06051af41 100644
--- a/smoketest/config-tests/igmp-pim-small
+++ b/smoketest/configs/assert/igmp-pim-small
diff --git a/smoketest/config-tests/ipoe-server b/smoketest/configs/assert/ipoe-server
index c21495ab2..c21495ab2 100644
--- a/smoketest/config-tests/ipoe-server
+++ b/smoketest/configs/assert/ipoe-server
diff --git a/smoketest/config-tests/ipv6-disable b/smoketest/configs/assert/ipv6-disable
index 5f906b5f7..5f906b5f7 100644
--- a/smoketest/config-tests/ipv6-disable
+++ b/smoketest/configs/assert/ipv6-disable
diff --git a/smoketest/config-tests/isis-small b/smoketest/configs/assert/isis-small
index e61d0362e..e61d0362e 100644
--- a/smoketest/config-tests/isis-small
+++ b/smoketest/configs/assert/isis-small
diff --git a/smoketest/config-tests/nat-basic b/smoketest/configs/assert/nat-basic
index f1cc0121d..f1cc0121d 100644
--- a/smoketest/config-tests/nat-basic
+++ b/smoketest/configs/assert/nat-basic
diff --git a/smoketest/config-tests/ospf-simple b/smoketest/configs/assert/ospf-simple
index 4273e4b8f..4273e4b8f 100644
--- a/smoketest/config-tests/ospf-simple
+++ b/smoketest/configs/assert/ospf-simple
diff --git a/smoketest/config-tests/ospf-small b/smoketest/configs/assert/ospf-small
index af69e5702..af69e5702 100644
--- a/smoketest/config-tests/ospf-small
+++ b/smoketest/configs/assert/ospf-small
diff --git a/smoketest/config-tests/pppoe-server b/smoketest/configs/assert/pppoe-server
index e488fc746..e488fc746 100644
--- a/smoketest/config-tests/pppoe-server
+++ b/smoketest/configs/assert/pppoe-server
diff --git a/smoketest/config-tests/qos-basic b/smoketest/configs/assert/qos-basic
index 655a5794e..655a5794e 100644
--- a/smoketest/config-tests/qos-basic
+++ b/smoketest/configs/assert/qos-basic
diff --git a/smoketest/config-tests/rip-router b/smoketest/configs/assert/rip-router
index d22f424a5..d22f424a5 100644
--- a/smoketest/config-tests/rip-router
+++ b/smoketest/configs/assert/rip-router
diff --git a/smoketest/config-tests/rpki-only b/smoketest/configs/assert/rpki-only
index f3e2a74b9..f3e2a74b9 100644
--- a/smoketest/config-tests/rpki-only
+++ b/smoketest/configs/assert/rpki-only
diff --git a/smoketest/config-tests/static-route-basic b/smoketest/configs/assert/static-route-basic
index a6135d2c4..a6135d2c4 100644
--- a/smoketest/config-tests/static-route-basic
+++ b/smoketest/configs/assert/static-route-basic
diff --git a/smoketest/config-tests/tunnel-broker b/smoketest/configs/assert/tunnel-broker
index 5518c303b..5518c303b 100644
--- a/smoketest/config-tests/tunnel-broker
+++ b/smoketest/configs/assert/tunnel-broker
diff --git a/smoketest/config-tests/vpn-openconnect-sstp b/smoketest/configs/assert/vpn-openconnect-sstp
index e7969f633..e7969f633 100644
--- a/smoketest/config-tests/vpn-openconnect-sstp
+++ b/smoketest/configs/assert/vpn-openconnect-sstp
diff --git a/smoketest/config-tests/vpn-openvpn b/smoketest/configs/assert/vpn-openvpn
index 55b1aa879..55b1aa879 100644
--- a/smoketest/config-tests/vpn-openvpn
+++ b/smoketest/configs/assert/vpn-openvpn
diff --git a/smoketest/config-tests/vpp b/smoketest/configs/assert/vpp
index aa8cd8978..aa8cd8978 100644
--- a/smoketest/config-tests/vpp
+++ b/smoketest/configs/assert/vpp
diff --git a/smoketest/config-tests/vrf-basic b/smoketest/configs/assert/vrf-basic
index 0c4e49c52..0c4e49c52 100644
--- a/smoketest/config-tests/vrf-basic
+++ b/smoketest/configs/assert/vrf-basic
diff --git a/smoketest/config-tests/vrf-bgp-pppoe-underlay b/smoketest/configs/assert/vrf-bgp-pppoe-underlay
index e3c765a9a..e3c765a9a 100644
--- a/smoketest/config-tests/vrf-bgp-pppoe-underlay
+++ b/smoketest/configs/assert/vrf-bgp-pppoe-underlay
diff --git a/smoketest/config-tests/vrf-ospf b/smoketest/configs/assert/vrf-ospf
index 53207d565..53207d565 100644
--- a/smoketest/config-tests/vrf-ospf
+++ b/smoketest/configs/assert/vrf-ospf
diff --git a/smoketest/config-tests/wireless-basic b/smoketest/configs/assert/wireless-basic
index e424b2b0f..e424b2b0f 100644
--- a/smoketest/config-tests/wireless-basic
+++ b/smoketest/configs/assert/wireless-basic
diff --git a/smoketest/configs.no-load/bgp-small-as b/smoketest/configs/no-load/bgp-small-as
index 6b953a3f6..6b953a3f6 100644
--- a/smoketest/configs.no-load/bgp-small-as
+++ b/smoketest/configs/no-load/bgp-small-as
diff --git a/smoketest/configs.no-load/firewall-big b/smoketest/configs/no-load/firewall-big
index 94b0c6dd5..94b0c6dd5 100644
--- a/smoketest/configs.no-load/firewall-big
+++ b/smoketest/configs/no-load/firewall-big
diff --git a/smoketest/configs.no-load/pki-ipsec b/smoketest/configs/no-load/pki-ipsec
index 6fc239d27..6fc239d27 100644
--- a/smoketest/configs.no-load/pki-ipsec
+++ b/smoketest/configs/no-load/pki-ipsec
diff --git a/smoketest/configs.no-load/vrf-bgp b/smoketest/configs/no-load/vrf-bgp
index 4ad372a36..4ad372a36 100644
--- a/smoketest/configs.no-load/vrf-bgp
+++ b/smoketest/configs/no-load/vrf-bgp