diff options
100 files changed, 2882 insertions, 789 deletions
diff --git a/.github/workflows/check-open-prs-conflict.yml b/.github/workflows/check-open-prs-conflict.yml new file mode 100644 index 000000000..52b11938e --- /dev/null +++ b/.github/workflows/check-open-prs-conflict.yml @@ -0,0 +1,17 @@ +name: "Open PRs Conflicts checker" +on: + push: + branches: + - current + - sagitta + - circinus + workflow_dispatch: + +permissions: + contents: read + pull-requests: write + +jobs: + check-pr-conflict-call: + uses: vyos/.github/.github/workflows/check-open-prs-conflict.yml@current + secrets: inherit diff --git a/.gitignore b/.gitignore index 839d2afff..7084332a8 100644 --- a/.gitignore +++ b/.gitignore @@ -152,9 +152,7 @@ data/reftree.cache data/configd-include.json # autogenerated vyos-commitd protobuf files -python/vyos/proto/*pb2.py python/vyos/proto/*.desc -python/vyos/proto/vyconf_proto.py # We do not use pip Pipfile @@ -100,7 +100,7 @@ clean: .PHONY: test test: generate-configd-include-json - set -e; python3 -m compileall -q -x '/vmware-tools/scripts/, /ppp/' . + set -e; python3 -m compileall -q -x '/vmware-tools/scripts/' -x '/ppp/' . PYTHONPATH=python/ python3 -m "nose" --with-xunit src --with-coverage --cover-erase --cover-xml --cover-package src/conf_mode,src/op_mode,src/completion,src/helpers,src/validators,src/tests --verbose .PHONY: check_migration_scripts_executable diff --git a/data/config.boot.default b/data/config.boot.default index db5d11ea1..02f56da8f 100644 --- a/data/config.boot.default +++ b/data/config.boot.default @@ -40,6 +40,9 @@ system { } } } + option { + reboot-on-upgrade-failure 5 + } syslog { local { facility all { diff --git a/data/templates/firewall/nftables-defines.j2 b/data/templates/firewall/nftables-defines.j2 index 3147b4c37..a1d1fa4f6 100644 --- a/data/templates/firewall/nftables-defines.j2 +++ b/data/templates/firewall/nftables-defines.j2 @@ -44,6 +44,15 @@ } {% endfor %} {% endif %} +{% if group.remote_group is vyos_defined and is_l3 and is_ipv6 %} +{% for name, name_config in group.remote_group.items() %} + set R6_{{ name }} { + type {{ ip_type }} + flags interval + auto-merge + } +{% endfor %} +{% endif %} {% if group.mac_group is vyos_defined %} {% for group_name, group_conf in group.mac_group.items() %} {% set includes = group_conf.include if group_conf.include is vyos_defined else [] %} diff --git a/data/templates/frr/daemons.frr.tmpl b/data/templates/frr/daemons.frr.tmpl index 835dc382b..afd888122 100644 --- a/data/templates/frr/daemons.frr.tmpl +++ b/data/templates/frr/daemons.frr.tmpl @@ -4,7 +4,6 @@ # Note: The following FRR-services must be kept disabled because they are replaced by other packages in VyOS: # # pimd Replaced by package igmpproxy. -# nhrpd Replaced by package opennhrp. # pbrd Replaced by PBR in nftables. # vrrpd Replaced by package keepalived. # diff --git a/data/templates/load-balancing/haproxy.cfg.j2 b/data/templates/load-balancing/haproxy.cfg.j2 index 70ea5d2b0..62934c612 100644 --- a/data/templates/load-balancing/haproxy.cfg.j2 +++ b/data/templates/load-balancing/haproxy.cfg.j2 @@ -50,9 +50,29 @@ defaults errorfile 503 /etc/haproxy/errors/503.http errorfile 504 /etc/haproxy/errors/504.http +# Default ACME backend +backend buildin_acme_certbot + server localhost 127.0.0.1:{{ get_default_port('certbot_haproxy') }} + # Frontend {% if service is vyos_defined %} {% for front, front_config in service.items() %} +{% if front_config.redirect_http_to_https is vyos_defined %} +{% set certbot_backend_name = 'certbot_' ~ front ~ '_backend' %} +frontend {{ front }}-http + mode http +{% if front_config.listen_address is vyos_defined %} +{% for address in front_config.listen_address %} + bind {{ address | bracketize_ipv6 }}:80 +{% endfor %} +{% else %} + bind [::]:80 v4v6 +{% endif %} + acl acme_acl path_beg /.well-known/acme-challenge/ + use_backend buildin_acme_certbot if acme_acl + redirect scheme https code 301 if !acme_acl +{% endif %} + frontend {{ front }} {% set ssl_front = [] %} {% if front_config.ssl.certificate is vyos_defined and front_config.ssl.certificate is iterable %} @@ -68,9 +88,6 @@ frontend {{ front }} {% else %} bind [::]:{{ front_config.port }} v4v6 {{ ssl_directive }} {{ ssl_front | join(' ') }} {% endif %} -{% if front_config.redirect_http_to_https is vyos_defined %} - http-request redirect scheme https unless { ssl_fc } -{% endif %} {% if front_config.logging is vyos_defined %} {% for facility, facility_config in front_config.logging.facility.items() %} log /dev/log {{ facility }} {{ facility_config.level }} @@ -237,6 +254,5 @@ backend {{ back }} {% if back_config.timeout.server is vyos_defined %} timeout server {{ back_config.timeout.server }}s {% endif %} - {% endfor %} {% endif %} diff --git a/debian/control b/debian/control index ec3147820..e58822e9f 100644 --- a/debian/control +++ b/debian/control @@ -120,7 +120,7 @@ Depends: dosfstools, grub-efi-amd64-signed [amd64], grub-efi-arm64-bin [arm64], - mokutil [amd64], + mokutil, shim-signed [amd64], sbsigntool [amd64], # Image signature verification tool @@ -385,7 +385,7 @@ Description: VyOS configuration scripts and data VyOS configuration scripts, interface definitions, and everything Package: vyos-1x-vmware -Architecture: amd64 +Architecture: all Depends: vyos-1x, open-vm-tools diff --git a/debian/vyos-1x.install b/debian/vyos-1x.install index 4e312a648..0fd5e3395 100644 --- a/debian/vyos-1x.install +++ b/debian/vyos-1x.install @@ -6,7 +6,6 @@ etc/dhcp etc/ipsec.d etc/logrotate.d etc/netplug -etc/opennhrp etc/modprobe.d etc/ppp etc/securetty diff --git a/debian/vyos-1x.postinst b/debian/vyos-1x.postinst index 798ecaa1b..9dd06d5e2 100644 --- a/debian/vyos-1x.postinst +++ b/debian/vyos-1x.postinst @@ -50,6 +50,10 @@ if [[ -e /usr/share/pam-configs/tacplus ]]; then rm /usr/share/pam-configs/tacplus fi +# Disable pam_nologin.so behavior for regular users +sed -i '/^auth[[:space:]]\+requisite[[:space:]]\+pam_nologin\.so$/s/^/#/' /etc/pam.d/login +sed -i '/^account[[:space:]]\+required[[:space:]]\+pam_nologin\.so$/s/^/#/' /etc/pam.d/sshd + # Add TACACS system users required for TACACS based system authentication if ! grep -q '^tacacs' /etc/passwd; then # Add the tacacs group and all 16 possible tacacs privilege-level users to diff --git a/interface-definitions/container.xml.in b/interface-definitions/container.xml.in index 2682c1b36..434bf7528 100644 --- a/interface-definitions/container.xml.in +++ b/interface-definitions/container.xml.in @@ -75,6 +75,12 @@ <multi/> </properties> </leafNode> + <leafNode name="privileged"> + <properties> + <help>Grant root capabilities to the container</help> + <valueless/> + </properties> + </leafNode> <node name="sysctl"> <properties> <help>Configure namespaced kernel parameters of the container</help> diff --git a/interface-definitions/include/accel-ppp/thread-count.xml.i b/interface-definitions/include/accel-ppp/thread-count.xml.i new file mode 100644 index 000000000..84d9224d0 --- /dev/null +++ b/interface-definitions/include/accel-ppp/thread-count.xml.i @@ -0,0 +1,27 @@ +<!-- include start from accel-ppp/thread-count.xml.i --> +<leafNode name="thread-count"> + <properties> + <help>Number of working threads</help> + <completionHelp> + <list>all half</list> + </completionHelp> + <valueHelp> + <format>all</format> + <description>Use all available CPU cores</description> + </valueHelp> + <valueHelp> + <format>half</format> + <description>Use half of available CPU cores</description> + </valueHelp> + <valueHelp> + <format>u32:1-512</format> + <description>Thread count</description> + </valueHelp> + <constraint> + <validator name="numeric" argument="--range 1-512"/> + <regex>(all|half)</regex> + </constraint> + </properties> + <defaultValue>all</defaultValue> +</leafNode> +<!-- include end --> diff --git a/interface-definitions/include/firewall/common-rule-ipv6.xml.i b/interface-definitions/include/firewall/common-rule-ipv6.xml.i index bb176fe71..65ec415fb 100644 --- a/interface-definitions/include/firewall/common-rule-ipv6.xml.i +++ b/interface-definitions/include/firewall/common-rule-ipv6.xml.i @@ -16,6 +16,7 @@ #include <include/firewall/port.xml.i> #include <include/firewall/source-destination-group-ipv6.xml.i> #include <include/firewall/source-destination-dynamic-group-ipv6.xml.i> + #include <include/firewall/source-destination-remote-group.xml.i> </children> </node> <leafNode name="jump-target"> @@ -39,6 +40,7 @@ #include <include/firewall/port.xml.i> #include <include/firewall/source-destination-group-ipv6.xml.i> #include <include/firewall/source-destination-dynamic-group-ipv6.xml.i> + #include <include/firewall/source-destination-remote-group.xml.i> </children> </node> -<!-- include end -->
\ No newline at end of file +<!-- include end --> diff --git a/interface-definitions/include/firewall/geoip.xml.i b/interface-definitions/include/firewall/geoip.xml.i index 9fb37a574..b8f2cbc45 100644 --- a/interface-definitions/include/firewall/geoip.xml.i +++ b/interface-definitions/include/firewall/geoip.xml.i @@ -12,7 +12,7 @@ <description>Country code (2 characters)</description> </valueHelp> <constraint> - <regex>^(ad|ae|af|ag|ai|al|am|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bl|bm|bn|bo|bq|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cu|cv|cw|cx|cy|cz|de|dj|dk|dm|do|dz|ec|ee|eg|eh|er|es|et|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mf|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|om|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ro|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|sk|sl|sm|sn|so|sr|ss|st|sv|sx|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tr|tt|tv|tw|tz|ua|ug|um|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|za|zm|zw)$</regex> + <regex>(ad|ae|af|ag|ai|al|am|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bl|bm|bn|bo|bq|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cu|cv|cw|cx|cy|cz|de|dj|dk|dm|do|dz|ec|ee|eg|eh|er|es|et|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mf|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|om|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ro|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|sk|sl|sm|sn|so|sr|ss|st|sv|sx|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tr|tt|tv|tw|tz|ua|ug|um|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|za|zm|zw)</regex> </constraint> <multi /> </properties> diff --git a/interface-definitions/include/haproxy/logging.xml.i b/interface-definitions/include/haproxy/logging.xml.i index e0af54fa4..315c959bf 100644 --- a/interface-definitions/include/haproxy/logging.xml.i +++ b/interface-definitions/include/haproxy/logging.xml.i @@ -4,7 +4,137 @@ <help>Logging parameters</help> </properties> <children> - #include <include/syslog-facility.xml.i> + <tagNode name="facility"> + <properties> + <help>Facility for logging</help> + <completionHelp> + <list>auth cron daemon kern lpr mail news syslog user uucp local0 local1 local2 local3 local4 local5 local6 local7</list> + </completionHelp> + <constraint> + <regex>(auth|cron|daemon|kern|lpr|mail|news|syslog|user|uucp|local0|local1|local2|local3|local4|local5|local6|local7)</regex> + </constraint> + <constraintErrorMessage>Invalid facility type</constraintErrorMessage> + <valueHelp> + <format>auth</format> + <description>Authentication and authorization</description> + </valueHelp> + <valueHelp> + <format>cron</format> + <description>Cron daemon</description> + </valueHelp> + <valueHelp> + <format>daemon</format> + <description>System daemons</description> + </valueHelp> + <valueHelp> + <format>kern</format> + <description>Kernel</description> + </valueHelp> + <valueHelp> + <format>lpr</format> + <description>Line printer spooler</description> + </valueHelp> + <valueHelp> + <format>mail</format> + <description>Mail subsystem</description> + </valueHelp> + <valueHelp> + <format>news</format> + <description>USENET subsystem</description> + </valueHelp> + <valueHelp> + <format>syslog</format> + <description>Authentication and authorization</description> + </valueHelp> + <valueHelp> + <format>user</format> + <description>Application processes</description> + </valueHelp> + <valueHelp> + <format>uucp</format> + <description>UUCP subsystem</description> + </valueHelp> + <valueHelp> + <format>local0</format> + <description>Local facility 0</description> + </valueHelp> + <valueHelp> + <format>local1</format> + <description>Local facility 1</description> + </valueHelp> + <valueHelp> + <format>local2</format> + <description>Local facility 2</description> + </valueHelp> + <valueHelp> + <format>local3</format> + <description>Local facility 3</description> + </valueHelp> + <valueHelp> + <format>local4</format> + <description>Local facility 4</description> + </valueHelp> + <valueHelp> + <format>local5</format> + <description>Local facility 5</description> + </valueHelp> + <valueHelp> + <format>local6</format> + <description>Local facility 6</description> + </valueHelp> + <valueHelp> + <format>local7</format> + <description>Local facility 7</description> + </valueHelp> + </properties> + <children> + <leafNode name="level"> + <properties> + <help>Logging level</help> + <completionHelp> + <list>emerg alert crit err warning notice info debug</list> + </completionHelp> + <valueHelp> + <format>emerg</format> + <description>Emergency messages</description> + </valueHelp> + <valueHelp> + <format>alert</format> + <description>Urgent messages</description> + </valueHelp> + <valueHelp> + <format>crit</format> + <description>Critical messages</description> + </valueHelp> + <valueHelp> + <format>err</format> + <description>Error messages</description> + </valueHelp> + <valueHelp> + <format>warning</format> + <description>Warning messages</description> + </valueHelp> + <valueHelp> + <format>notice</format> + <description>Messages for further investigation</description> + </valueHelp> + <valueHelp> + <format>info</format> + <description>Informational messages</description> + </valueHelp> + <valueHelp> + <format>debug</format> + <description>Debug messages</description> + </valueHelp> + <constraint> + <regex>(emerg|alert|crit|err|warning|notice|info|debug)</regex> + </constraint> + <constraintErrorMessage>Invalid loglevel</constraintErrorMessage> + </properties> + <defaultValue>err</defaultValue> + </leafNode> + </children> + </tagNode> </children> </node> <!-- include end --> diff --git a/interface-definitions/include/haproxy/rule-backend.xml.i b/interface-definitions/include/haproxy/rule-backend.xml.i index 1df9d5dcf..5faf09a96 100644 --- a/interface-definitions/include/haproxy/rule-backend.xml.i +++ b/interface-definitions/include/haproxy/rule-backend.xml.i @@ -38,7 +38,7 @@ <description>Set URL location</description> </valueHelp> <constraint> - <regex>^\/[\w\-.\/]+$</regex> + <regex>\/[\w\-.\/]+</regex> </constraint> <constraintErrorMessage>Incorrect URL format</constraintErrorMessage> </properties> @@ -90,7 +90,7 @@ <description>Begin URL</description> </valueHelp> <constraint> - <regex>^\/[\w\-.\/]+$</regex> + <regex>\/[\w\-.\/]+</regex> </constraint> <constraintErrorMessage>Incorrect URL format</constraintErrorMessage> <multi/> @@ -104,7 +104,7 @@ <description>End URL</description> </valueHelp> <constraint> - <regex>^\/[\w\-.\/]+$</regex> + <regex>\/[\w\-.\/]+</regex> </constraint> <constraintErrorMessage>Incorrect URL format</constraintErrorMessage> <multi/> @@ -118,7 +118,7 @@ <description>Exactly URL</description> </valueHelp> <constraint> - <regex>^\/[\w\-.\/]*$</regex> + <regex>\/[\w\-.\/]*</regex> </constraint> <constraintErrorMessage>Incorrect URL format</constraintErrorMessage> <multi/> diff --git a/interface-definitions/include/haproxy/rule-frontend.xml.i b/interface-definitions/include/haproxy/rule-frontend.xml.i index eabdd8632..d2e7a38c3 100644 --- a/interface-definitions/include/haproxy/rule-frontend.xml.i +++ b/interface-definitions/include/haproxy/rule-frontend.xml.i @@ -32,15 +32,15 @@ <children> <leafNode name="redirect-location"> <properties> - <help>Set URL location</help> + <help>Set path location</help> <valueHelp> <format>url</format> - <description>Set URL location</description> + <description>Set path location</description> </valueHelp> <constraint> - <regex>^\/[\w\-.\/]+$</regex> + <regex>\/[\w\-.\/]+</regex> </constraint> - <constraintErrorMessage>Incorrect URL format</constraintErrorMessage> + <constraintErrorMessage>Incorrect path format</constraintErrorMessage> </properties> </leafNode> <leafNode name="backend"> @@ -93,7 +93,7 @@ <description>Begin URL</description> </valueHelp> <constraint> - <regex>^\/[\w\-.\/]+$</regex> + <regex>\/[\w\-.\/]+</regex> </constraint> <constraintErrorMessage>Incorrect URL format</constraintErrorMessage> <multi/> @@ -107,7 +107,7 @@ <description>End URL</description> </valueHelp> <constraint> - <regex>^\/[\w\-.\/]+$</regex> + <regex>\/[\w\-.\/]+</regex> </constraint> <constraintErrorMessage>Incorrect URL format</constraintErrorMessage> <multi/> @@ -121,7 +121,7 @@ <description>Exactly URL</description> </valueHelp> <constraint> - <regex>^\/[\w\-.\/]+$</regex> + <regex>\/[\w\-.\/]+</regex> </constraint> <constraintErrorMessage>Incorrect URL format</constraintErrorMessage> <multi/> diff --git a/interface-definitions/include/version/reverseproxy-version.xml.i b/interface-definitions/include/version/reverseproxy-version.xml.i index 4f09f2848..71f7def1a 100644 --- a/interface-definitions/include/version/reverseproxy-version.xml.i +++ b/interface-definitions/include/version/reverseproxy-version.xml.i @@ -1,3 +1,3 @@ <!-- include start from include/version/reverseproxy-version.xml.i --> -<syntaxVersion component='reverse-proxy' version='2'></syntaxVersion> +<syntaxVersion component='reverse-proxy' version='3'></syntaxVersion> <!-- include end --> diff --git a/interface-definitions/interfaces_bonding.xml.in b/interface-definitions/interfaces_bonding.xml.in index cdacae2d0..9945fc15d 100644 --- a/interface-definitions/interfaces_bonding.xml.in +++ b/interface-definitions/interfaces_bonding.xml.in @@ -240,7 +240,7 @@ <description>Distribute based on MAC address</description> </valueHelp> <constraint> - <regex>(802.3ad|active-backup|broadcast|round-robin|transmit-load-balance|adaptive-load-balance|xor-hash)</regex> + <regex>(802\.3ad|active-backup|broadcast|round-robin|transmit-load-balance|adaptive-load-balance|xor-hash)</regex> </constraint> <constraintErrorMessage>mode must be 802.3ad, active-backup, broadcast, round-robin, transmit-load-balance, adaptive-load-balance, or xor</constraintErrorMessage> </properties> diff --git a/interface-definitions/load-balancing_haproxy.xml.in b/interface-definitions/load-balancing_haproxy.xml.in index b95e02337..61ff8bc81 100644 --- a/interface-definitions/load-balancing_haproxy.xml.in +++ b/interface-definitions/load-balancing_haproxy.xml.in @@ -4,7 +4,7 @@ <children> <node name="haproxy" owner="${vyos_conf_scripts_dir}/load-balancing_haproxy.py"> <properties> - <help>Configure haproxy</help> + <help>HAProxy TCP/HTTP Load Balancer</help> <priority>900</priority> </properties> <children> @@ -26,7 +26,7 @@ <constraintErrorMessage>Backend name must be alphanumeric and can contain hyphen and underscores</constraintErrorMessage> <valueHelp> <format>txt</format> - <description>Name of haproxy backend system</description> + <description>HAProxy backend system name</description> </valueHelp> <completionHelp> <path>load-balancing haproxy backend</path> @@ -159,7 +159,7 @@ <properties> <help>URI used for HTTP health check (Example: '/' or '/health')</help> <constraint> - <regex>^\/([^?#\s]*)(\?[^#\s]*)?$</regex> + <regex>\/([^?#\s]*)(\?[^#\s]*)?</regex> </constraint> </properties> </leafNode> diff --git a/interface-definitions/load-balancing_wan.xml.in b/interface-definitions/load-balancing_wan.xml.in index 310aa0343..f80440411 100644 --- a/interface-definitions/load-balancing_wan.xml.in +++ b/interface-definitions/load-balancing_wan.xml.in @@ -7,7 +7,7 @@ <children> <node name="wan" owner="${vyos_conf_scripts_dir}/load-balancing_wan.py"> <properties> - <help>Configure Wide Area Network (WAN) load-balancing</help> + <help>Wide Area Network (WAN) load-balancing</help> <priority>900</priority> </properties> <children> diff --git a/interface-definitions/nat66.xml.in b/interface-definitions/nat66.xml.in index c59725c53..2c1babd5a 100644 --- a/interface-definitions/nat66.xml.in +++ b/interface-definitions/nat66.xml.in @@ -53,6 +53,7 @@ </properties> </leafNode> #include <include/nat-port.xml.i> + #include <include/firewall/source-destination-group-ipv6.xml.i> </children> </node> <node name="source"> diff --git a/interface-definitions/policy.xml.in b/interface-definitions/policy.xml.in index 25dbf5581..31e01c68c 100644 --- a/interface-definitions/policy.xml.in +++ b/interface-definitions/policy.xml.in @@ -1519,7 +1519,7 @@ <constraint> <validator name="numeric" argument="--relative --"/> <validator name="numeric" argument="--range 0-4294967295"/> - <regex>^[+|-]?rtt$</regex> + <regex>[+|-]?rtt</regex> </constraint> </properties> </leafNode> diff --git a/interface-definitions/service_ipoe-server.xml.in b/interface-definitions/service_ipoe-server.xml.in index fe9d32bbd..3093151ea 100644 --- a/interface-definitions/service_ipoe-server.xml.in +++ b/interface-definitions/service_ipoe-server.xml.in @@ -237,6 +237,7 @@ #include <include/accel-ppp/max-concurrent-sessions.xml.i> #include <include/accel-ppp/shaper.xml.i> #include <include/accel-ppp/snmp.xml.i> + #include <include/accel-ppp/thread-count.xml.i> #include <include/generic-description.xml.i> #include <include/name-server-ipv4-ipv6.xml.i> #include <include/accel-ppp/log.xml.i> diff --git a/interface-definitions/service_pppoe-server.xml.in b/interface-definitions/service_pppoe-server.xml.in index 32215e9d2..81a4a95e3 100644 --- a/interface-definitions/service_pppoe-server.xml.in +++ b/interface-definitions/service_pppoe-server.xml.in @@ -175,6 +175,7 @@ </node> #include <include/accel-ppp/shaper.xml.i> #include <include/accel-ppp/snmp.xml.i> + #include <include/accel-ppp/thread-count.xml.i> #include <include/accel-ppp/wins-server.xml.i> #include <include/generic-description.xml.i> #include <include/name-server-ipv4-ipv6.xml.i> diff --git a/interface-definitions/service_snmp.xml.in b/interface-definitions/service_snmp.xml.in index cc21f5b8b..bdc9f88fe 100644 --- a/interface-definitions/service_snmp.xml.in +++ b/interface-definitions/service_snmp.xml.in @@ -13,7 +13,7 @@ <properties> <help>Community name</help> <constraint> - <regex>[[:alnum:]-_!@*#]{1,100}</regex> + <regex>[[:alnum:]\-_!@*#]{1,100}</regex> </constraint> <constraintErrorMessage>Community string is limited to alphanumerical characters, -, _, !, @, *, and # with a total lenght of 100</constraintErrorMessage> </properties> diff --git a/interface-definitions/system_option.xml.in b/interface-definitions/system_option.xml.in index c9240064f..5d385e3d0 100644 --- a/interface-definitions/system_option.xml.in +++ b/interface-definitions/system_option.xml.in @@ -37,7 +37,145 @@ <help>Kernel boot parameters</help> </properties> <children> - <leafNode name="disable-mitigations"> + <node name="cpu"> + <properties> + <help>CPU settings</help> + </properties> + <children> + <leafNode name="disable-nmi-watchdog"> + <properties> + <help>Disable the NMI watchdog for detecting hard CPU lockups</help> + <valueless/> + </properties> + </leafNode> + <leafNode name="isolate-cpus"> + <properties> + <help>Isolate specified CPUs from the scheduler</help> + <valueHelp> + <format>u32:0-511</format> + <description>CPU core</description> + </valueHelp> + <valueHelp> + <format><start-end></format> + <description>CPU core range (examples: "1", "4-7", "1,2-5,7")</description> + </valueHelp> + <constraint> + <validator name="cpu"/> + </constraint> + </properties> + </leafNode> + <leafNode name="nohz-full"> + <properties> + <help>Enable full tickless mode for specified CPUs</help> + <valueHelp> + <format>u32:0-511</format> + <description>CPU core</description> + </valueHelp> + <valueHelp> + <format><start-end></format> + <description>CPU core range (examples: "1", "4-7", "1,2-5,7")</description> + </valueHelp> + <constraint> + <validator name="cpu"/> + </constraint> + </properties> + </leafNode> + <leafNode name="rcu-no-cbs"> + <properties> + <help>Offload Read-Copy-Update (RCU) callback processing to specified CPUs</help> + <valueHelp> + <format>u32:0-511</format> + <description>CPU core</description> + </valueHelp> + <valueHelp> + <format><start-end></format> + <description>CPU core range (examples: "1", "4-7", "1,2-5,7")</description> + </valueHelp> + <constraint> + <validator name="cpu"/> + </constraint> + </properties> + </leafNode> + </children> + </node> + <node name="memory"> + <properties> + <help>Memory settings</help> + </properties> + <children> + <leafNode name="disable-numa-balancing"> + <properties> + <help>Disable automatic NUMA memory balancing</help> + <valueless/> + </properties> + </leafNode> + <leafNode name="default-hugepage-size"> + <properties> + <help>Set default hugepage size (e.g., 2M, 1G)</help> + <completionHelp> + <list>2M 1G</list> + </completionHelp> + <valueHelp> + <format>2M</format> + <description>2 megabytes</description> + </valueHelp> + <valueHelp> + <format>1G</format> + <description>1 gigabyte</description> + </valueHelp> + <constraint> + <regex>(2M|1G)</regex> + </constraint> + </properties> + </leafNode> + <tagNode name="hugepage-size"> + <properties> + <help>Set hugepage size for allocation (e.g., 2M, 1G)</help> + <completionHelp> + <list>2M 1G</list> + </completionHelp> + <valueHelp> + <format>2M</format> + <description>2 megabytes</description> + </valueHelp> + <valueHelp> + <format>1G</format> + <description>1 gigabyte</description> + </valueHelp> + <constraint> + <regex>(2M|1G)</regex> + </constraint> + </properties> + <children> + <leafNode name="hugepage-count"> + <properties> + <help>Allocate number of hugepages for system use</help> + <valueHelp> + <format>u32</format> + <description>Number of hugepages</description> + </valueHelp> + <constraint> + <validator name="numeric" argument="--range 1-100000"/> + </constraint> + </properties> + </leafNode> + </children> + </tagNode> + </children> + </node> + <leafNode name="disable-hpet"> + <properties> + <help>Disable High Precision Event Timer (HPET)</help> + <valueless/> + </properties> + </leafNode> + <leafNode name="disable-mce"> + <properties> + <help>Disable Machine Check Exceptions (MCE) reporting and handling</help> + <valueless/> + </properties> + </leafNode> + <leafNode name="disable-mitigations"> <properties> <help>Disable all optional CPU mitigations</help> <valueless/> @@ -69,6 +207,12 @@ </valueHelp> </properties> </leafNode> + <leafNode name="disable-softlockup"> + <properties> + <help>Disable soft lockup detector for kernel threads</help> + <valueless/> + </properties> + </leafNode> <leafNode name="quiet"> <properties> <help>Disable most log messages</help> @@ -198,6 +342,19 @@ <valueless/> </properties> </leafNode> + <leafNode name="reboot-on-upgrade-failure"> + <properties> + <help>Automatic reboot into previous running image on upgrade failure</help> + <valueHelp> + <format>u32:1-30</format> + <description>Timeout before automatic reboot (minutes)</description> + </valueHelp> + <constraint> + <validator name="numeric" argument="--range 5-30"/> + </constraint> + <constraintErrorMessage>Timeout out of range, must be 5 to 30 minutes</constraintErrorMessage> + </properties> + </leafNode> <node name="ssh-client"> <properties> <help>Global options used for SSH client</help> diff --git a/interface-definitions/vpn_l2tp.xml.in b/interface-definitions/vpn_l2tp.xml.in index c00e82534..d28f86653 100644 --- a/interface-definitions/vpn_l2tp.xml.in +++ b/interface-definitions/vpn_l2tp.xml.in @@ -137,6 +137,7 @@ #include <include/accel-ppp/ppp-options.xml.i> #include <include/accel-ppp/shaper.xml.i> #include <include/accel-ppp/snmp.xml.i> + #include <include/accel-ppp/thread-count.xml.i> #include <include/accel-ppp/wins-server.xml.i> #include <include/generic-description.xml.i> #include <include/name-server-ipv4-ipv6.xml.i> diff --git a/interface-definitions/vpn_pptp.xml.in b/interface-definitions/vpn_pptp.xml.in index 8aec0cb1c..3e985486d 100644 --- a/interface-definitions/vpn_pptp.xml.in +++ b/interface-definitions/vpn_pptp.xml.in @@ -53,6 +53,7 @@ #include <include/accel-ppp/ppp-options.xml.i> #include <include/accel-ppp/shaper.xml.i> #include <include/accel-ppp/snmp.xml.i> + #include <include/accel-ppp/thread-count.xml.i> #include <include/accel-ppp/wins-server.xml.i> #include <include/generic-description.xml.i> #include <include/name-server-ipv4-ipv6.xml.i> diff --git a/interface-definitions/vpn_sstp.xml.in b/interface-definitions/vpn_sstp.xml.in index 5fd5c95ca..851a202dc 100644 --- a/interface-definitions/vpn_sstp.xml.in +++ b/interface-definitions/vpn_sstp.xml.in @@ -50,6 +50,7 @@ #include <include/accel-ppp/ppp-options.xml.i> #include <include/accel-ppp/shaper.xml.i> #include <include/accel-ppp/snmp.xml.i> + #include <include/accel-ppp/thread-count.xml.i> #include <include/accel-ppp/wins-server.xml.i> #include <include/generic-description.xml.i> #include <include/name-server-ipv4-ipv6.xml.i> diff --git a/libvyosconfig b/libvyosconfig -Subproject 1dedc69476d707718031c45b53b626da8badf86 +Subproject 3e295c0c0256746f0811a9cb90d2489fd76a465 diff --git a/op-mode-definitions/monitor-log.xml.in b/op-mode-definitions/monitor-log.xml.in index 91e1c93ef..b6784d9ea 100644 --- a/op-mode-definitions/monitor-log.xml.in +++ b/op-mode-definitions/monitor-log.xml.in @@ -107,6 +107,12 @@ </properties> <command>journalctl --no-hostname --boot --follow --unit frr.service</command> </leafNode> + <leafNode name="haproxy"> + <properties> + <help>Monitor last lines of HAProxy log</help> + </properties> + <command>journalctl --no-hostname --boot --follow --unit haproxy.service</command> + </leafNode> <leafNode name="ipoe-server"> <properties> <help>Monitor last lines of IP over Ethernet server log</help> @@ -125,12 +131,6 @@ </properties> <command>journalctl --no-hostname --boot --follow --unit ndppd.service</command> </leafNode> - <leafNode name="nhrp"> - <properties> - <help>Monitor last lines of Next Hop Resolution Protocol log</help> - </properties> - <command>journalctl --no-hostname --boot --follow --unit opennhrp.service</command> - </leafNode> <leafNode name="ntp"> <properties> <help>Monitor last lines of Network Time Protocol log</help> diff --git a/op-mode-definitions/show-interfaces.xml.in b/op-mode-definitions/show-interfaces.xml.in index 09466647d..2d94080c7 100644 --- a/op-mode-definitions/show-interfaces.xml.in +++ b/op-mode-definitions/show-interfaces.xml.in @@ -26,6 +26,48 @@ </properties> <command>${vyos_op_scripts_dir}/interfaces.py show_summary</command> </leafNode> + <tagNode name="kernel"> + <properties> + <completionHelp> + <script>ip -j link show | jq -r '.[].ifname'</script> + </completionHelp> + </properties> + <command>${vyos_op_scripts_dir}/interfaces.py show_kernel --intf-name=$4</command> + <children> + <leafNode name="detail"> + <properties> + <help>Show system interface in JSON format</help> + </properties> + <command>${vyos_op_scripts_dir}/interfaces.py show_kernel --intf-name=$4 --detail</command> + </leafNode> + <leafNode name="json"> + <properties> + <help>Show system interface in JSON format</help> + </properties> + <command>${vyos_op_scripts_dir}/interfaces.py show_kernel --intf-name=$4 --raw</command> + </leafNode> + </children> + </tagNode> + <node name="kernel"> + <properties> + <help>Show all interfaces on this system</help> + </properties> + <command>${vyos_op_scripts_dir}/interfaces.py show_kernel</command> + <children> + <leafNode name="detail"> + <properties> + <help>Show system interface in JSON format</help> + </properties> + <command>${vyos_op_scripts_dir}/interfaces.py show_kernel --detail</command> + </leafNode> + <leafNode name="json"> + <properties> + <help>Show all interfaces in JSON format</help> + </properties> + <command>${vyos_op_scripts_dir}/interfaces.py show_kernel --raw</command> + </leafNode> + </children> + </node> </children> </node> </children> diff --git a/op-mode-definitions/show-log.xml.in b/op-mode-definitions/show-log.xml.in index b09d3c68b..fcde76e60 100755 --- a/op-mode-definitions/show-log.xml.in +++ b/op-mode-definitions/show-log.xml.in @@ -559,6 +559,12 @@ </properties> <command>journalctl --no-hostname --boot --unit frr.service</command> </leafNode> + <leafNode name="haproxy"> + <properties> + <help>Show log for HAProxy</help> + </properties> + <command>journalctl --no-hostname --boot --unit haproxy.service</command> + </leafNode> <leafNode name="https"> <properties> <help>Show log for HTTPs</help> @@ -671,12 +677,6 @@ </properties> <command>journalctl --no-hostname --boot --unit ndppd.service</command> </leafNode> - <leafNode name="nhrp"> - <properties> - <help>Show log for Next Hop Resolution Protocol (NHRP)</help> - </properties> - <command>journalctl --no-hostname --boot --unit opennhrp.service</command> - </leafNode> <leafNode name="ntp"> <properties> <help>Show log for Network Time Protocol (NTP)</help> diff --git a/python/vyos/accel_ppp_util.py b/python/vyos/accel_ppp_util.py index ae75e6654..49c0e3ede 100644 --- a/python/vyos/accel_ppp_util.py +++ b/python/vyos/accel_ppp_util.py @@ -221,10 +221,12 @@ def verify_accel_ppp_ip_pool(vpn_config): for interface, interface_config in vpn_config['interface'].items(): if dict_search('client_subnet', interface_config): break + if dict_search('external_dhcp.dhcp_relay', interface_config): + break else: raise ConfigError( 'Local auth and noauth mode requires local client-ip-pool \ - or client-ipv6-pool or client-subnet to be configured!') + or client-ipv6-pool or client-subnet or dhcp-relay to be configured!') else: raise ConfigError( "Local auth mode requires local client-ip-pool \ diff --git a/python/vyos/base.py b/python/vyos/base.py index ca96d96ce..3173ddc20 100644 --- a/python/vyos/base.py +++ b/python/vyos/base.py @@ -1,4 +1,4 @@ -# Copyright 2018-2022 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2018-2025 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -15,8 +15,7 @@ from textwrap import fill - -class BaseWarning: +class UserMessage: def __init__(self, header, message, **kwargs): self.message = message self.kwargs = kwargs @@ -33,7 +32,6 @@ class BaseWarning: messages = self.message.split('\n') isfirstmessage = True initial_indent = self.textinitindent - print('') for mes in messages: mes = fill(mes, initial_indent=initial_indent, subsequent_indent=self.standardindent, **self.kwargs) @@ -44,17 +42,24 @@ class BaseWarning: print('', flush=True) +class Message(): + def __init__(self, message, **kwargs): + self.Message = UserMessage('', message, **kwargs) + self.Message.print() + class Warning(): def __init__(self, message, **kwargs): - self.BaseWarn = BaseWarning('WARNING: ', message, **kwargs) - self.BaseWarn.print() + print('') + self.UserMessage = UserMessage('WARNING: ', message, **kwargs) + self.UserMessage.print() class DeprecationWarning(): def __init__(self, message, **kwargs): # Reformat the message and trim it to 72 characters in length - self.BaseWarn = BaseWarning('DEPRECATION WARNING: ', message, **kwargs) - self.BaseWarn.print() + print('') + self.UserMessage = UserMessage('DEPRECATION WARNING: ', message, **kwargs) + self.UserMessage.print() class ConfigError(Exception): diff --git a/python/vyos/config.py b/python/vyos/config.py index 546eeceab..9ae0467d4 100644 --- a/python/vyos/config.py +++ b/python/vyos/config.py @@ -1,4 +1,4 @@ -# Copyright 2017-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2017-2025 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -62,6 +62,7 @@ while functions prefixed "effective" return values from the running config. In operational mode, all functions return values from the running config. """ +import os import re import json from typing import Union @@ -73,8 +74,11 @@ from vyos.xml_ref import ext_dict_merge from vyos.xml_ref import relative_defaults from vyos.utils.dict import get_sub_dict from vyos.utils.dict import mangle_dict_keys +from vyos.utils.boot import boot_configuration_complete +from vyos.utils.backend import vyconf_backend from vyos.configsource import ConfigSource from vyos.configsource import ConfigSourceSession +from vyos.configsource import ConfigSourceVyconfSession class ConfigDict(dict): _from_defaults = {} @@ -132,7 +136,10 @@ class Config(object): """ def __init__(self, session_env=None, config_source=None): if config_source is None: - self._config_source = ConfigSourceSession(session_env) + if vyconf_backend() and boot_configuration_complete(): + self._config_source = ConfigSourceVyconfSession(session_env) + else: + self._config_source = ConfigSourceSession(session_env) else: if not isinstance(config_source, ConfigSource): raise TypeError("config_source not of type ConfigSource") diff --git a/python/vyos/configdict.py b/python/vyos/configdict.py index ff0a15933..a34b0176a 100644 --- a/python/vyos/configdict.py +++ b/python/vyos/configdict.py @@ -661,6 +661,7 @@ def get_accel_dict(config, base, chap_secrets, with_pki=False): Return a dictionary with the necessary interface config keys. """ from vyos.utils.cpu import get_core_count + from vyos.utils.cpu import get_half_cpus from vyos.template import is_ipv4 dict = config.get_config_dict(base, key_mangling=('-', '_'), @@ -670,7 +671,16 @@ def get_accel_dict(config, base, chap_secrets, with_pki=False): with_pki=with_pki) # set CPUs cores to process requests - dict.update({'thread_count' : get_core_count()}) + match dict.get('thread_count'): + case 'all': + dict['thread_count'] = get_core_count() + case 'half': + dict['thread_count'] = get_half_cpus() + case str(x) if x.isdigit(): + dict['thread_count'] = int(x) + case _: + dict['thread_count'] = get_core_count() + # we need to store the path to the secrets file dict.update({'chap_secrets_file' : chap_secrets}) diff --git a/python/vyos/configsession.py b/python/vyos/configsession.py index a3be29881..1b19c68b4 100644 --- a/python/vyos/configsession.py +++ b/python/vyos/configsession.py @@ -1,4 +1,4 @@ -# Copyright (C) 2019-2024 VyOS maintainers and contributors +# Copyright (C) 2019-2025 VyOS maintainers and contributors # # This library is free software; you can redistribute it and/or modify it under the terms of # the GNU Lesser General Public License as published by the Free Software Foundation; @@ -22,9 +22,10 @@ from vyos.defaults import directories from vyos.utils.process import is_systemd_service_running from vyos.utils.dict import dict_to_paths from vyos.utils.boot import boot_configuration_complete +from vyos.utils.backend import vyconf_backend from vyos.vyconf_session import VyconfSession +from vyos.base import Warning as Warn -vyconf_backend = False CLI_SHELL_API = '/bin/cli-shell-api' SET = '/opt/vyatta/sbin/my_set' @@ -120,6 +121,10 @@ def inject_vyos_env(env): env['vyos_sbin_dir'] = '/usr/sbin' env['vyos_validators_dir'] = '/usr/libexec/vyos/validators' + # with the retirement of the Cstore backend, this will remain as the + # sole indication of legacy CLI config mode, as checked by VyconfSession + env['_OFR_CONFIGURE'] = 'ok' + # if running the vyos-configd daemon, inject the vyshim env var if is_systemd_service_running('vyos-configd.service'): env['vyshim'] = '/usr/sbin/vyshim' @@ -164,37 +169,47 @@ class ConfigSession(object): for k, v in env_list: session_env[k] = v + session_env['CONFIGSESSION_PID'] = str(session_id) + self.__session_env = session_env self.__session_env['COMMIT_VIA'] = app self.__run_command([CLI_SHELL_API, 'setupSession']) - if vyconf_backend and boot_configuration_complete(): - self._vyconf_session = VyconfSession(on_error=ConfigSessionError) + if vyconf_backend() and boot_configuration_complete(): + self._vyconf_session = VyconfSession(pid=session_id, + on_error=ConfigSessionError) else: self._vyconf_session = None def __del__(self): - try: - output = ( - subprocess.check_output( - [CLI_SHELL_API, 'teardownSession'], env=self.__session_env + if self._vyconf_session is None: + try: + output = ( + subprocess.check_output( + [CLI_SHELL_API, 'teardownSession'], env=self.__session_env + ) + .decode() + .strip() ) - .decode() - .strip() - ) - if output: + if output: + print( + 'cli-shell-api teardownSession output for sesion {0}: {1}'.format( + self.__session_id, output + ), + file=sys.stderr, + ) + except Exception as e: print( - 'cli-shell-api teardownSession output for sesion {0}: {1}'.format( - self.__session_id, output - ), + 'Could not tear down session {0}: {1}'.format(self.__session_id, e), file=sys.stderr, ) - except Exception as e: - print( - 'Could not tear down session {0}: {1}'.format(self.__session_id, e), - file=sys.stderr, - ) + else: + if self._vyconf_session.session_changed(): + Warn('Exiting with uncommitted changes') + self._vyconf_session.discard() + self._vyconf_session.exit_config_mode() + self._vyconf_session.teardown() def __run_command(self, cmd_list): p = subprocess.Popen( diff --git a/python/vyos/configsource.py b/python/vyos/configsource.py index 65cef5333..e4ced6305 100644 --- a/python/vyos/configsource.py +++ b/python/vyos/configsource.py @@ -1,5 +1,5 @@ -# Copyright 2020-2023 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2020-2025 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -17,9 +17,16 @@ import os import re import subprocess +from typing import Union from vyos.configtree import ConfigTree from vyos.utils.boot import boot_configuration_complete +from vyos.vyconf_session import VyconfSession +from vyos.vyconf_session import VyconfSessionError +from vyos.defaults import directories +from vyos.xml_ref import is_tag +from vyos.xml_ref import is_leaf +from vyos.xml_ref import is_multi class VyOSError(Exception): """ @@ -310,6 +317,104 @@ class ConfigSourceSession(ConfigSource): except VyOSError: return False +class ConfigSourceVyconfSession(ConfigSource): + def __init__(self, session_env=None): + super().__init__() + + if session_env: + self.__session_env = session_env + else: + self.__session_env = None + + if session_env and 'CONFIGSESSION_PID' in session_env: + self.pid = int(session_env['CONFIGSESSION_PID']) + else: + self.pid = os.getppid() + + self._vyconf_session = VyconfSession(pid=self.pid) + try: + out = self._vyconf_session.get_config() + except VyconfSessionError as e: + raise ConfigSourceError(f'Init error in {type(self)}: {e}') + + session_dir = directories['vyconf_session_dir'] + + self.running_cache_path = os.path.join(session_dir, f'running_cache_{out}') + self.session_cache_path = os.path.join(session_dir, f'session_cache_{out}') + + self._running_config = ConfigTree(internal=self.running_cache_path) + self._session_config = ConfigTree(internal=self.session_cache_path) + + # N.B. level not yet implemented pending integration with legacy CLI + # cf. T7374 + self._level = [] + + def get_level(self): + return self._level + + def set_level(self): + pass + + def session_changed(self): + """ + Returns: + True if the config session has uncommited changes, False otherwise. + """ + try: + return self._vyconf_session.session_changed() + except VyconfSessionError: + # no actionable session info on error + return False + + def in_session(self): + """ + Returns: + True if called from a configuration session, False otherwise. + """ + return self._vyconf_session.in_session() + + def show_config(self, path: Union[str,list] = None, default: str = None, + effective: bool = False): + """ + Args: + path (str|list): Configuration tree path, or empty + default (str): Default value to return + + Returns: + str: working configuration + """ + + if path is None: + path = [] + if isinstance(path, str): + path = path.split() + + ct = self._running_config if effective else self._session_config + with_node = True if self.is_tag(path) else False + ct_at_path = ct.get_subtree(path, with_node=with_node) if path else ct + + res = ct_at_path.to_string().strip() + + return res if res else default + + def is_tag(self, path): + try: + return is_tag(path) + except ValueError: + return False + + def is_leaf(self, path): + try: + return is_leaf(path) + except ValueError: + return False + + def is_multi(self, path): + try: + return is_multi(path) + except ValueError: + return False + class ConfigSourceString(ConfigSource): def __init__(self, running_config_text=None, session_config_text=None): super().__init__() diff --git a/python/vyos/configtree.py b/python/vyos/configtree.py index ff40fbad0..faf124480 100644 --- a/python/vyos/configtree.py +++ b/python/vyos/configtree.py @@ -194,6 +194,7 @@ class ConfigTree(object): raise ValueError('Failed to read internal rep: {0}'.format(msg)) else: self.__config = config + self.__version = '' elif config_string is not None: config_section, version_section = extract_version(config_string) config_section = escape_backslash(config_section) diff --git a/python/vyos/defaults.py b/python/vyos/defaults.py index 7efccded6..fbde0298b 100644 --- a/python/vyos/defaults.py +++ b/python/vyos/defaults.py @@ -39,14 +39,20 @@ directories = { 'completion_dir' : f'{base_dir}/completion', 'ca_certificates' : '/usr/local/share/ca-certificates/vyos', 'ppp_nexthop_dir' : '/run/ppp_nexthop', - 'proto_path' : '/usr/share/vyos/vyconf' + 'proto_path' : '/usr/share/vyos/vyconf', + 'vyconf_session_dir' : f'{base_dir}/vyconf/session' } systemd_services = { + 'haproxy' : 'haproxy.service', 'syslog' : 'syslog.service', 'snmpd' : 'snmpd.service', } +internal_ports = { + 'certbot_haproxy' : 65080, # Certbot running behing haproxy +} + config_status = '/tmp/vyos-config-status' api_config_state = '/run/http-api-state' frr_debug_enable = '/tmp/vyos.frr.debug' diff --git a/python/vyos/firewall.py b/python/vyos/firewall.py index 9c320c82d..64022db84 100755 --- a/python/vyos/firewall.py +++ b/python/vyos/firewall.py @@ -319,7 +319,10 @@ def parse_rule(rule_conf, hook, fw_name, rule_id, ip_name): if group_name[0] == '!': operator = '!=' group_name = group_name[1:] - output.append(f'{ip_name} {prefix}addr {operator} @R_{group_name}') + if ip_name == 'ip': + output.append(f'{ip_name} {prefix}addr {operator} @R_{group_name}') + elif ip_name == 'ip6': + output.append(f'{ip_name} {prefix}addr {operator} @R6_{group_name}') if 'mac_group' in group: group_name = group['mac_group'] operator = '' @@ -471,14 +474,14 @@ def parse_rule(rule_conf, hook, fw_name, rule_id, ip_name): output.append('gre version 1') if gre_key: - # The offset of the key within the packet shifts depending on the C-flag. - # nftables cannot handle complex enough expressions to match multiple + # The offset of the key within the packet shifts depending on the C-flag. + # nftables cannot handle complex enough expressions to match multiple # offsets based on bitfields elsewhere. - # We enforce a specific match for the checksum flag in validation, so the - # gre_flags dict will always have a 'checksum' key when gre_key is populated. - if not gre_flags['checksum']: + # We enforce a specific match for the checksum flag in validation, so the + # gre_flags dict will always have a 'checksum' key when gre_key is populated. + if not gre_flags['checksum']: # No "unset" child node means C is set, we offset key lookup +32 bits - output.append(f'@th,64,32 == {gre_key}') + output.append(f'@th,64,32 == {gre_key}') else: output.append(f'@th,32,32 == {gre_key}') @@ -637,7 +640,7 @@ def parse_rule(rule_conf, hook, fw_name, rule_id, ip_name): return " ".join(output) def parse_gre_flags(flags, force_keyed=False): - flag_map = { # nft does not have symbolic names for these. + flag_map = { # nft does not have symbolic names for these. 'checksum': 1<<0, 'routing': 1<<1, 'key': 1<<2, @@ -648,7 +651,7 @@ def parse_gre_flags(flags, force_keyed=False): include = 0 exclude = 0 for fl_name, fl_state in flags.items(): - if not fl_state: + if not fl_state: include |= flag_map[fl_name] else: # 'unset' child tag exclude |= flag_map[fl_name] diff --git a/python/vyos/frrender.py b/python/vyos/frrender.py index 524167d8b..73d6dd5f0 100644 --- a/python/vyos/frrender.py +++ b/python/vyos/frrender.py @@ -697,6 +697,9 @@ class FRRender: debug('FRR: START CONFIGURATION RENDERING') # we can not reload an empty file, thus we always embed the marker output = '!\n' + # Enable FRR logging + output += 'log syslog\n' + output += 'log facility local7\n' # Enable SNMP agentx support # SNMP AgentX support cannot be disabled once enabled if 'snmp' in config_dict: diff --git a/python/vyos/ifconfig/wireguard.py b/python/vyos/ifconfig/wireguard.py index f5217aecb..3a28723b3 100644 --- a/python/vyos/ifconfig/wireguard.py +++ b/python/vyos/ifconfig/wireguard.py @@ -22,12 +22,13 @@ from tempfile import NamedTemporaryFile from hurry.filesize import size from hurry.filesize import alternative +from vyos.base import Warning from vyos.configquery import ConfigTreeQuery from vyos.ifconfig import Interface from vyos.ifconfig import Operational from vyos.template import is_ipv6 from vyos.template import is_ipv4 - +from vyos.utils.network import get_wireguard_peers class WireGuardOperational(Operational): def _dump(self): """Dump wireguard data in a python friendly way.""" @@ -251,92 +252,131 @@ class WireGuardIf(Interface): """Get a synthetic MAC address.""" return self.get_mac_synthetic() + def get_peer_public_keys(self, config, disabled=False): + """Get list of configured peer public keys""" + if 'peer' not in config: + return [] + + public_keys = [] + + for _, peer_config in config['peer'].items(): + if disabled == ('disable' in peer_config): + public_keys.append(peer_config['public_key']) + + return public_keys + def update(self, config): """General helper function which works on a dictionary retrived by get_config_dict(). It's main intention is to consolidate the scattered interface setup code and provide a single point of entry when workin on any interface.""" - tmp_file = NamedTemporaryFile('w') - tmp_file.write(config['private_key']) - tmp_file.flush() # Wireguard base command is identical for every peer base_cmd = f'wg set {self.ifname}' + interface_cmd = base_cmd if 'port' in config: interface_cmd += ' listen-port {port}' if 'fwmark' in config: interface_cmd += ' fwmark {fwmark}' - interface_cmd += f' private-key {tmp_file.name}' - interface_cmd = interface_cmd.format(**config) - # T6490: execute command to ensure interface configured - self._cmd(interface_cmd) + with NamedTemporaryFile('w') as tmp_file: + tmp_file.write(config['private_key']) + tmp_file.flush() - # If no PSK is given remove it by using /dev/null - passing keys via - # the shell (usually bash) is considered insecure, thus we use a file - no_psk_file = '/dev/null' + interface_cmd += f' private-key {tmp_file.name}' + interface_cmd = interface_cmd.format(**config) + # T6490: execute command to ensure interface configured + self._cmd(interface_cmd) + + current_peer_public_keys = get_wireguard_peers(self.ifname) + + if 'rebuild_required' in config: + # Remove all existing peers that no longer exist in config + current_public_keys = self.get_peer_public_keys(config) + cmd_remove_peers = [f' peer {public_key} remove' + for public_key in current_peer_public_keys + if public_key not in current_public_keys] + if cmd_remove_peers: + self._cmd(base_cmd + ''.join(cmd_remove_peers)) if 'peer' in config: + # Group removal of disabled peers in one command + current_disabled_peers = self.get_peer_public_keys(config, disabled=True) + cmd_disabled_peers = [f' peer {public_key} remove' + for public_key in current_disabled_peers] + if cmd_disabled_peers: + self._cmd(base_cmd + ''.join(cmd_disabled_peers)) + + peer_cmds = [] + peer_domain_cmds = [] + peer_psk_files = [] + for peer, peer_config in config['peer'].items(): # T4702: No need to configure this peer when it was explicitly # marked as disabled - also active sessions are terminated as # the public key was already removed when entering this method! if 'disable' in peer_config: - # remove peer if disabled, no error report even if peer not exists - cmd = base_cmd + ' peer {public_key} remove' - self._cmd(cmd.format(**peer_config)) continue - psk_file = no_psk_file - # start of with a fresh 'wg' command - peer_cmd = base_cmd + ' peer {public_key}' + peer_cmd = ' peer {public_key}' - try: - cmd = peer_cmd - - if 'preshared_key' in peer_config: - psk_file = '/tmp/tmp.wireguard.psk' - with open(psk_file, 'w') as f: - f.write(peer_config['preshared_key']) - cmd += f' preshared-key {psk_file}' - - # Persistent keepalive is optional - if 'persistent_keepalive' in peer_config: - cmd += ' persistent-keepalive {persistent_keepalive}' - - # Multiple allowed-ip ranges can be defined - ensure we are always - # dealing with a list - if isinstance(peer_config['allowed_ips'], str): - peer_config['allowed_ips'] = [peer_config['allowed_ips']] - cmd += ' allowed-ips ' + ','.join(peer_config['allowed_ips']) - - self._cmd(cmd.format(**peer_config)) - - cmd = peer_cmd - - # Ensure peer is created even if dns not working - if {'address', 'port'} <= set(peer_config): - if is_ipv6(peer_config['address']): - cmd += ' endpoint [{address}]:{port}' - elif is_ipv4(peer_config['address']): - cmd += ' endpoint {address}:{port}' - else: - # don't set endpoint if address uses domain name - continue - elif {'host_name', 'port'} <= set(peer_config): - cmd += ' endpoint {host_name}:{port}' - - self._cmd(cmd.format(**peer_config), env={ + cmd = peer_cmd + + if 'preshared_key' in peer_config: + with NamedTemporaryFile(mode='w', delete=False) as tmp_file: + tmp_file.write(peer_config['preshared_key']) + tmp_file.flush() + cmd += f' preshared-key {tmp_file.name}' + peer_psk_files.append(tmp_file.name) + else: + # If no PSK is given remove it by using /dev/null - passing keys via + # the shell (usually bash) is considered insecure, thus we use a file + cmd += f' preshared-key /dev/null' + + # Persistent keepalive is optional + if 'persistent_keepalive' in peer_config: + cmd += ' persistent-keepalive {persistent_keepalive}' + + # Multiple allowed-ip ranges can be defined - ensure we are always + # dealing with a list + if isinstance(peer_config['allowed_ips'], str): + peer_config['allowed_ips'] = [peer_config['allowed_ips']] + cmd += ' allowed-ips ' + ','.join(peer_config['allowed_ips']) + + peer_cmds.append(cmd.format(**peer_config)) + + cmd = peer_cmd + + # Ensure peer is created even if dns not working + if {'address', 'port'} <= set(peer_config): + if is_ipv6(peer_config['address']): + cmd += ' endpoint [{address}]:{port}' + elif is_ipv4(peer_config['address']): + cmd += ' endpoint {address}:{port}' + else: + # don't set endpoint if address uses domain name + continue + elif {'host_name', 'port'} <= set(peer_config): + cmd += ' endpoint {host_name}:{port}' + else: + continue + + peer_domain_cmds.append(cmd.format(**peer_config)) + + try: + if peer_cmds: + self._cmd(base_cmd + ''.join(peer_cmds)) + + if peer_domain_cmds: + self._cmd(base_cmd + ''.join(peer_domain_cmds), env={ 'WG_ENDPOINT_RESOLUTION_RETRIES': config['max_dns_retry']}) - except: - # todo: logging - pass - finally: - # PSK key file is not required to be stored persistently as its backed by CLI - if psk_file != no_psk_file and os.path.exists(psk_file): - os.remove(psk_file) + except Exception as e: + Warning(f'Failed to apply Wireguard peers on {self.ifname}: {e}') + finally: + for tmp in peer_psk_files: + os.unlink(tmp) # call base class super().update(config) diff --git a/python/vyos/proto/vycall_pb2.py b/python/vyos/proto/vycall_pb2.py new file mode 100644 index 000000000..95214d2a6 --- /dev/null +++ b/python/vyos/proto/vycall_pb2.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: vycall.proto +"""Generated protocol buffer code.""" +from google.protobuf.internal import builder as _builder +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0cvycall.proto\"&\n\x06Status\x12\x0f\n\x07success\x18\x01 \x02(\x08\x12\x0b\n\x03out\x18\x02 \x02(\t\"Y\n\x04\x43\x61ll\x12\x13\n\x0bscript_name\x18\x01 \x02(\t\x12\x11\n\ttag_value\x18\x02 \x01(\t\x12\x11\n\targ_value\x18\x03 \x01(\t\x12\x16\n\x05reply\x18\x04 \x01(\x0b\x32\x07.Status\"~\n\x06\x43ommit\x12\x12\n\nsession_id\x18\x01 \x02(\t\x12\x0f\n\x07\x64ry_run\x18\x04 \x02(\x08\x12\x0e\n\x06\x61tomic\x18\x05 \x02(\x08\x12\x12\n\nbackground\x18\x06 \x02(\x08\x12\x15\n\x04init\x18\x07 \x01(\x0b\x32\x07.Status\x12\x14\n\x05\x63\x61lls\x18\x08 \x03(\x0b\x32\x05.Call') + +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'vycall_pb2', globals()) +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _STATUS._serialized_start=16 + _STATUS._serialized_end=54 + _CALL._serialized_start=56 + _CALL._serialized_end=145 + _COMMIT._serialized_start=147 + _COMMIT._serialized_end=273 +# @@protoc_insertion_point(module_scope) diff --git a/python/vyos/proto/vyconf_pb2.py b/python/vyos/proto/vyconf_pb2.py new file mode 100644 index 000000000..3d5042888 --- /dev/null +++ b/python/vyos/proto/vyconf_pb2.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: vyconf.proto +"""Generated protocol buffer code.""" +from google.protobuf.internal import builder as _builder +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0cvyconf.proto\"\x89\x15\n\x07Request\x12!\n\x06prompt\x18\x01 \x01(\x0b\x32\x0f.Request.PromptH\x00\x12.\n\rsetup_session\x18\x02 \x01(\x0b\x32\x15.Request.SetupSessionH\x00\x12\x1b\n\x03set\x18\x03 \x01(\x0b\x32\x0c.Request.SetH\x00\x12!\n\x06\x64\x65lete\x18\x04 \x01(\x0b\x32\x0f.Request.DeleteH\x00\x12!\n\x06rename\x18\x05 \x01(\x0b\x32\x0f.Request.RenameH\x00\x12\x1d\n\x04\x63opy\x18\x06 \x01(\x0b\x32\r.Request.CopyH\x00\x12#\n\x07\x63omment\x18\x07 \x01(\x0b\x32\x10.Request.CommentH\x00\x12!\n\x06\x63ommit\x18\x08 \x01(\x0b\x32\x0f.Request.CommitH\x00\x12%\n\x08rollback\x18\t \x01(\x0b\x32\x11.Request.RollbackH\x00\x12\x1f\n\x05merge\x18\n \x01(\x0b\x32\x0e.Request.MergeH\x00\x12\x1d\n\x04save\x18\x0b \x01(\x0b\x32\r.Request.SaveH\x00\x12*\n\x0bshow_config\x18\x0c \x01(\x0b\x32\x13.Request.ShowConfigH\x00\x12!\n\x06\x65xists\x18\r \x01(\x0b\x32\x0f.Request.ExistsH\x00\x12&\n\tget_value\x18\x0e \x01(\x0b\x32\x11.Request.GetValueH\x00\x12(\n\nget_values\x18\x0f \x01(\x0b\x32\x12.Request.GetValuesH\x00\x12.\n\rlist_children\x18\x10 \x01(\x0b\x32\x15.Request.ListChildrenH\x00\x12)\n\x0brun_op_mode\x18\x11 \x01(\x0b\x32\x12.Request.RunOpModeH\x00\x12#\n\x07\x63onfirm\x18\x12 \x01(\x0b\x32\x10.Request.ConfirmH\x00\x12\x43\n\x18\x65nter_configuration_mode\x18\x13 \x01(\x0b\x32\x1f.Request.EnterConfigurationModeH\x00\x12\x41\n\x17\x65xit_configuration_mode\x18\x14 \x01(\x0b\x32\x1e.Request.ExitConfigurationModeH\x00\x12%\n\x08validate\x18\x15 \x01(\x0b\x32\x11.Request.ValidateH\x00\x12%\n\x08teardown\x18\x16 \x01(\x0b\x32\x11.Request.TeardownH\x00\x12\x30\n\x0ereload_reftree\x18\x17 \x01(\x0b\x32\x16.Request.ReloadReftreeH\x00\x12\x1d\n\x04load\x18\x18 \x01(\x0b\x32\r.Request.LoadH\x00\x12#\n\x07\x64iscard\x18\x19 \x01(\x0b\x32\x10.Request.DiscardH\x00\x12\x32\n\x0fsession_changed\x18\x1a \x01(\x0b\x32\x17.Request.SessionChangedH\x00\x12/\n\x0esession_of_pid\x18\x1b \x01(\x0b\x32\x15.Request.SessionOfPidH\x00\x12\x37\n\x12session_update_pid\x18\x1c \x01(\x0b\x32\x19.Request.SessionUpdatePidH\x00\x12(\n\nget_config\x18\x1d \x01(\x0b\x32\x12.Request.GetConfigH\x00\x1a\x08\n\x06Prompt\x1aP\n\x0cSetupSession\x12\x11\n\tClientPid\x18\x01 \x02(\x05\x12\x19\n\x11\x43lientApplication\x18\x02 \x01(\t\x12\x12\n\nOnBehalfOf\x18\x03 \x01(\x05\x1a!\n\x0cSessionOfPid\x12\x11\n\tClientPid\x18\x01 \x02(\x05\x1a%\n\x10SessionUpdatePid\x12\x11\n\tClientPid\x18\x01 \x02(\x05\x1a\x1a\n\tGetConfig\x12\r\n\x05\x64ummy\x18\x01 \x01(\x05\x1a\x1e\n\x08Teardown\x12\x12\n\nOnBehalfOf\x18\x01 \x01(\x05\x1a\x46\n\x08Validate\x12\x0c\n\x04Path\x18\x01 \x03(\t\x12,\n\routput_format\x18\x02 \x01(\x0e\x32\x15.Request.OutputFormat\x1a\x13\n\x03Set\x12\x0c\n\x04Path\x18\x01 \x03(\t\x1a\x16\n\x06\x44\x65lete\x12\x0c\n\x04Path\x18\x01 \x03(\t\x1a\x18\n\x07\x44iscard\x12\r\n\x05\x64ummy\x18\x01 \x01(\x05\x1a\x1f\n\x0eSessionChanged\x12\r\n\x05\x64ummy\x18\x01 \x01(\x05\x1a\x35\n\x06Rename\x12\x11\n\tEditLevel\x18\x01 \x03(\t\x12\x0c\n\x04\x46rom\x18\x02 \x02(\t\x12\n\n\x02To\x18\x03 \x02(\t\x1a\x33\n\x04\x43opy\x12\x11\n\tEditLevel\x18\x01 \x03(\t\x12\x0c\n\x04\x46rom\x18\x02 \x02(\t\x12\n\n\x02To\x18\x03 \x02(\t\x1a(\n\x07\x43omment\x12\x0c\n\x04Path\x18\x01 \x03(\t\x12\x0f\n\x07\x43omment\x18\x02 \x02(\t\x1aR\n\x06\x43ommit\x12\x0f\n\x07\x43onfirm\x18\x01 \x01(\x08\x12\x16\n\x0e\x43onfirmTimeout\x18\x02 \x01(\x05\x12\x0f\n\x07\x43omment\x18\x03 \x01(\t\x12\x0e\n\x06\x44ryRun\x18\x04 \x01(\x08\x1a\x1c\n\x08Rollback\x12\x10\n\x08Revision\x18\x01 \x02(\x05\x1a?\n\x04Load\x12\x10\n\x08Location\x18\x01 \x02(\t\x12%\n\x06\x66ormat\x18\x02 \x01(\x0e\x32\x15.Request.ConfigFormat\x1a@\n\x05Merge\x12\x10\n\x08Location\x18\x01 \x02(\t\x12%\n\x06\x66ormat\x18\x02 \x01(\x0e\x32\x15.Request.ConfigFormat\x1a?\n\x04Save\x12\x10\n\x08Location\x18\x01 \x02(\t\x12%\n\x06\x66ormat\x18\x02 \x01(\x0e\x32\x15.Request.ConfigFormat\x1a\x41\n\nShowConfig\x12\x0c\n\x04Path\x18\x01 \x03(\t\x12%\n\x06\x66ormat\x18\x02 \x01(\x0e\x32\x15.Request.ConfigFormat\x1a\x16\n\x06\x45xists\x12\x0c\n\x04Path\x18\x01 \x03(\t\x1a\x46\n\x08GetValue\x12\x0c\n\x04Path\x18\x01 \x03(\t\x12,\n\routput_format\x18\x02 \x01(\x0e\x32\x15.Request.OutputFormat\x1aG\n\tGetValues\x12\x0c\n\x04Path\x18\x01 \x03(\t\x12,\n\routput_format\x18\x02 \x01(\x0e\x32\x15.Request.OutputFormat\x1aJ\n\x0cListChildren\x12\x0c\n\x04Path\x18\x01 \x03(\t\x12,\n\routput_format\x18\x02 \x01(\x0e\x32\x15.Request.OutputFormat\x1aG\n\tRunOpMode\x12\x0c\n\x04Path\x18\x01 \x03(\t\x12,\n\routput_format\x18\x02 \x01(\x0e\x32\x15.Request.OutputFormat\x1a\t\n\x07\x43onfirm\x1a\x46\n\x16\x45nterConfigurationMode\x12\x11\n\tExclusive\x18\x01 \x02(\x08\x12\x19\n\x11OverrideExclusive\x18\x02 \x02(\x08\x1a\x17\n\x15\x45xitConfigurationMode\x1a#\n\rReloadReftree\x12\x12\n\nOnBehalfOf\x18\x01 \x01(\x05\"#\n\x0c\x43onfigFormat\x12\t\n\x05\x43URLY\x10\x00\x12\x08\n\x04JSON\x10\x01\")\n\x0cOutputFormat\x12\x0c\n\x08OutPlain\x10\x00\x12\x0b\n\x07OutJSON\x10\x01\x42\x05\n\x03msg\";\n\x0fRequestEnvelope\x12\r\n\x05token\x18\x01 \x01(\t\x12\x19\n\x07request\x18\x02 \x02(\x0b\x32\x08.Request\"S\n\x08Response\x12\x17\n\x06status\x18\x01 \x02(\x0e\x32\x07.Errnum\x12\x0e\n\x06output\x18\x02 \x01(\t\x12\r\n\x05\x65rror\x18\x03 \x01(\t\x12\x0f\n\x07warning\x18\x04 \x01(\t*\xd2\x01\n\x06\x45rrnum\x12\x0b\n\x07SUCCESS\x10\x00\x12\x08\n\x04\x46\x41IL\x10\x01\x12\x10\n\x0cINVALID_PATH\x10\x02\x12\x11\n\rINVALID_VALUE\x10\x03\x12\x16\n\x12\x43OMMIT_IN_PROGRESS\x10\x04\x12\x18\n\x14\x43ONFIGURATION_LOCKED\x10\x05\x12\x12\n\x0eINTERNAL_ERROR\x10\x06\x12\x15\n\x11PERMISSION_DENIED\x10\x07\x12\x17\n\x13PATH_ALREADY_EXISTS\x10\x08\x12\x16\n\x12UNCOMMITED_CHANGES\x10\t') + +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'vyconf_pb2', globals()) +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _ERRNUM._serialized_start=2863 + _ERRNUM._serialized_end=3073 + _REQUEST._serialized_start=17 + _REQUEST._serialized_end=2714 + _REQUEST_PROMPT._serialized_start=1237 + _REQUEST_PROMPT._serialized_end=1245 + _REQUEST_SETUPSESSION._serialized_start=1247 + _REQUEST_SETUPSESSION._serialized_end=1327 + _REQUEST_SESSIONOFPID._serialized_start=1329 + _REQUEST_SESSIONOFPID._serialized_end=1362 + _REQUEST_SESSIONUPDATEPID._serialized_start=1364 + _REQUEST_SESSIONUPDATEPID._serialized_end=1401 + _REQUEST_GETCONFIG._serialized_start=1403 + _REQUEST_GETCONFIG._serialized_end=1429 + _REQUEST_TEARDOWN._serialized_start=1431 + _REQUEST_TEARDOWN._serialized_end=1461 + _REQUEST_VALIDATE._serialized_start=1463 + _REQUEST_VALIDATE._serialized_end=1533 + _REQUEST_SET._serialized_start=1535 + _REQUEST_SET._serialized_end=1554 + _REQUEST_DELETE._serialized_start=1556 + _REQUEST_DELETE._serialized_end=1578 + _REQUEST_DISCARD._serialized_start=1580 + _REQUEST_DISCARD._serialized_end=1604 + _REQUEST_SESSIONCHANGED._serialized_start=1606 + _REQUEST_SESSIONCHANGED._serialized_end=1637 + _REQUEST_RENAME._serialized_start=1639 + _REQUEST_RENAME._serialized_end=1692 + _REQUEST_COPY._serialized_start=1694 + _REQUEST_COPY._serialized_end=1745 + _REQUEST_COMMENT._serialized_start=1747 + _REQUEST_COMMENT._serialized_end=1787 + _REQUEST_COMMIT._serialized_start=1789 + _REQUEST_COMMIT._serialized_end=1871 + _REQUEST_ROLLBACK._serialized_start=1873 + _REQUEST_ROLLBACK._serialized_end=1901 + _REQUEST_LOAD._serialized_start=1903 + _REQUEST_LOAD._serialized_end=1966 + _REQUEST_MERGE._serialized_start=1968 + _REQUEST_MERGE._serialized_end=2032 + _REQUEST_SAVE._serialized_start=2034 + _REQUEST_SAVE._serialized_end=2097 + _REQUEST_SHOWCONFIG._serialized_start=2099 + _REQUEST_SHOWCONFIG._serialized_end=2164 + _REQUEST_EXISTS._serialized_start=2166 + _REQUEST_EXISTS._serialized_end=2188 + _REQUEST_GETVALUE._serialized_start=2190 + _REQUEST_GETVALUE._serialized_end=2260 + _REQUEST_GETVALUES._serialized_start=2262 + _REQUEST_GETVALUES._serialized_end=2333 + _REQUEST_LISTCHILDREN._serialized_start=2335 + _REQUEST_LISTCHILDREN._serialized_end=2409 + _REQUEST_RUNOPMODE._serialized_start=2411 + _REQUEST_RUNOPMODE._serialized_end=2482 + _REQUEST_CONFIRM._serialized_start=1799 + _REQUEST_CONFIRM._serialized_end=1808 + _REQUEST_ENTERCONFIGURATIONMODE._serialized_start=2495 + _REQUEST_ENTERCONFIGURATIONMODE._serialized_end=2565 + _REQUEST_EXITCONFIGURATIONMODE._serialized_start=2567 + _REQUEST_EXITCONFIGURATIONMODE._serialized_end=2590 + _REQUEST_RELOADREFTREE._serialized_start=2592 + _REQUEST_RELOADREFTREE._serialized_end=2627 + _REQUEST_CONFIGFORMAT._serialized_start=2629 + _REQUEST_CONFIGFORMAT._serialized_end=2664 + _REQUEST_OUTPUTFORMAT._serialized_start=2666 + _REQUEST_OUTPUTFORMAT._serialized_end=2707 + _REQUESTENVELOPE._serialized_start=2716 + _REQUESTENVELOPE._serialized_end=2775 + _RESPONSE._serialized_start=2777 + _RESPONSE._serialized_end=2860 +# @@protoc_insertion_point(module_scope) diff --git a/python/vyos/proto/vyconf_proto.py b/python/vyos/proto/vyconf_proto.py new file mode 100644 index 000000000..404ef2f27 --- /dev/null +++ b/python/vyos/proto/vyconf_proto.py @@ -0,0 +1,377 @@ +from enum import IntEnum +from dataclasses import dataclass +from dataclasses import field + +class Errnum(IntEnum): + SUCCESS = 0 + FAIL = 1 + INVALID_PATH = 2 + INVALID_VALUE = 3 + COMMIT_IN_PROGRESS = 4 + CONFIGURATION_LOCKED = 5 + INTERNAL_ERROR = 6 + PERMISSION_DENIED = 7 + PATH_ALREADY_EXISTS = 8 + UNCOMMITED_CHANGES = 9 + +class ConfigFormat(IntEnum): + CURLY = 0 + JSON = 1 + +class OutputFormat(IntEnum): + OutPlain = 0 + OutJSON = 1 + +@dataclass +class Prompt: + pass + +@dataclass +class SetupSession: + ClientPid: int = 0 + ClientApplication: str = None + OnBehalfOf: int = None + +@dataclass +class SessionOfPid: + ClientPid: int = 0 + +@dataclass +class SessionUpdatePid: + ClientPid: int = 0 + +@dataclass +class GetConfig: + dummy: int = None + +@dataclass +class Teardown: + OnBehalfOf: int = None + +@dataclass +class Validate: + Path: list[str] = field(default_factory=list) + output_format: OutputFormat = None + +@dataclass +class Set: + Path: list[str] = field(default_factory=list) + +@dataclass +class Delete: + Path: list[str] = field(default_factory=list) + +@dataclass +class Discard: + dummy: int = None + +@dataclass +class SessionChanged: + dummy: int = None + +@dataclass +class Rename: + EditLevel: list[str] = field(default_factory=list) + From: str = "" + To: str = "" + +@dataclass +class Copy: + EditLevel: list[str] = field(default_factory=list) + From: str = "" + To: str = "" + +@dataclass +class Comment: + Path: list[str] = field(default_factory=list) + Comment: str = "" + +@dataclass +class Commit: + Confirm: bool = None + ConfirmTimeout: int = None + Comment: str = None + DryRun: bool = None + +@dataclass +class Rollback: + Revision: int = 0 + +@dataclass +class Load: + Location: str = "" + format: ConfigFormat = None + +@dataclass +class Merge: + Location: str = "" + format: ConfigFormat = None + +@dataclass +class Save: + Location: str = "" + format: ConfigFormat = None + +@dataclass +class ShowConfig: + Path: list[str] = field(default_factory=list) + format: ConfigFormat = None + +@dataclass +class Exists: + Path: list[str] = field(default_factory=list) + +@dataclass +class GetValue: + Path: list[str] = field(default_factory=list) + output_format: OutputFormat = None + +@dataclass +class GetValues: + Path: list[str] = field(default_factory=list) + output_format: OutputFormat = None + +@dataclass +class ListChildren: + Path: list[str] = field(default_factory=list) + output_format: OutputFormat = None + +@dataclass +class RunOpMode: + Path: list[str] = field(default_factory=list) + output_format: OutputFormat = None + +@dataclass +class Confirm: + pass + +@dataclass +class EnterConfigurationMode: + Exclusive: bool = False + OverrideExclusive: bool = False + +@dataclass +class ExitConfigurationMode: + pass + +@dataclass +class ReloadReftree: + OnBehalfOf: int = None + +@dataclass +class Request: + prompt: Prompt = None + setup_session: SetupSession = None + set: Set = None + delete: Delete = None + rename: Rename = None + copy: Copy = None + comment: Comment = None + commit: Commit = None + rollback: Rollback = None + merge: Merge = None + save: Save = None + show_config: ShowConfig = None + exists: Exists = None + get_value: GetValue = None + get_values: GetValues = None + list_children: ListChildren = None + run_op_mode: RunOpMode = None + confirm: Confirm = None + enter_configuration_mode: EnterConfigurationMode = None + exit_configuration_mode: ExitConfigurationMode = None + validate: Validate = None + teardown: Teardown = None + reload_reftree: ReloadReftree = None + load: Load = None + discard: Discard = None + session_changed: SessionChanged = None + session_of_pid: SessionOfPid = None + session_update_pid: SessionUpdatePid = None + get_config: GetConfig = None + +@dataclass +class RequestEnvelope: + token: str = None + request: Request = None + +@dataclass +class Response: + status: Errnum = None + output: str = None + error: str = None + warning: str = None + +def set_request_prompt(token: str = None): + reqi = Prompt () + req = Request(prompt=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_setup_session(token: str = None, client_pid: int = 0, client_application: str = None, on_behalf_of: int = None): + reqi = SetupSession (client_pid, client_application, on_behalf_of) + req = Request(setup_session=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_session_of_pid(token: str = None, client_pid: int = 0): + reqi = SessionOfPid (client_pid) + req = Request(session_of_pid=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_session_update_pid(token: str = None, client_pid: int = 0): + reqi = SessionUpdatePid (client_pid) + req = Request(session_update_pid=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_get_config(token: str = None, dummy: int = None): + reqi = GetConfig (dummy) + req = Request(get_config=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_teardown(token: str = None, on_behalf_of: int = None): + reqi = Teardown (on_behalf_of) + req = Request(teardown=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_validate(token: str = None, path: list[str] = [], output_format: OutputFormat = None): + reqi = Validate (path, output_format) + req = Request(validate=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_set(token: str = None, path: list[str] = []): + reqi = Set (path) + req = Request(set=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_delete(token: str = None, path: list[str] = []): + reqi = Delete (path) + req = Request(delete=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_discard(token: str = None, dummy: int = None): + reqi = Discard (dummy) + req = Request(discard=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_session_changed(token: str = None, dummy: int = None): + reqi = SessionChanged (dummy) + req = Request(session_changed=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_rename(token: str = None, edit_level: list[str] = [], from_: str = "", to: str = ""): + reqi = Rename (edit_level, from_, to) + req = Request(rename=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_copy(token: str = None, edit_level: list[str] = [], from_: str = "", to: str = ""): + reqi = Copy (edit_level, from_, to) + req = Request(copy=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_comment(token: str = None, path: list[str] = [], comment: str = ""): + reqi = Comment (path, comment) + req = Request(comment=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_commit(token: str = None, confirm: bool = None, confirm_timeout: int = None, comment: str = None, dry_run: bool = None): + reqi = Commit (confirm, confirm_timeout, comment, dry_run) + req = Request(commit=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_rollback(token: str = None, revision: int = 0): + reqi = Rollback (revision) + req = Request(rollback=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_load(token: str = None, location: str = "", format: ConfigFormat = None): + reqi = Load (location, format) + req = Request(load=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_merge(token: str = None, location: str = "", format: ConfigFormat = None): + reqi = Merge (location, format) + req = Request(merge=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_save(token: str = None, location: str = "", format: ConfigFormat = None): + reqi = Save (location, format) + req = Request(save=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_show_config(token: str = None, path: list[str] = [], format: ConfigFormat = None): + reqi = ShowConfig (path, format) + req = Request(show_config=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_exists(token: str = None, path: list[str] = []): + reqi = Exists (path) + req = Request(exists=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_get_value(token: str = None, path: list[str] = [], output_format: OutputFormat = None): + reqi = GetValue (path, output_format) + req = Request(get_value=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_get_values(token: str = None, path: list[str] = [], output_format: OutputFormat = None): + reqi = GetValues (path, output_format) + req = Request(get_values=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_list_children(token: str = None, path: list[str] = [], output_format: OutputFormat = None): + reqi = ListChildren (path, output_format) + req = Request(list_children=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_run_op_mode(token: str = None, path: list[str] = [], output_format: OutputFormat = None): + reqi = RunOpMode (path, output_format) + req = Request(run_op_mode=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_confirm(token: str = None): + reqi = Confirm () + req = Request(confirm=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_enter_configuration_mode(token: str = None, exclusive: bool = False, override_exclusive: bool = False): + reqi = EnterConfigurationMode (exclusive, override_exclusive) + req = Request(enter_configuration_mode=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_exit_configuration_mode(token: str = None): + reqi = ExitConfigurationMode () + req = Request(exit_configuration_mode=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_reload_reftree(token: str = None, on_behalf_of: int = None): + reqi = ReloadReftree (on_behalf_of) + req = Request(reload_reftree=reqi) + req_env = RequestEnvelope(token, req) + return req_env diff --git a/python/vyos/template.py b/python/vyos/template.py index d79e1183f..aa215db95 100755 --- a/python/vyos/template.py +++ b/python/vyos/template.py @@ -36,6 +36,7 @@ DEFAULT_TEMPLATE_DIR = directories["templates"] # Holds template filters registered via register_filter() _FILTERS = {} _TESTS = {} +_CLEVER_FUNCTIONS = {} # reuse Environments with identical settings to improve performance @functools.lru_cache(maxsize=2) @@ -58,6 +59,7 @@ def _get_environment(location=None): ) env.filters.update(_FILTERS) env.tests.update(_TESTS) + env.globals.update(_CLEVER_FUNCTIONS) return env @@ -77,7 +79,7 @@ def register_filter(name, func=None): "Filters can only be registered before rendering the first template" ) if name in _FILTERS: - raise ValueError(f"A filter with name {name!r} was registered already") + raise ValueError(f"A filter with name {name!r} was already registered") _FILTERS[name] = func return func @@ -97,10 +99,30 @@ def register_test(name, func=None): "Tests can only be registered before rendering the first template" ) if name in _TESTS: - raise ValueError(f"A test with name {name!r} was registered already") + raise ValueError(f"A test with name {name!r} was already registered") _TESTS[name] = func return func +def register_clever_function(name, func=None): + """Register a function to be available as test in templates under given name. + + It can also be used as a decorator, see below in this module for examples. + + :raise RuntimeError: + when trying to register a test after a template has been rendered already + :raise ValueError: when trying to register a name which was taken already + """ + if func is None: + return functools.partial(register_clever_function, name) + if _get_environment.cache_info().currsize: + raise RuntimeError( + "Clever functions can only be registered before rendering the" \ + "first template") + if name in _CLEVER_FUNCTIONS: + raise ValueError(f"A clever function with name {name!r} was already "\ + "registered") + _CLEVER_FUNCTIONS[name] = func + return func def render_to_string(template, content, formater=None, location=None): """Render a template from the template directory, raise on any errors. @@ -150,6 +172,8 @@ def render( # As we are opening the file with 'w', we are performing the rendering before # calling open() to not accidentally erase the file if rendering fails rendered = render_to_string(template, content, formater, location) + # Remove any trailing character and always add a new line at the end + rendered = rendered.rstrip() + "\n" # Write to file with open(destination, "w") as file: @@ -704,7 +728,7 @@ def conntrack_rule(rule_conf, rule_id, action, ipv6=False): if port[0] == '!': operator = '!=' port = port[1:] - output.append(f'th {prefix}port {operator} {port}') + output.append(f'th {prefix}port {operator} {{ {port} }}') if 'group' in side_conf: group = side_conf['group'] @@ -1050,3 +1074,21 @@ def vyos_defined(value, test_value=None, var_type=None): else: # Valid value and is matching optional argument if provided - return true return True + +@register_clever_function('get_default_port') +def get_default_port(service): + """ + Jinja2 plugin to retrieve common service port number from vyos.defaults + class form a Jinja2 template. This removes the need to hardcode, or pass in + the data using the general dictionary. + + Added to remove code complexity and make it easier to read. + + Example: + {{ get_default_port('certbot_haproxy') }} + """ + from vyos.defaults import internal_ports + if service not in internal_ports: + raise RuntimeError(f'Service "{service}" not found in internal ' \ + 'vyos.defaults.internal_ports dict!') + return internal_ports[service] diff --git a/python/vyos/utils/backend.py b/python/vyos/utils/backend.py new file mode 100644 index 000000000..400ea9b69 --- /dev/null +++ b/python/vyos/utils/backend.py @@ -0,0 +1,88 @@ +# Copyright 2025 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see <http://www.gnu.org/licenses/>. + +# N.B. the following is a temporary addition for running smoketests under +# vyconf and is not to be called explicitly, at the risk of catastophe. + +# pylint: disable=wrong-import-position + +from pathlib import Path + +from vyos.utils.io import ask_yes_no +from vyos.utils.process import call + +VYCONF_SENTINEL = '/run/vyconf_backend' + +MSG_ENABLE_VYCONF = 'This will enable the vyconf backend for testing. Proceed?' +MSG_DISABLE_VYCONF = ( + 'This will restore the legacy backend; it requires a reboot. Proceed?' +) + +# read/set immutable file attribute without popen: +# https://www.geeklab.info/2021/04/chattr-and-lsattr-in-python/ +import fcntl # pylint: disable=C0411 # noqa: E402 +from array import array # pylint: disable=C0411 # noqa: E402 + +# FS constants - see /uapi/linux/fs.h in kernel source +# or <elixir.free-electrons.com/linux/latest/source/include/uapi/linux/fs.h> +FS_IOC_GETFLAGS = 0x80086601 +FS_IOC_SETFLAGS = 0x40086602 +FS_IMMUTABLE_FL = 0x010 + + +def chattri(filename: str, value: bool): + with open(filename, 'r') as f: + arg = array('L', [0]) + fcntl.ioctl(f.fileno(), FS_IOC_GETFLAGS, arg, True) + if value: + arg[0] = arg[0] | FS_IMMUTABLE_FL + else: + arg[0] = arg[0] & ~FS_IMMUTABLE_FL + fcntl.ioctl(f.fileno(), FS_IOC_SETFLAGS, arg, True) + + +def lsattri(filename: str) -> bool: + with open(filename, 'r') as f: + arg = array('L', [0]) + fcntl.ioctl(f.fileno(), FS_IOC_GETFLAGS, arg, True) + return bool(arg[0] & FS_IMMUTABLE_FL) + + +# End: read/set immutable file attribute without popen + + +def vyconf_backend() -> bool: + return Path(VYCONF_SENTINEL).exists() and lsattri(VYCONF_SENTINEL) + + +def set_vyconf_backend(value: bool, no_prompt: bool = False): + vyconfd_service = 'vyconfd.service' + match value: + case True: + if vyconf_backend(): + return + if not no_prompt and not ask_yes_no(MSG_ENABLE_VYCONF): + return + Path(VYCONF_SENTINEL).touch() + chattri(VYCONF_SENTINEL, True) + call(f'systemctl restart {vyconfd_service}') + case False: + if not vyconf_backend(): + return + if not no_prompt and not ask_yes_no(MSG_DISABLE_VYCONF): + return + chattri(VYCONF_SENTINEL, False) + Path(VYCONF_SENTINEL).unlink() + call('/sbin/shutdown -r now') diff --git a/python/vyos/utils/commit.py b/python/vyos/utils/commit.py index 105aed8c2..9167c78d2 100644 --- a/python/vyos/utils/commit.py +++ b/python/vyos/utils/commit.py @@ -1,4 +1,4 @@ -# Copyright 2023 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2023-2025 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -13,8 +13,13 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library. If not, see <http://www.gnu.org/licenses/>. +# pylint: disable=import-outside-toplevel + +from typing import IO + + def commit_in_progress(): - """ Not to be used in normal op mode scripts! """ + """Not to be used in normal op mode scripts!""" # The CStore backend locks the config by opening a file # The file is not removed after commit, so just checking @@ -36,7 +41,9 @@ def commit_in_progress(): from vyos.defaults import commit_lock if getuser() != 'root': - raise OSError('This functions needs to be run as root to return correct results!') + raise OSError( + 'This functions needs to be run as root to return correct results!' + ) for proc in process_iter(): try: @@ -45,7 +52,7 @@ def commit_in_progress(): for f in files: if f.path == commit_lock: return True - except NoSuchProcess as err: + except NoSuchProcess: # Process died before we could examine it pass # Default case @@ -53,8 +60,44 @@ def commit_in_progress(): def wait_for_commit_lock(): - """ Not to be used in normal op mode scripts! """ + """Not to be used in normal op mode scripts!""" from time import sleep + # Very synchronous approach to multiprocessing while commit_in_progress(): sleep(1) + + +# For transitional compatibility with the legacy commit locking mechanism, +# we require a lockf/fcntl (POSIX-type) lock, hence the following in place +# of vyos.utils.locking + + +def acquire_commit_lock_file() -> tuple[IO, str]: + import fcntl + from pathlib import Path + from vyos.defaults import commit_lock + + try: + # pylint: disable=consider-using-with + lock_fd = Path(commit_lock).open('w') + except IOError as e: + out = f'Critical error opening commit lock file {e}' + return None, out + + try: + fcntl.lockf(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + return lock_fd, '' + except IOError: + out = 'Configuration system locked by another commit in progress' + lock_fd.close() + return None, out + + +def release_commit_lock_file(file_descr): + import fcntl + + if file_descr is None: + return + fcntl.lockf(file_descr, fcntl.LOCK_UN) + file_descr.close() diff --git a/python/vyos/utils/cpu.py b/python/vyos/utils/cpu.py index 8ace77d15..6f21eb526 100644 --- a/python/vyos/utils/cpu.py +++ b/python/vyos/utils/cpu.py @@ -26,6 +26,7 @@ It has special cases for x86_64 and MAY work correctly on other architectures, but nothing is certain. """ +import os import re def _read_cpuinfo(): @@ -114,3 +115,8 @@ def get_available_cpus(): out = json.loads(cmd('lscpu --extended -b --json')) return out['cpus'] + + +def get_half_cpus(): + """ return 1/2 of the numbers of available CPUs """ + return max(1, os.cpu_count() // 2) diff --git a/python/vyos/utils/network.py b/python/vyos/utils/network.py index 2f666f0ee..0a84be478 100644 --- a/python/vyos/utils/network.py +++ b/python/vyos/utils/network.py @@ -256,40 +256,60 @@ def mac2eui64(mac, prefix=None): except: # pylint: disable=bare-except return -def check_port_availability(ipaddress, port, protocol): +def check_port_availability(address: str=None, port: int=0, protocol: str='tcp') -> bool: """ - Check if port is available and not used by any service - Return False if a port is busy or IP address does not exists + Check if given port is available and not used by any service. + Should be used carefully for services that can start listening dynamically, because IP address may be dynamic too + + Args: + address: IPv4 or IPv6 address - if None, checks on all interfaces + port: TCP/UDP port number. + + + Returns: + False if a port is busy or IP address does not exists + True if a port is free and IP address exists """ - from socketserver import TCPServer, UDPServer + import socket from ipaddress import ip_address + # treat None as "any address" + address = address or '::' + # verify arguments try: - ipaddress = ip_address(ipaddress).compressed - except: - raise ValueError(f'The {ipaddress} is not a valid IPv4 or IPv6 address') + address = ip_address(address).compressed + except ValueError: + raise ValueError(f'{address} is not a valid IPv4 or IPv6 address') if port not in range(1, 65536): - raise ValueError(f'The port number {port} is not in the 1-65535 range') + raise ValueError(f'Port {port} is not in range 1-65535') if protocol not in ['tcp', 'udp']: - raise ValueError(f'The protocol {protocol} is not supported. Only tcp and udp are allowed') + raise ValueError(f'{protocol} is not supported - only tcp and udp are allowed') - # check port availability + protocol = socket.SOCK_STREAM if protocol == 'tcp' else socket.SOCK_DGRAM try: - if protocol == 'tcp': - server = TCPServer((ipaddress, port), None, bind_and_activate=True) - if protocol == 'udp': - server = UDPServer((ipaddress, port), None, bind_and_activate=True) - server.server_close() - except Exception as e: - # errno.h: - #define EADDRINUSE 98 /* Address already in use */ - if e.errno == 98: + addr_info = socket.getaddrinfo(address, port, socket.AF_UNSPEC, protocol) + except socket.gaierror as e: + print(f'Invalid address: {address}') + return False + + for family, socktype, proto, canonname, sockaddr in addr_info: + try: + with socket.socket(family, socktype, proto) as s: + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind(sockaddr) + # port is free to use + return True + except OSError: + # port is already in use return False - return True + # if we reach this point, no socket was tested and we assume the port is + # already in use - better safe then sorry + return False + def is_listen_port_bind_service(port: int, service: str) -> bool: """Check if listen port bound to expected program name @@ -396,6 +416,21 @@ def is_wireguard_key_pair(private_key: str, public_key:str) -> bool: else: return False +def get_wireguard_peers(ifname: str) -> list: + """ + Return list of configured Wireguard peers for interface + :param ifname: Interface name + :type ifname: str + :return: list of public keys + :rtype: list + """ + if not interface_exists(ifname): + return [] + + from vyos.utils.process import cmd + peers = cmd(f'wg show {ifname} peers') + return peers.splitlines() + def is_subnet_connected(subnet, primary=False): """ Verify is the given IPv4/IPv6 subnet is connected to any interface on this @@ -615,3 +650,19 @@ def is_valid_ipv4_address_or_range(addr: str) -> bool: return ip_network(addr).version == 4 except: return False + +def is_valid_ipv6_address_or_range(addr: str) -> bool: + """ + Validates if the provided address is a valid IPv4, CIDR or IPv4 range + :param addr: address to test + :return: bool: True if provided address is valid + """ + from ipaddress import ip_network + try: + if '-' in addr: # If we are checking a range, validate both address's individually + split = addr.split('-') + return is_valid_ipv6_address_or_range(split[0]) and is_valid_ipv6_address_or_range(split[1]) + else: + return ip_network(addr).version == 6 + except: + return False diff --git a/python/vyos/utils/process.py b/python/vyos/utils/process.py index 121b6e240..21335e6b3 100644 --- a/python/vyos/utils/process.py +++ b/python/vyos/utils/process.py @@ -14,6 +14,7 @@ # License along with this library. If not, see <http://www.gnu.org/licenses/>. import os +import shlex from subprocess import Popen from subprocess import PIPE @@ -21,20 +22,17 @@ from subprocess import STDOUT from subprocess import DEVNULL -def get_wrapper(vrf, netns, auth): - wrapper = '' +def get_wrapper(vrf, netns): + wrapper = None if vrf: - wrapper = f'ip vrf exec {vrf} ' + wrapper = ['ip', 'vrf', 'exec', vrf] elif netns: - wrapper = f'ip netns exec {netns} ' - if auth: - wrapper = f'{auth} {wrapper}' + wrapper = ['ip', 'netns', 'exec', netns] return wrapper def popen(command, flag='', shell=None, input=None, timeout=None, env=None, - stdout=PIPE, stderr=PIPE, decode='utf-8', auth='', vrf=None, - netns=None): + stdout=PIPE, stderr=PIPE, decode='utf-8', vrf=None, netns=None): """ popen is a wrapper helper around subprocess.Popen with it default setting it will return a tuple (out, err) @@ -75,28 +73,33 @@ def popen(command, flag='', shell=None, input=None, timeout=None, env=None, if not debug.enabled(flag): flag = 'command' + use_shell = shell + stdin = None + if shell is None: + use_shell = False + if ' ' in command: + use_shell = True + if env: + use_shell = True + # Must be run as root to execute command in VRF or network namespace + wrapper = get_wrapper(vrf, netns) if vrf or netns: if os.getuid() != 0: raise OSError( 'Permission denied: cannot execute commands in VRF and netns contexts as an unprivileged user' ) - wrapper = get_wrapper(vrf, netns, auth) - command = f'{wrapper} {command}' if wrapper else command + if use_shell: + command = f'{shlex.join(wrapper)} {command}' + else: + if type(command) is not list: + command = [command] + command = wrapper + command - cmd_msg = f"cmd '{command}'" + cmd_msg = f"cmd '{command}'" if use_shell else f"cmd '{shlex.join(command)}'" debug.message(cmd_msg, flag) - use_shell = shell - stdin = None - if shell is None: - use_shell = False - if ' ' in command: - use_shell = True - if env: - use_shell = True - if input: stdin = PIPE input = input.encode() if type(input) is str else input @@ -155,7 +158,7 @@ def run(command, flag='', shell=None, input=None, timeout=None, env=None, def cmd(command, flag='', shell=None, input=None, timeout=None, env=None, stdout=PIPE, stderr=PIPE, decode='utf-8', raising=None, message='', - expect=[0], auth='', vrf=None, netns=None): + expect=[0], vrf=None, netns=None): """ A wrapper around popen, which returns the stdout and will raise the error code of a command @@ -171,12 +174,11 @@ def cmd(command, flag='', shell=None, input=None, timeout=None, env=None, input=input, timeout=timeout, env=env, shell=shell, decode=decode, - auth=auth, vrf=vrf, netns=netns, ) if code not in expect: - wrapper = get_wrapper(vrf, netns, auth='') + wrapper = get_wrapper(vrf, netns) command = f'{wrapper} {command}' feedback = message + '\n' if message else '' feedback += f'failed to run command: {command}\n' diff --git a/python/vyos/utils/session.py b/python/vyos/utils/session.py new file mode 100644 index 000000000..28559dc59 --- /dev/null +++ b/python/vyos/utils/session.py @@ -0,0 +1,25 @@ +# Copyright 2025 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see <http://www.gnu.org/licenses/>. + +# pylint: disable=import-outside-toplevel + + +def in_config_session(): + """Vyatta bash completion uses the following environment variable for + indication of the config mode environment, independent of legacy backend + initialization of Cstore""" + from os import environ + + return '_OFR_CONFIGURE' in environ diff --git a/python/vyos/vyconf_session.py b/python/vyos/vyconf_session.py index 506095625..4250f0cfb 100644 --- a/python/vyos/vyconf_session.py +++ b/python/vyos/vyconf_session.py @@ -15,6 +15,7 @@ # # +import os import tempfile import shutil from functools import wraps @@ -24,26 +25,86 @@ from vyos.proto import vyconf_client from vyos.migrate import ConfigMigrate from vyos.migrate import ConfigMigrateError from vyos.component_version import append_system_version +from vyos.utils.session import in_config_session +from vyos.proto.vyconf_proto import Errnum +from vyos.utils.commit import acquire_commit_lock_file +from vyos.utils.commit import release_commit_lock_file -def output(o): - out = '' - for res in (o.output, o.error, o.warning): - if res is not None: - out = out + res - return out +class VyconfSessionError(Exception): + pass class VyconfSession: - def __init__(self, token: str = None, on_error: Type[Exception] = None): + def __init__( + self, token: str = None, pid: int = None, on_error: Type[Exception] = None + ): + self.pid = os.getpid() if pid is None else pid if token is None: - out = vyconf_client.send_request('setup_session') + # CLI applications with arg pid=getppid() allow coordination + # with the ambient session; other uses (such as ConfigSession) + # may default to self pid + out = vyconf_client.send_request('session_of_pid', client_pid=self.pid) + if out.output is None: + out = vyconf_client.send_request('setup_session', client_pid=self.pid) self.__token = out.output else: + out = vyconf_client.send_request( + 'session_update_pid', token=token, client_pid=self.pid + ) + if out.status: + raise ValueError(f'No existing session for token: {token}') self.__token = token + self.in_config_session = in_config_session() + if self.in_config_session: + out = vyconf_client.send_request( + 'enter_configuration_mode', token=self.__token + ) + if out.status: + raise VyconfSessionError(self.output(out)) + self.on_error = on_error + def __del__(self): + if not self.in_config_session: + self.teardown() + + def teardown(self): + vyconf_client.send_request('teardown', token=self.__token) + + def exit_config_mode(self): + if self.session_changed(): + return 'Uncommited changes', Errnum.UNCOMMITED_CHANGES + out = vyconf_client.send_request('exit_configuration_mode', token=self.__token) + return self.output(out), out.status + + def in_session(self) -> bool: + return self.in_config_session + + def session_changed(self) -> bool: + out = vyconf_client.send_request('session_changed', token=self.__token) + return not bool(out.status) + + def get_config(self): + out = vyconf_client.send_request('get_config', token=self.__token) + if out.status: + raise VyconfSessionError(self.output(out)) + return out.output + + @staticmethod + def config_mode(f): + @wraps(f) + def wrapped(self, *args, **kwargs): + msg = 'operation not available outside of config mode' + if not self.in_config_session: + if self.on_error is None: + raise VyconfSessionError(msg) + raise self.on_error(msg) + return f(self, *args, **kwargs) + + return wrapped + @staticmethod def raise_exception(f): @wraps(f) @@ -57,31 +118,46 @@ class VyconfSession: return wrapped + @staticmethod + def output(o): + out = '' + for res in (o.output, o.error, o.warning): + if res is not None: + out = out + res + return out + @raise_exception + @config_mode def set(self, path: list[str]) -> tuple[str, int]: out = vyconf_client.send_request('set', token=self.__token, path=path) - return output(out), out.status + return self.output(out), out.status @raise_exception + @config_mode def delete(self, path: list[str]) -> tuple[str, int]: out = vyconf_client.send_request('delete', token=self.__token, path=path) - return output(out), out.status + return self.output(out), out.status @raise_exception + @config_mode def commit(self) -> tuple[str, int]: + lock_fd, out = acquire_commit_lock_file() + if lock_fd is None: + return out, Errnum.COMMIT_IN_PROGRESS + out = vyconf_client.send_request('commit', token=self.__token) - return output(out), out.status + release_commit_lock_file(lock_fd) + + return self.output(out), out.status @raise_exception + @config_mode def discard(self) -> tuple[str, int]: out = vyconf_client.send_request('discard', token=self.__token) - return output(out), out.status - - def session_changed(self) -> bool: - out = vyconf_client.send_request('session_changed', token=self.__token) - return not bool(out.status) + return self.output(out), out.status @raise_exception + @config_mode def load_config(self, file: str, migrate: bool = False) -> tuple[str, int]: # pylint: disable=consider-using-with if migrate: @@ -101,23 +177,18 @@ class VyconfSession: if tmp: tmp.close() - return output(out), out.status + return self.output(out), out.status @raise_exception def save_config(self, file: str, append_version: bool = False) -> tuple[str, int]: out = vyconf_client.send_request('save', token=self.__token, location=file) if append_version: append_system_version(file) - return output(out), out.status + return self.output(out), out.status @raise_exception def show_config(self, path: list[str] = None) -> tuple[str, int]: if path is None: path = [] out = vyconf_client.send_request('show_config', token=self.__token, path=path) - return output(out), out.status - - def __del__(self): - out = vyconf_client.send_request('teardown', token=self.__token) - if out.status: - print(f'Could not tear down session {self.__token}: {output(out)}') + return self.output(out), out.status diff --git a/smoketest/config-tests/basic-haproxy b/smoketest/config-tests/basic-haproxy new file mode 100644 index 000000000..7755fc4ea --- /dev/null +++ b/smoketest/config-tests/basic-haproxy @@ -0,0 +1,46 @@ +set interfaces dummy dum0 address '172.18.254.203/32' +set interfaces ethernet eth0 duplex 'auto' +set interfaces ethernet eth0 speed 'auto' +set interfaces ethernet eth0 vif 203 address '172.18.203.10/24' +set interfaces ethernet eth1 duplex 'auto' +set interfaces ethernet eth1 speed 'auto' +set interfaces ethernet eth2 duplex 'auto' +set interfaces ethernet eth2 speed 'auto' +set load-balancing haproxy backend webserver logging facility daemon +set load-balancing haproxy backend webserver logging facility user level 'info' +set load-balancing haproxy backend webserver server web01 address '192.0.2.1' +set load-balancing haproxy backend webserver server web01 port '443' +set load-balancing haproxy backend webserver ssl no-verify +set load-balancing haproxy global-parameters logging facility daemon +set load-balancing haproxy global-parameters logging facility user level 'info' +set load-balancing haproxy service frontend backend 'webserver' +set load-balancing haproxy service frontend logging facility daemon +set load-balancing haproxy service frontend logging facility user level 'info' +set load-balancing haproxy service frontend port '443' +set load-balancing haproxy service frontend ssl certificate 'dummy' +set pki certificate dummy certificate 'MIIDsTCCApmgAwIBAgIUegVgO1wIN2v44trXZ+Kb1t48uL0wDQYJKoZIhvcNAQELBQAwVzELMAkGA1UEBhMCR0IxEzARBgNVBAgMClNvbWUtU3RhdGUxEjAQBgNVBAcMCVNvbWUtQ2l0eTENMAsGA1UECgwEVnlPUzEQMA4GA1UEAwwHdnlvcy5pbzAeFw0yNTA1MDUxODIzMTdaFw0yNjA1MDUxODIzMTdaMFcxCzAJBgNVBAYTAkdCMRMwEQYDVQQIDApTb21lLVN0YXRlMRIwEAYDVQQHDAlTb21lLUNpdHkxDTALBgNVBAoMBFZ5T1MxEDAOBgNVBAMMB3Z5b3MuaW8wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDEfMAwYLKKVhGlUXr9gkVC7uBi+0O9yyEgd5QzPByePXYw0FrSLWmLRfQuByFDPIVANcEa3FgIXIAeKmxItw7IhFRsG5soSOXXgBxdAH/qzEbWhwzgafnxZKJkmrQr8YA3IFtkFPr2+5s26WdjtwEM0tzIFkq6hmWSX1axUgvYlF2uCxjututMZ6I5JCa0uR3gBRuNONuGPH3Ko9zUEATffv53j9DbYVEM0lfVNewefPoVJmWz+oT0wP/kNx6tREf+aUAF4m+eBsqnggITftW2fyeFnoBPCcPp3HUgSwZhesunqz+YeW6Pk+WWb5vl+2QbMKKtz5qK6dI3q0z9yp4FAgMBAAGjdTBzMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgeAMBMGA1UdJQQMMAoGCCsGAQUFBwMBMB0GA1UdDgQWBBSr4OYIWkb8UGuQnEFnjSbmvR+4vDAfBgNVHSMEGDAWgBSr4OYIWkb8UGuQnEFnjSbmvR+4vDANBgkqhkiG9w0BAQsFAAOCAQEAUmRWRPGXsvfuRT+53id3EufH1IJAdowrt6yBZsHobvqCXO2+YhG7oG6/UqUYiv5bHN5xEMQyWd7nyrLOUeFo2bpcMIOlpl6AoUIY65Gm2BqQ7FuPxLLO25RdpZ5WkMGX5kJsKY0/PcpamRKNz1khgFcRyxf9WGhCAIjDCWIWs8lkvPN3m75SFCW7MTuzzQOrzvI6nqqcHO4k8hRBznp26WLUW1rQKpNN09nZGOkeNYK5QbzKN/RUmtEHQZhlgLAIr09jUaA4RDLI1SdD6LR5nvpa9RJBTyS/kISF8BXKMgvUbDHN2nP+VUUrut2ZwoU+pxV4RVT2pS760HuYj4+sYQ==' +set pki certificate dummy private key 'MIIEuwIBADANBgkqhkiG9w0BAQEFAASCBKUwggShAgEAAoIBAQDEfMAwYLKKVhGlUXr9gkVC7uBi+0O9yyEgd5QzPByePXYw0FrSLWmLRfQuByFDPIVANcEa3FgIXIAeKmxItw7IhFRsG5soSOXXgBxdAH/qzEbWhwzgafnxZKJkmrQr8YA3IFtkFPr2+5s26WdjtwEM0tzIFkq6hmWSX1axUgvYlF2uCxjututMZ6I5JCa0uR3gBRuNONuGPH3Ko9zUEATffv53j9DbYVEM0lfVNewefPoVJmWz+oT0wP/kNx6tREf+aUAF4m+eBsqnggITftW2fyeFnoBPCcPp3HUgSwZhesunqz+YeW6Pk+WWb5vl+2QbMKKtz5qK6dI3q0z9yp4FAgMBAAECgf9plqCMg2pKEWRFl183bqWAm7lnLnsUOfABFNPYa3U+uKQUKZpboTBfzDfZqNak3XNQV0mTAR8pFfoMhQjQU9hUxH7ivjw1RUCHjixCF0vLBkTB34gL7FUbiEIFhR1NW3pCJY73OXOkIZG1Obh6Syb8KubDeu4bTmb90/TnDDAs6OYXJ5yo7ZDZLvLu5a3Dli+H4K5Qb5VJ74o/vtodBo3wmKBgy2Ey6JqF+y7/3HeE66rVhYNft5pURgemWnNYqh3oDTJASqpA/8n90o8ceYPVJugQ7029UiyTp0xgBRXFszgiYPkBlsNWB+9+ospopOmYU0owBykH+RtD/bQ0mwECgYEA4gxPdYbHg/GLihHrkv0t5V0pSEhBeBeTBdWG+P/6K90vXofpp7qISdOeYMGkh8mY+PfZNHksdu67d2ks5on5/dNf5YXWCm+LMMRiSsfOo11NISNdNHS6afqs18Wq1aKawv/rwotfxrakM4Gar692+jgz/l/X+FdOQwmE6uEus8ECgYEA3oW4eGtzGq28TiqyhTHduTkas0ckPWX8ulasyPnLxBKDNNohbXXpFIJIcrnl24QFJw3MJbo+R+OxZHgzPK8r64gIGVa7vLCR2fiU/RFoUa1Jo1pPOqXXWqf/Mvdokm2p0atrjRUX9VhjoFsDLcqTgAmfSCsVSqGucX6ER7Gy60UCgYAvmhoNjNFtFquk6rsqHAjTOTgdUaH/0S8T1nBy9SzQmeaEyKhKuvxCV78NbxnfwnNlUoQ6CZ50eTefINXkwn+TlTSnl/SIBA9SuLheOQ9p1ZcNeG4DQuWStcg6NBUSoghnMg+Ky2Di7slLU2qovpGWhcllMve/A1umwFVuRPdZwQKBgHS7mYYyd/Oq6HnpFDWjbzlXp5Yc3/oFooruJT5ZLHfzbjkvpRGTJW7I2dC1jMuXekx+hHXWOg3keI7IL7jJ/DRW7Ei+o0XdKuY57Y7ErwEJ8vNq0N1nWo4IS2wlNgp61PdVAdrFEgh3EexxUj2XY8FrSs/FKio4nxaS1Dn4EnAxAoGBAKqLvuPpmCMVwlIu57WGxL5d9i7EjIGY65l6HTKQYoHCzE51rkowH1La2fuUYz0IpExq2lcrLbOUtSyhXH7Zlktiz//Gu/P90SUfR/ZGcLeZi+EDyK2OctpnWBjs2Dmfg4D6vxk39yV8AB97pYG073GcJ/P54qRUuEitbpJwH+fB' +set service ntp allow-client address '0.0.0.0/0' +set service ntp allow-client address '::/0' +set service ntp server 172.16.100.10 +set service ntp server 172.16.100.20 +set service ntp server 172.16.110.30 +set service ssh disable-host-validation +set service ssh port '22' +set system config-management commit-revisions '200' +set system conntrack modules ftp +set system conntrack modules h323 +set system conntrack modules nfs +set system conntrack modules pptp +set system conntrack modules sip +set system conntrack modules sqlnet +set system conntrack modules tftp +set system console device ttyS0 speed '115200' +set system host-name 'vyos' +set system login user vyos authentication encrypted-password '$6$O5gJRlDYQpj$MtrCV9lxMnZPMbcxlU7.FI793MImNHznxGoMFgm3Q6QP3vfKJyOSRCt3Ka/GzFQyW1yZS4NS616NLHaIPPFHc0' +set system login user vyos authentication plaintext-password '' +set system name-server '172.16.254.30' +set system option kernel disable-mitigations +set system syslog local facility all level 'info' +set system syslog local facility local7 level 'debug' +set system time-zone 'Europe/Berlin' diff --git a/smoketest/configs/basic-haproxy b/smoketest/configs/basic-haproxy new file mode 100644 index 000000000..83fffbac6 --- /dev/null +++ b/smoketest/configs/basic-haproxy @@ -0,0 +1,153 @@ +interfaces { + dummy dum0 { + address "172.18.254.203/32" + } + ethernet eth0 { + duplex "auto" + speed "auto" + vif 203 { + address "172.18.203.10/24" + } + } + ethernet eth1 { + duplex "auto" + speed "auto" + } + ethernet eth2 { + duplex "auto" + speed "auto" + } +} +load-balancing { + reverse-proxy { + backend webserver { + logging { + facility all { + level "all" + } + facility daemon { + level "all" + } + facility user { + level "info" + } + } + server web01 { + address "192.0.2.1" + port "443" + } + ssl { + no-verify + } + } + global-parameters { + logging { + facility all { + level "all" + } + facility daemon { + level "all" + } + facility user { + level "info" + } + } + } + service frontend { + backend "webserver" + logging { + facility all { + level "all" + } + facility daemon { + level "all" + } + facility user { + level "info" + } + } + port "443" + ssl { + certificate "dummy" + } + } + } +} +pki { + certificate dummy { + certificate "MIIDsTCCApmgAwIBAgIUegVgO1wIN2v44trXZ+Kb1t48uL0wDQYJKoZIhvcNAQELBQAwVzELMAkGA1UEBhMCR0IxEzARBgNVBAgMClNvbWUtU3RhdGUxEjAQBgNVBAcMCVNvbWUtQ2l0eTENMAsGA1UECgwEVnlPUzEQMA4GA1UEAwwHdnlvcy5pbzAeFw0yNTA1MDUxODIzMTdaFw0yNjA1MDUxODIzMTdaMFcxCzAJBgNVBAYTAkdCMRMwEQYDVQQIDApTb21lLVN0YXRlMRIwEAYDVQQHDAlTb21lLUNpdHkxDTALBgNVBAoMBFZ5T1MxEDAOBgNVBAMMB3Z5b3MuaW8wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDEfMAwYLKKVhGlUXr9gkVC7uBi+0O9yyEgd5QzPByePXYw0FrSLWmLRfQuByFDPIVANcEa3FgIXIAeKmxItw7IhFRsG5soSOXXgBxdAH/qzEbWhwzgafnxZKJkmrQr8YA3IFtkFPr2+5s26WdjtwEM0tzIFkq6hmWSX1axUgvYlF2uCxjututMZ6I5JCa0uR3gBRuNONuGPH3Ko9zUEATffv53j9DbYVEM0lfVNewefPoVJmWz+oT0wP/kNx6tREf+aUAF4m+eBsqnggITftW2fyeFnoBPCcPp3HUgSwZhesunqz+YeW6Pk+WWb5vl+2QbMKKtz5qK6dI3q0z9yp4FAgMBAAGjdTBzMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgeAMBMGA1UdJQQMMAoGCCsGAQUFBwMBMB0GA1UdDgQWBBSr4OYIWkb8UGuQnEFnjSbmvR+4vDAfBgNVHSMEGDAWgBSr4OYIWkb8UGuQnEFnjSbmvR+4vDANBgkqhkiG9w0BAQsFAAOCAQEAUmRWRPGXsvfuRT+53id3EufH1IJAdowrt6yBZsHobvqCXO2+YhG7oG6/UqUYiv5bHN5xEMQyWd7nyrLOUeFo2bpcMIOlpl6AoUIY65Gm2BqQ7FuPxLLO25RdpZ5WkMGX5kJsKY0/PcpamRKNz1khgFcRyxf9WGhCAIjDCWIWs8lkvPN3m75SFCW7MTuzzQOrzvI6nqqcHO4k8hRBznp26WLUW1rQKpNN09nZGOkeNYK5QbzKN/RUmtEHQZhlgLAIr09jUaA4RDLI1SdD6LR5nvpa9RJBTyS/kISF8BXKMgvUbDHN2nP+VUUrut2ZwoU+pxV4RVT2pS760HuYj4+sYQ==" + private { + key "MIIEuwIBADANBgkqhkiG9w0BAQEFAASCBKUwggShAgEAAoIBAQDEfMAwYLKKVhGlUXr9gkVC7uBi+0O9yyEgd5QzPByePXYw0FrSLWmLRfQuByFDPIVANcEa3FgIXIAeKmxItw7IhFRsG5soSOXXgBxdAH/qzEbWhwzgafnxZKJkmrQr8YA3IFtkFPr2+5s26WdjtwEM0tzIFkq6hmWSX1axUgvYlF2uCxjututMZ6I5JCa0uR3gBRuNONuGPH3Ko9zUEATffv53j9DbYVEM0lfVNewefPoVJmWz+oT0wP/kNx6tREf+aUAF4m+eBsqnggITftW2fyeFnoBPCcPp3HUgSwZhesunqz+YeW6Pk+WWb5vl+2QbMKKtz5qK6dI3q0z9yp4FAgMBAAECgf9plqCMg2pKEWRFl183bqWAm7lnLnsUOfABFNPYa3U+uKQUKZpboTBfzDfZqNak3XNQV0mTAR8pFfoMhQjQU9hUxH7ivjw1RUCHjixCF0vLBkTB34gL7FUbiEIFhR1NW3pCJY73OXOkIZG1Obh6Syb8KubDeu4bTmb90/TnDDAs6OYXJ5yo7ZDZLvLu5a3Dli+H4K5Qb5VJ74o/vtodBo3wmKBgy2Ey6JqF+y7/3HeE66rVhYNft5pURgemWnNYqh3oDTJASqpA/8n90o8ceYPVJugQ7029UiyTp0xgBRXFszgiYPkBlsNWB+9+ospopOmYU0owBykH+RtD/bQ0mwECgYEA4gxPdYbHg/GLihHrkv0t5V0pSEhBeBeTBdWG+P/6K90vXofpp7qISdOeYMGkh8mY+PfZNHksdu67d2ks5on5/dNf5YXWCm+LMMRiSsfOo11NISNdNHS6afqs18Wq1aKawv/rwotfxrakM4Gar692+jgz/l/X+FdOQwmE6uEus8ECgYEA3oW4eGtzGq28TiqyhTHduTkas0ckPWX8ulasyPnLxBKDNNohbXXpFIJIcrnl24QFJw3MJbo+R+OxZHgzPK8r64gIGVa7vLCR2fiU/RFoUa1Jo1pPOqXXWqf/Mvdokm2p0atrjRUX9VhjoFsDLcqTgAmfSCsVSqGucX6ER7Gy60UCgYAvmhoNjNFtFquk6rsqHAjTOTgdUaH/0S8T1nBy9SzQmeaEyKhKuvxCV78NbxnfwnNlUoQ6CZ50eTefINXkwn+TlTSnl/SIBA9SuLheOQ9p1ZcNeG4DQuWStcg6NBUSoghnMg+Ky2Di7slLU2qovpGWhcllMve/A1umwFVuRPdZwQKBgHS7mYYyd/Oq6HnpFDWjbzlXp5Yc3/oFooruJT5ZLHfzbjkvpRGTJW7I2dC1jMuXekx+hHXWOg3keI7IL7jJ/DRW7Ei+o0XdKuY57Y7ErwEJ8vNq0N1nWo4IS2wlNgp61PdVAdrFEgh3EexxUj2XY8FrSs/FKio4nxaS1Dn4EnAxAoGBAKqLvuPpmCMVwlIu57WGxL5d9i7EjIGY65l6HTKQYoHCzE51rkowH1La2fuUYz0IpExq2lcrLbOUtSyhXH7Zlktiz//Gu/P90SUfR/ZGcLeZi+EDyK2OctpnWBjs2Dmfg4D6vxk39yV8AB97pYG073GcJ/P54qRUuEitbpJwH+fB" + } + } +} +service { + ntp { + allow-client { + address "0.0.0.0/0" + address "::/0" + } + server 172.16.100.10 { + } + server 172.16.100.20 { + } + server 172.16.110.30 { + } + } + ssh { + disable-host-validation + port "22" + } +} +system { + config-management { + commit-revisions "200" + } + console { + device ttyS0 { + speed 115200 + } + } + conntrack { + modules { + ftp + h323 + nfs + pptp + sip + sqlnet + tftp + } + } + host-name vyos + login { + user vyos { + authentication { + encrypted-password $6$O5gJRlDYQpj$MtrCV9lxMnZPMbcxlU7.FI793MImNHznxGoMFgm3Q6QP3vfKJyOSRCt3Ka/GzFQyW1yZS4NS616NLHaIPPFHc0 + plaintext-password "" + } + } + } + name-server "172.16.254.30" + option { + kernel { + disable-mitigations + } + } + syslog { + global { + facility all { + level "info" + } + facility local7 { + level "debug" + } + } + } + time-zone "Europe/Berlin" +} + + +// Warning: Do not remove the following line. +// vyos-config-version: "bgp@5:broadcast-relay@1:cluster@2:config-management@1:conntrack@5:conntrack-sync@2:container@2:dhcp-relay@2:dhcp-server@8:dhcpv6-server@1:dns-dynamic@4:dns-forwarding@4:firewall@15:flow-accounting@1:https@6:ids@1:interfaces@32:ipoe-server@3:ipsec@13:isis@3:l2tp@9:lldp@2:mdns@1:monitoring@1:nat@8:nat66@3:ntp@3:openconnect@3:ospf@2:pim@1:policy@8:pppoe-server@10:pptp@5:qos@2:quagga@11:reverse-proxy@1:rip@1:rpki@2:salt@1:snmp@3:ssh@2:sstp@6:system@27:vrf@3:vrrp@4:vyos-accel-ppp@2:wanloadbalance@3:webproxy@2" +// Release version: 1.4.1 diff --git a/smoketest/scripts/cli/test_firewall.py b/smoketest/scripts/cli/test_firewall.py index 69de0c326..851a15f16 100755 --- a/smoketest/scripts/cli/test_firewall.py +++ b/smoketest/scripts/cli/test_firewall.py @@ -1295,7 +1295,7 @@ class TestFirewall(VyOSUnitTestSHIM.TestCase): ['R_group01'], ['type ipv4_addr'], ['flags interval'], - ['meta l4proto', 'daddr @R_group01', "ipv4-INP-filter-10"] + ['meta l4proto', 'daddr @R_group01', 'ipv4-INP-filter-10'] ] self.verify_nftables(nftables_search, 'ip vyos_filter') @@ -1314,5 +1314,79 @@ class TestFirewall(VyOSUnitTestSHIM.TestCase): self.cli_discard() + def test_ipv6_remote_group(self): + # Setup base config for test + self.cli_set(['firewall', 'group', 'remote-group', 'group01', 'url', 'http://127.0.0.1:80/list.txt']) + self.cli_set(['firewall', 'group', 'remote-group', 'group01', 'description', 'Example Group 01']) + self.cli_set(['firewall', 'ipv6', 'input', 'filter', 'rule', '10', 'action', 'drop']) + self.cli_set(['firewall', 'ipv6', 'input', 'filter', 'rule', '10', 'protocol', 'tcp']) + self.cli_set(['firewall', 'ipv6', 'input', 'filter', 'rule', '10', 'destination', 'group', 'remote-group', 'group01']) + + self.cli_commit() + + # Test remote-group had been loaded correctly in nft + nftables_search = [ + ['R6_group01'], + ['type ipv6_addr'], + ['flags interval'], + ['meta l4proto', 'daddr @R6_group01', 'ipv6-INP-filter-10'] + ] + self.verify_nftables(nftables_search, 'ip6 vyos_filter') + + # Test remote-group cannot be configured without a URL + self.cli_delete(['firewall', 'group', 'remote-group', 'group01', 'url']) + + with self.assertRaises(ConfigSessionError): + self.cli_commit() + self.cli_discard() + + # Test remote-group cannot be set alongside address in rules + self.cli_set(['firewall', 'ipv6', 'input', 'filter', 'rule', '10', 'destination', 'address', '2001:db8::1']) + + with self.assertRaises(ConfigSessionError): + self.cli_commit() + self.cli_discard() + + + def test_remote_group(self): + # Setup base config for test adding remote group to both ipv4 and ipv6 rules + self.cli_set(['firewall', 'group', 'remote-group', 'group01', 'url', 'http://127.0.0.1:80/list.txt']) + self.cli_set(['firewall', 'group', 'remote-group', 'group01', 'description', 'Example Group 01']) + self.cli_set(['firewall', 'ipv4', 'output', 'filter', 'rule', '10', 'action', 'drop']) + self.cli_set(['firewall', 'ipv4', 'output', 'filter', 'rule', '10', 'protocol', 'tcp']) + self.cli_set(['firewall', 'ipv4', 'output', 'filter', 'rule', '10', 'destination', 'group', 'remote-group', 'group01']) + self.cli_set(['firewall', 'ipv4', 'input', 'filter', 'rule', '10', 'action', 'drop']) + self.cli_set(['firewall', 'ipv4', 'input', 'filter', 'rule', '10', 'protocol', 'tcp']) + self.cli_set(['firewall', 'ipv4', 'input', 'filter', 'rule', '10', 'source', 'group', 'remote-group', 'group01']) + self.cli_set(['firewall', 'ipv6', 'output', 'filter', 'rule', '10', 'action', 'drop']) + self.cli_set(['firewall', 'ipv6', 'output', 'filter', 'rule', '10', 'protocol', 'tcp']) + self.cli_set(['firewall', 'ipv6', 'output', 'filter', 'rule', '10', 'destination', 'group', 'remote-group', 'group01']) + self.cli_set(['firewall', 'ipv6', 'input', 'filter', 'rule', '10', 'action', 'drop']) + self.cli_set(['firewall', 'ipv6', 'input', 'filter', 'rule', '10', 'protocol', 'tcp']) + self.cli_set(['firewall', 'ipv6', 'input', 'filter', 'rule', '10', 'source', 'group', 'remote-group', 'group01']) + + self.cli_commit() + + # Test remote-group had been loaded correctly in nft ip table + nftables_v4_search = [ + ['R_group01'], + ['type ipv4_addr'], + ['flags interval'], + ['meta l4proto', 'daddr @R_group01', 'ipv4-OUT-filter-10'], + ['meta l4proto', 'saddr @R_group01', 'ipv4-INP-filter-10'], + ] + self.verify_nftables(nftables_v4_search, 'ip vyos_filter') + + # Test remote-group had been loaded correctly in nft ip6 table + nftables_v6_search = [ + ['R6_group01'], + ['type ipv6_addr'], + ['flags interval'], + ['meta l4proto', 'daddr @R6_group01', 'ipv6-OUT-filter-10'], + ['meta l4proto', 'saddr @R6_group01', 'ipv6-INP-filter-10'], + ] + self.verify_nftables(nftables_v6_search, 'ip6 vyos_filter') + + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/smoketest/scripts/cli/test_interfaces_wireguard.py b/smoketest/scripts/cli/test_interfaces_wireguard.py index f8cd18cf2..7bc82c187 100755 --- a/smoketest/scripts/cli/test_interfaces_wireguard.py +++ b/smoketest/scripts/cli/test_interfaces_wireguard.py @@ -154,13 +154,15 @@ class WireGuardInterfaceTest(BasicInterfaceTest.TestCase): tmp = read_file(f'/sys/class/net/{intf}/threaded') self.assertTrue(tmp, "1") - def test_wireguard_peer_pubkey_change(self): + def test_wireguard_peer_change(self): # T5707 changing WireGuard CLI public key of a peer - it's not removed + # Also check if allowed-ips update - def get_peers(interface) -> list: + def get_peers(interface) -> list[tuple]: tmp = cmd(f'sudo wg show {interface} dump') first_line = True peers = [] + allowed_ips = [] for line in tmp.split('\n'): if not line: continue # Skip empty lines and last line @@ -170,24 +172,27 @@ class WireGuardInterfaceTest(BasicInterfaceTest.TestCase): first_line = False else: peers.append(items[0]) - return peers + allowed_ips.append(items[3]) + return peers, allowed_ips interface = 'wg1337' port = '1337' privkey = 'iJi4lb2HhkLx2KSAGOjji2alKkYsJjSPkHkrcpxgEVU=' pubkey_1 = 'srQ8VF6z/LDjKCzpxBzFpmaNUOeuHYzIfc2dcmoc/h4=' pubkey_2 = '8pbMHiQ7NECVP7F65Mb2W8+4ldGG2oaGvDSpSEsOBn8=' + allowed_ips_1 = '10.205.212.10/32' + allowed_ips_2 = '10.205.212.11/32' self.cli_set(base_path + [interface, 'address', '172.16.0.1/24']) self.cli_set(base_path + [interface, 'port', port]) self.cli_set(base_path + [interface, 'private-key', privkey]) self.cli_set(base_path + [interface, 'peer', 'VyOS', 'public-key', pubkey_1]) - self.cli_set(base_path + [interface, 'peer', 'VyOS', 'allowed-ips', '10.205.212.10/32']) + self.cli_set(base_path + [interface, 'peer', 'VyOS', 'allowed-ips', allowed_ips_1]) self.cli_commit() - peers = get_peers(interface) + peers, _ = get_peers(interface) self.assertIn(pubkey_1, peers) self.assertNotIn(pubkey_2, peers) @@ -196,10 +201,20 @@ class WireGuardInterfaceTest(BasicInterfaceTest.TestCase): self.cli_commit() # Verify config - peers = get_peers(interface) + peers, _ = get_peers(interface) self.assertNotIn(pubkey_1, peers) self.assertIn(pubkey_2, peers) + # Update allowed-ips + self.cli_delete(base_path + [interface, 'peer', 'VyOS', 'allowed-ips', allowed_ips_1]) + self.cli_set(base_path + [interface, 'peer', 'VyOS', 'allowed-ips', allowed_ips_2]) + self.cli_commit() + + # Verify config + _, allowed_ips = get_peers(interface) + self.assertNotIn(allowed_ips_1, allowed_ips) + self.assertIn(allowed_ips_2, allowed_ips) + def test_wireguard_hostname(self): # T4930: Test dynamic endpoint support interface = 'wg1234' diff --git a/smoketest/scripts/cli/test_load-balancing_haproxy.py b/smoketest/scripts/cli/test_load-balancing_haproxy.py index 077f1974f..833e0a92b 100755 --- a/smoketest/scripts/cli/test_load-balancing_haproxy.py +++ b/smoketest/scripts/cli/test_load-balancing_haproxy.py @@ -14,11 +14,14 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +import re +import textwrap import unittest from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError +from vyos.template import get_default_port from vyos.utils.process import process_named_running from vyos.utils.file import read_file @@ -131,7 +134,25 @@ ZXLrtgVJR9W020qTurO2f91qfU8646n11hR9ObBB1IYbagOU0Pw1Nrq/FRp/u2tx 7i7xFz2WEiQeSCPaKYOiqM3t """ +haproxy_service_name = 'https_front' +haproxy_backend_name = 'bk-01' +def parse_haproxy_config() -> dict: + config_str = read_file(HAPROXY_CONF) + section_pattern = re.compile(r'^(global|defaults|frontend\s+\S+|backend\s+\S+)', re.MULTILINE) + sections = {} + + matches = list(section_pattern.finditer(config_str)) + + for i, match in enumerate(matches): + section_name = match.group(1).strip() + start = match.end() + end = matches[i + 1].start() if i + 1 < len(matches) else len(config_str) + section_body = config_str[start:end] + dedented_body = textwrap.dedent(section_body).strip() + sections[section_name] = dedented_body + + return sections class TestLoadBalancingReverseProxy(VyOSUnitTestSHIM.TestCase): def tearDown(self): # Check for running process @@ -146,14 +167,14 @@ class TestLoadBalancingReverseProxy(VyOSUnitTestSHIM.TestCase): self.assertFalse(process_named_running(PROCESS_NAME)) def base_config(self): - self.cli_set(base_path + ['service', 'https_front', 'mode', 'http']) - self.cli_set(base_path + ['service', 'https_front', 'port', '4433']) - self.cli_set(base_path + ['service', 'https_front', 'backend', 'bk-01']) + self.cli_set(base_path + ['service', haproxy_service_name, 'mode', 'http']) + self.cli_set(base_path + ['service', haproxy_service_name, 'port', '4433']) + self.cli_set(base_path + ['service', haproxy_service_name, 'backend', haproxy_backend_name]) - self.cli_set(base_path + ['backend', 'bk-01', 'mode', 'http']) - self.cli_set(base_path + ['backend', 'bk-01', 'server', 'bk-01', 'address', '192.0.2.11']) - self.cli_set(base_path + ['backend', 'bk-01', 'server', 'bk-01', 'port', '9090']) - self.cli_set(base_path + ['backend', 'bk-01', 'server', 'bk-01', 'send-proxy']) + self.cli_set(base_path + ['backend', haproxy_backend_name, 'mode', 'http']) + self.cli_set(base_path + ['backend', haproxy_backend_name, 'server', haproxy_backend_name, 'address', '192.0.2.11']) + self.cli_set(base_path + ['backend', haproxy_backend_name, 'server', haproxy_backend_name, 'port', '9090']) + self.cli_set(base_path + ['backend', haproxy_backend_name, 'server', haproxy_backend_name, 'send-proxy']) self.cli_set(base_path + ['global-parameters', 'max-connections', '1000']) @@ -167,15 +188,15 @@ class TestLoadBalancingReverseProxy(VyOSUnitTestSHIM.TestCase): self.cli_set(['pki', 'certificate', 'smoketest', 'certificate', valid_cert.replace('\n','')]) self.cli_set(['pki', 'certificate', 'smoketest', 'private', 'key', valid_cert_private_key.replace('\n','')]) - def test_01_lb_reverse_proxy_domain(self): + def test_reverse_proxy_domain(self): domains_bk_first = ['n1.example.com', 'n2.example.com', 'n3.example.com'] domain_bk_second = 'n5.example.com' - frontend = 'https_front' + frontend = 'vyos_smoketest' front_port = '4433' bk_server_first = '192.0.2.11' bk_server_second = '192.0.2.12' - bk_first_name = 'bk-01' - bk_second_name = 'bk-02' + bk_first_name = 'vyosbk-01' + bk_second_name = 'vyosbk-02' bk_server_port = '9090' mode = 'http' rule_ten = '10' @@ -241,9 +262,9 @@ class TestLoadBalancingReverseProxy(VyOSUnitTestSHIM.TestCase): self.assertIn(f'server {bk_second_name} {bk_server_second}:{bk_server_port}', config) self.assertIn(f'server {bk_second_name} {bk_server_second}:{bk_server_port} backup', config) - def test_02_lb_reverse_proxy_cert_not_exists(self): + def test_reverse_proxy_cert_not_exists(self): self.base_config() - self.cli_set(base_path + ['service', 'https_front', 'ssl', 'certificate', 'cert']) + self.cli_set(base_path + ['service', haproxy_service_name, 'ssl', 'certificate', 'cert']) with self.assertRaises(ConfigSessionError) as e: self.cli_commit() @@ -253,19 +274,19 @@ class TestLoadBalancingReverseProxy(VyOSUnitTestSHIM.TestCase): self.configure_pki() self.base_config() - self.cli_set(base_path + ['service', 'https_front', 'ssl', 'certificate', 'cert']) + self.cli_set(base_path + ['service', haproxy_service_name, 'ssl', 'certificate', 'cert']) with self.assertRaises(ConfigSessionError) as e: self.cli_commit() # self.assertIn('\nCertificate "cert" does not exist\n', str(e.exception)) - self.cli_delete(base_path + ['service', 'https_front', 'ssl', 'certificate', 'cert']) - self.cli_set(base_path + ['service', 'https_front', 'ssl', 'certificate', 'smoketest']) + self.cli_delete(base_path + ['service', haproxy_service_name, 'ssl', 'certificate', 'cert']) + self.cli_set(base_path + ['service', haproxy_service_name, 'ssl', 'certificate', 'smoketest']) self.cli_commit() - def test_03_lb_reverse_proxy_ca_not_exists(self): + def test_reverse_proxy_ca_not_exists(self): self.base_config() - self.cli_set(base_path + ['backend', 'bk-01', 'ssl', 'ca-certificate', 'ca-test']) + self.cli_set(base_path + ['backend', haproxy_backend_name, 'ssl', 'ca-certificate', 'ca-test']) with self.assertRaises(ConfigSessionError) as e: self.cli_commit() @@ -275,40 +296,40 @@ class TestLoadBalancingReverseProxy(VyOSUnitTestSHIM.TestCase): self.configure_pki() self.base_config() - self.cli_set(base_path + ['backend', 'bk-01', 'ssl', 'ca-certificate', 'ca-test']) + self.cli_set(base_path + ['backend', haproxy_backend_name, 'ssl', 'ca-certificate', 'ca-test']) with self.assertRaises(ConfigSessionError) as e: self.cli_commit() # self.assertIn('\nCA certificate "ca-test" does not exist\n', str(e.exception)) - self.cli_delete(base_path + ['backend', 'bk-01', 'ssl', 'ca-certificate', 'ca-test']) - self.cli_set(base_path + ['backend', 'bk-01', 'ssl', 'ca-certificate', 'smoketest']) + self.cli_delete(base_path + ['backend', haproxy_backend_name, 'ssl', 'ca-certificate', 'ca-test']) + self.cli_set(base_path + ['backend', haproxy_backend_name, 'ssl', 'ca-certificate', 'smoketest']) self.cli_commit() - def test_04_lb_reverse_proxy_backend_ssl_no_verify(self): + def test_reverse_proxy_backend_ssl_no_verify(self): # Setup base self.configure_pki() self.base_config() # Set no-verify option - self.cli_set(base_path + ['backend', 'bk-01', 'ssl', 'no-verify']) + self.cli_set(base_path + ['backend', haproxy_backend_name, 'ssl', 'no-verify']) self.cli_commit() # Test no-verify option config = read_file(HAPROXY_CONF) - self.assertIn('server bk-01 192.0.2.11:9090 send-proxy ssl verify none', config) + self.assertIn(f'server {haproxy_backend_name} 192.0.2.11:9090 send-proxy ssl verify none', config) # Test setting ca-certificate alongside no-verify option fails, to test config validation - self.cli_set(base_path + ['backend', 'bk-01', 'ssl', 'ca-certificate', 'smoketest']) + self.cli_set(base_path + ['backend', haproxy_backend_name, 'ssl', 'ca-certificate', 'smoketest']) with self.assertRaises(ConfigSessionError) as e: self.cli_commit() - def test_05_lb_reverse_proxy_backend_http_check(self): + def test_reverse_proxy_backend_http_check(self): # Setup base self.base_config() # Set http-check - self.cli_set(base_path + ['backend', 'bk-01', 'http-check', 'method', 'get']) + self.cli_set(base_path + ['backend', haproxy_backend_name, 'http-check', 'method', 'get']) self.cli_commit() # Test http-check @@ -317,8 +338,8 @@ class TestLoadBalancingReverseProxy(VyOSUnitTestSHIM.TestCase): self.assertIn('http-check send meth GET', config) # Set http-check with uri and status - self.cli_set(base_path + ['backend', 'bk-01', 'http-check', 'uri', '/health']) - self.cli_set(base_path + ['backend', 'bk-01', 'http-check', 'expect', 'status', '200']) + self.cli_set(base_path + ['backend', haproxy_backend_name, 'http-check', 'uri', '/health']) + self.cli_set(base_path + ['backend', haproxy_backend_name, 'http-check', 'expect', 'status', '200']) self.cli_commit() # Test http-check with uri and status @@ -328,8 +349,8 @@ class TestLoadBalancingReverseProxy(VyOSUnitTestSHIM.TestCase): self.assertIn('http-check expect status 200', config) # Set http-check with string - self.cli_delete(base_path + ['backend', 'bk-01', 'http-check', 'expect', 'status', '200']) - self.cli_set(base_path + ['backend', 'bk-01', 'http-check', 'expect', 'string', 'success']) + self.cli_delete(base_path + ['backend', haproxy_backend_name, 'http-check', 'expect', 'status', '200']) + self.cli_set(base_path + ['backend', haproxy_backend_name, 'http-check', 'expect', 'string', 'success']) self.cli_commit() # Test http-check with string @@ -339,11 +360,11 @@ class TestLoadBalancingReverseProxy(VyOSUnitTestSHIM.TestCase): self.assertIn('http-check expect string success', config) # Test configuring both http-check & health-check fails validation script - self.cli_set(base_path + ['backend', 'bk-01', 'health-check', 'ldap']) + self.cli_set(base_path + ['backend', haproxy_backend_name, 'health-check', 'ldap']) with self.assertRaises(ConfigSessionError) as e: self.cli_commit() - def test_06_lb_reverse_proxy_tcp_mode(self): + def test_reverse_proxy_tcp_mode(self): frontend = 'tcp_8443' mode = 'tcp' front_port = '8433' @@ -390,27 +411,27 @@ class TestLoadBalancingReverseProxy(VyOSUnitTestSHIM.TestCase): self.assertIn(f'mode {mode}', config) self.assertIn(f'server {bk_name} {bk_server}:{bk_server_port}', config) - def test_07_lb_reverse_proxy_http_response_headers(self): + def test_reverse_proxy_http_response_headers(self): # Setup base self.configure_pki() self.base_config() # Set example headers in both frontend and backend - self.cli_set(base_path + ['service', 'https_front', 'http-response-headers', 'Cache-Control', 'value', 'max-age=604800']) - self.cli_set(base_path + ['backend', 'bk-01', 'http-response-headers', 'Proxy-Backend-ID', 'value', 'bk-01']) + self.cli_set(base_path + ['service', haproxy_service_name, 'http-response-headers', 'Cache-Control', 'value', 'max-age=604800']) + self.cli_set(base_path + ['backend', haproxy_backend_name, 'http-response-headers', 'Proxy-Backend-ID', 'value', haproxy_backend_name]) self.cli_commit() # Test headers are present in generated configuration file config = read_file(HAPROXY_CONF) self.assertIn('http-response set-header Cache-Control \'max-age=604800\'', config) - self.assertIn('http-response set-header Proxy-Backend-ID \'bk-01\'', config) + self.assertIn(f'http-response set-header Proxy-Backend-ID \'{haproxy_backend_name}\'', config) # Test setting alongside modes other than http is blocked by validation conditions - self.cli_set(base_path + ['service', 'https_front', 'mode', 'tcp']) + self.cli_set(base_path + ['service', haproxy_service_name, 'mode', 'tcp']) with self.assertRaises(ConfigSessionError) as e: self.cli_commit() - def test_08_lb_reverse_proxy_tcp_health_checks(self): + def test_reverse_proxy_tcp_health_checks(self): # Setup PKI self.configure_pki() @@ -458,7 +479,7 @@ class TestLoadBalancingReverseProxy(VyOSUnitTestSHIM.TestCase): config = read_file(HAPROXY_CONF) self.assertIn(f'option smtpchk', config) - def test_09_lb_reverse_proxy_logging(self): + def test_reverse_proxy_logging(self): # Setup base self.base_config() self.cli_commit() @@ -477,7 +498,7 @@ class TestLoadBalancingReverseProxy(VyOSUnitTestSHIM.TestCase): self.assertIn('log /dev/log local2 warning', config) # Test backend logging options - backend_path = base_path + ['backend', 'bk-01'] + backend_path = base_path + ['backend', haproxy_backend_name] self.cli_set(backend_path + ['logging', 'facility', 'local3', 'level', 'debug']) self.cli_set(backend_path + ['logging', 'facility', 'local4', 'level', 'info']) self.cli_commit() @@ -488,7 +509,7 @@ class TestLoadBalancingReverseProxy(VyOSUnitTestSHIM.TestCase): self.assertIn('log /dev/log local4 info', config) # Test service logging options - service_path = base_path + ['service', 'https_front'] + service_path = base_path + ['service', haproxy_service_name] self.cli_set(service_path + ['logging', 'facility', 'local5', 'level', 'notice']) self.cli_set(service_path + ['logging', 'facility', 'local6', 'level', 'crit']) self.cli_commit() @@ -498,16 +519,17 @@ class TestLoadBalancingReverseProxy(VyOSUnitTestSHIM.TestCase): self.assertIn('log /dev/log local5 notice', config) self.assertIn('log /dev/log local6 crit', config) - def test_10_lb_reverse_proxy_http_compression(self): + def test_reverse_proxy_http_compression(self): # Setup base self.configure_pki() self.base_config() # Configure compression in frontend - self.cli_set(base_path + ['service', 'https_front', 'http-compression', 'algorithm', 'gzip']) - self.cli_set(base_path + ['service', 'https_front', 'http-compression', 'mime-type', 'text/html']) - self.cli_set(base_path + ['service', 'https_front', 'http-compression', 'mime-type', 'text/javascript']) - self.cli_set(base_path + ['service', 'https_front', 'http-compression', 'mime-type', 'text/plain']) + http_comp_path = base_path + ['service', haproxy_service_name, 'http-compression'] + self.cli_set(http_comp_path + ['algorithm', 'gzip']) + self.cli_set(http_comp_path + ['mime-type', 'text/html']) + self.cli_set(http_comp_path + ['mime-type', 'text/javascript']) + self.cli_set(http_comp_path + ['mime-type', 'text/plain']) self.cli_commit() # Test compression is present in generated configuration file @@ -517,11 +539,11 @@ class TestLoadBalancingReverseProxy(VyOSUnitTestSHIM.TestCase): self.assertIn('compression type text/html text/javascript text/plain', config) # Test setting compression without specifying any mime-types fails verification - self.cli_delete(base_path + ['service', 'https_front', 'http-compression', 'mime-type']) + self.cli_delete(base_path + ['service', haproxy_service_name, 'http-compression', 'mime-type']) with self.assertRaises(ConfigSessionError) as e: self.cli_commit() - def test_11_lb_haproxy_timeout(self): + def test_reverse_proxy_timeout(self): t_default_check = '5' t_default_client = '50' t_default_connect = '10' @@ -551,7 +573,7 @@ class TestLoadBalancingReverseProxy(VyOSUnitTestSHIM.TestCase): self.cli_set(base_path + ['timeout', 'client', t_client]) self.cli_set(base_path + ['timeout', 'connect', t_connect]) self.cli_set(base_path + ['timeout', 'server', t_server]) - self.cli_set(base_path + ['service', 'https_front', 'timeout', 'client', t_front_client]) + self.cli_set(base_path + ['service', haproxy_service_name, 'timeout', 'client', t_front_client]) self.cli_commit() @@ -569,5 +591,25 @@ class TestLoadBalancingReverseProxy(VyOSUnitTestSHIM.TestCase): for config_entry in config_entries: self.assertIn(config_entry, config) + def test_reverse_proxy_http_redirect(self): + self.base_config() + self.cli_set(base_path + ['service', haproxy_service_name, 'redirect-http-to-https']) + + self.cli_commit() + + config = parse_haproxy_config() + frontend_name = f'frontend {haproxy_service_name}-http' + self.assertIn(frontend_name, config.keys()) + self.assertIn('mode http', config[frontend_name]) + self.assertIn('bind [::]:80 v4v6', config[frontend_name]) + self.assertIn('acl acme_acl path_beg /.well-known/acme-challenge/', config[frontend_name]) + self.assertIn('use_backend buildin_acme_certbot if acme_acl', config[frontend_name]) + self.assertIn('redirect scheme https code 301 if !acme_acl', config[frontend_name]) + + backend_name = 'backend buildin_acme_certbot' + self.assertIn(backend_name, config.keys()) + port = get_default_port('certbot_haproxy') + self.assertIn(f'server localhost 127.0.0.1:{port}', config[backend_name]) + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/smoketest/scripts/cli/test_nat66.py b/smoketest/scripts/cli/test_nat66.py index 52ad8e3ef..d4b5d6aa4 100755 --- a/smoketest/scripts/cli/test_nat66.py +++ b/smoketest/scripts/cli/test_nat66.py @@ -227,6 +227,35 @@ class TestNAT66(VyOSUnitTestSHIM.TestCase): self.verify_nftables(nftables_search, 'ip6 vyos_nat') + def test_source_nat66_network_group(self): + address_group = 'smoketest_addr' + address_group_member = 'fc00::1' + network_group = 'smoketest_net' + network_group_member = 'fc00::/64' + translation_prefix = 'fc01::/64' + + self.cli_set(['firewall', 'group', 'ipv6-address-group', address_group, 'address', address_group_member]) + self.cli_set(['firewall', 'group', 'ipv6-network-group', network_group, 'network', network_group_member]) + + self.cli_set(src_path + ['rule', '1', 'destination', 'group', 'address-group', address_group]) + self.cli_set(src_path + ['rule', '1', 'translation', 'address', translation_prefix]) + + self.cli_set(src_path + ['rule', '2', 'destination', 'group', 'network-group', network_group]) + self.cli_set(src_path + ['rule', '2', 'translation', 'address', translation_prefix]) + + self.cli_commit() + + nftables_search = [ + [f'set A6_{address_group}'], + [f'elements = {{ {address_group_member} }}'], + [f'set N6_{network_group}'], + [f'elements = {{ {network_group_member} }}'], + ['ip6 daddr', f'@A6_{address_group}', 'snat prefix to fc01::/64'], + ['ip6 daddr', f'@N6_{network_group}', 'snat prefix to fc01::/64'] + ] + + self.verify_nftables(nftables_search, 'ip6 vyos_nat') + def test_nat66_no_rules(self): # T3206: deleting all rules but keep the direction 'destination' or # 'source' resulteds in KeyError: 'rule'. diff --git a/smoketest/scripts/cli/test_system_conntrack.py b/smoketest/scripts/cli/test_system_conntrack.py index 72deb7525..f6bb3cf7c 100755 --- a/smoketest/scripts/cli/test_system_conntrack.py +++ b/smoketest/scripts/cli/test_system_conntrack.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2021-2024 VyOS maintainers and contributors +# Copyright (C) 2021-2025 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -195,6 +195,8 @@ class TestSystemConntrack(VyOSUnitTestSHIM.TestCase): def test_conntrack_ignore(self): address_group = 'conntracktest' address_group_member = '192.168.0.1' + port_single = '53' + ports_multi = '500,4500' ipv6_address_group = 'conntracktest6' ipv6_address_group_member = 'dead:beef::1' @@ -211,6 +213,14 @@ class TestSystemConntrack(VyOSUnitTestSHIM.TestCase): self.cli_set(base_path + ['ignore', 'ipv4', 'rule', '2', 'destination', 'group', 'address-group', address_group]) self.cli_set(base_path + ['ignore', 'ipv4', 'rule', '2', 'protocol', 'all']) + self.cli_set(base_path + ['ignore', 'ipv4', 'rule', '3', 'source', 'address', '192.0.2.1']) + self.cli_set(base_path + ['ignore', 'ipv4', 'rule', '3', 'destination', 'port', ports_multi]) + self.cli_set(base_path + ['ignore', 'ipv4', 'rule', '3', 'protocol', 'udp']) + + self.cli_set(base_path + ['ignore', 'ipv4', 'rule', '4', 'source', 'address', '192.0.2.1']) + self.cli_set(base_path + ['ignore', 'ipv4', 'rule', '4', 'destination', 'port', port_single]) + self.cli_set(base_path + ['ignore', 'ipv4', 'rule', '4', 'protocol', 'udp']) + self.cli_set(base_path + ['ignore', 'ipv6', 'rule', '11', 'source', 'address', 'fe80::1']) self.cli_set(base_path + ['ignore', 'ipv6', 'rule', '11', 'destination', 'address', 'fe80::2']) self.cli_set(base_path + ['ignore', 'ipv6', 'rule', '11', 'destination', 'port', '22']) @@ -226,7 +236,9 @@ class TestSystemConntrack(VyOSUnitTestSHIM.TestCase): nftables_search = [ ['ip saddr 192.0.2.1', 'ip daddr 192.0.2.2', 'tcp dport 22', 'tcp flags & syn == syn', 'notrack'], - ['ip saddr 192.0.2.1', 'ip daddr @A_conntracktest', 'notrack'] + ['ip saddr 192.0.2.1', 'ip daddr @A_conntracktest', 'notrack'], + ['ip saddr 192.0.2.1', 'udp dport { 500, 4500 }', 'notrack'], + ['ip saddr 192.0.2.1', 'udp dport 53', 'notrack'] ] nftables6_search = [ diff --git a/smoketest/scripts/cli/test_system_login.py b/smoketest/scripts/cli/test_system_login.py index 71dec68d8..fd5af12ba 100755 --- a/smoketest/scripts/cli/test_system_login.py +++ b/smoketest/scripts/cli/test_system_login.py @@ -548,5 +548,34 @@ class TestSystemLogin(VyOSUnitTestSHIM.TestCase): self.cli_commit() self.cli_discard() + def test_pam_nologin(self): + # Testcase for T7443, test if we can login with a non-privileged user + # when there are only 5 minutes left until the system reboots + username = users[0] + password = f'{username}-pSWd-t3st' + + self.cli_set(base_path + ['user', username, 'authentication', 'plaintext-password', password]) + self.cli_commit() + + # Login with proper credentials + out, err = self.ssh_send_cmd(ssh_test_command, username, password) + # verify login + self.assertFalse(err) + self.assertEqual(out, self.ssh_test_command_result) + + # Request system reboot in 5 minutes - this will activate pam_nologin.so + # and prevent any login - but we have this disabled, so we must be able + # to login to the router + self.op_mode(['reboot', 'in', '4']) + + # verify login + # Login with proper credentials - after reboot is pending + out, err = self.ssh_send_cmd(ssh_test_command, username, password) + self.assertFalse(err) + self.assertEqual(out, self.ssh_test_command_result) + + # Cancel pending reboot - we do wan't to preceed with the remaining tests + self.op_mode(['reboot', 'cancel']) + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/smoketest/scripts/cli/test_system_option.py b/smoketest/scripts/cli/test_system_option.py index 4daa812c0..c7f8c1f3e 100755 --- a/smoketest/scripts/cli/test_system_option.py +++ b/smoketest/scripts/cli/test_system_option.py @@ -102,9 +102,28 @@ class TestSystemOption(VyOSUnitTestSHIM.TestCase): def test_kernel_options(self): amd_pstate_mode = 'active' - + isolate_cpus = '1,2,3' + nohz_full = '2' + rcu_no_cbs = '1,2,4-5' + default_hp_size = '2M' + hp_size_1g = '1G' + hp_size_2m = '2M' + hp_count_1g = '2' + hp_count_2m = '512' + + self.cli_set(['system', 'option', 'kernel', 'cpu', 'disable-nmi-watchdog']) + self.cli_set(['system', 'option', 'kernel', 'cpu', 'isolate-cpus', isolate_cpus]) + self.cli_set(['system', 'option', 'kernel', 'cpu', 'nohz-full', nohz_full]) + self.cli_set(['system', 'option', 'kernel', 'cpu', 'rcu-no-cbs', rcu_no_cbs]) + self.cli_set(['system', 'option', 'kernel', 'disable-hpet']) + self.cli_set(['system', 'option', 'kernel', 'disable-mce']) self.cli_set(['system', 'option', 'kernel', 'disable-mitigations']) self.cli_set(['system', 'option', 'kernel', 'disable-power-saving']) + self.cli_set(['system', 'option', 'kernel', 'disable-softlockup']) + self.cli_set(['system', 'option', 'kernel', 'memory', 'disable-numa-balancing']) + self.cli_set(['system', 'option', 'kernel', 'memory', 'default-hugepage-size', default_hp_size]) + self.cli_set(['system', 'option', 'kernel', 'memory', 'hugepage-size', hp_size_1g, 'hugepage-count', hp_count_1g]) + self.cli_set(['system', 'option', 'kernel', 'memory', 'hugepage-size', hp_size_2m, 'hugepage-count', hp_count_2m]) self.cli_set(['system', 'option', 'kernel', 'quiet']) self.cli_set(['system', 'option', 'kernel', 'amd-pstate-driver', amd_pstate_mode]) @@ -121,6 +140,17 @@ class TestSystemOption(VyOSUnitTestSHIM.TestCase): self.assertIn(' mitigations=off', tmp) self.assertIn(' intel_idle.max_cstate=0 processor.max_cstate=1', tmp) self.assertIn(' quiet', tmp) + self.assertIn(' nmi_watchdog=0', tmp) + self.assertIn(' hpet=disable', tmp) + self.assertIn(' mce=off', tmp) + self.assertIn(' nosoftlockup', tmp) + self.assertIn(f' isolcpus={isolate_cpus}', tmp) + self.assertIn(f' nohz_full={nohz_full}', tmp) + self.assertIn(f' rcu_nocbs={rcu_no_cbs}', tmp) + self.assertIn(f' default_hugepagesz={default_hp_size}', tmp) + self.assertIn(f' hugepagesz={hp_size_1g} hugepages={hp_count_1g}', tmp) + self.assertIn(f' hugepagesz={hp_size_2m} hugepages={hp_count_2m}', tmp) + self.assertIn(' numa_balancing=disable', tmp) if cpu_vendor == 'AuthenticAMD': self.assertIn(f' initcall_blacklist=acpi_cpufreq_init amd_pstate={amd_pstate_mode}', tmp) diff --git a/src/conf_mode/container.py b/src/conf_mode/container.py index 18d660a4e..94882fc14 100755 --- a/src/conf_mode/container.py +++ b/src/conf_mode/container.py @@ -324,6 +324,11 @@ def generate_run_arguments(name, container_config): cap = cap.upper().replace('-', '_') capabilities += f' --cap-add={cap}' + # Grant root capabilities to the container + privileged = '' + if 'privileged' in container_config: + privileged = '--privileged' + # Add a host device to the container /dev/x:/dev/x device = '' if 'device' in container_config: @@ -402,7 +407,7 @@ def generate_run_arguments(name, container_config): for ns in container_config['name_server']: name_server += f'--dns {ns}' - container_base_cmd = f'--detach --interactive --tty --replace {capabilities} --cpus {cpu_quota} {sysctl_opt} ' \ + container_base_cmd = f'--detach --interactive --tty --replace {capabilities} {privileged} --cpus {cpu_quota} {sysctl_opt} ' \ f'--memory {memory}m --shm-size {shared_memory}m --memory-swap 0 --restart {restart} ' \ f'--name {name} {hostname} {device} {port} {name_server} {volume} {tmpfs} {env_opt} {label} {uid} {host_pid}' diff --git a/src/conf_mode/firewall.py b/src/conf_mode/firewall.py index 274ca2ce6..348eaeba3 100755 --- a/src/conf_mode/firewall.py +++ b/src/conf_mode/firewall.py @@ -17,6 +17,8 @@ import os import re +from glob import glob + from sys import exit from vyos.base import Warning from vyos.config import Config @@ -30,6 +32,7 @@ from vyos.firewall import geoip_update from vyos.template import render from vyos.utils.dict import dict_search_args from vyos.utils.dict import dict_search_recursive +from vyos.utils.file import write_file from vyos.utils.process import call from vyos.utils.process import cmd from vyos.utils.process import rc_cmd @@ -37,7 +40,6 @@ from vyos.utils.network import get_vrf_members from vyos.utils.network import get_interface_vrf from vyos import ConfigError from vyos import airbag -from pathlib import Path from subprocess import run as subp_run airbag.enable() @@ -626,10 +628,11 @@ def apply(firewall): domain_action = 'restart' if dict_search_args(firewall, 'group', 'remote_group') or dict_search_args(firewall, 'group', 'domain_group') or firewall['ip_fqdn'].items() or firewall['ip6_fqdn'].items(): text = f'# Automatically generated by firewall.py\nThis file indicates that vyos-domain-resolver service is used by the firewall.\n' - Path(domain_resolver_usage).write_text(text) + write_file(domain_resolver_usage, text) else: - Path(domain_resolver_usage).unlink(missing_ok=True) - if not Path('/run').glob('use-vyos-domain-resolver*'): + if os.path.exists(domain_resolver_usage): + os.unlink(domain_resolver_usage) + if not glob('/run/use-vyos-domain-resolver*'): domain_action = 'stop' call(f'systemctl {domain_action} vyos-domain-resolver.service') diff --git a/src/conf_mode/interfaces_wireguard.py b/src/conf_mode/interfaces_wireguard.py index 192937dba..770667df1 100755 --- a/src/conf_mode/interfaces_wireguard.py +++ b/src/conf_mode/interfaces_wireguard.py @@ -14,6 +14,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +import os + +from glob import glob from sys import exit from vyos.config import Config @@ -35,7 +38,6 @@ from vyos.utils.network import is_wireguard_key_pair from vyos.utils.process import call from vyos import ConfigError from vyos import airbag -from pathlib import Path airbag.enable() @@ -97,7 +99,7 @@ def verify(wireguard): if 'port' in wireguard and 'port_changed' in wireguard: listen_port = int(wireguard['port']) - if check_port_availability('0.0.0.0', listen_port, 'udp') is not True: + if check_port_availability(None, listen_port, protocol='udp') is not True: raise ConfigError(f'UDP port {listen_port} is busy or unavailable and ' 'cannot be used for the interface!') @@ -145,19 +147,11 @@ def generate(wireguard): def apply(wireguard): check_kmod('wireguard') - if 'rebuild_required' in wireguard or 'deleted' in wireguard: - wg = WireGuardIf(**wireguard) - # WireGuard only supports peer removal based on the configured public-key, - # by deleting the entire interface this is the shortcut instead of parsing - # out all peers and removing them one by one. - # - # Peer reconfiguration will always come with a short downtime while the - # WireGuard interface is recreated (see below) - wg.remove() + wg = WireGuardIf(**wireguard) - # Create the new interface if required - if 'deleted' not in wireguard: - wg = WireGuardIf(**wireguard) + if 'deleted' in wireguard: + wg.remove() + else: wg.update(wireguard) domain_resolver_usage = '/run/use-vyos-domain-resolver-interfaces-wireguard-' + wireguard['ifname'] @@ -168,12 +162,12 @@ def apply(wireguard): from vyos.utils.file import write_file text = f'# Automatically generated by interfaces_wireguard.py\nThis file indicates that vyos-domain-resolver service is used by the interfaces_wireguard.\n' - text += "intefaces:\n" + "".join([f" - {peer}\n" for peer in wireguard['peers_need_resolve']]) - Path(domain_resolver_usage).write_text(text) + text += "interfaces:\n" + "".join([f" - {peer}\n" for peer in wireguard['peers_need_resolve']]) write_file(domain_resolver_usage, text) else: - Path(domain_resolver_usage).unlink(missing_ok=True) - if not Path('/run').glob('use-vyos-domain-resolver*'): + if os.path.exists(domain_resolver_usage): + os.unlink(domain_resolver_usage) + if not glob('/run/use-vyos-domain-resolver*'): domain_action = 'stop' call(f'systemctl {domain_action} vyos-domain-resolver.service') diff --git a/src/conf_mode/load-balancing_haproxy.py b/src/conf_mode/load-balancing_haproxy.py index 5fd1beec9..504a90596 100644 --- a/src/conf_mode/load-balancing_haproxy.py +++ b/src/conf_mode/load-balancing_haproxy.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2023-2024 VyOS maintainers and contributors +# Copyright (C) 2023-2025 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -19,6 +19,7 @@ import os from sys import exit from shutil import rmtree +from vyos.defaults import systemd_services from vyos.config import Config from vyos.configverify import verify_pki_certificate from vyos.configverify import verify_pki_ca_certificate @@ -39,7 +40,6 @@ airbag.enable() load_balancing_dir = '/run/haproxy' load_balancing_conf_file = f'{load_balancing_dir}/haproxy.cfg' -systemd_service = 'haproxy.service' systemd_override = '/run/systemd/system/haproxy.service.d/10-override.conf' def get_config(config=None): @@ -65,18 +65,18 @@ def verify(lb): return None if 'backend' not in lb or 'service' not in lb: - raise ConfigError(f'"service" and "backend" must be configured!') + raise ConfigError('Both "service" and "backend" must be configured!') for front, front_config in lb['service'].items(): if 'port' not in front_config: raise ConfigError(f'"{front} service port" must be configured!') # Check if bind address:port are used by another service - tmp_address = front_config.get('address', '0.0.0.0') + tmp_address = front_config.get('address', None) tmp_port = front_config['port'] if check_port_availability(tmp_address, int(tmp_port), 'tcp') is not True and \ not is_listen_port_bind_service(int(tmp_port), 'haproxy'): - raise ConfigError(f'"TCP" port "{tmp_port}" is used by another service') + raise ConfigError(f'TCP port "{tmp_port}" is used by another service') if 'http_compression' in front_config: if front_config['mode'] != 'http': @@ -85,16 +85,19 @@ def verify(lb): raise ConfigError(f'service {front} must have at least one mime-type configured to use' f'http_compression!') + for cert in dict_search('ssl.certificate', front_config) or []: + verify_pki_certificate(lb, cert) + for back, back_config in lb['backend'].items(): if 'http_check' in back_config: http_check = back_config['http_check'] if 'expect' in http_check and 'status' in http_check['expect'] and 'string' in http_check['expect']: - raise ConfigError(f'"expect status" and "expect string" can not be configured together!') + raise ConfigError('"expect status" and "expect string" can not be configured together!') if 'health_check' in back_config: if back_config['mode'] != 'tcp': raise ConfigError(f'backend "{back}" can only be configured with {back_config["health_check"]} ' + - f'health-check whilst in TCP mode!') + 'health-check whilst in TCP mode!') if 'http_check' in back_config: raise ConfigError(f'backend "{back}" cannot be configured with both http-check and health-check!') @@ -112,20 +115,15 @@ def verify(lb): if {'no_verify', 'ca_certificate'} <= set(back_config['ssl']): raise ConfigError(f'backend {back} cannot have both ssl options no-verify and ca-certificate set!') + tmp = dict_search('ssl.ca_certificate', back_config) + if tmp: verify_pki_ca_certificate(lb, tmp) + # Check if http-response-headers are configured in any frontend/backend where mode != http for group in ['service', 'backend']: for config_name, config in lb[group].items(): if 'http_response_headers' in config and config['mode'] != 'http': raise ConfigError(f'{group} {config_name} must be set to http mode to use http_response_headers!') - for front, front_config in lb['service'].items(): - for cert in dict_search('ssl.certificate', front_config) or []: - verify_pki_certificate(lb, cert) - - for back, back_config in lb['backend'].items(): - tmp = dict_search('ssl.ca_certificate', back_config) - if tmp: verify_pki_ca_certificate(lb, tmp) - def generate(lb): if not lb: @@ -193,12 +191,11 @@ def generate(lb): return None def apply(lb): + action = 'stop' + if lb: + action = 'reload-or-restart' call('systemctl daemon-reload') - if not lb: - call(f'systemctl stop {systemd_service}') - else: - call(f'systemctl reload-or-restart {systemd_service}') - + call(f'systemctl {action} {systemd_services["haproxy"]}') return None diff --git a/src/conf_mode/nat.py b/src/conf_mode/nat.py index 504b3e82a..6c88e5cfd 100755 --- a/src/conf_mode/nat.py +++ b/src/conf_mode/nat.py @@ -16,8 +16,8 @@ import os +from glob import glob from sys import exit -from pathlib import Path from vyos.base import Warning from vyos.config import Config @@ -265,9 +265,9 @@ def apply(nat): text = f'# Automatically generated by nat.py\nThis file indicates that vyos-domain-resolver service is used by nat.\n' write_file(domain_resolver_usage, text) elif os.path.exists(domain_resolver_usage): - Path(domain_resolver_usage).unlink(missing_ok=True) + os.unlink(domain_resolver_usage) - if not Path('/run').glob('use-vyos-domain-resolver*'): + if not glob('/run/use-vyos-domain-resolver*'): domain_action = 'stop' call(f'systemctl {domain_action} vyos-domain-resolver.service') diff --git a/src/conf_mode/nat66.py b/src/conf_mode/nat66.py index 95dfae3a5..c65950c9e 100755 --- a/src/conf_mode/nat66.py +++ b/src/conf_mode/nat66.py @@ -92,6 +92,10 @@ def verify(nat): if prefix != None: if not is_ipv6(prefix): raise ConfigError(f'{err_msg} source-prefix not specified') + + if 'destination' in config and 'group' in config['destination']: + if len({'address_group', 'network_group', 'domain_group'} & set(config['destination']['group'])) > 1: + raise ConfigError('Only one address-group, network-group or domain-group can be specified') if dict_search('destination.rule', nat): for rule, config in dict_search('destination.rule', nat).items(): diff --git a/src/conf_mode/pki.py b/src/conf_mode/pki.py index 724f97555..869518dd9 100755 --- a/src/conf_mode/pki.py +++ b/src/conf_mode/pki.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2021-2024 VyOS maintainers and contributors +# Copyright (C) 2021-2025 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -19,6 +19,7 @@ import os from sys import argv from sys import exit +from vyos.base import Message from vyos.config import Config from vyos.config import config_dict_merge from vyos.configdep import set_dependents @@ -27,6 +28,8 @@ from vyos.configdict import node_changed from vyos.configdiff import Diff from vyos.configdiff import get_config_diff from vyos.defaults import directories +from vyos.defaults import internal_ports +from vyos.defaults import systemd_services from vyos.pki import encode_certificate from vyos.pki import is_ca_certificate from vyos.pki import load_certificate @@ -42,9 +45,11 @@ from vyos.utils.dict import dict_search from vyos.utils.dict import dict_search_args from vyos.utils.dict import dict_search_recursive from vyos.utils.file import read_file +from vyos.utils.network import check_port_availability from vyos.utils.process import call from vyos.utils.process import cmd from vyos.utils.process import is_systemd_service_active +from vyos.utils.process import is_systemd_service_running from vyos import ConfigError from vyos import airbag airbag.enable() @@ -128,8 +133,20 @@ def certbot_request(name: str, config: dict, dry_run: bool=True): f'--standalone --agree-tos --no-eff-email --expand --server {config["url"]} '\ f'--email {config["email"]} --key-type rsa --rsa-key-size {config["rsa_key_size"]} '\ f'{domains}' + + listen_address = None if 'listen_address' in config: - tmp += f' --http-01-address {config["listen_address"]}' + listen_address = config['listen_address'] + + # When ACME is used behind a reverse proxy, we always bind to localhost + # whatever the CLI listen-address is configured for. + if ('haproxy' in dict_search('used_by', config) and + is_systemd_service_running(systemd_services['haproxy']) and + not check_port_availability(listen_address, 80)): + tmp += f' --http-01-address 127.0.0.1 --http-01-port {internal_ports["certbot_haproxy"]}' + elif listen_address: + tmp += f' --http-01-address {listen_address}' + # verify() does not need to actually request a cert but only test for plausability if dry_run: tmp += ' --dry-run' @@ -150,14 +167,18 @@ def get_config(config=None): if len(argv) > 1 and argv[1] == 'certbot_renew': pki['certbot_renew'] = {} - changed_keys = ['ca', 'certificate', 'dh', 'key-pair', 'openssh', 'openvpn'] + # Walk through the list of sync_translate mapping and build a list + # which is later used to check if the node was changed in the CLI config + changed_keys = [] + for value in sync_translate.values(): + if value not in changed_keys: + changed_keys.append(value) + # Check for changes to said given keys in the CLI config for key in changed_keys: tmp = node_changed(conf, base + [key], recursive=True, expand_nodes=Diff.DELETE | Diff.ADD) - if 'changed' not in pki: pki.update({'changed':{}}) - pki['changed'].update({key.replace('-', '_') : tmp}) # We only merge on the defaults of there is a configuration at all @@ -219,8 +240,8 @@ def get_config(config=None): continue path = search['path'] - path_str = ' '.join(path + found_path) - #print(f'PKI: Updating config: {path_str} {item_name}') + path_str = ' '.join(path + found_path).replace('_','-') + Message(f'Updating configuration: "{path_str} {item_name}"') if path[0] == 'interfaces': ifname = found_path[0] @@ -230,6 +251,24 @@ def get_config(config=None): if not D.node_changed_presence(path): set_dependents(path[1], conf) + # Check PKI certificates if they are auto-generated by ACME. If they are, + # traverse the current configuration and determine the service where the + # certificate is used by. + # Required to check if we might need to run certbot behing a reverse proxy. + if 'certificate' in pki: + for name, cert_config in pki['certificate'].items(): + if 'acme' not in cert_config: + continue + if not dict_search('system.load_balancing.haproxy', pki): + continue + used_by = [] + for cert_list, _ in dict_search_recursive( + pki['system']['load_balancing']['haproxy'], 'certificate'): + if name in cert_list: + used_by.append('haproxy') + if used_by: + pki['certificate'][name]['acme'].update({'used_by': used_by}) + return pki def is_valid_certificate(raw_data): @@ -321,6 +360,15 @@ def verify(pki): raise ConfigError(f'An email address is required to request '\ f'certificate for "{name}" via ACME!') + listen_address = None + if 'listen_address' in cert_conf['acme']: + listen_address = cert_conf['acme']['listen_address'] + + if 'used_by' not in cert_conf['acme']: + if not check_port_availability(listen_address, 80): + raise ConfigError('Port 80 is already in use and not available '\ + f'to provide ACME challenge for "{name}"!') + if 'certbot_renew' not in pki: # Only run the ACME command if something on this entity changed, # as this is time intensive @@ -374,27 +422,35 @@ def verify(pki): for search in sync_search: for key in search['keys']: changed_key = sync_translate[key] - if changed_key not in pki['changed']: continue - for item_name in pki['changed'][changed_key]: node_present = False if changed_key == 'openvpn': node_present = dict_search_args(pki, 'openvpn', 'shared_secret', item_name) else: node_present = dict_search_args(pki, changed_key, item_name) + # If the node is still present, we can skip the check + # as we are not deleting it + if node_present: + continue - if not node_present: - search_dict = dict_search_args(pki['system'], *search['path']) - - if not search_dict: - continue + search_dict = dict_search_args(pki['system'], *search['path']) + if not search_dict: + continue - for found_name, found_path in dict_search_recursive(search_dict, key): - if found_name == item_name: - path_str = " ".join(search['path'] + found_path) - raise ConfigError(f'PKI object "{item_name}" still in use by "{path_str}"') + for found_name, found_path in dict_search_recursive(search_dict, key): + # Check if the name matches either by string compare, or beeing + # part of a list + if ((isinstance(found_name, str) and found_name == item_name) or + (isinstance(found_name, list) and item_name in found_name)): + # We do not support _ in CLI paths - this is only a convenience + # as we mangle all - to _, now it's time to reverse this! + path_str = ' '.join(search['path'] + found_path).replace('_','-') + object = changed_key.replace('_','-') + tmp = f'Embedded PKI {object} with name "{item_name}" is still '\ + f'in use by CLI path "{path_str}"' + raise ConfigError(tmp) return None @@ -490,7 +546,7 @@ def generate(pki): if not ca_cert_present: tmp = dict_search_args(pki, 'ca', f'{autochain_prefix}{cert}', 'certificate') if not bool(tmp) or tmp != cert_chain_base64: - print(f'Adding/replacing automatically imported CA certificate for "{cert}" ...') + Message(f'Add/replace automatically imported CA certificate for "{cert}"...') add_cli_node(['pki', 'ca', f'{autochain_prefix}{cert}', 'certificate'], value=cert_chain_base64) return None diff --git a/src/conf_mode/policy.py b/src/conf_mode/policy.py index a90e33e81..ec9005890 100755 --- a/src/conf_mode/policy.py +++ b/src/conf_mode/policy.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +import re from sys import exit from vyos.config import Config @@ -24,9 +25,20 @@ from vyos.frrender import get_frrender_dict from vyos.utils.dict import dict_search from vyos.utils.process import is_systemd_service_running from vyos import ConfigError +from vyos.base import Warning from vyos import airbag airbag.enable() +# Sanity checks for large-community-list regex: +# * Require complete 3-tuples, no blank members. Catch missed & doubled colons. +# * Permit appropriate community separators (whitespace, underscore) +# * Permit common regex between tuples while requiring at least one separator +# (eg, "1:1:1_.*_4:4:4", matching "1:1:1 4:4:4" and "1:1:1 2:2:2 4:4:4", +# but not "1:1:13 24:4:4") +# Best practice: stick with basic patterns, mind your wildcards and whitespace. +# Regex that doesn't match this pattern will be allowed with a warning. +large_community_regex_pattern = r'([^: _]+):([^: _]+):([^: _]+)([ _]([^:]+):([^: _]+):([^: _]+))*' + def community_action_compatibility(actions: dict) -> bool: """ Check compatibility of values in community and large community sections @@ -147,6 +159,10 @@ def verify(config_dict): if 'regex' not in rule_config: raise ConfigError(f'A regex {mandatory_error}') + if policy_type == 'large_community_list': + if not re.fullmatch(large_community_regex_pattern, rule_config['regex']): + Warning(f'"policy large-community-list {instance} rule {rule} regex" does not follow expected form and may not match as expected.') + if policy_type in ['prefix_list', 'prefix_list6']: if 'prefix' not in rule_config: raise ConfigError(f'A prefix {mandatory_error}') diff --git a/src/conf_mode/protocols_bgp.py b/src/conf_mode/protocols_bgp.py index a0c853bce..e29f3358a 100755 --- a/src/conf_mode/protocols_bgp.py +++ b/src/conf_mode/protocols_bgp.py @@ -527,11 +527,15 @@ def verify(config_dict): raise ConfigError( 'Please unconfigure import vrf commands before using vpn commands in dependent VRFs!') + # Verify if the route-map exists + if dict_search('route_map.vrf.import', afi_config) is not None: + verify_route_map(afi_config['route_map']['vrf']['import'], bgp) + if (dict_search('route_map.vrf.import', afi_config) is not None or dict_search('import.vrf', afi_config) is not None): # FRR error: please unconfigure vpn to vrf commands before # using import vrf commands - if ('vpn' in afi_config['import'] + if (dict_search('import.vpn', afi_config) is not None or dict_search('export.vpn', afi_config) is not None): raise ConfigError('Please unconfigure VPN to VRF commands before '\ 'using "import vrf" commands!') @@ -541,7 +545,6 @@ def verify(config_dict): raise ConfigError('Please unconfigure route-map VPN to VRF commands before '\ 'using "import vrf" commands!') - # Verify that the export/import route-maps do exist for export_import in ['export', 'import']: tmp = dict_search(f'route_map.vpn.{export_import}', afi_config) diff --git a/src/conf_mode/service_ipoe-server.py b/src/conf_mode/service_ipoe-server.py index a14d4b5b6..083c27523 100755 --- a/src/conf_mode/service_ipoe-server.py +++ b/src/conf_mode/service_ipoe-server.py @@ -88,6 +88,12 @@ def verify(ipoe): 'Can configure username with Lua script only for RADIUS authentication' ) + if dict_search('external_dhcp.dhcp_relay', iface_config): + if not dict_search('external_dhcp.giaddr', iface_config): + raise ConfigError( + f'"external-dhcp dhcp-relay" requires "giaddr" to be set for interface {interface}' + ) + verify_accel_ppp_authentication(ipoe, local_users=False) verify_accel_ppp_ip_pool(ipoe) verify_accel_ppp_name_servers(ipoe) diff --git a/src/conf_mode/service_monitoring_prometheus.py b/src/conf_mode/service_monitoring_prometheus.py index 9a07d8593..f55b09f6c 100755 --- a/src/conf_mode/service_monitoring_prometheus.py +++ b/src/conf_mode/service_monitoring_prometheus.py @@ -48,9 +48,21 @@ def get_config(config=None): if not conf.exists(base): return None - monitoring = conf.get_config_dict( - base, key_mangling=('-', '_'), get_first_key=True, with_recursive_defaults=True - ) + monitoring = {} + exporters = { + 'node_exporter': base + ['node-exporter'], + 'frr_exporter': base + ['frr-exporter'], + 'blackbox_exporter': base + ['blackbox-exporter'], + } + + for exporter_name, exporter_base in exporters.items(): + if conf.exists(exporter_base): + monitoring[exporter_name] = conf.get_config_dict( + exporter_base, + key_mangling=('-', '_'), + get_first_key=True, + with_recursive_defaults=True, + ) tmp = is_node_changed(conf, base + ['node-exporter', 'vrf']) if tmp: diff --git a/src/conf_mode/service_pppoe-server.py b/src/conf_mode/service_pppoe-server.py index ac697c509..f7cb3dcba 100755 --- a/src/conf_mode/service_pppoe-server.py +++ b/src/conf_mode/service_pppoe-server.py @@ -73,7 +73,9 @@ def get_config(config=None): # https://phabricator.accel-ppp.org/T3 conditions = [is_node_changed(conf, base + ['client-ip-pool']), is_node_changed(conf, base + ['client-ipv6-pool']), - is_node_changed(conf, base + ['interface'])] + is_node_changed(conf, base + ['interface']), + is_node_changed(conf, base + ['authentication','radius','dynamic-author']), + is_node_changed(conf, base + ['authentication','mode'])] if any(conditions): pppoe.update({'restart_required': {}}) pppoe['server_type'] = 'pppoe' diff --git a/src/conf_mode/system_option.py b/src/conf_mode/system_option.py index 3d76a1eaa..5acad6599 100755 --- a/src/conf_mode/system_option.py +++ b/src/conf_mode/system_option.py @@ -127,6 +127,9 @@ def generate(options): # occurance is used for having the appropriate options passed to GRUB # when re-configuring options on the CLI. cmdline_options = [] + kernel_opts = options.get('kernel', {}) + k_cpu_opts = kernel_opts.get('cpu', {}) + k_memory_opts = kernel_opts.get('memory', {}) if 'kernel' in options: if 'disable_mitigations' in options['kernel']: cmdline_options.append('mitigations=off') @@ -138,6 +141,48 @@ def generate(options): f'initcall_blacklist=acpi_cpufreq_init amd_pstate={mode}') if 'quiet' in options['kernel']: cmdline_options.append('quiet') + + if 'disable_hpet' in kernel_opts: + cmdline_options.append('hpet=disable') + + if 'disable_mce' in kernel_opts: + cmdline_options.append('mce=off') + + if 'disable_softlockup' in kernel_opts: + cmdline_options.append('nosoftlockup') + + # CPU options + isol_cpus = k_cpu_opts.get('isolate_cpus') + if isol_cpus: + cmdline_options.append(f'isolcpus={isol_cpus}') + + nohz_full = k_cpu_opts.get('nohz_full') + if nohz_full: + cmdline_options.append(f'nohz_full={nohz_full}') + + rcu_nocbs = k_cpu_opts.get('rcu_no_cbs') + if rcu_nocbs: + cmdline_options.append(f'rcu_nocbs={rcu_nocbs}') + + if 'disable_nmi_watchdog' in k_cpu_opts: + cmdline_options.append('nmi_watchdog=0') + + # Memory options + if 'disable_numa_balancing' in k_memory_opts: + cmdline_options.append('numa_balancing=disable') + + default_hp_size = k_memory_opts.get('default_hugepage_size') + if default_hp_size: + cmdline_options.append(f'default_hugepagesz={default_hp_size}') + + hp_sizes = k_memory_opts.get('hugepage_size') + if hp_sizes: + for size, settings in hp_sizes.items(): + cmdline_options.append(f'hugepagesz={size}') + count = settings.get('hugepage_count') + if count: + cmdline_options.append(f'hugepages={count}') + grub_util.update_kernel_cmdline_options(' '.join(cmdline_options)) return None diff --git a/src/conf_mode/vpn_ipsec.py b/src/conf_mode/vpn_ipsec.py index 2754314f7..ac25cd671 100755 --- a/src/conf_mode/vpn_ipsec.py +++ b/src/conf_mode/vpn_ipsec.py @@ -727,7 +727,7 @@ def generate(ipsec): for remote_prefix in remote_prefixes: local_net = ipaddress.ip_network(local_prefix) remote_net = ipaddress.ip_network(remote_prefix) - if local_net.overlaps(remote_net): + if local_net.subnet_of(remote_net): if passthrough is None: passthrough = [] passthrough.append(local_prefix) diff --git a/src/conf_mode/vpn_openconnect.py b/src/conf_mode/vpn_openconnect.py index 42785134f..0346c7819 100755 --- a/src/conf_mode/vpn_openconnect.py +++ b/src/conf_mode/vpn_openconnect.py @@ -93,7 +93,7 @@ def verify(ocserv): "radius" in ocserv["authentication"]["mode"]): raise ConfigError('OpenConnect authentication modes are mutually-exclusive, remove either local or radius from your configuration') if "radius" in ocserv["authentication"]["mode"]: - if not ocserv["authentication"]['radius']['server']: + if 'server' not in ocserv['authentication']['radius']: raise ConfigError('OpenConnect authentication mode radius requires at least one RADIUS server') if "local" in ocserv["authentication"]["mode"]: if not ocserv.get("authentication", {}).get("local_users"): diff --git a/src/etc/opennhrp/opennhrp-script.py b/src/etc/opennhrp/opennhrp-script.py deleted file mode 100755 index f6f6d075c..000000000 --- a/src/etc/opennhrp/opennhrp-script.py +++ /dev/null @@ -1,371 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2021-2023 VyOS maintainers and contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 or later as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -import os -import re -import sys -import vyos.ipsec - -from json import loads -from pathlib import Path - -from vyos.logger import getLogger -from vyos.utils.process import cmd -from vyos.utils.process import process_named_running - -NHRP_CONFIG: str = '/run/opennhrp/opennhrp.conf' - - -def vici_get_ipsec_uniqueid(conn: str, src_nbma: str, - dst_nbma: str) -> list[str]: - """ Find and return IKE SAs by src nbma and dst nbma - - Args: - conn (str): a connection name - src_nbma (str): an IP address of NBMA source - dst_nbma (str): an IP address of NBMA destination - - Returns: - list: a list of IKE connections that match a criteria - """ - if not conn or not src_nbma or not dst_nbma: - logger.error( - f'Incomplete input data for resolving IKE unique ids: ' - f'conn: {conn}, src_nbma: {src_nbma}, dst_nbma: {dst_nbma}') - return [] - - try: - logger.info( - f'Resolving IKE unique ids for: conn: {conn}, ' - f'src_nbma: {src_nbma}, dst_nbma: {dst_nbma}') - list_ikeid: list[str] = [] - list_sa: list = vyos.ipsec.get_vici_sas_by_name(conn, None) - for sa in list_sa: - if sa[conn]['local-host'].decode('ascii') == src_nbma \ - and sa[conn]['remote-host'].decode('ascii') == dst_nbma: - list_ikeid.append(sa[conn]['uniqueid'].decode('ascii')) - return list_ikeid - except Exception as err: - logger.error(f'Unable to find unique ids for IKE: {err}') - return [] - - -def vici_ike_terminate(list_ikeid: list[str]) -> bool: - """Terminating IKE SAs by list of IKE IDs - - Args: - list_ikeid (list[str]): a list of IKE ids to terminate - - Returns: - bool: result of termination action - """ - if not list: - logger.warning('An empty list for termination was provided') - return False - - try: - vyos.ipsec.terminate_vici_ikeid_list(list_ikeid) - return True - except Exception as err: - logger.error(f'Failed to terminate SA for IKE ids {list_ikeid}: {err}') - return False - - -def parse_type_ipsec(interface: str) -> tuple[str, str]: - """Get DMVPN Type and NHRP Profile from the configuration - - Args: - interface (str): a name of interface - - Returns: - tuple[str, str]: `peer_type` and `profile_name` - """ - if not interface: - logger.error('Cannot find peer type - no input provided') - return '', '' - - config_file: str = Path(NHRP_CONFIG).read_text() - regex: str = rf'^interface {interface} #(?P<peer_type>hub|spoke) ?(?P<profile_name>[^\n]*)$' - match = re.search(regex, config_file, re.M) - if match: - return match.groupdict()['peer_type'], match.groupdict()[ - 'profile_name'] - return '', '' - - -def add_peer_route(nbma_src: str, nbma_dst: str, mtu: str) -> None: - """Add a route to a NBMA peer - - Args: - nbma_src (str): a local IP address - nbma_dst (str): a remote IP address - mtu (str): a MTU for a route - """ - logger.info(f'Adding route from {nbma_src} to {nbma_dst} with MTU {mtu}') - # Find routes to a peer - route_get_cmd: str = f'sudo ip --json route get {nbma_dst} from {nbma_src}' - try: - route_info_data = loads(cmd(route_get_cmd)) - except Exception as err: - logger.error(f'Unable to find a route to {nbma_dst}: {err}') - return - - # Check if an output has an expected format - if not isinstance(route_info_data, list): - logger.error( - f'Garbage returned from the "{route_get_cmd}" ' - f'command: {route_info_data}') - return - - # Add static routes to a peer - for route_item in route_info_data: - route_dev = route_item.get('dev') - route_dst = route_item.get('dst') - route_gateway = route_item.get('gateway') - # Prepare a command to add a route - route_add_cmd = 'sudo ip route add' - if route_dst: - route_add_cmd = f'{route_add_cmd} {route_dst}' - if route_gateway: - route_add_cmd = f'{route_add_cmd} via {route_gateway}' - if route_dev: - route_add_cmd = f'{route_add_cmd} dev {route_dev}' - route_add_cmd = f'{route_add_cmd} proto 42 mtu {mtu}' - # Add a route - try: - cmd(route_add_cmd) - except Exception as err: - logger.error( - f'Unable to add a route using command "{route_add_cmd}": ' - f'{err}') - - -def vici_initiate(conn: str, child_sa: str, src_addr: str, - dest_addr: str) -> bool: - """Initiate IKE SA connection with specific peer - - Args: - conn (str): an IKE connection name - child_sa (str): a child SA profile name - src_addr (str): NBMA local address - dest_addr (str): NBMA address of a peer - - Returns: - bool: a result of initiation command - """ - logger.info( - f'Trying to initiate connection. Name: {conn}, child sa: {child_sa}, ' - f'src_addr: {src_addr}, dst_addr: {dest_addr}') - try: - vyos.ipsec.vici_initiate(conn, child_sa, src_addr, dest_addr) - return True - except Exception as err: - logger.error(f'Unable to initiate connection {err}') - return False - - -def vici_terminate(conn: str, src_addr: str, dest_addr: str) -> None: - """Find and terminate IKE SAs by local NBMA and remote NBMA addresses - - Args: - conn (str): IKE connection name - src_addr (str): NBMA local address - dest_addr (str): NBMA address of a peer - """ - logger.info( - f'Terminating IKE connection {conn} between {src_addr} ' - f'and {dest_addr}') - - ikeid_list: list[str] = vici_get_ipsec_uniqueid(conn, src_addr, dest_addr) - - if not ikeid_list: - logger.warning( - f'No active sessions found for IKE profile {conn}, ' - f'local NBMA {src_addr}, remote NBMA {dest_addr}') - else: - try: - vyos.ipsec.terminate_vici_ikeid_list(ikeid_list) - except Exception as err: - logger.error( - f'Failed to terminate SA for IKE ids {ikeid_list}: {err}') - -def iface_up(interface: str) -> None: - """Proceed tunnel interface UP event - - Args: - interface (str): an interface name - """ - if not interface: - logger.warning('No interface name provided for UP event') - - logger.info(f'Turning up interface {interface}') - try: - cmd(f'sudo ip route flush proto 42 dev {interface}') - cmd(f'sudo ip neigh flush dev {interface}') - except Exception as err: - logger.error( - f'Unable to flush route on interface "{interface}": {err}') - - -def peer_up(dmvpn_type: str, conn: str) -> None: - """Proceed NHRP peer UP event - - Args: - dmvpn_type (str): a type of peer - conn (str): an IKE profile name - """ - logger.info(f'Peer UP event for {dmvpn_type} using IKE profile {conn}') - src_nbma = os.getenv('NHRP_SRCNBMA') - dest_nbma = os.getenv('NHRP_DESTNBMA') - dest_mtu = os.getenv('NHRP_DESTMTU') - - if not src_nbma or not dest_nbma: - logger.error( - f'Can not get NHRP NBMA addresses: local {src_nbma}, ' - f'remote {dest_nbma}') - return - - logger.info(f'NBMA addresses: local {src_nbma}, remote {dest_nbma}') - if dest_mtu: - add_peer_route(src_nbma, dest_nbma, dest_mtu) - if conn and dmvpn_type == 'spoke' and process_named_running('charon'): - vici_terminate(conn, src_nbma, dest_nbma) - vici_initiate(conn, 'dmvpn', src_nbma, dest_nbma) - - -def peer_down(dmvpn_type: str, conn: str) -> None: - """Proceed NHRP peer DOWN event - - Args: - dmvpn_type (str): a type of peer - conn (str): an IKE profile name - """ - logger.info(f'Peer DOWN event for {dmvpn_type} using IKE profile {conn}') - - src_nbma = os.getenv('NHRP_SRCNBMA') - dest_nbma = os.getenv('NHRP_DESTNBMA') - - if not src_nbma or not dest_nbma: - logger.error( - f'Can not get NHRP NBMA addresses: local {src_nbma}, ' - f'remote {dest_nbma}') - return - - logger.info(f'NBMA addresses: local {src_nbma}, remote {dest_nbma}') - if conn and dmvpn_type == 'spoke' and process_named_running('charon'): - vici_terminate(conn, src_nbma, dest_nbma) - try: - cmd(f'sudo ip route del {dest_nbma} src {src_nbma} proto 42') - except Exception as err: - logger.error( - f'Unable to del route from {src_nbma} to {dest_nbma}: {err}') - - -def route_up(interface: str) -> None: - """Proceed NHRP route UP event - - Args: - interface (str): an interface name - """ - logger.info(f'Route UP event for interface {interface}') - - dest_addr = os.getenv('NHRP_DESTADDR') - dest_prefix = os.getenv('NHRP_DESTPREFIX') - next_hop = os.getenv('NHRP_NEXTHOP') - - if not dest_addr or not dest_prefix or not next_hop: - logger.error( - f'Can not get route details: dest_addr {dest_addr}, ' - f'dest_prefix {dest_prefix}, next_hop {next_hop}') - return - - logger.info( - f'Route details: dest_addr {dest_addr}, dest_prefix {dest_prefix}, ' - f'next_hop {next_hop}') - - try: - cmd(f'sudo ip route replace {dest_addr}/{dest_prefix} proto 42 \ - via {next_hop} dev {interface}') - cmd('sudo ip route flush cache') - except Exception as err: - logger.error( - f'Unable replace or flush route to {dest_addr}/{dest_prefix} ' - f'via {next_hop} dev {interface}: {err}') - - -def route_down(interface: str) -> None: - """Proceed NHRP route DOWN event - - Args: - interface (str): an interface name - """ - logger.info(f'Route DOWN event for interface {interface}') - - dest_addr = os.getenv('NHRP_DESTADDR') - dest_prefix = os.getenv('NHRP_DESTPREFIX') - - if not dest_addr or not dest_prefix: - logger.error( - f'Can not get route details: dest_addr {dest_addr}, ' - f'dest_prefix {dest_prefix}') - return - - logger.info( - f'Route details: dest_addr {dest_addr}, dest_prefix {dest_prefix}') - try: - cmd(f'sudo ip route del {dest_addr}/{dest_prefix} proto 42') - cmd('sudo ip route flush cache') - except Exception as err: - logger.error( - f'Unable delete or flush route to {dest_addr}/{dest_prefix}: ' - f'{err}') - - -if __name__ == '__main__': - logger = getLogger('opennhrp-script', syslog=True) - logger.debug( - f'Running script with arguments: {sys.argv}, ' - f'environment: {os.environ}') - - action = sys.argv[1] - interface = os.getenv('NHRP_INTERFACE') - - if not interface: - logger.error('Can not get NHRP interface name') - sys.exit(1) - - dmvpn_type, profile_name = parse_type_ipsec(interface) - if not dmvpn_type: - logger.info(f'Interface {interface} is not NHRP tunnel') - sys.exit() - - dmvpn_conn: str = '' - if profile_name: - dmvpn_conn: str = f'dmvpn-{profile_name}-{interface}' - if action == 'interface-up': - iface_up(interface) - elif action == 'peer-register': - pass - elif action == 'peer-up': - peer_up(dmvpn_type, dmvpn_conn) - elif action == 'peer-down': - peer_down(dmvpn_type, dmvpn_conn) - elif action == 'route-up': - route_up(interface) - elif action == 'route-down': - route_down(interface) - - sys.exit() diff --git a/src/etc/systemd/system/frr.service.d/override.conf b/src/etc/systemd/system/frr.service.d/override.conf index 614b4f7ed..a4a73ecd9 100644 --- a/src/etc/systemd/system/frr.service.d/override.conf +++ b/src/etc/systemd/system/frr.service.d/override.conf @@ -3,9 +3,11 @@ After=vyos-router.service [Service] LimitNOFILE=4096 -ExecStartPre=/bin/bash -c 'mkdir -p /run/frr/config; \ +ExecStartPre=/bin/bash -c 'if [ ! -f /run/frr/config/frr.conf ]; then \ + mkdir -p /run/frr/config; \ echo "log syslog" > /run/frr/config/frr.conf; \ echo "log facility local7" >> /run/frr/config/frr.conf; \ chown frr:frr /run/frr/config/frr.conf; \ chmod 664 /run/frr/config/frr.conf; \ - mount --bind /run/frr/config/frr.conf /etc/frr/frr.conf' + mount --bind /run/frr/config/frr.conf /etc/frr/frr.conf; \ +fi;' diff --git a/src/helpers/run-config-migration.py b/src/helpers/run-config-migration.py index e6ce97363..8e0d56150 100755 --- a/src/helpers/run-config-migration.py +++ b/src/helpers/run-config-migration.py @@ -19,6 +19,7 @@ import sys import time from argparse import ArgumentParser from shutil import copyfile +from vyos.utils.file import read_file from vyos.migrate import ConfigMigrate from vyos.migrate import ConfigMigrateError @@ -76,3 +77,9 @@ except ConfigMigrateError as e: if backup is not None and not config_migrate.config_modified: os.unlink(backup) + +# T1771: add knob on Kernel command-line to simulate failed config migrator run +# used to test if the automatic image reboot works. +kernel_cmdline = read_file('/proc/cmdline') +if 'vyos-fail-migration' in kernel_cmdline.split(): + sys.exit(1) diff --git a/src/helpers/set_vyconf_backend.py b/src/helpers/set_vyconf_backend.py new file mode 100755 index 000000000..6747e51c3 --- /dev/null +++ b/src/helpers/set_vyconf_backend.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2025 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +# + +# N.B. only for use within testing framework; explicit invocation will leave +# system in inconsistent state. + +from argparse import ArgumentParser + +from vyos.utils.backend import set_vyconf_backend + + +parser = ArgumentParser() +parser.add_argument('--disable', action='store_true', + help='enable/disable vyconf backend') +parser.add_argument('--no-prompt', action='store_true', + help='confirm without prompt') + +args = parser.parse_args() + +match args.disable: + case False: + set_vyconf_backend(True, no_prompt=args.no_prompt) + case True: + set_vyconf_backend(False, no_prompt=args.no_prompt) diff --git a/src/helpers/teardown-config-session.py b/src/helpers/teardown-config-session.py new file mode 100755 index 000000000..c94876924 --- /dev/null +++ b/src/helpers/teardown-config-session.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2025 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +# + +import sys +import os + +from vyos.vyconf_session import VyconfSession + +if len(sys.argv) < 2: + sys.exit('session pid is required') + +pid = sys.argv[1] + +vc = VyconfSession(pid=pid) +vc.teardown() diff --git a/src/init/vyos-router b/src/init/vyos-router index 8584234b3..5c88c0665 100755 --- a/src/init/vyos-router +++ b/src/init/vyos-router @@ -67,37 +67,50 @@ disabled () { grep -q -w no-vyos-$1 /proc/cmdline } +motd_helper() { + MOTD_DIR="/run/motd.d" + MOTD_FILE="${MOTD_DIR}/99-vyos-update-failed" + + if [[ ! -d ${MOTD_DIR} ]]; then + mkdir -p ${MOTD_DIR} + fi + + echo "" > ${MOTD_FILE} + echo "WARNING: Image update to \"$1\" failed." >> ${MOTD_FILE} + echo "Please check the logs:" >> ${MOTD_FILE} + echo "/usr/lib/live/mount/persistence/boot/$1/rw/var/log" >> ${MOTD_FILE} + echo "Message is cleared on next reboot!" >> ${MOTD_FILE} + echo "" >> ${MOTD_FILE} +} + # Load encrypted config volume mount_encrypted_config() { persist_path=$(/opt/vyatta/sbin/vyos-persistpath) if [ $? == 0 ]; then if [ -e $persist_path/boot ]; then image_name=$(cat /proc/cmdline | sed -e s+^.*vyos-union=/boot/++ | sed -e 's/ .*$//') - if [ -z "$image_name" ]; then - return + return 0 fi if [ ! -f $persist_path/luks/$image_name ]; then - return + return 0 fi vyos_tpm_key=$(python3 -c 'from vyos.tpm import read_tpm_key; print(read_tpm_key().decode())' 2>/dev/null) - if [ $? -ne 0 ]; then echo "ERROR: Failed to fetch encryption key from TPM. Encrypted config volume has not been mounted" echo "Use 'encryption load' to load volume with recovery key" echo "or 'encryption disable' to decrypt volume with recovery key" - return + return 1 fi echo $vyos_tpm_key | tr -d '\r\n' | cryptsetup open $persist_path/luks/$image_name vyos_config --key-file=- - if [ $? -ne 0 ]; then echo "ERROR: Failed to decrypt config volume. Encrypted config volume has not been mounted" echo "Use 'encryption load' to load volume with recovery key" echo "or 'encryption disable' to decrypt volume with recovery key" - return + return 1 fi mount /dev/mapper/vyos_config /config @@ -106,6 +119,7 @@ mount_encrypted_config() { echo "Mounted encrypted config volume" fi fi + return 0 } unmount_encrypted_config() { @@ -160,11 +174,16 @@ migrate_bootfile () if [ -x $vyos_libexec_dir/run-config-migration.py ]; then log_progress_msg migrate sg ${GROUP} -c "$vyos_libexec_dir/run-config-migration.py $BOOTFILE" + STATUS=$? + if [[ "$STATUS" != "0" ]]; then + return 1 + fi # update vyconf copy after migration if [ -d $VYCONF_CONFIG_DIR ] ; then cp -f $BOOTFILE $VYCONF_CONFIG_DIR/config.boot fi fi + return 0 } # configure system-specific settings @@ -187,8 +206,13 @@ load_bootfile () fi if [ -x $vyos_libexec_dir/vyos-boot-config-loader.py ]; then sg ${GROUP} -c "$vyos_libexec_dir/vyos-boot-config-loader.py $BOOTFILE" + STATUS=$? + if [[ "$STATUS" != "0" ]]; then + return 1 + fi fi ) + return 0 } # restore if missing pre-config script @@ -289,10 +313,10 @@ clear_or_override_config_files () keepalived/keepalived.conf cron.d/vyos-crontab \ ipvsadm.rules default/ipvsadm resolv.conf do - if [ -s /etc/$conf ] ; then - empty /etc/$conf - chmod 0644 /etc/$conf - fi + if [ -s /etc/$conf ] ; then + empty /etc/$conf + chmod 0644 /etc/$conf + fi done } @@ -417,7 +441,8 @@ gen_duid () start () { - echo -e "Initializing VyOS router\033[0m" + log_success_msg "Starting VyOS router" + # reset and clean config files security_reset || log_failure_msg "security reset failed" @@ -483,7 +508,7 @@ start () # enable some debugging before loading the configuration if grep -q vyos-debug /proc/cmdline; then - log_action_begin_msg "Enable runtime debugging options" + log_success_msg "Enable runtime debugging options" FRR_DEBUG=$(python3 -c "from vyos.defaults import frr_debug_enable; print(frr_debug_enable)") touch $FRR_DEBUG touch /tmp/vyos.container.debug @@ -510,7 +535,7 @@ start () && chgrp ${GROUP} ${vyatta_configdir} log_action_end_msg $? - mount_encrypted_config + mount_encrypted_config || overall_status=1 # T5239: early read of system hostname as this value is read-only once during # FRR initialisation @@ -526,7 +551,7 @@ start () cleanup_post_commit_hooks - disabled migrate || migrate_bootfile + disabled migrate || migrate_bootfile || overall_status=1 restore_if_missing_preconfig_script @@ -534,27 +559,66 @@ start () run_postupgrade_script - update_interface_config + update_interface_config || overall_status=1 - disabled system_config || system_config + disabled system_config || system_config || overall_status=1 systemctl start vyconfd.service for s in ${subinit[@]} ; do - if ! disabled $s; then - log_progress_msg $s - if ! ${vyatta_sbindir}/${s}.init start - then log_failure_msg - exit 1 + if ! disabled $s; then + log_progress_msg $s + if ! ${vyatta_sbindir}/${s}.init start + then log_failure_msg + exit 1 + fi fi - fi done bind_mount_boot - disabled configure || load_bootfile + disabled configure || load_bootfile || overall_status=1 log_end_msg $? + FIRST_BOOT_FILE="/config/first_boot" + UPDATE_FAILED_BOOT_FILE="/config/update_failed" + AUTOMATIC_REBOOT_TMO=$(${vyos_libexec_dir}/read-saved-value.py --path "system option reboot-on-upgrade-failure") + # Image upgrade failed - get previous image name, re-set it as default image + # and perform an automatic reboot. Automatic reboot timeout can be set via CLI + if [[ -n $AUTOMATIC_REBOOT_TMO ]] && [[ -f ${FIRST_BOOT_FILE} ]] && [[ ${overall_status} -ne 0 ]]; then + previous_image=$(jq -r '.previous_image' ${FIRST_BOOT_FILE}) + + # If the image update failed, we need to inform the image we will revert + # to about this + running_image=$(${vyos_op_scripts_dir}/image_info.py show_images_current --raw | jq -r '.image_running') + echo "{\"failed_image_update\": \"${running_image}\"}" \ + > /usr/lib/live/mount/persistence/boot/${previous_image}/rw/${UPDATE_FAILED_BOOT_FILE} + + ${vyos_op_scripts_dir}/image_manager.py --action set --image-name "${previous_image}" >/dev/null 2>&1 + motd_helper "${running_image}" + + log_daemon_msg "Booting failed, reverting to previous image" + log_progress_msg ${previous_image} + log_end_msg 0 + log_daemon_msg "Automatic reboot in ${AUTOMATIC_REBOOT_TMO} minutes" + sync ; shutdown --reboot --no-wall ${AUTOMATIC_REBOOT_TMO} >/dev/null 2>&1 + log_progress_msg "Use \"reboot cancel\" to cancel" + log_end_msg 0 + fi + # After image upgrade failure and once booted into the previous working + # image, inform the user via MOTD about the failure + if [[ -n $AUTOMATIC_REBOOT_TMO ]] && [[ -f ${UPDATE_FAILED_BOOT_FILE} ]] ; then + failed_image_update=$(jq -r '.failed_image_update' ${UPDATE_FAILED_BOOT_FILE}) + motd_helper "${failed_image_update}" + fi + # Clear marker files used by automatic reboot on image upgrade mechanism + if [[ -f ${FIRST_BOOT_FILE} ]]; then + rm -f ${FIRST_BOOT_FILE} + fi + if [[ -f ${UPDATE_FAILED_BOOT_FILE} ]] ; then + rm -f ${UPDATE_FAILED_BOOT_FILE} + fi + telinit q chmod g-w,o-w / diff --git a/src/migration-scripts/reverse-proxy/2-to-3 b/src/migration-scripts/reverse-proxy/2-to-3 new file mode 100755 index 000000000..ac539618e --- /dev/null +++ b/src/migration-scripts/reverse-proxy/2-to-3 @@ -0,0 +1,66 @@ +# Copyright 2025 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see <http://www.gnu.org/licenses/>. + +# T7429: logging facility "all" unavailable in code + +from vyos.configtree import ConfigTree + +base = ['load-balancing', 'haproxy'] +unsupported_facilities = ['all', 'authpriv', 'mark'] + +def config_migrator(config, config_path: list) -> None: + if not config.exists(config_path): + return + # Remove unsupported backend HAProxy syslog facilities form CLI + # Works for both backend and service CLI nodes + for service_backend in config.list_nodes(config_path): + log_path = config_path + [service_backend, 'logging', 'facility'] + if not config.exists(log_path): + continue + # Remove unsupported syslog facilities form CLI + for facility in config.list_nodes(log_path): + if facility in unsupported_facilities: + config.delete(log_path + [facility]) + continue + # Remove unsupported facility log level form CLI. VyOS will fallback + # to default log level if not set + if config.exists(log_path + [facility, 'level']): + tmp = config.return_value(log_path + [facility, 'level']) + if tmp == 'all': + config.delete(log_path + [facility, 'level']) + +def migrate(config: ConfigTree) -> None: + if not config.exists(base): + # Nothing to do + return + + # Remove unsupported syslog facilities form CLI + global_path = base + ['global-parameters', 'logging', 'facility'] + if config.exists(global_path): + for facility in config.list_nodes(global_path): + if facility in unsupported_facilities: + config.delete(global_path + [facility]) + continue + # Remove unsupported facility log level form CLI. VyOS will fallback + # to default log level if not set + if config.exists(global_path + [facility, 'level']): + tmp = config.return_value(global_path + [facility, 'level']) + if tmp == 'all': + config.delete(global_path + [facility, 'level']) + + # Remove unsupported backend HAProxy syslog facilities from CLI + config_migrator(config, base + ['backend']) + # Remove unsupported service HAProxy syslog facilities from CLI + config_migrator(config, base + ['service']) diff --git a/src/migration-scripts/vrf/1-to-2 b/src/migration-scripts/vrf/1-to-2 index 557a9ec58..89b0f708a 100644 --- a/src/migration-scripts/vrf/1-to-2 +++ b/src/migration-scripts/vrf/1-to-2 @@ -37,7 +37,10 @@ def migrate(config: ConfigTree) -> None: new_static_base = vrf_base + [vrf, 'protocols'] config.set(new_static_base) config.copy(static_base, new_static_base + ['static']) - config.set_tag(new_static_base + ['static', 'route']) + if config.exists(new_static_base + ['static', 'route']): + config.set_tag(new_static_base + ['static', 'route']) + if config.exists(new_static_base + ['static', 'route6']): + config.set_tag(new_static_base + ['static', 'route6']) # Now delete the old configuration config.delete(base) diff --git a/src/migration-scripts/vrf/2-to-3 b/src/migration-scripts/vrf/2-to-3 index acacffb41..5f396e7ed 100644 --- a/src/migration-scripts/vrf/2-to-3 +++ b/src/migration-scripts/vrf/2-to-3 @@ -76,7 +76,8 @@ def migrate(config: ConfigTree) -> None: # Get a list of all currently used VRFs and tables vrfs_current = {} for vrf in config.list_nodes(base): - vrfs_current[vrf] = int(config.return_value(base + [vrf, 'table'])) + if config.exists(base + [vrf, 'table']): + vrfs_current[vrf] = int(config.return_value(base + [vrf, 'table'])) # Check VRF names and table numbers name_regex = re.compile(r'^\d.*$') diff --git a/src/op_mode/firewall.py b/src/op_mode/firewall.py index ac47e3273..f3309ee34 100755 --- a/src/op_mode/firewall.py +++ b/src/op_mode/firewall.py @@ -18,6 +18,7 @@ import argparse import ipaddress import json import re +from signal import signal, SIGPIPE, SIG_DFL import tabulate import textwrap @@ -25,6 +26,9 @@ from vyos.config import Config from vyos.utils.process import cmd from vyos.utils.dict import dict_search_args +signal(SIGPIPE, SIG_DFL) + + def get_config_node(conf, node=None, family=None, hook=None, priority=None): if node == 'nat': if family == 'ipv6': @@ -648,12 +652,14 @@ def show_firewall_group(name=None): references = find_references(group_type, remote_name) row = [remote_name, textwrap.fill(remote_conf.get('description') or '', 50), group_type, '\n'.join(references) or 'N/D'] members = get_nftables_remote_group_members("ipv4", 'vyos_filter', f'R_{remote_name}') + members6 = get_nftables_remote_group_members("ipv6", 'vyos_filter', f'R6_{remote_name}') if 'url' in remote_conf: # display only the url if no members are found for both views - if not members: + if not members and not members6: if args.detail: - header_tail = ['Remote URL'] + header_tail = ['IPv6 Members', 'Remote URL'] + row.append('N/D') row.append('N/D') row.append(remote_conf['url']) else: @@ -662,8 +668,15 @@ def show_firewall_group(name=None): else: # display all table elements in detail view if args.detail: - header_tail = ['Remote URL'] - row += [' '.join(members)] + header_tail = ['IPv6 Members', 'Remote URL'] + if members: + row.append(' '.join(members)) + else: + row.append('N/D') + if members6: + row.append(' '.join(members6)) + else: + row.append('N/D') row.append(remote_conf['url']) rows.append(row) else: diff --git a/src/op_mode/image_info.py b/src/op_mode/image_info.py index 56aefcd6e..0ec930543 100755 --- a/src/op_mode/image_info.py +++ b/src/op_mode/image_info.py @@ -72,6 +72,14 @@ def _format_show_images_details( return tabulated +def show_images_current(raw: bool) -> Union[image.BootDetails, str]: + + images_summary = show_images_summary(raw=True) + if raw: + return {'image_running' : images_summary['image_running']} + else: + return images_summary['image_running'] + def show_images_summary(raw: bool) -> Union[image.BootDetails, str]: images_available: list[str] = grub.version_list() diff --git a/src/op_mode/image_installer.py b/src/op_mode/image_installer.py index 150ae57d3..27371a18f 100755 --- a/src/op_mode/image_installer.py +++ b/src/op_mode/image_installer.py @@ -27,6 +27,7 @@ from os import readlink from os import getpid from os import getppid from json import loads +from json import dumps from typing import Union from urllib.parse import urlparse from passlib.hosts import linux_context @@ -54,6 +55,7 @@ from vyos.utils.dict import dict_search from vyos.utils.io import ask_input, ask_yes_no, select_entry from vyos.utils.file import chmod_2775 from vyos.utils.file import read_file +from vyos.utils.file import write_file from vyos.utils.process import cmd, run, rc_cmd from vyos.version import get_version_data @@ -922,8 +924,7 @@ def install_image() -> None: for disk_target in l: disk.partition_mount(disk_target.partition['efi'], f'{DIR_DST_ROOT}/boot/efi') grub.install(disk_target.name, f'{DIR_DST_ROOT}/boot/', - f'{DIR_DST_ROOT}/boot/efi', - id=f'VyOS (RAID disk {l.index(disk_target) + 1})') + f'{DIR_DST_ROOT}/boot/efi') disk.partition_umount(disk_target.partition['efi']) else: print('Installing GRUB to the drive') @@ -1041,6 +1042,12 @@ def add_image(image_path: str, vrf: str = None, username: str = '', chmod_2775(target_config_dir) copytree('/opt/vyatta/etc/config/', target_config_dir, symlinks=True, copy_function=copy_preserve_owner, dirs_exist_ok=True) + + # Record information from which image we upgraded to the new one. + # This can be used for a future automatic rollback into the old image. + tmp = {'previous_image' : image.get_running_image()} + write_file(f'{target_config_dir}/first_boot', dumps(tmp)) + else: Path(target_config_dir).mkdir(parents=True) chown(target_config_dir, group='vyattacfg') diff --git a/src/op_mode/interfaces.py b/src/op_mode/interfaces.py index e7afc4caa..c97f3b129 100755 --- a/src/op_mode/interfaces.py +++ b/src/op_mode/interfaces.py @@ -29,6 +29,7 @@ from vyos.ifconfig import Section from vyos.ifconfig import Interface from vyos.ifconfig import VRRP from vyos.utils.process import cmd +from vyos.utils.network import interface_exists from vyos.utils.process import rc_cmd from vyos.utils.process import call @@ -84,6 +85,14 @@ def filtered_interfaces(ifnames: typing.Union[str, list], yield interface +def detailed_output(dataset, headers): + for data in dataset: + adjusted_rule = data + [""] * (len(headers) - len(data)) # account for different header length, like default-action + transformed_rule = [[header, adjusted_rule[i]] for i, header in enumerate(headers) if i < len(adjusted_rule)] # create key-pair list from headers and rules lists; wrap at 100 char + + print(tabulate(transformed_rule, tablefmt="presto")) + print() + def _split_text(text, used=0): """ take a string and attempt to split it to fit with the width of the screen @@ -296,6 +305,114 @@ def _get_counter_data(ifname: typing.Optional[str], return ret +def _get_kernel_data(raw, ifname = None, detail = False): + if ifname: + # Check if the interface exists + if not interface_exists(ifname): + raise vyos.opmode.IncorrectValue(f"{ifname} does not exist!") + int_name = f'dev {ifname}' + else: + int_name = '' + + kernel_interface = json.loads(cmd(f'ip -j -d -s address show {int_name}')) + + # Return early if raw + if raw: + return kernel_interface, None + + # Format the kernel data + kernel_interface_out = _format_kernel_data(kernel_interface, detail) + + return kernel_interface, kernel_interface_out + +def _format_kernel_data(data, detail): + output_list = [] + tmpInfo = {} + + # Sort interfaces by name + for interface in sorted(data, key=lambda x: x.get('ifname', '')): + if interface.get('linkinfo', {}).get('info_kind') == 'vrf': + continue + + # Get the device model; ex. Intel Corporation Ethernet Controller I225-V + dev_model = interface.get('parentdev', '') + if 'parentdev' in interface: + parentdev = interface['parentdev'] + if re.match(r'^[0-9a-fA-F]{4}:', parentdev): + dev_model = cmd(f'lspci -nn -s {parentdev}').split(']:')[1].strip() + + # Get the IP addresses on interface + ip_list = [] + has_global = False + + for ip in interface['addr_info']: + if ip.get('scope') in ('global', 'host'): + has_global = True + local = ip.get('local', '-') + prefixlen = ip.get('prefixlen', '') + ip_list.append(f"{local}/{prefixlen}") + + + # If no global IP address, add '-'; indicates no IP address on interface + if not has_global: + ip_list.append('-') + + sl_status = ('A' if not 'UP' in interface['flags'] else 'u') + '/' + ('D' if interface['operstate'] == 'DOWN' else 'u') + + # Generate temporary dict to hold data + tmpInfo['ifname'] = interface.get('ifname', '') + tmpInfo['ip'] = ip_list + tmpInfo['mac'] = interface.get('address', '') + tmpInfo['mtu'] = interface.get('mtu', '') + tmpInfo['vrf'] = interface.get('master', 'default') + tmpInfo['status'] = sl_status + tmpInfo['description'] = interface.get('ifalias', '') + tmpInfo['device'] = dev_model + tmpInfo['alternate_names'] = interface.get('altnames', '') + tmpInfo['minimum_mtu'] = interface.get('min_mtu', '') + tmpInfo['maximum_mtu'] = interface.get('max_mtu', '') + rx_stats = interface.get('stats64', {}).get('rx') + tx_stats = interface.get('stats64', {}).get('tx') + tmpInfo['rx_packets'] = rx_stats.get('packets', "") + tmpInfo['rx_bytes'] = rx_stats.get('bytes', "") + tmpInfo['rx_errors'] = rx_stats.get('errors', "") + tmpInfo['rx_dropped'] = rx_stats.get('dropped', "") + tmpInfo['rx_over_errors'] = rx_stats.get('over_errors', '') + tmpInfo['multicast'] = rx_stats.get('multicast', "") + tmpInfo['tx_packets'] = tx_stats.get('packets', "") + tmpInfo['tx_bytes'] = tx_stats.get('bytes', "") + tmpInfo['tx_errors'] = tx_stats.get('errors', "") + tmpInfo['tx_dropped'] = tx_stats.get('dropped', "") + tmpInfo['tx_carrier_errors'] = tx_stats.get('carrier_errors', "") + tmpInfo['tx_collisions'] = tx_stats.get('collisions', "") + + # Generate output list; detail adds more fields + output_list.append([tmpInfo['ifname'], + '\n'.join(tmpInfo['ip']), + tmpInfo['mac'], + tmpInfo['vrf'], + tmpInfo['mtu'], + tmpInfo['status'], + tmpInfo['description'], + *([tmpInfo['device']] if detail else []), + *(['\n'.join(tmpInfo['alternate_names'])] if detail else []), + *([tmpInfo['minimum_mtu']] if detail else []), + *([tmpInfo['maximum_mtu']] if detail else []), + *([tmpInfo['rx_packets']] if detail else []), + *([tmpInfo['rx_bytes']] if detail else []), + *([tmpInfo['rx_errors']] if detail else []), + *([tmpInfo['rx_dropped']] if detail else []), + *([tmpInfo['rx_over_errors']] if detail else []), + *([tmpInfo['multicast']] if detail else []), + *([tmpInfo['tx_packets']] if detail else []), + *([tmpInfo['tx_bytes']] if detail else []), + *([tmpInfo['tx_errors']] if detail else []), + *([tmpInfo['tx_dropped']] if detail else []), + *([tmpInfo['tx_carrier_errors']] if detail else []), + *([tmpInfo['tx_collisions']] if detail else [])]) + + return output_list + @catch_broken_pipe def _format_show_data(data: list): unhandled = [] @@ -445,6 +562,27 @@ def _format_show_counters(data: list): print (output) return output +def show_kernel(raw: bool, intf_name: typing.Optional[str], detail: bool): + raw_data, data = _get_kernel_data(raw, intf_name, detail) + + # Return early if raw + if raw: + return raw_data + + # Normal headers; show interfaces kernel + headers = ['Interface', 'IP Address', 'MAC', 'VRF', 'MTU', 'S/L', 'Description'] + + # Detail headers; show interfaces kernel detail + detail_header = ['Interface', 'IP Address', 'MAC', 'VRF', 'MTU', 'S/L', 'Description', + 'Device', 'Alternate Names','Minimum MTU', 'Maximum MTU', 'RX_Packets', + 'RX_Bytes', 'RX_Errors', 'RX_Dropped', 'Receive Overrun Errors', 'Received Multicast', + 'TX_Packets', 'TX_Bytes', 'TX_Errors', 'TX_Dropped', 'Transmit Carrier Errors', + 'Transmit Collisions'] + + if detail: + detailed_output(data, detail_header) + else: + print(tabulate(data, headers)) def _show_raw(data: list, intf_name: str): if intf_name is not None and len(data) <= 1: diff --git a/src/services/vyos-domain-resolver b/src/services/vyos-domain-resolver index 4419fc4a7..fb18724af 100755 --- a/src/services/vyos-domain-resolver +++ b/src/services/vyos-domain-resolver @@ -28,7 +28,7 @@ from vyos.utils.commit import commit_in_progress from vyos.utils.dict import dict_search_args from vyos.utils.kernel import WIREGUARD_REKEY_AFTER_TIME from vyos.utils.file import makedir, chmod_775, write_file, read_file -from vyos.utils.network import is_valid_ipv4_address_or_range +from vyos.utils.network import is_valid_ipv4_address_or_range, is_valid_ipv6_address_or_range from vyos.utils.process import cmd from vyos.utils.process import run from vyos.xml_ref import get_defaults @@ -143,10 +143,11 @@ def update_remote_group(config): for set_name, remote_config in remote_groups.items(): if 'url' not in remote_config: continue - nft_set_name = f'R_{set_name}' + nft_ip_set_name = f'R_{set_name}' + nft_ip6_set_name = f'R6_{set_name}' # Create list file if necessary - list_file = os.path.join(firewall_config_dir, f"{nft_set_name}.txt") + list_file = os.path.join(firewall_config_dir, f"{nft_ip_set_name}.txt") if not os.path.exists(list_file): write_file(list_file, '', user="root", group="vyattacfg", mode=0o644) @@ -159,16 +160,32 @@ def update_remote_group(config): # Read list file ip_list = [] + ip6_list = [] + invalid_list = [] for line in read_file(list_file).splitlines(): line_first_word = line.strip().partition(' ')[0] if is_valid_ipv4_address_or_range(line_first_word): ip_list.append(line_first_word) + elif is_valid_ipv6_address_or_range(line_first_word): + ip6_list.append(line_first_word) + else: + if line_first_word[0].isalnum(): + invalid_list.append(line_first_word) - # Load tables + # Load ip tables for table in ipv4_tables: - if (table, nft_set_name) in valid_sets: - conf_lines += nft_output(table, nft_set_name, ip_list) + if (table, nft_ip_set_name) in valid_sets: + conf_lines += nft_output(table, nft_ip_set_name, ip_list) + + # Load ip6 tables + for table in ipv6_tables: + if (table, nft_ip6_set_name) in valid_sets: + conf_lines += nft_output(table, nft_ip6_set_name, ip6_list) + + invalid_str = ", ".join(invalid_list) + if invalid_str: + logger.info(f'Invalid address for set {set_name}: {invalid_str}') count += 1 diff --git a/src/systemd/opennhrp.service b/src/systemd/opennhrp.service deleted file mode 100644 index c9a44de29..000000000 --- a/src/systemd/opennhrp.service +++ /dev/null @@ -1,13 +0,0 @@ -[Unit] -Description=OpenNHRP -After=vyos-router.service -ConditionPathExists=/run/opennhrp/opennhrp.conf -StartLimitIntervalSec=0 - -[Service] -Type=forking -ExecStart=/usr/sbin/opennhrp -d -v -a /run/opennhrp.socket -c /run/opennhrp/opennhrp.conf -s /etc/opennhrp/opennhrp-script.py -p /run/opennhrp/opennhrp.pid -ExecReload=/usr/bin/kill -HUP $MAINPID -PIDFile=/run/opennhrp/opennhrp.pid -Restart=on-failure -RestartSec=20 diff --git a/src/systemd/vyconfd.service b/src/systemd/vyconfd.service index ab2280263..d23ca2202 100644 --- a/src/systemd/vyconfd.service +++ b/src/systemd/vyconfd.service @@ -8,7 +8,7 @@ DefaultDependencies=no After=systemd-remount-fs.service [Service] -ExecStart=/usr/libexec/vyos/vyconf/vyconfd --log-file /var/run/log/vyconfd.log +ExecStart=/usr/libexec/vyos/vyconf/vyconfd --log-file /var/run/log/vyconfd.log --legacy-config-path Type=exec SyslogIdentifier=vyconfd SyslogFacility=daemon diff --git a/src/tests/test_template.py b/src/tests/test_template.py index 6377f6da5..7cae867a0 100644 --- a/src/tests/test_template.py +++ b/src/tests/test_template.py @@ -190,3 +190,12 @@ class TestVyOSTemplate(TestCase): for group_name, group_config in data['ike_group'].items(): ciphers = vyos.template.get_esp_ike_cipher(group_config) self.assertIn(IKEv2_DEFAULT, ','.join(ciphers)) + + def test_get_default_port(self): + from vyos.defaults import internal_ports + + with self.assertRaises(RuntimeError): + vyos.template.get_default_port('UNKNOWN') + + self.assertEqual(vyos.template.get_default_port('certbot_haproxy'), + internal_ports['certbot_haproxy']) diff --git a/src/tests/test_utils_network.py b/src/tests/test_utils_network.py index d68dec16f..92fde447d 100644 --- a/src/tests/test_utils_network.py +++ b/src/tests/test_utils_network.py @@ -1,4 +1,4 @@ -# Copyright (C) 2020-2024 VyOS maintainers and contributors +# Copyright (C) 2020-2025 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -43,3 +43,12 @@ class TestVyOSUtilsNetwork(TestCase): self.assertFalse(vyos.utils.network.is_loopback_addr('::2')) self.assertFalse(vyos.utils.network.is_loopback_addr('192.0.2.1')) + + def test_check_port_availability(self): + self.assertTrue(vyos.utils.network.check_port_availability('::1', 8080)) + self.assertTrue(vyos.utils.network.check_port_availability('127.0.0.1', 8080)) + self.assertTrue(vyos.utils.network.check_port_availability(None, 8080, protocol='udp')) + # We do not have 192.0.2.1 configured on this system + self.assertFalse(vyos.utils.network.check_port_availability('192.0.2.1', 443)) + # We do not have 2001:db8::1 configured on this system + self.assertFalse(vyos.utils.network.check_port_availability('2001:db8::1', 80, protocol='udp')) diff --git a/src/validators/bgp-large-community-list b/src/validators/bgp-large-community-list index 9ba5b27eb..75276630c 100755 --- a/src/validators/bgp-large-community-list +++ b/src/validators/bgp-large-community-list @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2021-2023 VyOS maintainers and contributors +# Copyright (C) 2021-2025 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -17,18 +17,27 @@ import re import sys -pattern = '(.*):(.*):(.*)' -allowedChars = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '.', '+', '*', '?', '^', '$', '(', ')', '[', ']', '{', '}', '|', '\\', ':', '-' } +allowedChars = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '.', '+', '*', '?', '^', '$', '(', ')', '[', ']', '{', '}', '|', '\\', ':', '-', '_', ' ' } if __name__ == '__main__': if len(sys.argv) != 2: sys.exit(1) - value = sys.argv[1].split(':') - if not len(value) == 3: + value = sys.argv[1] + + # Require at least one well-formed large-community tuple in the pattern. + tmp = value.split(':') + if len(tmp) < 3: + sys.exit(1) + + # Simple guard against invalid community & 1003.2 pattern chars + if not set(value).issubset(allowedChars): sys.exit(1) - if not (re.match(pattern, sys.argv[1]) and set(sys.argv[1]).issubset(allowedChars)): + # Don't feed FRR badly formed regex + try: + re.compile(value) + except re.error: sys.exit(1) sys.exit(0) diff --git a/src/validators/cpu b/src/validators/cpu new file mode 100755 index 000000000..959a49248 --- /dev/null +++ b/src/validators/cpu @@ -0,0 +1,43 @@ +#!/usr/bin/python3 + +import re +import sys + +MAX_CPU = 511 + + +def validate_isolcpus(value): + pattern = re.compile(r'^(\d{1,3}(-\d{1,3})?)(,(\d{1,3}(-\d{1,3})?))*$') + if not pattern.fullmatch(value): + return False + + flat_list = [] + for part in value.split(','): + if '-' in part: + start, end = map(int, part.split('-')) + if start > end or start < 0 or end > MAX_CPU: + return False + flat_list.extend(range(start, end + 1)) + else: + num = int(part) + if num < 0 or num > MAX_CPU: + return False + flat_list.append(num) + + for i in range(1, len(flat_list)): + if flat_list[i] <= flat_list[i - 1]: + return False + + return True + + +if __name__ == "__main__": + if len(sys.argv) != 2: + print("Usage: python3 cpu.py <cpu_list>") + sys.exit(1) + + input_value = sys.argv[1] + if validate_isolcpus(input_value): + sys.exit(0) + else: + sys.exit(1) |