diff options
391 files changed, 3680 insertions, 2287 deletions
diff --git a/.gitignore b/.gitignore index fe92f5b9d..e766a2c27 100644 --- a/.gitignore +++ b/.gitignore @@ -133,6 +133,10 @@ debian/*.substvars *.vpwhist *.vtg +# VS Code +.vscode/* +!.vscode/settings.json + # VIM *.swp diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000..caa87ba4a --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,18 @@ +{ + "files.trimTrailingWhitespace": true, + "editor.tabSize": 4, + "editor.insertSpaces": true, + "files.insertFinalNewline": true, + "files.eol": "\n", + # https://code.visualstudio.com/docs/languages/identifiers#_known-language-identifiers + "files.associations": { + "*.xml.i": "xml", + "*.xml.in": "xml", + "*.j2": "jinja", + }, + "editor.indentSize": "tabSize", + "[jinja]": { + "editor.tabSize": 4, + "editor.wordBasedSuggestions": false + } +} diff --git a/data/templates/frr/bgpd.frr.j2 b/data/templates/frr/bgpd.frr.j2 index 3e101820c..ddfba2306 100644 --- a/data/templates/frr/bgpd.frr.j2 +++ b/data/templates/frr/bgpd.frr.j2 @@ -565,3 +565,14 @@ bgp route-reflector allow-outbound-policy timers bgp {{ timers.keepalive }} {{ timers.holdtime }} {% endif %} exit +! +{% if interface is vyos_defined %} +{% for iface, iface_config in interface.items() %} +interface {{ iface }} +{% if iface_config.mpls.forwarding is vyos_defined %} + mpls bgp forwarding +{% endif %} +exit +! +{% endfor %} +{% endif %} diff --git a/data/templates/frr/ospf6d.frr.j2 b/data/templates/frr/ospf6d.frr.j2 index 84394ed1a..b0b5663dd 100644 --- a/data/templates/frr/ospf6d.frr.j2 +++ b/data/templates/frr/ospf6d.frr.j2 @@ -80,6 +80,27 @@ router ospf6 {{ 'vrf ' ~ vrf if vrf is vyos_defined }} {% if distance.ospfv3 is vyos_defined %} distance ospf6 {{ 'intra-area ' ~ distance.ospfv3.intra_area if distance.ospfv3.intra_area is vyos_defined }} {{ 'inter-area ' ~ distance.ospfv3.inter_area if distance.ospfv3.inter_area is vyos_defined }} {{ 'external ' ~ distance.ospfv3.external if distance.ospfv3.external is vyos_defined }} {% endif %} +{% if graceful_restart is vyos_defined %} +{% if graceful_restart.grace_period is vyos_defined %} + graceful-restart grace-period {{ graceful_restart.grace_period }} +{% endif %} +{% if graceful_restart.helper.enable.router_id is vyos_defined %} +{% for router_id in graceful_restart.helper.enable.router_id %} + graceful-restart helper enable {{ router_id }} +{% endfor %} +{% elif graceful_restart.helper.enable is vyos_defined %} + graceful-restart helper enable +{% endif %} +{% if graceful_restart.helper.planned_only is vyos_defined %} + graceful-restart helper planned-only +{% endif %} +{% if graceful_restart.helper.lsa_check_disable is vyos_defined %} + graceful-restart helper lsa-check-disable +{% endif %} +{% if graceful_restart.helper.supported_grace_time is vyos_defined %} + graceful-restart helper supported-grace-time {{ graceful_restart.helper.supported_grace_time }} +{% endif %} +{% endif %} {% if log_adjacency_changes is vyos_defined %} log-adjacency-changes {{ "detail" if log_adjacency_changes.detail is vyos_defined }} {% endif %} diff --git a/data/templates/frr/ospfd.frr.j2 b/data/templates/frr/ospfd.frr.j2 index 1ee8d8752..040628e82 100644 --- a/data/templates/frr/ospfd.frr.j2 +++ b/data/templates/frr/ospfd.frr.j2 @@ -133,6 +133,9 @@ router ospf {{ 'vrf ' ~ vrf if vrf is vyos_defined }} {% if auto_cost.reference_bandwidth is vyos_defined %} auto-cost reference-bandwidth {{ auto_cost.reference_bandwidth }} {% endif %} +{% if capability.opaque is vyos_defined %} + capability opaque +{% endif %} {% if default_information.originate is vyos_defined %} default-information originate {{ 'always' if default_information.originate.always is vyos_defined }} {{ 'metric ' + default_information.originate.metric if default_information.originate.metric is vyos_defined }} {{ 'metric-type ' + default_information.originate.metric_type if default_information.originate.metric_type is vyos_defined }} {{ 'route-map ' + default_information.originate.route_map if default_information.originate.route_map is vyos_defined }} {% endif %} @@ -153,6 +156,27 @@ router ospf {{ 'vrf ' ~ vrf if vrf is vyos_defined }} {% if distance.ospf is vyos_defined %} distance ospf {{ 'intra-area ' + distance.ospf.intra_area if distance.ospf.intra_area is vyos_defined }} {{ 'inter-area ' + distance.ospf.inter_area if distance.ospf.inter_area is vyos_defined }} {{ 'external ' + distance.ospf.external if distance.ospf.external is vyos_defined }} {% endif %} +{% if graceful_restart is vyos_defined %} +{% if graceful_restart.grace_period is vyos_defined %} + graceful-restart grace-period {{ graceful_restart.grace_period }} +{% endif %} +{% if graceful_restart.helper.enable.router_id is vyos_defined %} +{% for router_id in graceful_restart.helper.enable.router_id %} + graceful-restart helper enable {{ router_id }} +{% endfor %} +{% elif graceful_restart.helper.enable is vyos_defined %} + graceful-restart helper enable +{% endif %} +{% if graceful_restart.helper.planned_only is vyos_defined %} + graceful-restart helper planned-only +{% endif %} +{% if graceful_restart.helper.no_strict_lsa_checking is vyos_defined %} + no graceful-restart helper strict-lsa-checking +{% endif %} +{% if graceful_restart.helper.supported_grace_time is vyos_defined %} + graceful-restart helper supported-grace-time {{ graceful_restart.helper.supported_grace_time }} +{% endif %} +{% endif %} {% if log_adjacency_changes is vyos_defined %} log-adjacency-changes {{ "detail" if log_adjacency_changes.detail is vyos_defined }} {% endif %} diff --git a/data/templates/high-availability/keepalived.conf.j2 b/data/templates/high-availability/keepalived.conf.j2 index bcd92358f..d54f575b5 100644 --- a/data/templates/high-availability/keepalived.conf.j2 +++ b/data/templates/high-availability/keepalived.conf.j2 @@ -25,9 +25,9 @@ global_defs { {% if vrrp.global_parameters.garp.master_repeat is vyos_defined %} vrrp_garp_master_repeat {{ vrrp.global_parameters.garp.master_repeat }} {% endif %} -{% if vrrp.global_parameters.version is vyos_defined %} +{% endif %} +{% if vrrp.global_parameters.version is vyos_defined %} vrrp_version {{ vrrp.global_parameters.version }} -{% endif %} {% endif %} notify_fifo /run/keepalived/keepalived_notify_fifo notify_fifo_script /usr/libexec/vyos/system/keepalived-fifo.py @@ -173,7 +173,7 @@ vrrp_sync_group {{ name }} { {% for vserver, vserver_config in virtual_server.items() %} # Vserver {{ vserver }} {% if vserver_config.port is vyos_defined %} -virtual_server {{ vserver }} {{ vserver_config.port }} { +virtual_server {{ vserver_config.address }} {{ vserver_config.port }} { {% else %} virtual_server fwmark {{ vserver_config.fwmark }} { {% endif %} diff --git a/data/templates/lldp/vyos.conf.j2 b/data/templates/lldp/vyos.conf.j2 index ec84231d8..dfa422ab8 100644 --- a/data/templates/lldp/vyos.conf.j2 +++ b/data/templates/lldp/vyos.conf.j2 @@ -4,7 +4,7 @@ configure system platform VyOS configure system description "VyOS {{ version }}" {% if interface is vyos_defined %} {% set tmp = [] %} -{% for iface, iface_options in interface.items() if not iface_options.disable %} +{% for iface, iface_options in interface.items() if iface_options.disable is not vyos_defined %} {% if iface == 'all' %} {% set iface = '*' %} {% endif %} diff --git a/data/templates/openvpn/server.conf.j2 b/data/templates/openvpn/server.conf.j2 index 6332ed9c2..525605240 100644 --- a/data/templates/openvpn/server.conf.j2 +++ b/data/templates/openvpn/server.conf.j2 @@ -48,6 +48,10 @@ push "redirect-gateway def1" {% if use_lzo_compression is vyos_defined %} compress lzo {% endif %} +{% if enable_dco is not vyos_defined %} +disable-dco +{% endif %} + {% if mode is vyos_defined('client') %} # diff --git a/data/templates/rsyslog/logrotate.j2 b/data/templates/rsyslog/logrotate.j2 index 89d1a8a50..cc535c48f 100644 --- a/data/templates/rsyslog/logrotate.j2 +++ b/data/templates/rsyslog/logrotate.j2 @@ -1,4 +1,15 @@ ### Autogenerated by system-syslog.py ### +/var/log/messages { + missingok + notifempty + create + rotate 5 + size=256k + postrotate + invoke-rc.d rsyslog rotate > /dev/null + endscript +} + {% if file is vyos_defined %} {% for file_name, file_options in file.items() %} /var/log/user/{{ file_name }} { diff --git a/data/templates/rsyslog/rsyslog.conf.j2 b/data/templates/rsyslog/rsyslog.conf.j2 index 5352fc367..dff904129 100644 --- a/data/templates/rsyslog/rsyslog.conf.j2 +++ b/data/templates/rsyslog/rsyslog.conf.j2 @@ -23,11 +23,13 @@ $outchannel global,/var/log/messages,262144,/usr/sbin/logrotate {{ logrotate }} {% if file is vyos_defined %} # File based configuration section {% for file_name, file_options in file.items() %} -$outchannel {{ file_name }},/var/log/user/{{ file_name }},{{ file_options.archive.size }},/usr/sbin/logrotate {{ logrotate }} {% set tmp = [] %} -{% for facility, facility_options in file_options.facility.items() %} -{% set _ = tmp.append(facility.replace('all', '*') + '.' + facility_options.level) %} -{% endfor %} +$outchannel {{ file_name }},/var/log/user/{{ file_name }},{{ file_options.archive.size }},/usr/sbin/logrotate {{ logrotate }} +{% if file_options.facility is vyos_defined %} +{% for facility, facility_options in file_options.facility.items() %} +{% set _ = tmp.append(facility.replace('all', '*') + '.' + facility_options.level) %} +{% endfor %} +{% endif %} {{ tmp | join(';') }} :omfile:${{ file }} {% endfor %} {% endif %} @@ -45,9 +47,11 @@ $outchannel {{ file_name }},/var/log/user/{{ file_name }},{{ file_options.archiv # Remote logging {% for host_name, host_options in host.items() %} {% set tmp = [] %} -{% for facility, facility_options in host_options.facility.items() %} -{% set _ = tmp.append(facility.replace('all', '*') + '.' + facility_options.level) %} -{% endfor %} +{% if host_options.facility is vyos_defined %} +{% for facility, facility_options in host_options.facility.items() %} +{% set _ = tmp.append(facility.replace('all', '*') + '.' + facility_options.level) %} +{% endfor %} +{% endif %} {% if host_options.protocol is vyos_defined('tcp') %} {% if host_options.format.octet_counted is vyos_defined %} {{ tmp | join(';') }} @@(o){{ host_name | bracketize_ipv6 }}:{{ host_options.port }};RSYSLOG_SyslogProtocol23Format @@ -63,9 +67,12 @@ $outchannel {{ file_name }},/var/log/user/{{ file_name }},{{ file_options.archiv {% if user is defined and user is not none %} # Log to user terminal {% for username, user_options in user.items() %} -{% for facility, facility_options in user_options.facility.items() %} -{% set _ = tmp.append(facility.replace('all', '*') + '.' + facility_options.level) %} -{% endfor %} +{% set tmp = [] %} +{% if user_options.facility is vyos_defined %} +{% for facility, facility_options in user_options.facility.items() %} +{% set _ = tmp.append(facility.replace('all', '*') + '.' + facility_options.level) %} +{% endfor %} +{% endif %} {{ tmp | join(';') }} :omusrmsg:{{ username }} {% endfor %} {% endif %} diff --git a/data/templates/vpp/sysctl.conf.j2 b/data/templates/vpp/sysctl.conf.j2 deleted file mode 100644 index 2207e2e38..000000000 --- a/data/templates/vpp/sysctl.conf.j2 +++ /dev/null @@ -1,15 +0,0 @@ -# Number of 2MB hugepages desired -vm.nr_hugepages=1024 - -# Must be greater than or equal to (2 * vm.nr_hugepages). -vm.max_map_count=3096 - -# All groups allowed to access hugepages -vm.hugetlb_shm_group=0 - -# Shared Memory Max must be greater or equal to the total size of hugepages. -# For 2MB pages, TotalHugepageSize = vm.nr_hugepages * 2 * 1024 * 1024 -# If the existing kernel.shmmax setting (cat /proc/sys/kernel/shmmax) -# is greater than the calculated TotalHugepageSize then set this parameter -# to current shmmax value. -kernel.shmmax=2147483648 diff --git a/debian/control b/debian/control index 40920cadc..8e9aaa702 100644 --- a/debian/control +++ b/debian/control @@ -72,6 +72,7 @@ Depends: iperf, iperf3, iproute2 (>= 6.0.0), + iptables, iputils-arping, isc-dhcp-client, isc-dhcp-relay, @@ -92,6 +93,7 @@ Depends: libstrongswan-standard-plugins (>=5.9), libvppinfra, libvyosconfig0, + linux-cpupower, lldpd, lm-sensors, lsscsi, diff --git a/debian/rules b/debian/rules index e613f1e0a..9ada2bf87 100755 --- a/debian/rules +++ b/debian/rules @@ -51,6 +51,10 @@ override_dh_auto_install: mkdir -p $(DIR)/$(VYOS_LIBEXEC_DIR)/op_mode cp -r src/op_mode/* $(DIR)/$(VYOS_LIBEXEC_DIR)/op_mode + # Install op mode scripts + mkdir -p $(DIR)/$(VYOS_LIBEXEC_DIR)/init + cp -r src/init/* $(DIR)/$(VYOS_LIBEXEC_DIR)/init + # Install validators mkdir -p $(DIR)/$(VYOS_LIBEXEC_DIR)/validators cp -r src/validators/* $(DIR)/$(VYOS_LIBEXEC_DIR)/validators @@ -123,3 +127,7 @@ override_dh_auto_install: # Install udev script mkdir -p $(DIR)/usr/lib/udev cp src/helpers/vyos_net_name $(DIR)/usr/lib/udev + +override_dh_installsystemd: + dh_installsystemd -pvyos-1x --name vyos-router vyos-router.service + dh_installsystemd -pvyos-1x --name vyos vyos.target diff --git a/debian/vyos-1x.install b/debian/vyos-1x.install index 07cdf8c74..9e43669be 100644 --- a/debian/vyos-1x.install +++ b/debian/vyos-1x.install @@ -30,6 +30,7 @@ usr/bin/vyos-hostsd-client usr/lib usr/libexec/vyos/completion usr/libexec/vyos/conf_mode +usr/libexec/vyos/init usr/libexec/vyos/op_mode usr/libexec/vyos/services usr/libexec/vyos/system diff --git a/debian/vyos-1x.postinst b/debian/vyos-1x.postinst index 2958afd0a..93e7ced9b 100644 --- a/debian/vyos-1x.postinst +++ b/debian/vyos-1x.postinst @@ -186,5 +186,8 @@ systemctl enable vyos-config-cloud-init.service # # sudo mv /opt/vyatta/share/vyatta-cfg/vpp /opt/vyatta/share/vyatta-cfg/templates/vpp if [ -d /opt/vyatta/share/vyatta-cfg/templates/vpp ]; then + if [ -d /opt/vyatta/share/vyatta-cfg/vpp ]; then + rm -rf /opt/vyatta/share/vyatta-cfg/vpp + fi mv /opt/vyatta/share/vyatta-cfg/templates/vpp /opt/vyatta/share/vyatta-cfg/vpp fi diff --git a/interface-definitions/dhcp-relay.xml.in b/interface-definitions/dhcp-relay.xml.in index 2a2597dd5..42715c9bb 100644 --- a/interface-definitions/dhcp-relay.xml.in +++ b/interface-definitions/dhcp-relay.xml.in @@ -9,6 +9,7 @@ <priority>910</priority> </properties> <children> + #include <include/generic-disable-node.xml.i> #include <include/generic-interface-multi-broadcast.xml.i> <leafNode name="listen-interface"> <properties> diff --git a/interface-definitions/dhcpv6-relay.xml.in b/interface-definitions/dhcpv6-relay.xml.in index 947adef75..a80317609 100644 --- a/interface-definitions/dhcpv6-relay.xml.in +++ b/interface-definitions/dhcpv6-relay.xml.in @@ -9,6 +9,7 @@ <priority>900</priority> </properties> <children> + #include <include/generic-disable-node.xml.i> <tagNode name="listen-interface"> <properties> <help>Interface for DHCPv6 Relay Agent to listen for requests</help> diff --git a/interface-definitions/high-availability.xml.in b/interface-definitions/high-availability.xml.in index d1bbcc365..4f55916fa 100644 --- a/interface-definitions/high-availability.xml.in +++ b/interface-definitions/high-availability.xml.in @@ -336,9 +336,10 @@ </node> <tagNode name="virtual-server"> <properties> - <help>Load-balancing virtual server address</help> + <help>Load-balancing virtual server alias</help> </properties> <children> + #include <include/address-ipv4-ipv6-single.xml.i> <leafNode name="algorithm"> <properties> <help>Schedule algorithm (default - least-connection)</help> diff --git a/interface-definitions/include/address-ipv4-ipv6-single.xml.i b/interface-definitions/include/address-ipv4-ipv6-single.xml.i new file mode 100644 index 000000000..dc3d6fc1b --- /dev/null +++ b/interface-definitions/include/address-ipv4-ipv6-single.xml.i @@ -0,0 +1,18 @@ +<!-- include start from interface/address-ipv4-ipv6.xml.i --> +<leafNode name="address"> + <properties> + <help>IP address</help> + <valueHelp> + <format>ipv4</format> + <description>IPv4 address</description> + </valueHelp> + <valueHelp> + <format>ipv6</format> + <description>IPv6 address</description> + </valueHelp> + <constraint> + <validator name="ip-address"/> + </constraint> + </properties> +</leafNode> +<!-- include end --> diff --git a/interface-definitions/include/bgp/afi-vpn-label.xml.i b/interface-definitions/include/bgp/afi-vpn-label.xml.i new file mode 100644 index 000000000..6c7e73d9b --- /dev/null +++ b/interface-definitions/include/bgp/afi-vpn-label.xml.i @@ -0,0 +1,14 @@ +<!-- include start from bgp/afi-vpn-label.xml.i --> +<leafNode name="label"> + <properties> + <help>MPLS label value assigned to route</help> + <valueHelp> + <format>u32:0-1048575</format> + <description>MPLS label value</description> + </valueHelp> + <constraint> + <validator name="numeric" argument="--range 0-1048575"/> + </constraint> + </properties> +</leafNode> +<!-- include end --> diff --git a/interface-definitions/include/bgp/protocol-common-config.xml.i b/interface-definitions/include/bgp/protocol-common-config.xml.i index d69fd7dab..12024ed8b 100644 --- a/interface-definitions/include/bgp/protocol-common-config.xml.i +++ b/interface-definitions/include/bgp/protocol-common-config.xml.i @@ -355,15 +355,7 @@ <help>Apply local policy routing to interface</help> </properties> <children> - <leafNode name="interface"> - <properties> - <help>Interface</help> - <completionHelp> - <script>${vyos_completion_dir}/list_interfaces</script> - </completionHelp> - <multi/> - </properties> - </leafNode> + #include <include/generic-interface-multi.xml.i> </children> </node> </children> @@ -386,18 +378,7 @@ </properties> <children> #include <include/bgp/route-distinguisher.xml.i> - <leafNode name="label"> - <properties> - <help>MPLS label value assigned to route</help> - <valueHelp> - <format>u32:0-1048575</format> - <description>MPLS label value</description> - </valueHelp> - <constraint> - <validator name="numeric" argument="--range 0-1048575"/> - </constraint> - </properties> - </leafNode> + #include <include/bgp/afi-vpn-label.xml.i> </children> </tagNode> </children> @@ -766,18 +747,7 @@ </properties> <children> #include <include/bgp/route-distinguisher.xml.i> - <leafNode name="label"> - <properties> - <help>MPLS label value assigned to route</help> - <valueHelp> - <format>u32:0-1048575</format> - <description>MPLS label value</description> - </valueHelp> - <constraint> - <validator name="numeric" argument="--range 0-1048575"/> - </constraint> - </properties> - </leafNode> + #include <include/bgp/afi-vpn-label.xml.i> </children> </tagNode> </children> @@ -840,12 +810,7 @@ <help>Specify handling for BUM packets</help> </properties> <children> - <leafNode name="disable"> - <properties> - <help>Do not flood any BUM packets</help> - <valueless/> - </properties> - </leafNode> + #include <include/generic-disable-node.xml.i> <leafNode name="head-end-replication"> <properties> <help>Flood BUM packets using head-end replication</help> @@ -873,6 +838,36 @@ </node> </children> </node> +<tagNode name="interface"> + <properties> + <help>Configure interface related parameters, e.g. MPLS</help> + <completionHelp> + <script>${vyos_completion_dir}/list_interfaces</script> + </completionHelp> + <valueHelp> + <format>txt</format> + <description>Interface name</description> + </valueHelp> + <constraint> + #include <include/constraint/interface-name.xml.i> + </constraint> + </properties> + <children> + <node name="mpls"> + <properties> + <help>MPLS options</help> + </properties> + <children> + <leafNode name="forwarding"> + <properties> + <help>Enable MPLS forwarding for eBGP directly connected peers</help> + <valueless/> + </properties> + </leafNode> + </children> + </node> + </children> +</tagNode> <node name="listen"> <properties> <help>Listen for and accept BGP dynamic neighbors from range</help> @@ -1545,7 +1540,9 @@ </properties> <children> #include <include/bgp/neighbor-afi-ipv4-unicast.xml.i> + #include <include/bgp/neighbor-afi-ipv4-vpn.xml.i> #include <include/bgp/neighbor-afi-ipv6-unicast.xml.i> + #include <include/bgp/neighbor-afi-ipv6-vpn.xml.i> #include <include/bgp/neighbor-afi-l2vpn-evpn.xml.i> </children> </node> diff --git a/interface-definitions/include/ospf/graceful-restart.xml.i b/interface-definitions/include/ospf/graceful-restart.xml.i new file mode 100644 index 000000000..37d9a7f13 --- /dev/null +++ b/interface-definitions/include/ospf/graceful-restart.xml.i @@ -0,0 +1,67 @@ +<!-- include start from ospf/graceful-restart.xml.i --> +<node name="graceful-restart"> + <properties> + <help>Graceful Restart</help> + </properties> + <children> + <leafNode name="grace-period"> + <properties> + <help>Maximum length of the grace period</help> + <valueHelp> + <format>u32:1-1800</format> + <description>Maximum length of the grace period in seconds</description> + </valueHelp> + <constraint> + <validator name="numeric" argument="--range 5-1800"/> + </constraint> + </properties> + <defaultValue>120</defaultValue> + </leafNode> + <node name="helper"> + <properties> + <help>OSPF graceful-restart helpers</help> + </properties> + <children> + <node name="enable"> + <properties> + <help>Enable helper support</help> + </properties> + <children> + <leafNode name="router-id"> + <properties> + <help>Advertising Router-ID</help> + <valueHelp> + <format>ipv4</format> + <description>Router-ID in IP address format</description> + </valueHelp> + <constraint> + <validator name="ipv4-address"/> + </constraint> + <multi/> + </properties> + </leafNode> + </children> + </node> + <leafNode name="planned-only"> + <properties> + <help>Supported only planned restart</help> + <valueless/> + </properties> + </leafNode> + <leafNode name="supported-grace-time"> + <properties> + <help>Supported grace timer</help> + <valueHelp> + <format>u32:10-1800</format> + <description>Grace interval in seconds</description> + </valueHelp> + <constraint> + <validator name="numeric" argument="--range 10-1800"/> + </constraint> + </properties> + </leafNode> + </children> + </node> + </children> +</node> +<!-- include end --> diff --git a/interface-definitions/include/ospf/protocol-common-config.xml.i b/interface-definitions/include/ospf/protocol-common-config.xml.i index 3492b873f..c4778e126 100644 --- a/interface-definitions/include/ospf/protocol-common-config.xml.i +++ b/interface-definitions/include/ospf/protocol-common-config.xml.i @@ -326,6 +326,19 @@ </children> </tagNode> #include <include/ospf/auto-cost.xml.i> +<node name="capability"> + <properties> + <help>Enable specific OSPF features</help> + </properties> + <children> + <leafNode name="opaque"> + <properties> + <help>Opaque LSA</help> + <valueless/> + </properties> + </leafNode> + </children> +</node> #include <include/ospf/default-information.xml.i> <leafNode name="default-metric"> <properties> @@ -339,6 +352,21 @@ </constraint> </properties> </leafNode> +#include <include/ospf/graceful-restart.xml.i> +<node name="graceful-restart"> + <children> + <node name="helper"> + <children> + <leafNode name="no-strict-lsa-checking"> + <properties> + <help>Disable strict LSA check</help> + <valueless/> + </properties> + </leafNode> + </children> + </node> + </children> +</node> <leafNode name="maximum-paths"> <properties> <help>Maximum multiple paths (ECMP)</help> @@ -928,4 +956,4 @@ </node> </children> </node> -<!-- include end -->
\ No newline at end of file +<!-- include end --> diff --git a/interface-definitions/include/ospfv3/protocol-common-config.xml.i b/interface-definitions/include/ospfv3/protocol-common-config.xml.i index a7de50638..4c3ca68e1 100644 --- a/interface-definitions/include/ospfv3/protocol-common-config.xml.i +++ b/interface-definitions/include/ospfv3/protocol-common-config.xml.i @@ -107,6 +107,21 @@ </node> </children> </node> +#include <include/ospf/graceful-restart.xml.i> +<node name="graceful-restart"> + <children> + <node name="helper"> + <children> + <leafNode name="lsa-check-disable"> + <properties> + <help>Disable strict LSA check</help> + <valueless/> + </properties> + </leafNode> + </children> + </node> + </children> +</node> <tagNode name="interface"> <properties> <help>Enable routing on an IPv6 interface</help> diff --git a/interface-definitions/include/version/vrrp-version.xml.i b/interface-definitions/include/version/vrrp-version.xml.i index 626dd6cbc..1514b19ab 100644 --- a/interface-definitions/include/version/vrrp-version.xml.i +++ b/interface-definitions/include/version/vrrp-version.xml.i @@ -1,3 +1,3 @@ <!-- include start from include/version/vrrp-version.xml.i --> -<syntaxVersion component='vrrp' version='3'></syntaxVersion> +<syntaxVersion component='vrrp' version='4'></syntaxVersion> <!-- include end --> diff --git a/interface-definitions/interfaces-openvpn.xml.in b/interface-definitions/interfaces-openvpn.xml.in index 4e061c3e6..ca6d80f8b 100644 --- a/interface-definitions/interfaces-openvpn.xml.in +++ b/interface-definitions/interfaces-openvpn.xml.in @@ -793,6 +793,12 @@ <valueless/> </properties> </leafNode> + <leafNode name="enable-dco"> + <properties> + <help>Use to enable OpenVPN data channel offload on this TUN interface</help> + <valueless/> + </properties> + </leafNode> #include <include/interface/redirect.xml.i> #include <include/interface/vrf.xml.i> </children> diff --git a/interface-definitions/nat.xml.in b/interface-definitions/nat.xml.in index 501ff05d3..a06ceefb6 100644 --- a/interface-definitions/nat.xml.in +++ b/interface-definitions/nat.xml.in @@ -44,6 +44,14 @@ </leafNode> #include <include/nat-translation-port.xml.i> #include <include/nat-translation-options.xml.i> + <node name="redirect"> + <properties> + <help>Redirect to local host</help> + </properties> + <children> + #include <include/nat-translation-port.xml.i> + </children> + </node> </children> </node> </children> diff --git a/interface-definitions/protocols-mpls.xml.in b/interface-definitions/protocols-mpls.xml.in index 43ca659e9..831601fc6 100644 --- a/interface-definitions/protocols-mpls.xml.in +++ b/interface-definitions/protocols-mpls.xml.in @@ -6,7 +6,7 @@ <node name="mpls" owner="${vyos_conf_scripts_dir}/protocols_mpls.py"> <properties> <help>Multiprotocol Label Switching (MPLS)</help> - <priority>400</priority> + <priority>490</priority> </properties> <children> <node name="ldp"> diff --git a/interface-definitions/service-config-sync.xml.in b/interface-definitions/service-config-sync.xml.in new file mode 100644 index 000000000..e804e17f7 --- /dev/null +++ b/interface-definitions/service-config-sync.xml.in @@ -0,0 +1,104 @@ +<?xml version="1.0" encoding="UTF-8"?> +<interfaceDefinition> + <node name="service"> + <children> + <node name="config-sync" owner="${vyos_conf_scripts_dir}/service_config_sync.py"> + <properties> + <help>Configuration synchronization</help> + </properties> + <children> + <node name="secondary"> + <properties> + <help>Secondary server parameters</help> + </properties> + <children> + <leafNode name="address"> + <properties> + <help>IP address</help> + <valueHelp> + <format>ipv4</format> + <description>IPv4 address to match</description> + </valueHelp> + <valueHelp> + <format>ipv6</format> + <description>IPv6 address to match</description> + </valueHelp> + <valueHelp> + <format>hostname</format> + <description>FQDN address to match</description> + </valueHelp> + <constraint> + <validator name="ipv4-address"/> + <validator name="ipv6-address"/> + <validator name="fqdn"/> + </constraint> + </properties> + </leafNode> + <leafNode name="timeout"> + <properties> + <help>Connection API timeout</help> + <valueHelp> + <format>u32:1-300</format> + <description>Connection API timeout</description> + </valueHelp> + <constraint> + <validator name="numeric" argument="--range 1-300"/> + </constraint> + </properties> + <defaultValue>60</defaultValue> + </leafNode> + <leafNode name="key"> + <properties> + <help>HTTP API key</help> + </properties> + </leafNode> + </children> + </node> + <leafNode name="mode"> + <properties> + <help>Synchronization mode</help> + <completionHelp> + <list>load set</list> + </completionHelp> + <valueHelp> + <format>load</format> + <description>Load and replace configuration section</description> + </valueHelp> + <valueHelp> + <format>set</format> + <description>Set configuration section</description> + </valueHelp> + <constraint> + <regex>(load|set)</regex> + </constraint> + </properties> + </leafNode> + <leafNode name="section"> + <properties> + <help>Section for synchronization</help> + <completionHelp> + <list>nat nat66 firewall</list> + </completionHelp> + <valueHelp> + <format>nat</format> + <description>NAT</description> + </valueHelp> + <valueHelp> + <format>nat66</format> + <description>NAT66</description> + </valueHelp> + <valueHelp> + <format>firewall</format> + <description>firewall</description> + </valueHelp> + <constraint> + <regex>(nat|nat66|firewall)</regex> + </constraint> + <multi/> + </properties> + </leafNode> + </children> + </node> + </children> + </node> +</interfaceDefinition> diff --git a/op-mode-definitions/container.xml.in b/op-mode-definitions/container.xml.in index ada9a4d59..f581d39fa 100644 --- a/op-mode-definitions/container.xml.in +++ b/op-mode-definitions/container.xml.in @@ -83,7 +83,7 @@ <children> <tagNode name="container"> <properties> - <help>Monitor last lines of container logs</help> + <help>Monitor last lines of container log</help> <completionHelp> <path>container name</path> </completionHelp> diff --git a/op-mode-definitions/dns-dynamic.xml.in b/op-mode-definitions/dns-dynamic.xml.in index 4f0399964..8f32f63f9 100644 --- a/op-mode-definitions/dns-dynamic.xml.in +++ b/op-mode-definitions/dns-dynamic.xml.in @@ -6,7 +6,7 @@ <children> <node name="dns"> <properties> - <help>Monitor last lines of Domain Name System (DNS) related services</help> + <help>Monitor last lines of Domain Name System related services</help> </properties> <children> <node name="dynamic"> @@ -27,7 +27,7 @@ <children> <node name="dns"> <properties> - <help>Show log for Domain Name System (DNS) related services</help> + <help>Show log for Domain Name System related services</help> </properties> <children> <node name="dynamic"> @@ -42,7 +42,7 @@ </node> <node name="dns"> <properties> - <help>Show Domain Name System (DNS) related information</help> + <help>Show Domain Name System related information</help> </properties> <children> <node name="dynamic"> @@ -66,7 +66,7 @@ <children> <node name="dns"> <properties> - <help>Restart specific Domain Name System (DNS) related service</help> + <help>Restart specific Domain Name System related service</help> </properties> <children> <node name="dynamic"> @@ -86,7 +86,7 @@ <children> <node name="dns"> <properties> - <help>Update Domain Name System (DNS) related information</help> + <help>Update Domain Name System related information</help> </properties> <children> <node name="dynamic"> diff --git a/op-mode-definitions/include/ospf-common.xml.i b/op-mode-definitions/include/ospf/common.xml.i index 979ffb07e..c8341bd3e 100644 --- a/op-mode-definitions/include/ospf-common.xml.i +++ b/op-mode-definitions/include/ospf/common.xml.i @@ -502,6 +502,7 @@ </tagNode> </children> </node> +#include <include/ospf/graceful-restart.xml.i> <node name="interface"> <properties> <help>Show IPv4 OSPF interface information</help> @@ -556,4 +557,3 @@ </children> </node> <!-- included end --> - diff --git a/op-mode-definitions/include/ospf/graceful-restart.xml.i b/op-mode-definitions/include/ospf/graceful-restart.xml.i new file mode 100644 index 000000000..736d8f951 --- /dev/null +++ b/op-mode-definitions/include/ospf/graceful-restart.xml.i @@ -0,0 +1,13 @@ +<node name="graceful-restart"> + <properties> + <help>Show IPv4 OSPF Graceful Restart</help> + </properties> + <children> + <leafNode name="helper"> + <properties> + <help>OSPF Graceful Restart helper details</help> + </properties> + <command>${vyos_op_scripts_dir}/vtysh_wrapper.sh $@</command> + </leafNode> + </children> + </node> diff --git a/op-mode-definitions/monitor-log.xml.in b/op-mode-definitions/monitor-log.xml.in index c577c5a39..8f3f73478 100644 --- a/op-mode-definitions/monitor-log.xml.in +++ b/op-mode-definitions/monitor-log.xml.in @@ -16,7 +16,7 @@ </node> <node name="ids"> <properties> - <help>Monitor log for Intrusion Detection System</help> + <help>Monitor Intrusion Detection System log</help> </properties> <children> <leafNode name="ddos-protection"> @@ -29,18 +29,18 @@ </node> <node name="dhcp"> <properties> - <help>Monitor last lines of Dynamic Host Control Protocol (DHCP)</help> + <help>Monitor last lines of Dynamic Host Control Protocol log</help> </properties> <children> <node name="server"> <properties> - <help>Monitor last lines of DHCP server</help> + <help>Monitor last lines of DHCP server log</help> </properties> <command>journalctl --no-hostname --follow --boot --unit isc-dhcp-server.service</command> </node> <node name="client"> <properties> - <help>Monitor last lines of DHCP client</help> + <help>Monitor last lines of DHCP client log</help> </properties> <command>journalctl --no-hostname --follow --boot --unit "dhclient@*.service"</command> <children> @@ -59,18 +59,18 @@ </node> <node name="dhcpv6"> <properties> - <help>Monitor last lines of Dynamic Host Control Protocol IPv6 (DHCPv6)</help> + <help>Monitor last lines of Dynamic Host Control Protocol IPv6 log</help> </properties> <children> <node name="server"> <properties> - <help>Monitor last lines of DHCPv6 server</help> + <help>Monitor last lines of DHCPv6 server log</help> </properties> <command>journalctl --no-hostname --follow --boot --unit isc-dhcp-server6.service</command> </node> <node name="client"> <properties> - <help>Monitor last lines of DHCPv6 client</help> + <help>Monitor last lines of DHCPv6 client log</help> </properties> <command>journalctl --no-hostname --follow --boot --unit "dhcp6c@*.service"</command> <children> @@ -95,7 +95,7 @@ </leafNode> <leafNode name="ipoe-server"> <properties> - <help>Monitor last lines of IPoE server log</help> + <help>Monitor last lines of IP over Ethernet server log</help> </properties> <command>journalctl --no-hostname --boot --follow --unit accel-ppp@ipoe.service</command> </leafNode> @@ -107,13 +107,13 @@ </leafNode> <leafNode name="nhrp"> <properties> - <help>Monitor last lines of Next Hop Resolution Protocol (NHRP) log</help> + <help>Monitor last lines of Next Hop Resolution Protocol log</help> </properties> <command>journalctl --no-hostname --boot --follow --unit opennhrp.service</command> </leafNode> <leafNode name="ntp"> <properties> - <help>Monitor last lines of Network Time Protocol (NTP) log</help> + <help>Monitor last lines of Network Time Protocol log</help> </properties> <command>journalctl --no-hostname --boot --follow --unit chrony.service</command> </leafNode> @@ -142,7 +142,7 @@ </leafNode> <node name="protocol"> <properties> - <help>Monitor log for Routing Protocol</help> + <help>Monitor routing protocol logs</help> </properties> <children> <leafNode name="ospf"> @@ -232,25 +232,25 @@ </node> <leafNode name="router-advert"> <properties> - <help>Monitor last lines of Router Advertisement Daemon (radvd)</help> + <help>Monitor last lines of Router Advertisement Daemon log</help> </properties> <command>journalctl --no-hostname --boot --follow --unit radvd.service</command> </leafNode> <leafNode name="snmp"> <properties> - <help>Monitor last lines of Simple Network Monitoring Protocol (SNMP)</help> + <help>Monitor last lines of Simple Network Monitoring Protocol log</help> </properties> <command>journalctl --no-hostname --boot --follow --unit snmpd.service</command> </leafNode> <leafNode name="ssh"> <properties> - <help>Monitor last lines of Secure Shell (SSH)</help> + <help>Monitor last lines of Secure Shell log</help> </properties> <command>journalctl --no-hostname --boot --follow --unit ssh.service</command> </leafNode> <node name="sstpc"> <properties> - <help>Monitor last lines of SSTP client log</help> + <help>Monitor last lines of Secure Socket Tunneling Protocol log</help> </properties> <command>journalctl --no-hostname --boot --follow --unit "ppp@sstpc*.service"</command> <children> @@ -267,7 +267,7 @@ </node> <node name="vpn"> <properties> - <help>Monitor Virtual Private Network (VPN) services</help> + <help>Monitor Virtual Private Network services</help> </properties> <children> <leafNode name="all"> @@ -278,36 +278,54 @@ </leafNode> <leafNode name="ipsec"> <properties> - <help>Monitor last lines of IPsec</help> + <help>Monitor last lines of IPsec log</help> </properties> <command>journalctl --no-hostname --boot --follow --unit strongswan.service</command> </leafNode> <leafNode name="l2tp"> <properties> - <help>Monitor last lines of L2TP</help> + <help>Monitor last lines of L2TP log</help> </properties> <command>journalctl --no-hostname --boot --follow --unit accel-ppp@l2tp.service</command> </leafNode> <leafNode name="openconnect"> <properties> - <help>Monitor last lines of OpenConnect</help> + <help>Monitor last lines of OpenConnect log</help> </properties> <command>journalctl --no-hostname --boot --follow --unit ocserv.service</command> </leafNode> <leafNode name="pptp"> <properties> - <help>Monitor last lines of PPTP</help> + <help>Monitor last lines of PPTP log</help> </properties> <command>journalctl --no-hostname --boot --follow --unit accel-ppp@pptp.service</command> </leafNode> <leafNode name="sstp"> <properties> - <help>Monitor last lines of SSTP</help> + <help>Monitor last lines of SSTP log</help> </properties> <command>journalctl --no-hostname --boot --follow --unit accel-ppp@sstp.service</command> </leafNode> </children> </node> + <leafNode name="vpp"> + <properties> + <help>Monitor last lines of Vector Packet Processor log</help> + </properties> + <command>journalctl --no-hostname --boot --follow --unit vpp.service</command> + </leafNode> + <leafNode name="vrrp"> + <properties> + <help>Monitor last lines of Virtual Router Redundancy Protocol log</help> + </properties> + <command>journalctl --no-hostname --boot --follow --unit keepalived.service</command> + </leafNode> + <leafNode name="webproxy"> + <properties> + <help>Monitor last lines of Webproxy log</help> + </properties> + <command>journalctl --no-hostname --boot --follow --unit squid.service</command> + </leafNode> </children> </node> </children> diff --git a/op-mode-definitions/pki.xml.in b/op-mode-definitions/pki.xml.in index 346febec0..c5abf86cd 100644 --- a/op-mode-definitions/pki.xml.in +++ b/op-mode-definitions/pki.xml.in @@ -505,6 +505,14 @@ </completionHelp> </properties> <command>sudo ${vyos_op_scripts_dir}/pki.py --action show --ca "$4"</command> + <children> + <leafNode name="pem"> + <properties> + <help>Show x509 CA certificate in PEM format</help> + </properties> + <command>sudo ${vyos_op_scripts_dir}/pki.py --action show --ca "$4" --pem</command> + </leafNode> + </children> </tagNode> <leafNode name="certificate"> <properties> @@ -520,6 +528,14 @@ </completionHelp> </properties> <command>sudo ${vyos_op_scripts_dir}/pki.py --action show --certificate "$4"</command> + <children> + <leafNode name="pem"> + <properties> + <help>Show x509 certificate in PEM format</help> + </properties> + <command>sudo ${vyos_op_scripts_dir}/pki.py --action show --certificate "$4" --pem</command> + </leafNode> + </children> </tagNode> <leafNode name="crl"> <properties> @@ -527,6 +543,23 @@ </properties> <command>sudo ${vyos_op_scripts_dir}/pki.py --action show --crl "all"</command> </leafNode> + <tagNode name="crl"> + <properties> + <help>Show x509 certificate revocation lists by CA name</help> + <completionHelp> + <path>pki ca</path> + </completionHelp> + </properties> + <command>sudo ${vyos_op_scripts_dir}/pki.py --action show --crl "$4"</command> + <children> + <leafNode name="pem"> + <properties> + <help>Show x509 certificate revocation lists by CA name in PEM format</help> + </properties> + <command>sudo ${vyos_op_scripts_dir}/pki.py --action show --crl "$4" --pem</command> + </leafNode> + </children> + </tagNode> </children> <command>sudo ${vyos_op_scripts_dir}/pki.py --action show</command> </node> diff --git a/op-mode-definitions/show-bridge.xml.in b/op-mode-definitions/show-bridge.xml.in index acf3a00c7..a272ea204 100644 --- a/op-mode-definitions/show-bridge.xml.in +++ b/op-mode-definitions/show-bridge.xml.in @@ -42,6 +42,18 @@ </properties> <command>${vyos_op_scripts_dir}/bridge.py show_fdb --interface=$3</command> </leafNode> + <leafNode name="detail"> + <properties> + <help>Display bridge interface details</help> + </properties> + <command>${vyos_op_scripts_dir}/bridge.py show_detail --interface=$3</command> + </leafNode> + <leafNode name="nexthop-group"> + <properties> + <help>Display bridge interface nexthop-group</help> + </properties> + <command>${vyos_op_scripts_dir}/bridge.py show_detail --nexthop_group --interface=$3</command> + </leafNode> </children> </tagNode> </children> diff --git a/op-mode-definitions/show-ip-ospf.xml.in b/op-mode-definitions/show-ip-ospf.xml.in index 704ed984f..f3b9da90c 100644 --- a/op-mode-definitions/show-ip-ospf.xml.in +++ b/op-mode-definitions/show-ip-ospf.xml.in @@ -13,7 +13,7 @@ </properties> <command>${vyos_op_scripts_dir}/vtysh_wrapper.sh $@</command> <children> - #include <include/ospf-common.xml.i> + #include <include/ospf/common.xml.i> <tagNode name="vrf"> <properties> <help>Show OSPF routing protocol for given VRF</help> @@ -24,7 +24,7 @@ </properties> <command>${vyos_op_scripts_dir}/vtysh_wrapper.sh $@</command> <children> - #include <include/ospf-common.xml.i> + #include <include/ospf/common.xml.i> </children> </tagNode> </children> diff --git a/op-mode-definitions/show-ipv6-ospfv3.xml.in b/op-mode-definitions/show-ipv6-ospfv3.xml.in index a63465472..e1fcf470f 100644 --- a/op-mode-definitions/show-ipv6-ospfv3.xml.in +++ b/op-mode-definitions/show-ipv6-ospfv3.xml.in @@ -41,6 +41,7 @@ </tagNode> #include <include/ospfv3/border-routers.xml.i> #include <include/ospfv3/database.xml.i> + #include <include/ospf/graceful-restart.xml.i> #include <include/ospfv3/interface.xml.i> #include <include/ospfv3/linkstate.xml.i> #include <include/ospfv3/neighbor.xml.i> @@ -94,6 +95,7 @@ </tagNode> #include <include/ospfv3/border-routers.xml.i> #include <include/ospfv3/database.xml.i> + #include <include/ospf/graceful-restart.xml.i> #include <include/ospfv3/interface.xml.i> #include <include/ospfv3/linkstate.xml.i> #include <include/ospfv3/neighbor.xml.i> diff --git a/op-mode-definitions/show-log.xml.in b/op-mode-definitions/show-log.xml.in index 7663e4c00..579e348f7 100644 --- a/op-mode-definitions/show-log.xml.in +++ b/op-mode-definitions/show-log.xml.in @@ -2,9 +2,18 @@ <interfaceDefinition> <node name="show"> <children> + <tagNode name="log"> + <properties> + <help>Show last number of messages in master logging buffer</help> + <completionHelp> + <list><1-9999></list> + </completionHelp> + </properties> + <command>if ${vyos_validators_dir}/numeric --range 1-9999 "$3"; then journalctl --no-hostname --boot --lines "$3"; fi</command> + </tagNode> <node name="log"> <properties> - <help>Show contents of current master log file</help> + <help>Show contents of current master logging buffer</help> </properties> <command>journalctl --no-hostname --boot</command> <children> @@ -461,6 +470,12 @@ </leafNode> </children> </node> + <leafNode name="vpp"> + <properties> + <help>Show log for Vector Packet Processor (VPP)</help> + </properties> + <command>journalctl --no-hostname --boot --unit vpp.service</command> + </leafNode> <leafNode name="vrrp"> <properties> <help>Show log for Virtual Router Redundancy Protocol (VRRP)</help> diff --git a/python/vyos/accel_ppp.py b/python/vyos/accel_ppp.py index 0af311e57..0b4f8a9fe 100644 --- a/python/vyos/accel_ppp.py +++ b/python/vyos/accel_ppp.py @@ -18,7 +18,7 @@ import sys import vyos.opmode -from vyos.util import rc_cmd +from vyos.utils.process import rc_cmd def get_server_statistics(accel_statistics, pattern, sep=':') -> dict: diff --git a/python/vyos/config.py b/python/vyos/config.py index c3bb68373..179f60c43 100644 --- a/python/vyos/config.py +++ b/python/vyos/config.py @@ -68,10 +68,16 @@ import json from copy import deepcopy import vyos.configtree -from vyos.xml_ref import multi_to_list, merge_defaults, relative_defaults +from vyos.xml_ref import multi_to_list, from_source +from vyos.xml_ref import merge_defaults, relative_defaults from vyos.utils.dict import get_sub_dict, mangle_dict_keys from vyos.configsource import ConfigSource, ConfigSourceSession +class ConfigDict(dict): + _from_defaults = {} + def from_defaults(self, path: list[str]): + return from_source(self._from_defaults, path) + class Config(object): """ The class of config access objects. @@ -239,9 +245,9 @@ class Config(object): """ lpath = self._make_path(path) root_dict = self.get_cached_root_dict(effective) - conf_dict = get_sub_dict(root_dict, lpath, get_first_key) + conf_dict = get_sub_dict(root_dict, lpath, get_first_key=get_first_key) - if key_mangling is None and no_multi_convert and not with_defaults: + if key_mangling is None and no_multi_convert and not (with_defaults or with_recursive_defaults): return deepcopy(conf_dict) rpath = lpath if get_first_key else lpath[:-1] @@ -250,6 +256,7 @@ class Config(object): conf_dict = multi_to_list(rpath, conf_dict) if with_defaults or with_recursive_defaults: + conf_dict = ConfigDict(conf_dict) conf_dict = merge_defaults(lpath, conf_dict, get_first_key=get_first_key, recursive=with_recursive_defaults) @@ -263,7 +270,17 @@ class Config(object): isinstance(key_mangling[1], str)): raise ValueError("key_mangling must be a tuple of two strings") - conf_dict = mangle_dict_keys(conf_dict, key_mangling[0], key_mangling[1], abs_path=rpath, no_tag_node_value_mangle=no_tag_node_value_mangle) + def mangle(obj): + return mangle_dict_keys(obj, key_mangling[0], key_mangling[1], + abs_path=rpath, + no_tag_node_value_mangle=no_tag_node_value_mangle) + + if isinstance(conf_dict, ConfigDict): + from_defaults = mangle(conf_dict._from_defaults) + conf_dict = mangle(conf_dict) + conf_dict._from_defaults = from_defaults + else: + conf_dict = mangle(conf_dict) return conf_dict diff --git a/python/vyos/config_mgmt.py b/python/vyos/config_mgmt.py index 26114149f..4ddabd6c2 100644 --- a/python/vyos/config_mgmt.py +++ b/python/vyos/config_mgmt.py @@ -18,16 +18,21 @@ import re import sys import gzip import logging + from typing import Optional, Tuple, Union from filecmp import cmp from datetime import datetime +from textwrap import dedent +from pathlib import Path from tabulate import tabulate from vyos.config import Config from vyos.configtree import ConfigTree, ConfigTreeError, show_diff from vyos.defaults import directories from vyos.version import get_full_version_data -from vyos.util import is_systemd_service_active, ask_yes_no, rc_cmd +from vyos.utils.io import ask_yes_no +from vyos.utils.process import is_systemd_service_active +from vyos.utils.process import rc_cmd SAVE_CONFIG = '/opt/vyatta/sbin/vyatta-save-config.pl' @@ -456,19 +461,18 @@ Proceed ?''' return ConfigTree(c) def _add_logrotate_conf(self): - conf = f"""{archive_config_file} {{ - su root vyattacfg - rotate {self.max_revisions} - start 0 - compress - copy -}}""" - mask = os.umask(0o133) - - with open(logrotate_conf, 'w') as f: - f.write(conf) - - os.umask(mask) + conf: str = dedent(f"""\ + {archive_config_file} {{ + su root vyattacfg + rotate {self.max_revisions} + start 0 + compress + copy + }} + """) + conf_file = Path(logrotate_conf) + conf_file.write_text(conf) + conf_file.chmod(0o644) def _archive_active_config(self) -> bool: mask = os.umask(0o113) diff --git a/python/vyos/configdep.py b/python/vyos/configdep.py index d4b2cc78f..7a8559839 100644 --- a/python/vyos/configdep.py +++ b/python/vyos/configdep.py @@ -18,7 +18,7 @@ import json import typing from inspect import stack -from vyos.util import load_as_module +from vyos.utils.system import load_as_module from vyos.defaults import directories from vyos.configsource import VyOSError from vyos import ConfigError diff --git a/python/vyos/configdict.py b/python/vyos/configdict.py index 1205342df..f642d38f2 100644 --- a/python/vyos/configdict.py +++ b/python/vyos/configdict.py @@ -19,9 +19,9 @@ A library for retrieving value dicts from VyOS configs in a declarative fashion. import os import json -from vyos.util import dict_search +from vyos.utils.dict import dict_search from vyos.xml import defaults -from vyos.util import cmd +from vyos.utils.process import cmd def retrieve_config(path_hash, base_path, config): """ @@ -590,7 +590,7 @@ def get_accel_dict(config, base, chap_secrets): Return a dictionary with the necessary interface config keys. """ - from vyos.util import get_half_cpus + from vyos.utils.system import get_half_cpus from vyos.template import is_ipv4 dict = config.get_config_dict(base, key_mangling=('-', '_'), diff --git a/python/vyos/configdiff.py b/python/vyos/configdiff.py index ac86af09c..0caa204c3 100644 --- a/python/vyos/configdiff.py +++ b/python/vyos/configdiff.py @@ -19,8 +19,9 @@ from vyos.config import Config from vyos.configtree import DiffTree from vyos.configdict import dict_merge from vyos.configdict import list_diff -from vyos.util import get_sub_dict, mangle_dict_keys -from vyos.util import dict_search_args +from vyos.utils.dict import get_sub_dict +from vyos.utils.dict import mangle_dict_keys +from vyos.utils.dict import dict_search_args from vyos.xml import defaults class ConfigDiffError(Exception): diff --git a/python/vyos/configquery.py b/python/vyos/configquery.py index 85fef8777..71ad5b4f0 100644 --- a/python/vyos/configquery.py +++ b/python/vyos/configquery.py @@ -1,4 +1,4 @@ -# Copyright 2021 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2021-2023 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -19,9 +19,11 @@ settings from op mode, and execution of arbitrary op mode commands. ''' import os -from subprocess import STDOUT -from vyos.util import popen, boot_configuration_complete +from vyos.utils.process import STDOUT +from vyos.utils.process import popen + +from vyos.utils.boot import boot_configuration_complete from vyos.config import Config from vyos.configsource import ConfigSourceSession, ConfigSourceString from vyos.defaults import directories diff --git a/python/vyos/configsession.py b/python/vyos/configsession.py index decb82437..e8918d577 100644 --- a/python/vyos/configsession.py +++ b/python/vyos/configsession.py @@ -17,7 +17,7 @@ import re import sys import subprocess -from vyos.util import is_systemd_service_running +from vyos.utils.process import is_systemd_service_running from vyos.utils.dict import dict_to_paths CLI_SHELL_API = '/bin/cli-shell-api' diff --git a/python/vyos/configsource.py b/python/vyos/configsource.py index 510b5b65a..f582bdfab 100644 --- a/python/vyos/configsource.py +++ b/python/vyos/configsource.py @@ -1,5 +1,5 @@ -# Copyright 2020 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2020-2023 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -19,7 +19,7 @@ import re import subprocess from vyos.configtree import ConfigTree -from vyos.util import boot_configuration_complete +from vyos.utils.boot import boot_configuration_complete class VyOSError(Exception): """ diff --git a/python/vyos/configverify.py b/python/vyos/configverify.py index 94dcdf4d9..5b94bd98b 100644 --- a/python/vyos/configverify.py +++ b/python/vyos/configverify.py @@ -22,8 +22,8 @@ # makes use of it! from vyos import ConfigError -from vyos.util import dict_search -from vyos.util import dict_search_recursive +from vyos.utils.dict import dict_search +from vyos.utils.dict import dict_search_recursive def verify_mtu(config): """ @@ -314,8 +314,6 @@ def verify_dhcpv6(config): recurring validation of DHCPv6 options which are mutually exclusive. """ if 'dhcpv6_options' in config: - from vyos.util import dict_search - if {'parameters_only', 'temporary'} <= set(config['dhcpv6_options']): raise ConfigError('DHCPv6 temporary and parameters-only options ' 'are mutually exclusive!') @@ -460,7 +458,7 @@ def verify_diffie_hellman_length(file, min_keysize): then or equal to min_keysize """ import os import re - from vyos.util import cmd + from vyos.utils.process import cmd try: keysize = str(min_keysize) diff --git a/python/vyos/dicts.py b/python/vyos/dicts.py deleted file mode 100644 index b12cda40f..000000000 --- a/python/vyos/dicts.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2019 VyOS maintainers and contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 or later as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -from vyos import ConfigError - - -class FixedDict(dict): - """ - FixedDict: A dictionnary not allowing new keys to be created after initialisation. - - >>> f = FixedDict(**{'count':1}) - >>> f['count'] = 2 - >>> f['king'] = 3 - File "...", line ..., in __setitem__ - raise ConfigError(f'Option "{k}" has no defined default') - """ - - def __init__(self, **options): - self._allowed = options.keys() - super().__init__(**options) - - def __setitem__(self, k, v): - """ - __setitem__ is a builtin which is called by python when setting dict values: - >>> d = dict() - >>> d['key'] = 'value' - >>> d - {'key': 'value'} - - is syntaxic sugar for - - >>> d = dict() - >>> d.__setitem__('key','value') - >>> d - {'key': 'value'} - """ - if k not in self._allowed: - raise ConfigError(f'Option "{k}" has no defined default') - super().__setitem__(k, v) diff --git a/python/vyos/ethtool.py b/python/vyos/ethtool.py index 9b7da89fa..ca3bcfc3d 100644 --- a/python/vyos/ethtool.py +++ b/python/vyos/ethtool.py @@ -16,7 +16,7 @@ import os import re -from vyos.util import popen +from vyos.utils.process import popen # These drivers do not support using ethtool to change the speed, duplex, or # flow control settings diff --git a/python/vyos/firewall.py b/python/vyos/firewall.py index 919032a41..2793b201c 100644 --- a/python/vyos/firewall.py +++ b/python/vyos/firewall.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2021-2022 VyOS maintainers and contributors +# Copyright (C) 2021-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -28,11 +28,11 @@ from time import strftime from vyos.remote import download from vyos.template import is_ipv4 from vyos.template import render -from vyos.util import call -from vyos.util import cmd -from vyos.util import dict_search_args -from vyos.util import dict_search_recursive -from vyos.util import run +from vyos.utils.dict import dict_search_args +from vyos.utils.dict import dict_search_recursive +from vyos.utils.process import call +from vyos.utils.process import cmd +from vyos.utils.process import run # Domain Resolver @@ -45,7 +45,7 @@ def fqdn_config_parse(firewall): rule = path[3] # rule id suffix = path[4][0] # source/destination (1 char) set_name = f'{fw_name}_{rule}_{suffix}' - + if path[0] == 'name': firewall['ip_fqdn'][set_name] = domain elif path[0] == 'ipv6_name': diff --git a/python/vyos/frr.py b/python/vyos/frr.py index a84f183ef..2e3c8a271 100644 --- a/python/vyos/frr.py +++ b/python/vyos/frr.py @@ -67,9 +67,12 @@ Apply the new configuration: import tempfile import re -from vyos import util -from vyos.util import chown -from vyos.util import cmd + +from vyos.utils.permission import chown +from vyos.utils.process import cmd +from vyos.utils.process import popen +from vyos.utils.process import STDOUT + import logging from logging.handlers import SysLogHandler import os @@ -144,7 +147,7 @@ def get_configuration(daemon=None, marked=False): if daemon: cmd += f' -d {daemon}' - output, code = util.popen(cmd, stderr=util.STDOUT) + output, code = popen(cmd, stderr=STDOUT) if code: raise OSError(code, output) @@ -166,7 +169,7 @@ def mark_configuration(config): config: The configuration string to mark/test return: The marked configuration from FRR """ - output, code = util.popen(f"{path_vtysh} -m -f -", stderr=util.STDOUT, input=config) + output, code = popen(f"{path_vtysh} -m -f -", stderr=STDOUT, input=config) if code == 2: raise ConfigurationNotValid(str(output)) @@ -206,7 +209,7 @@ def reload_configuration(config, daemon=None): cmd += f' {f.name}' LOG.debug(f'reload_configuration: Executing command against frr-reload: "{cmd}"') - output, code = util.popen(cmd, stderr=util.STDOUT) + output, code = popen(cmd, stderr=STDOUT) f.close() for i, e in enumerate(output.split('\n')): LOG.debug(f'frr-reload output: {i:3} {e}') @@ -235,7 +238,7 @@ def execute(command): cmd = f"{path_vtysh} -c '{command}'" - output, code = util.popen(cmd, stderr=util.STDOUT) + output, code = popen(cmd, stderr=STDOUT) if code: raise OSError(code, output) @@ -267,7 +270,7 @@ def configure(lines, daemon=False): for x in lines: cmd += f" -c '{x}'" - output, code = util.popen(cmd, stderr=util.STDOUT) + output, code = popen(cmd, stderr=STDOUT) if code == 1: raise ConfigurationNotValid(f'Configuration FRR failed: {repr(output)}') elif code: diff --git a/python/vyos/ifconfig/bond.py b/python/vyos/ifconfig/bond.py index 0edd17055..e88f860be 100644 --- a/python/vyos/ifconfig/bond.py +++ b/python/vyos/ifconfig/bond.py @@ -16,8 +16,8 @@ import os from vyos.ifconfig.interface import Interface -from vyos.util import cmd -from vyos.util import dict_search +from vyos.utils.process import cmd +from vyos.utils.dict import dict_search from vyos.validate import assert_list from vyos.validate import assert_positive diff --git a/python/vyos/ifconfig/bridge.py b/python/vyos/ifconfig/bridge.py index aa818bc5f..b103b49d8 100644 --- a/python/vyos/ifconfig/bridge.py +++ b/python/vyos/ifconfig/bridge.py @@ -19,8 +19,8 @@ import json from vyos.ifconfig.interface import Interface from vyos.validate import assert_boolean from vyos.validate import assert_positive -from vyos.util import cmd -from vyos.util import dict_search +from vyos.utils.process import cmd +from vyos.utils.dict import dict_search from vyos.configdict import get_vlan_ids from vyos.configdict import list_diff diff --git a/python/vyos/ifconfig/control.py b/python/vyos/ifconfig/control.py index 7a6b36e7c..c8366cb58 100644 --- a/python/vyos/ifconfig/control.py +++ b/python/vyos/ifconfig/control.py @@ -19,10 +19,10 @@ from inspect import signature from inspect import _empty from vyos.ifconfig.section import Section -from vyos.util import popen -from vyos.util import cmd -from vyos.util import read_file -from vyos.util import write_file +from vyos.utils.process import popen +from vyos.utils.process import cmd +from vyos.utils.file import read_file +from vyos.utils.file import write_file from vyos import debug class Control(Section): diff --git a/python/vyos/ifconfig/ethernet.py b/python/vyos/ifconfig/ethernet.py index 30bea3b86..4ff044c23 100644 --- a/python/vyos/ifconfig/ethernet.py +++ b/python/vyos/ifconfig/ethernet.py @@ -20,9 +20,9 @@ from glob import glob from vyos.base import Warning from vyos.ethtool import Ethtool from vyos.ifconfig.interface import Interface -from vyos.util import run -from vyos.util import dict_search -from vyos.util import read_file +from vyos.utils.dict import dict_search +from vyos.utils.file import read_file +from vyos.utils.process import run from vyos.validate import assert_list @Interface.register diff --git a/python/vyos/ifconfig/geneve.py b/python/vyos/ifconfig/geneve.py index 7a05e47a7..fbb261a35 100644 --- a/python/vyos/ifconfig/geneve.py +++ b/python/vyos/ifconfig/geneve.py @@ -14,7 +14,7 @@ # License along with this library. If not, see <http://www.gnu.org/licenses/>. from vyos.ifconfig import Interface -from vyos.util import dict_search +from vyos.utils.dict import dict_search @Interface.register class GeneveIf(Interface): diff --git a/python/vyos/ifconfig/interface.py b/python/vyos/ifconfig/interface.py index f6289a6e6..120f2131b 100644 --- a/python/vyos/ifconfig/interface.py +++ b/python/vyos/ifconfig/interface.py @@ -32,12 +32,12 @@ from vyos.configdict import list_diff from vyos.configdict import dict_merge from vyos.configdict import get_vlan_ids from vyos.template import render -from vyos.util import mac2eui64 -from vyos.util import dict_search -from vyos.util import read_file -from vyos.util import get_interface_config -from vyos.util import get_interface_namespace -from vyos.util import is_systemd_service_active +from vyos.utils.network import mac2eui64 +from vyos.utils.dict import dict_search +from vyos.utils.file import read_file +from vyos.utils.network import get_interface_config +from vyos.utils.network import get_interface_namespace +from vyos.utils.process import is_systemd_service_active from vyos.template import is_ipv4 from vyos.template import is_ipv6 from vyos.validate import is_intf_addr_assigned diff --git a/python/vyos/ifconfig/l2tpv3.py b/python/vyos/ifconfig/l2tpv3.py index fcd1fbf81..85a89ef8b 100644 --- a/python/vyos/ifconfig/l2tpv3.py +++ b/python/vyos/ifconfig/l2tpv3.py @@ -1,4 +1,4 @@ -# Copyright 2019-2021 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2019-2023 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -15,7 +15,8 @@ from time import sleep from time import time -from vyos.util import run + +from vyos.utils.process import run from vyos.ifconfig.interface import Interface def wait_for_add_l2tpv3(timeout=10, sleep_interval=1, cmd=None): diff --git a/python/vyos/ifconfig/pppoe.py b/python/vyos/ifconfig/pppoe.py index 437fe0cae..fd4590beb 100644 --- a/python/vyos/ifconfig/pppoe.py +++ b/python/vyos/ifconfig/pppoe.py @@ -15,7 +15,7 @@ from vyos.ifconfig.interface import Interface from vyos.validate import assert_range -from vyos.util import get_interface_config +from vyos.utils.network import get_interface_config @Interface.register class PPPoEIf(Interface): diff --git a/python/vyos/ifconfig/tunnel.py b/python/vyos/ifconfig/tunnel.py index b7bf7d982..fb2f38e2b 100644 --- a/python/vyos/ifconfig/tunnel.py +++ b/python/vyos/ifconfig/tunnel.py @@ -17,7 +17,7 @@ # https://community.hetzner.com/tutorials/linux-setup-gre-tunnel from vyos.ifconfig.interface import Interface -from vyos.util import dict_search +from vyos.utils.dict import dict_search from vyos.validate import assert_list def enable_to_on(value): diff --git a/python/vyos/ifconfig/vrrp.py b/python/vyos/ifconfig/vrrp.py index 47aaadecd..fde903a53 100644 --- a/python/vyos/ifconfig/vrrp.py +++ b/python/vyos/ifconfig/vrrp.py @@ -1,4 +1,4 @@ -# Copyright 2019 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2019-2023 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -21,8 +21,11 @@ from time import time from time import sleep from tabulate import tabulate -from vyos import util from vyos.configquery import ConfigTreeQuery +from vyos.utils.convert import seconds_to_human +from vyos.utils.file import read_file +from vyos.utils.file import wait_for_file_write_complete +from vyos.utils.process import process_running class VRRPError(Exception): pass @@ -84,21 +87,21 @@ class VRRP(object): def is_running(cls): if not os.path.exists(cls.location['pid']): return False - return util.process_running(cls.location['pid']) + return process_running(cls.location['pid']) @classmethod def collect(cls, what): fname = cls.location[what] try: # send signal to generate the configuration file - pid = util.read_file(cls.location['pid']) - util.wait_for_file_write_complete(fname, + pid = read_file(cls.location['pid']) + wait_for_file_write_complete(fname, pre_hook=(lambda: os.kill(int(pid), cls._signal[what])), timeout=30) - return util.read_file(fname) + return read_file(fname) except OSError: - # raised by vyos.util.read_file + # raised by vyos.utils.file.read_file raise VRRPNoData("VRRP data is not available (wait time exceeded)") except FileNotFoundError: raise VRRPNoData("VRRP data is not available (process not running or no active groups)") @@ -145,7 +148,7 @@ class VRRP(object): priority = data['effective_priority'] since = int(time() - float(data['last_transition'])) - last = util.seconds_to_human(since) + last = seconds_to_human(since) groups.append([name, intf, vrid, state, priority, last]) diff --git a/python/vyos/ifconfig/vti.py b/python/vyos/ifconfig/vti.py index dc99d365a..9ebbeb9ed 100644 --- a/python/vyos/ifconfig/vti.py +++ b/python/vyos/ifconfig/vti.py @@ -14,7 +14,7 @@ # License along with this library. If not, see <http://www.gnu.org/licenses/>. from vyos.ifconfig.interface import Interface -from vyos.util import dict_search +from vyos.utils.dict import dict_search @Interface.register class VTIIf(Interface): diff --git a/python/vyos/ifconfig/vxlan.py b/python/vyos/ifconfig/vxlan.py index 5baff10a9..6a9911588 100644 --- a/python/vyos/ifconfig/vxlan.py +++ b/python/vyos/ifconfig/vxlan.py @@ -15,7 +15,7 @@ from vyos import ConfigError from vyos.ifconfig import Interface -from vyos.util import dict_search +from vyos.utils.dict import dict_search @Interface.register class VXLANIf(Interface): diff --git a/python/vyos/initialsetup.py b/python/vyos/initialsetup.py index 574e7892d..3b280dc6b 100644 --- a/python/vyos/initialsetup.py +++ b/python/vyos/initialsetup.py @@ -1,7 +1,7 @@ # initialsetup -- functions for setting common values in config file, # for use in installation and first boot scripts # -# Copyright (C) 2018 VyOS maintainers and contributors +# Copyright (C) 2018-2023 VyOS maintainers and contributors # # This library is free software; you can redistribute it and/or modify it under the terms of # the GNU Lesser General Public License as published by the Free Software Foundation; @@ -12,10 +12,12 @@ # See the GNU Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public License along with this library; -# if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA import vyos.configtree -import vyos.authutils + +from vyos.utils.auth import make_password_hash +from vyos.utils.auth import split_ssh_public_key def set_interface_address(config, intf, addr, intf_type="ethernet"): config.set(["interfaces", intf_type, intf, "address"], value=addr) @@ -35,8 +37,8 @@ def set_default_gateway(config, gateway): def set_user_password(config, user, password): # Make a password hash - hash = vyos.authutils.make_password_hash(password) - + hash = make_password_hash(password) + config.set(["system", "login", "user", user, "authentication", "encrypted-password"], value=hash) config.set(["system", "login", "user", user, "authentication", "plaintext-password"], value="") @@ -48,7 +50,7 @@ def set_user_level(config, user, level): config.set(["system", "login", "user", user, "level"], value=level) def set_user_ssh_key(config, user, key_string): - key = vyos.authutils.split_ssh_public_key(key_string, defaultname=user) + key = split_ssh_public_key(key_string, defaultname=user) config.set(["system", "login", "user", user, "authentication", "public-keys", key["name"], "key"], value=key["data"]) config.set(["system", "login", "user", user, "authentication", "public-keys", key["name"], "type"], value=key["type"]) diff --git a/python/vyos/migrator.py b/python/vyos/migrator.py index 87c74e1ea..872682bc0 100644 --- a/python/vyos/migrator.py +++ b/python/vyos/migrator.py @@ -20,7 +20,7 @@ import logging import vyos.defaults import vyos.component_version as component_version -from vyos.util import cmd +from vyos.utils.process import cmd log_file = os.path.join(vyos.defaults.directories['config'], 'vyos-migrate.log') diff --git a/python/vyos/nat.py b/python/vyos/nat.py index 53fd7fb33..603fedb9b 100644 --- a/python/vyos/nat.py +++ b/python/vyos/nat.py @@ -15,7 +15,7 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. from vyos.template import is_ip_network -from vyos.util import dict_search_args +from vyos.utils.dict import dict_search_args from vyos.template import bracketize_ipv6 @@ -54,28 +54,32 @@ def parse_nat_rule(rule_conf, rule_id, nat_type, ipv6=False): translation_str = 'return' log_suffix = '-EXCL' elif 'translation' in rule_conf: - translation_prefix = nat_type[:1] - translation_output = [f'{translation_prefix}nat'] addr = dict_search_args(rule_conf, 'translation', 'address') port = dict_search_args(rule_conf, 'translation', 'port') - - if addr and is_ip_network(addr): - if not ipv6: - map_addr = dict_search_args(rule_conf, nat_type, 'address') - translation_output.append(f'{ip_prefix} prefix to {ip_prefix} {translation_prefix}addr map {{ {map_addr} : {addr} }}') - ignore_type_addr = True - else: - translation_output.append(f'prefix to {addr}') - elif addr == 'masquerade': - if port: - addr = f'{addr} to ' - translation_output = [addr] - log_suffix = '-MASQ' + redirect_port = dict_search_args(rule_conf, 'translation', 'redirect', 'port') + if redirect_port: + translation_output = [f'redirect to {redirect_port}'] else: - translation_output.append('to') - if addr: - addr = bracketize_ipv6(addr) - translation_output.append(addr) + translation_prefix = nat_type[:1] + translation_output = [f'{translation_prefix}nat'] + + if addr and is_ip_network(addr): + if not ipv6: + map_addr = dict_search_args(rule_conf, nat_type, 'address') + translation_output.append(f'{ip_prefix} prefix to {ip_prefix} {translation_prefix}addr map {{ {map_addr} : {addr} }}') + ignore_type_addr = True + else: + translation_output.append(f'prefix to {addr}') + elif addr == 'masquerade': + if port: + addr = f'{addr} to ' + translation_output = [addr] + log_suffix = '-MASQ' + else: + translation_output.append('to') + if addr: + addr = bracketize_ipv6(addr) + translation_output.append(addr) options = [] addr_mapping = dict_search_args(rule_conf, 'translation', 'options', 'address_mapping') diff --git a/python/vyos/qos/base.py b/python/vyos/qos/base.py index 717e3c214..6c5a3d79c 100644 --- a/python/vyos/qos/base.py +++ b/python/vyos/qos/base.py @@ -16,9 +16,9 @@ import os from vyos.base import Warning -from vyos.util import cmd -from vyos.util import dict_search -from vyos.util import read_file +from vyos.utils.process import cmd +from vyos.utils.dict import dict_search +from vyos.utils.file import read_file from vyos.utils.network import get_protocol_by_name @@ -331,13 +331,15 @@ class QoSBase: # burst = cls_config['burst'] # filter_cmd += f' burst {burst}' + if 'default' in config: + default_cls_id = 1 + if 'class' in config: + class_id_max = self._get_class_max_id(config) + default_cls_id = int(class_id_max) +1 + self._build_base_qdisc(config['default'], default_cls_id) + if self.qostype == 'limiter': if 'default' in config: - if 'class' in config: - class_id_max = self._get_class_max_id(config) - default_cls_id = int(class_id_max) + 1 - self._build_base_qdisc(config['default'], default_cls_id) - filter_cmd = f'tc filter replace dev {self._interface} parent {self._parent:x}: ' filter_cmd += 'prio 255 protocol all basic' diff --git a/python/vyos/qos/priority.py b/python/vyos/qos/priority.py index 6d4a60a43..8182400f9 100644 --- a/python/vyos/qos/priority.py +++ b/python/vyos/qos/priority.py @@ -14,7 +14,7 @@ # License along with this library. If not, see <http://www.gnu.org/licenses/>. from vyos.qos.base import QoSBase -from vyos.util import dict_search +from vyos.utils.dict import dict_search class Priority(QoSBase): _parent = 1 diff --git a/python/vyos/remote.py b/python/vyos/remote.py index 66044fa52..cf731c881 100644 --- a/python/vyos/remote.py +++ b/python/vyos/remote.py @@ -25,22 +25,21 @@ import urllib.parse from ftplib import FTP from ftplib import FTP_TLS -from paramiko import SSHClient +from paramiko import SSHClient, SSHException from paramiko import MissingHostKeyPolicy from requests import Session from requests.adapters import HTTPAdapter from requests.packages.urllib3 import PoolManager -from vyos.util import ask_yes_no -from vyos.util import begin -from vyos.util import cmd -from vyos.util import make_incremental_progressbar -from vyos.util import make_progressbar -from vyos.util import print_error +from vyos.utils.io import ask_yes_no +from vyos.utils.io import make_incremental_progressbar +from vyos.utils.io import make_progressbar +from vyos.utils.io import print_error +from vyos.utils.misc import begin +from vyos.utils.process import cmd from vyos.version import get_version - CHUNK_SIZE = 8192 class InteractivePolicy(MissingHostKeyPolicy): @@ -51,7 +50,7 @@ class InteractivePolicy(MissingHostKeyPolicy): def missing_host_key(self, client, hostname, key): print_error(f"Host '{hostname}' not found in known hosts.") print_error('Fingerprint: ' + key.get_fingerprint().hex()) - if ask_yes_no('Do you wish to continue?'): + if sys.stdout.isatty() and ask_yes_no('Do you wish to continue?'): if client._host_keys_filename\ and ask_yes_no('Do you wish to permanently add this host/key pair to known hosts?'): client._host_keys.add(hostname, key.get_name(), key) @@ -97,7 +96,13 @@ def check_storage(path, size): class FtpC: - def __init__(self, url, progressbar=False, check_space=False, source_host='', source_port=0): + def __init__(self, + url, + progressbar=False, + check_space=False, + source_host='', + source_port=0, + timeout=10): self.secure = url.scheme == 'ftps' self.hostname = url.hostname self.path = url.path @@ -107,12 +112,15 @@ class FtpC: self.source = (source_host, source_port) self.progressbar = progressbar self.check_space = check_space + self.timeout = timeout def _establish(self): if self.secure: - return FTP_TLS(source_address=self.source, context=ssl.create_default_context()) + return FTP_TLS(source_address=self.source, + context=ssl.create_default_context(), + timeout=self.timeout) else: - return FTP(source_address=self.source) + return FTP(source_address=self.source, timeout=self.timeout) def download(self, location: str): # Open the file upfront before establishing connection. @@ -151,7 +159,13 @@ class FtpC: class SshC: known_hosts = os.path.expanduser('~/.ssh/known_hosts') - def __init__(self, url, progressbar=False, check_space=False, source_host='', source_port=0): + def __init__(self, + url, + progressbar=False, + check_space=False, + source_host='', + source_port=0, + timeout=10.0): self.hostname = url.hostname self.path = url.path self.username = url.username or os.getenv('REMOTE_USERNAME') @@ -160,6 +174,7 @@ class SshC: self.source = (source_host, source_port) self.progressbar = progressbar self.check_space = check_space + self.timeout = timeout def _establish(self): ssh = SSHClient() @@ -170,7 +185,7 @@ class SshC: ssh.set_missing_host_key_policy(InteractivePolicy()) # `socket.create_connection()` automatically picks a NIC and an IPv4/IPv6 address family # for us on dual-stack systems. - sock = socket.create_connection((self.hostname, self.port), socket.getdefaulttimeout(), self.source) + sock = socket.create_connection((self.hostname, self.port), self.timeout, self.source) ssh.connect(self.hostname, self.port, self.username, self.password, sock=sock) return ssh @@ -199,13 +214,20 @@ class SshC: class HttpC: - def __init__(self, url, progressbar=False, check_space=False, source_host='', source_port=0): + def __init__(self, + url, + progressbar=False, + check_space=False, + source_host='', + source_port=0, + timeout=10.0): self.urlstring = urllib.parse.urlunsplit(url) self.progressbar = progressbar self.check_space = check_space self.source_pair = (source_host, source_port) self.username = url.username or os.getenv('REMOTE_USERNAME') self.password = url.password or os.getenv('REMOTE_PASSWORD') + self.timeout = timeout def _establish(self): session = Session() @@ -221,8 +243,11 @@ class HttpC: # Not only would it potentially mess up with the progress bar but # `shutil.copyfileobj(request.raw, file)` does not handle automatic decoding. s.headers.update({'Accept-Encoding': 'identity'}) - with s.head(self.urlstring, allow_redirects=True) as r: + with s.head(self.urlstring, + allow_redirects=True, + timeout=self.timeout) as r: # Abort early if the destination is inaccessible. + print('pre-3') r.raise_for_status() # If the request got redirected, keep the last URL we ended up with. final_urlstring = r.url @@ -236,7 +261,8 @@ class HttpC: size = None if self.check_space: check_storage(location, size) - with s.get(final_urlstring, stream=True) as r, open(location, 'wb') as f: + with s.get(final_urlstring, stream=True, + timeout=self.timeout) as r, open(location, 'wb') as f: if self.progressbar and size: progress = make_incremental_progressbar(CHUNK_SIZE / size) next(progress) @@ -250,7 +276,10 @@ class HttpC: def upload(self, location: str): # Does not yet support progressbars. with self._establish() as s, open(location, 'rb') as f: - s.post(self.urlstring, data=f, allow_redirects=True) + s.post(self.urlstring, + data=f, + allow_redirects=True, + timeout=self.timeout) class TftpC: @@ -259,10 +288,16 @@ class TftpC: # 2. Since there's no concept authentication, we don't need to deal with keys/passwords. # 3. It would be a waste to import, audit and maintain a third-party library for TFTP. # 4. I'd rather not implement the entire protocol here, no matter how simple it is. - def __init__(self, url, progressbar=False, check_space=False, source_host=None, source_port=0): + def __init__(self, + url, + progressbar=False, + check_space=False, + source_host=None, + source_port=0, + timeout=10): source_option = f'--interface {source_host} --local-port {source_port}' if source_host else '' progress_flag = '--progress-bar' if progressbar else '-s' - self.command = f'curl {source_option} {progress_flag}' + self.command = f'curl {source_option} {progress_flag} --connect-timeout {timeout}' self.urlstring = urllib.parse.urlunsplit(url) def download(self, location: str): @@ -287,10 +322,16 @@ def urlc(urlstring, *args, **kwargs): raise ValueError(f'Unsupported URL scheme: "{url.scheme}"') def download(local_path, urlstring, *args, **kwargs): - urlc(urlstring, *args, **kwargs).download(local_path) + try: + urlc(urlstring, *args, **kwargs).download(local_path) + except Exception as err: + print_error(f'Unable to download "{urlstring}": {err}') def upload(local_path, urlstring, *args, **kwargs): - urlc(urlstring, *args, **kwargs).upload(local_path) + try: + urlc(urlstring, *args, **kwargs).upload(local_path) + except Exception as err: + print_error(f'Unable to upload "{urlstring}": {err}') def get_remote_config(urlstring, source_host='', source_port=0): """ diff --git a/python/vyos/template.py b/python/vyos/template.py index 254a15e3a..7d1c3970f 100644 --- a/python/vyos/template.py +++ b/python/vyos/template.py @@ -20,10 +20,10 @@ from jinja2 import Environment from jinja2 import FileSystemLoader from jinja2 import ChainableUndefined from vyos.defaults import directories -from vyos.util import chmod -from vyos.util import chown -from vyos.util import dict_search_args -from vyos.util import makedir +from vyos.utils.dict import dict_search_args +from vyos.utils.file import makedir +from vyos.utils.permission import chmod +from vyos.utils.permission import chown # Holds template filters registered via register_filter() _FILTERS = {} @@ -162,19 +162,19 @@ def force_to_list(value): @register_filter('seconds_to_human') def seconds_to_human(seconds, separator=""): """ Convert seconds to human-readable values like 1d6h15m23s """ - from vyos.util import seconds_to_human + from vyos.utils.convert import seconds_to_human return seconds_to_human(seconds, separator=separator) @register_filter('bytes_to_human') def bytes_to_human(bytes, initial_exponent=0, precision=2): """ Convert bytes to human-readable values like 1.44M """ - from vyos.util import bytes_to_human + from vyos.utils.convert import bytes_to_human return bytes_to_human(bytes, initial_exponent=initial_exponent, precision=precision) @register_filter('human_to_bytes') def human_to_bytes(value): """ Convert a data amount with a unit suffix to bytes, like 2K to 2048 """ - from vyos.util import human_to_bytes + from vyos.utils.convert import human_to_bytes return human_to_bytes(value) @register_filter('ip_from_cidr') @@ -424,7 +424,7 @@ def get_dhcp_router(interface): if not os.path.exists(lease_file): return None - from vyos.util import read_file + from vyos.utils.file import read_file for line in read_file(lease_file).splitlines(): if 'option routers' in line: (_, _, address) = line.split() diff --git a/python/vyos/util.py b/python/vyos/util.py deleted file mode 100644 index 33da5da40..000000000 --- a/python/vyos/util.py +++ /dev/null @@ -1,1175 +0,0 @@ -# Copyright 2020-2022 VyOS maintainers and contributors <maintainers@vyos.io> -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see <http://www.gnu.org/licenses/>. - -import os -import re -import sys - -# -# NOTE: Do not import full classes here, move your import to the function -# where it is used so it is as local as possible to the execution -# - -from subprocess import Popen -from subprocess import PIPE -from subprocess import STDOUT -from subprocess import DEVNULL - -def popen(command, flag='', shell=None, input=None, timeout=None, env=None, - stdout=PIPE, stderr=PIPE, decode='utf-8'): - """ - popen is a wrapper helper aound subprocess.Popen - with it default setting it will return a tuple (out, err) - out: the output of the program run - err: the error code returned by the program - - it can be affected by the following flags: - shell: do not try to auto-detect if a shell is required - for example if a pipe (|) or redirection (>, >>) is used - input: data to sent to the child process via STDIN - the data should be bytes but string will be converted - timeout: time after which the command will be considered to have failed - env: mapping that defines the environment variables for the new process - stdout: define how the output of the program should be handled - - PIPE (default), sends stdout to the output - - DEVNULL, discard the output - stderr: define how the output of the program should be handled - - None (default), send/merge the data to/with stderr - - PIPE, popen will append it to output - - STDOUT, send the data to be merged with stdout - - DEVNULL, discard the output - decode: specify the expected text encoding (utf-8, ascii, ...) - the default is explicitely utf-8 which is python's own default - - usage: - get both stdout and stderr: popen('command', stdout=PIPE, stderr=STDOUT) - discard stdout and get stderr: popen('command', stdout=DEVNUL, stderr=PIPE) - """ - - # airbag must be left as an import in the function as otherwise we have a - # a circual import dependency - from vyos import debug - from vyos import airbag - - # log if the flag is set, otherwise log if command is set - if not debug.enabled(flag): - flag = 'command' - - cmd_msg = f"cmd '{command}'" - debug.message(cmd_msg, flag) - - use_shell = shell - stdin = None - if shell is None: - use_shell = False - if ' ' in command: - use_shell = True - if env: - use_shell = True - - if input: - stdin = PIPE - input = input.encode() if type(input) is str else input - - p = Popen(command, stdin=stdin, stdout=stdout, stderr=stderr, - env=env, shell=use_shell) - - pipe = p.communicate(input, timeout) - - pipe_out = b'' - if stdout == PIPE: - pipe_out = pipe[0] - - pipe_err = b'' - if stderr == PIPE: - pipe_err = pipe[1] - - str_out = pipe_out.decode(decode).replace('\r\n', '\n').strip() - str_err = pipe_err.decode(decode).replace('\r\n', '\n').strip() - - out_msg = f"returned (out):\n{str_out}" - if str_out: - debug.message(out_msg, flag) - - if str_err: - err_msg = f"returned (err):\n{str_err}" - # this message will also be send to syslog via airbag - debug.message(err_msg, flag, destination=sys.stderr) - - # should something go wrong, report this too via airbag - airbag.noteworthy(cmd_msg) - airbag.noteworthy(out_msg) - airbag.noteworthy(err_msg) - - return str_out, p.returncode - - -def run(command, flag='', shell=None, input=None, timeout=None, env=None, - stdout=DEVNULL, stderr=PIPE, decode='utf-8'): - """ - A wrapper around popen, which discard the stdout and - will return the error code of a command - """ - _, code = popen( - command, flag, - stdout=stdout, stderr=stderr, - input=input, timeout=timeout, - env=env, shell=shell, - decode=decode, - ) - return code - - -def cmd(command, flag='', shell=None, input=None, timeout=None, env=None, - stdout=PIPE, stderr=PIPE, decode='utf-8', raising=None, message='', - expect=[0]): - """ - A wrapper around popen, which returns the stdout and - will raise the error code of a command - - raising: specify which call should be used when raising - the class should only require a string as parameter - (default is OSError) with the error code - expect: a list of error codes to consider as normal - """ - decoded, code = popen( - command, flag, - stdout=stdout, stderr=stderr, - input=input, timeout=timeout, - env=env, shell=shell, - decode=decode, - ) - if code not in expect: - feedback = message + '\n' if message else '' - feedback += f'failed to run command: {command}\n' - feedback += f'returned: {decoded}\n' - feedback += f'exit code: {code}' - if raising is None: - # error code can be recovered with .errno - raise OSError(code, feedback) - else: - raise raising(feedback) - return decoded - - -def rc_cmd(command, flag='', shell=None, input=None, timeout=None, env=None, - stdout=PIPE, stderr=STDOUT, decode='utf-8'): - """ - A wrapper around popen, which returns the return code - of a command and stdout - - % rc_cmd('uname') - (0, 'Linux') - % rc_cmd('ip link show dev eth99') - (1, 'Device "eth99" does not exist.') - """ - out, code = popen( - command, flag, - stdout=stdout, stderr=stderr, - input=input, timeout=timeout, - env=env, shell=shell, - decode=decode, - ) - return code, out - - -def call(command, flag='', shell=None, input=None, timeout=None, env=None, - stdout=PIPE, stderr=PIPE, decode='utf-8'): - """ - A wrapper around popen, which print the stdout and - will return the error code of a command - """ - out, code = popen( - command, flag, - stdout=stdout, stderr=stderr, - input=input, timeout=timeout, - env=env, shell=shell, - decode=decode, - ) - if out: - print(out) - return code - - -def read_file(fname, defaultonfailure=None): - """ - read the content of a file, stripping any end characters (space, newlines) - should defaultonfailure be not None, it is returned on failure to read - """ - try: - """ Read a file to string """ - with open(fname, 'r') as f: - data = f.read().strip() - return data - except Exception as e: - if defaultonfailure is not None: - return defaultonfailure - raise e - -def write_file(fname, data, defaultonfailure=None, user=None, group=None, mode=None, append=False): - """ - Write content of data to given fname, should defaultonfailure be not None, - it is returned on failure to read. - - If directory of file is not present, it is auto-created. - """ - dirname = os.path.dirname(fname) - if not os.path.isdir(dirname): - os.makedirs(dirname, mode=0o755, exist_ok=False) - chown(dirname, user, group) - - try: - """ Write a file to string """ - bytes = 0 - with open(fname, 'w' if not append else 'a') as f: - bytes = f.write(data) - chown(fname, user, group) - chmod(fname, mode) - return bytes - except Exception as e: - if defaultonfailure is not None: - return defaultonfailure - raise e - -def read_json(fname, defaultonfailure=None): - """ - read and json decode the content of a file - should defaultonfailure be not None, it is returned on failure to read - """ - import json - try: - with open(fname, 'r') as f: - data = json.load(f) - return data - except Exception as e: - if defaultonfailure is not None: - return defaultonfailure - raise e - - -def chown(path, user, group): - """ change file/directory owner """ - from pwd import getpwnam - from grp import getgrnam - - if user is None or group is None: - return False - - # path may also be an open file descriptor - if not isinstance(path, int) and not os.path.exists(path): - return False - - uid = getpwnam(user).pw_uid - gid = getgrnam(group).gr_gid - os.chown(path, uid, gid) - return True - - -def chmod(path, bitmask): - # path may also be an open file descriptor - if not isinstance(path, int) and not os.path.exists(path): - return - if bitmask is None: - return - os.chmod(path, bitmask) - - -def chmod_600(path): - """ make file only read/writable by owner """ - from stat import S_IRUSR, S_IWUSR - - bitmask = S_IRUSR | S_IWUSR - chmod(path, bitmask) - - -def chmod_750(path): - """ make file/directory only executable to user and group """ - from stat import S_IRUSR, S_IWUSR, S_IXUSR, S_IRGRP, S_IXGRP - - bitmask = S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IXGRP - chmod(path, bitmask) - - -def chmod_755(path): - """ make file executable by all """ - from stat import S_IRUSR, S_IWUSR, S_IXUSR, S_IRGRP, S_IXGRP, S_IROTH, S_IXOTH - - bitmask = S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IXGRP | \ - S_IROTH | S_IXOTH - chmod(path, bitmask) - - -def makedir(path, user=None, group=None): - if os.path.exists(path): - return - os.makedirs(path, mode=0o755) - chown(path, user, group) - -def colon_separated_to_dict(data_string, uniquekeys=False): - """ Converts a string containing newline-separated entries - of colon-separated key-value pairs into a dict. - - Such files are common in Linux /proc filesystem - - Args: - data_string (str): data string - uniquekeys (bool): whether to insist that keys are unique or not - - Returns: dict - - Raises: - ValueError: if uniquekeys=True and the data string has - duplicate keys. - - Note: - If uniquekeys=True, then dict entries are always strings, - otherwise they are always lists of strings. - """ - import re - key_value_re = re.compile('([^:]+)\s*\:\s*(.*)') - - data_raw = re.split('\n', data_string) - - data = {} - - for l in data_raw: - l = l.strip() - if l: - match = re.match(key_value_re, l) - if match and (len(match.groups()) == 2): - key = match.groups()[0].strip() - value = match.groups()[1].strip() - else: - raise ValueError(f"""Line "{l}" could not be parsed a colon-separated pair """, l) - if key in data.keys(): - if uniquekeys: - raise ValueError("Data string has duplicate keys: {0}".format(key)) - else: - data[key].append(value) - else: - if uniquekeys: - data[key] = value - else: - data[key] = [value] - else: - pass - - return data - -def _mangle_dict_keys(data, regex, replacement, abs_path=[], no_tag_node_value_mangle=False, mod=0): - """ Mangles dict keys according to a regex and replacement character. - Some libraries like Jinja2 do not like certain characters in dict keys. - This function can be used for replacing all offending characters - with something acceptable. - - Args: - data (dict): Original dict to mangle - - Returns: dict - """ - from vyos.xml import is_tag - - new_dict = {} - - for key in data.keys(): - save_mod = mod - save_path = abs_path[:] - - abs_path.append(key) - - if not is_tag(abs_path): - new_key = re.sub(regex, replacement, key) - else: - if mod%2: - new_key = key - else: - new_key = re.sub(regex, replacement, key) - if no_tag_node_value_mangle: - mod += 1 - - value = data[key] - - if isinstance(value, dict): - new_dict[new_key] = _mangle_dict_keys(value, regex, replacement, abs_path=abs_path, mod=mod, no_tag_node_value_mangle=no_tag_node_value_mangle) - else: - new_dict[new_key] = value - - mod = save_mod - abs_path = save_path[:] - - return new_dict - -def mangle_dict_keys(data, regex, replacement, abs_path=[], no_tag_node_value_mangle=False): - return _mangle_dict_keys(data, regex, replacement, abs_path=abs_path, no_tag_node_value_mangle=no_tag_node_value_mangle, mod=0) - -def _get_sub_dict(d, lpath): - k = lpath[0] - if k not in d.keys(): - return {} - c = {k: d[k]} - lpath = lpath[1:] - if not lpath: - return c - elif not isinstance(c[k], dict): - return {} - return _get_sub_dict(c[k], lpath) - -def get_sub_dict(source, lpath, get_first_key=False): - """ Returns the sub-dict of a nested dict, defined by path of keys. - - Args: - source (dict): Source dict to extract from - lpath (list[str]): sequence of keys - - Returns: source, if lpath is empty, else - {key : source[..]..[key]} for key the last element of lpath, if exists - {} otherwise - """ - if not isinstance(source, dict): - raise TypeError("source must be of type dict") - if not isinstance(lpath, list): - raise TypeError("path must be of type list") - if not lpath: - return source - - ret = _get_sub_dict(source, lpath) - - if get_first_key and lpath and ret: - tmp = next(iter(ret.values())) - if not isinstance(tmp, dict): - raise TypeError("Data under node is not of type dict") - ret = tmp - - return ret - -def process_running(pid_file): - """ Checks if a process with PID in pid_file is running """ - from psutil import pid_exists - if not os.path.isfile(pid_file): - return False - with open(pid_file, 'r') as f: - pid = f.read().strip() - return pid_exists(int(pid)) - -def process_named_running(name, cmdline: str=None): - """ Checks if process with given name is running and returns its PID. - If Process is not running, return None - """ - from psutil import process_iter - for p in process_iter(['name', 'pid', 'cmdline']): - if cmdline: - if p.info['name'] == name and cmdline in p.info['cmdline']: - return p.info['pid'] - elif p.info['name'] == name: - return p.info['pid'] - return None - -def is_list_equal(first: list, second: list) -> bool: - """ Check if 2 lists are equal and list not empty """ - if len(first) != len(second) or len(first) == 0: - return False - return sorted(first) == sorted(second) - -def is_listen_port_bind_service(port: int, service: str) -> bool: - """Check if listen port bound to expected program name - :param port: Bind port - :param service: Program name - :return: bool - - Example: - % is_listen_port_bind_service(443, 'nginx') - True - % is_listen_port_bind_service(443, 'ocserv-main') - False - """ - from psutil import net_connections as connections - from psutil import Process as process - for connection in connections(): - addr = connection.laddr - pid = connection.pid - pid_name = process(pid).name() - pid_port = addr.port - if service == pid_name and port == pid_port: - return True - return False - -def seconds_to_human(s, separator=""): - """ Converts number of seconds passed to a human-readable - interval such as 1w4d18h35m59s - """ - s = int(s) - - week = 60 * 60 * 24 * 7 - day = 60 * 60 * 24 - hour = 60 * 60 - - remainder = 0 - result = "" - - weeks = s // week - if weeks > 0: - result = "{0}w".format(weeks) - s = s % week - - days = s // day - if days > 0: - result = "{0}{1}{2}d".format(result, separator, days) - s = s % day - - hours = s // hour - if hours > 0: - result = "{0}{1}{2}h".format(result, separator, hours) - s = s % hour - - minutes = s // 60 - if minutes > 0: - result = "{0}{1}{2}m".format(result, separator, minutes) - s = s % 60 - - seconds = s - if seconds > 0: - result = "{0}{1}{2}s".format(result, separator, seconds) - - return result - -def bytes_to_human(bytes, initial_exponent=0, precision=2): - """ Converts a value in bytes to a human-readable size string like 640 KB - - The initial_exponent parameter is the exponent of 2, - e.g. 10 (1024) for kilobytes, 20 (1024 * 1024) for megabytes. - """ - - if bytes == 0: - return "0 B" - - from math import log2 - - bytes = bytes * (2**initial_exponent) - - # log2 is a float, while range checking requires an int - exponent = int(log2(bytes)) - - if exponent < 10: - value = bytes - suffix = "B" - elif exponent in range(10, 20): - value = bytes / 1024 - suffix = "KB" - elif exponent in range(20, 30): - value = bytes / 1024**2 - suffix = "MB" - elif exponent in range(30, 40): - value = bytes / 1024**3 - suffix = "GB" - else: - value = bytes / 1024**4 - suffix = "TB" - # Add a new case when the first machine with petabyte RAM - # hits the market. - - size_string = "{0:.{1}f} {2}".format(value, precision, suffix) - return size_string - -def human_to_bytes(value): - """ Converts a data amount with a unit suffix to bytes, like 2K to 2048 """ - - from re import match as re_match - - res = re_match(r'^\s*(\d+(?:\.\d+)?)\s*([a-zA-Z]+)\s*$', value) - - if not res: - raise ValueError(f"'{value}' is not a valid data amount") - else: - amount = float(res.group(1)) - unit = res.group(2).lower() - - if unit == 'b': - res = amount - elif (unit == 'k') or (unit == 'kb'): - res = amount * 1024 - elif (unit == 'm') or (unit == 'mb'): - res = amount * 1024**2 - elif (unit == 'g') or (unit == 'gb'): - res = amount * 1024**3 - elif (unit == 't') or (unit == 'tb'): - res = amount * 1024**4 - else: - raise ValueError(f"Unsupported data unit '{unit}'") - - # There cannot be fractional bytes, so we convert them to integer. - # However, truncating causes problems with conversion back to human unit, - # so we round instead -- that seems to work well enough. - return round(res) - -def get_cfg_group_id(): - from grp import getgrnam - from vyos.defaults import cfg_group - - group_data = getgrnam(cfg_group) - return group_data.gr_gid - - -def file_is_persistent(path): - import re - location = r'^(/config|/opt/vyatta/etc/config)' - absolute = os.path.abspath(os.path.dirname(path)) - return re.match(location,absolute) - -def wait_for_inotify(file_path, pre_hook=None, event_type=None, timeout=None, sleep_interval=0.1): - """ Waits for an inotify event to occur """ - if not os.path.dirname(file_path): - raise ValueError( - "File path {} does not have a directory part (required for inotify watching)".format(file_path)) - if not os.path.basename(file_path): - raise ValueError( - "File path {} does not have a file part, do not know what to watch for".format(file_path)) - - from inotify.adapters import Inotify - from time import time - from time import sleep - - time_start = time() - - i = Inotify() - i.add_watch(os.path.dirname(file_path)) - - if pre_hook: - pre_hook() - - for event in i.event_gen(yield_nones=True): - if (timeout is not None) and ((time() - time_start) > timeout): - # If the function didn't return until this point, - # the file failed to have been written to and closed within the timeout - raise OSError("Waiting for file {} to be written has failed".format(file_path)) - - # Most such events don't take much time, so it's better to check right away - # and sleep later. - if event is not None: - (_, type_names, path, filename) = event - if filename == os.path.basename(file_path): - if event_type in type_names: - return - sleep(sleep_interval) - -def wait_for_file_write_complete(file_path, pre_hook=None, timeout=None, sleep_interval=0.1): - """ Waits for a process to close a file after opening it in write mode. """ - wait_for_inotify(file_path, - event_type='IN_CLOSE_WRITE', pre_hook=pre_hook, timeout=timeout, sleep_interval=sleep_interval) - -def commit_in_progress(): - """ Not to be used in normal op mode scripts! """ - - # The CStore backend locks the config by opening a file - # The file is not removed after commit, so just checking - # if it exists is insufficient, we need to know if it's open by anyone - - # There are two ways to check if any other process keeps a file open. - # The first one is to try opening it and see if the OS objects. - # That's faster but prone to race conditions and can be intrusive. - # The other one is to actually check if any process keeps it open. - # It's non-intrusive but needs root permissions, else you can't check - # processes of other users. - # - # Since this will be used in scripts that modify the config outside of the CLI - # framework, those knowingly have root permissions. - # For everything else, we add a safeguard. - from psutil import process_iter - from psutil import NoSuchProcess - from getpass import getuser - from vyos.defaults import commit_lock - - if getuser() != 'root': - raise OSError('This functions needs to be run as root to return correct results!') - - for proc in process_iter(): - try: - files = proc.open_files() - if files: - for f in files: - if f.path == commit_lock: - return True - except NoSuchProcess as err: - # Process died before we could examine it - pass - # Default case - return False - - -def wait_for_commit_lock(): - """ Not to be used in normal op mode scripts! """ - from time import sleep - # Very synchronous approach to multiprocessing - while commit_in_progress(): - sleep(1) - -def ask_input(question, default='', numeric_only=False, valid_responses=[]): - question_out = question - if default: - question_out += f' (Default: {default})' - response = '' - while True: - response = input(question_out + ' ').strip() - if not response and default: - return default - if numeric_only: - if not response.isnumeric(): - print("Invalid value, try again.") - continue - response = int(response) - if valid_responses and response not in valid_responses: - print("Invalid value, try again.") - continue - break - return response - -def ask_yes_no(question, default=False) -> bool: - """Ask a yes/no question via input() and return their answer.""" - from sys import stdout - default_msg = "[Y/n]" if default else "[y/N]" - while True: - try: - stdout.write("%s %s " % (question, default_msg)) - c = input().lower() - if c == '': - return default - elif c in ("y", "ye", "yes"): - return True - elif c in ("n", "no"): - return False - else: - stdout.write("Please respond with yes/y or no/n\n") - except EOFError: - stdout.write("\nPlease respond with yes/y or no/n\n") - -def is_admin() -> bool: - """Look if current user is in sudo group""" - from getpass import getuser - from grp import getgrnam - current_user = getuser() - (_, _, _, admin_group_members) = getgrnam('sudo') - return current_user in admin_group_members - - -def mac2eui64(mac, prefix=None): - """ - Convert a MAC address to a EUI64 address or, with prefix provided, a full - IPv6 address. - Thankfully copied from https://gist.github.com/wido/f5e32576bb57b5cc6f934e177a37a0d3 - """ - import re - from ipaddress import ip_network - # http://tools.ietf.org/html/rfc4291#section-2.5.1 - eui64 = re.sub(r'[.:-]', '', mac).lower() - eui64 = eui64[0:6] + 'fffe' + eui64[6:] - eui64 = hex(int(eui64[0:2], 16) ^ 2)[2:].zfill(2) + eui64[2:] - - if prefix is None: - return ':'.join(re.findall(r'.{4}', eui64)) - else: - try: - net = ip_network(prefix, strict=False) - euil = int('0x{0}'.format(eui64), 16) - return str(net[euil]) - except: # pylint: disable=bare-except - return - -def get_half_cpus(): - """ return 1/2 of the numbers of available CPUs """ - cpu = os.cpu_count() - if cpu > 1: - cpu /= 2 - return int(cpu) - -def check_kmod(k_mod): - """ Common utility function to load required kernel modules on demand """ - from vyos import ConfigError - if isinstance(k_mod, str): - k_mod = k_mod.split() - for module in k_mod: - if not os.path.exists(f'/sys/module/{module}'): - if call(f'modprobe {module}') != 0: - raise ConfigError(f'Loading Kernel module {module} failed') - -def find_device_file(device): - """ Recurively search /dev for the given device file and return its full path. - If no device file was found 'None' is returned """ - from fnmatch import fnmatch - - for root, dirs, files in os.walk('/dev'): - for basename in files: - if fnmatch(basename, device): - return os.path.join(root, basename) - - return None - -def dict_search(path, dict_object): - """ Traverse Python dictionary (dict_object) delimited by dot (.). - Return value of key if found, None otherwise. - - This is faster implementation then jmespath.search('foo.bar', dict_object)""" - if not isinstance(dict_object, dict) or not path: - return None - - parts = path.split('.') - inside = parts[:-1] - if not inside: - if path not in dict_object: - return None - return dict_object[path] - c = dict_object - for p in parts[:-1]: - c = c.get(p, {}) - return c.get(parts[-1], None) - -def dict_search_args(dict_object, *path): - # Traverse dictionary using variable arguments - # Added due to above function not allowing for '.' in the key names - # Example: dict_search_args(some_dict, 'key', 'subkey', 'subsubkey', ...) - if not isinstance(dict_object, dict) or not path: - return None - - for item in path: - if item not in dict_object: - return None - dict_object = dict_object[item] - return dict_object - -def dict_search_recursive(dict_object, key, path=[]): - """ Traverse a dictionary recurisvely and return the value of the key - we are looking for. - - Thankfully copied from https://stackoverflow.com/a/19871956 - - Modified to yield optional path to found keys - """ - if isinstance(dict_object, list): - for i in dict_object: - new_path = path + [i] - for x in dict_search_recursive(i, key, new_path): - yield x - elif isinstance(dict_object, dict): - if key in dict_object: - new_path = path + [key] - yield dict_object[key], new_path - for k, j in dict_object.items(): - new_path = path + [k] - for x in dict_search_recursive(j, key, new_path): - yield x - -def convert_data(data): - """Convert multiple types of data to types usable in CLI - - Args: - data (str | bytes | list | OrderedDict): input data - - Returns: - str | list | dict: converted data - """ - from base64 import b64encode - from collections import OrderedDict - - if isinstance(data, str): - return data - if isinstance(data, bytes): - try: - return data.decode() - except UnicodeDecodeError: - return b64encode(data).decode() - if isinstance(data, list): - list_tmp = [] - for item in data: - list_tmp.append(convert_data(item)) - return list_tmp - if isinstance(data, OrderedDict): - dict_tmp = {} - for key, value in data.items(): - dict_tmp[key] = convert_data(value) - return dict_tmp - -def get_bridge_fdb(interface): - """ Returns the forwarding database entries for a given interface """ - if not os.path.exists(f'/sys/class/net/{interface}'): - return None - from json import loads - tmp = loads(cmd(f'bridge -j fdb show dev {interface}')) - return tmp - -def get_interface_config(interface): - """ Returns the used encapsulation protocol for given interface. - If interface does not exist, None is returned. - """ - if not os.path.exists(f'/sys/class/net/{interface}'): - return None - from json import loads - tmp = loads(cmd(f'ip -d -j link show {interface}'))[0] - return tmp - -def get_interface_address(interface): - """ Returns the used encapsulation protocol for given interface. - If interface does not exist, None is returned. - """ - if not os.path.exists(f'/sys/class/net/{interface}'): - return None - from json import loads - tmp = loads(cmd(f'ip -d -j addr show {interface}'))[0] - return tmp - -def get_interface_namespace(iface): - """ - Returns wich netns the interface belongs to - """ - from json import loads - # Check if netns exist - tmp = loads(cmd(f'ip --json netns ls')) - if len(tmp) == 0: - return None - - for ns in tmp: - namespace = f'{ns["name"]}' - # Search interface in each netns - data = loads(cmd(f'ip netns exec {namespace} ip -j link show')) - for compare in data: - if iface == compare["ifname"]: - return namespace - -def get_all_vrfs(): - """ Return a dictionary of all system wide known VRF instances """ - from json import loads - tmp = loads(cmd('ip -j vrf list')) - # Result is of type [{"name":"red","table":1000},{"name":"blue","table":2000}] - # so we will re-arrange it to a more nicer representation: - # {'red': {'table': 1000}, 'blue': {'table': 2000}} - data = {} - for entry in tmp: - name = entry.pop('name') - data[name] = entry - return data - -def print_error(str='', end='\n'): - """ - Print `str` to stderr, terminated with `end`. - Used for warnings and out-of-band messages to avoid mangling precious - stdout output. - """ - sys.stderr.write(str) - sys.stderr.write(end) - sys.stderr.flush() - -def make_progressbar(): - """ - Make a procedure that takes two arguments `done` and `total` and prints a - progressbar based on the ratio thereof, whose length is determined by the - width of the terminal. - """ - import shutil, math - col, _ = shutil.get_terminal_size() - col = max(col - 15, 20) - def print_progressbar(done, total): - if done <= total: - increment = total / col - length = math.ceil(done / increment) - percentage = str(math.ceil(100 * done / total)).rjust(3) - print_error(f'[{length * "#"}{(col - length) * "_"}] {percentage}%', '\r') - # Print a newline so that the subsequent prints don't overwrite the full bar. - if done == total: - print_error() - return print_progressbar - -def make_incremental_progressbar(increment: float): - """ - Make a generator that displays a progressbar that grows monotonically with - every iteration. - First call displays it at 0% and every subsequent iteration displays it - at `increment` increments where 0.0 < `increment` < 1.0. - Intended for FTP and HTTP transfers with stateless callbacks. - """ - print_progressbar = make_progressbar() - total = 0.0 - while total < 1.0: - print_progressbar(total, 1.0) - yield - total += increment - print_progressbar(1, 1) - # Ignore further calls. - while True: - yield - -def begin(*args): - """ - Evaluate arguments in order and return the result of the *last* argument. - For combining multiple expressions in one statement. Useful for lambdas. - """ - return args[-1] - -def begin0(*args): - """ - Evaluate arguments in order and return the result of the *first* argument. - For combining multiple expressions in one statement. Useful for lambdas. - """ - return args[0] - -def is_systemd_service_active(service): - """ Test is a specified systemd service is activated. - Returns True if service is active, false otherwise. - Copied from: https://unix.stackexchange.com/a/435317 """ - tmp = cmd(f'systemctl show --value -p ActiveState {service}') - return bool((tmp == 'active')) - -def is_systemd_service_running(service): - """ Test is a specified systemd service is actually running. - Returns True if service is running, false otherwise. - Copied from: https://unix.stackexchange.com/a/435317 """ - tmp = cmd(f'systemctl show --value -p SubState {service}') - return bool((tmp == 'running')) - -def check_port_availability(ipaddress, port, protocol): - """ - Check if port is available and not used by any service - Return False if a port is busy or IP address does not exists - Should be used carefully for services that can start listening - dynamically, because IP address may be dynamic too - """ - from socketserver import TCPServer, UDPServer - from ipaddress import ip_address - - # verify arguments - try: - ipaddress = ip_address(ipaddress).compressed - except: - raise ValueError(f'The {ipaddress} is not a valid IPv4 or IPv6 address') - if port not in range(1, 65536): - raise ValueError(f'The port number {port} is not in the 1-65535 range') - if protocol not in ['tcp', 'udp']: - raise ValueError( - f'The protocol {protocol} is not supported. Only tcp and udp are allowed' - ) - - # check port availability - try: - if protocol == 'tcp': - server = TCPServer((ipaddress, port), None, bind_and_activate=True) - if protocol == 'udp': - server = UDPServer((ipaddress, port), None, bind_and_activate=True) - server.server_close() - except Exception as e: - # errno.h: - #define EADDRINUSE 98 /* Address already in use */ - if e.errno == 98: - return False - - return True - -def install_into_config(conf, config_paths, override_prompt=True): - # Allows op-mode scripts to install values if called from an active config session - # config_paths: dict of config paths - # override_prompt: if True, user will be prompted before existing nodes are overwritten - - if not config_paths: - return None - - from vyos.config import Config - - if not Config().in_session(): - print('You are not in configure mode, commands to install manually from configure mode:') - for path in config_paths: - print(f'set {path}') - return None - - count = 0 - failed = [] - - for path in config_paths: - if override_prompt and conf.exists(path) and not conf.is_multi(path): - if not ask_yes_no(f'Config node "{node}" already exists. Do you want to overwrite it?'): - continue - - try: - cmd(f'/opt/vyatta/sbin/my_set {path}') - count += 1 - except: - failed.append(path) - - if failed: - print(f'Failed to install {len(failed)} value(s). Commands to manually install:') - for path in failed: - print(f'set {path}') - - if count > 0: - print(f'{count} value(s) installed. Use "compare" to see the pending changes, and "commit" to apply.') - -def is_wwan_connected(interface): - """ Determine if a given WWAN interface, e.g. wwan0 is connected to the - carrier network or not """ - import json - - if not interface.startswith('wwan'): - raise ValueError(f'Specified interface "{interface}" is not a WWAN interface') - - # ModemManager is required for connection(s) - if service is not running, - # there won't be any connection at all! - if not is_systemd_service_active('ModemManager.service'): - return False - - modem = interface.lstrip('wwan') - - tmp = cmd(f'mmcli --modem {modem} --output-json') - tmp = json.loads(tmp) - - # return True/False if interface is in connected state - return dict_search('modem.generic.state', tmp) == 'connected' - -def boot_configuration_complete() -> bool: - """ Check if the boot config loader has completed - """ - from vyos.defaults import config_status - - if os.path.isfile(config_status): - return True - return False - -def boot_configuration_success() -> bool: - from vyos.defaults import config_status - - try: - with open(config_status) as f: - res = f.read().strip() - except FileNotFoundError: - return False - - if int(res) == 0: - return True - return False - -def sysctl_read(name): - """ Read and return current value of sysctl() option """ - tmp = cmd(f'sysctl {name}') - return tmp.split()[-1] - -def sysctl_write(name, value): - """ Change value via sysctl() - return True if changed, False otherwise """ - tmp = cmd(f'sysctl {name}') - # last list index contains the actual value - only write if value differs - if sysctl_read(name) != str(value): - call(f'sysctl -wq {name}={value}') - return True - return False - -def load_as_module(name: str, path: str): - import importlib.util - - spec = importlib.util.spec_from_file_location(name, path) - mod = importlib.util.module_from_spec(spec) - spec.loader.exec_module(mod) - return mod diff --git a/python/vyos/utils/__init__.py b/python/vyos/utils/__init__.py index 0d3998053..f2783113a 100644 --- a/python/vyos/utils/__init__.py +++ b/python/vyos/utils/__init__.py @@ -13,4 +13,17 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library. If not, see <http://www.gnu.org/licenses/>. +from vyos.utils import auth +from vyos.utils import boot +from vyos.utils import commit +from vyos.utils import convert +from vyos.utils import dict +from vyos.utils import file +from vyos.utils import io +from vyos.utils import kernel +from vyos.utils import list +from vyos.utils import misc from vyos.utils import network +from vyos.utils import permission +from vyos.utils import process +from vyos.utils import system diff --git a/python/vyos/authutils.py b/python/vyos/utils/auth.py index 66b5f4a74..a59858d72 100644 --- a/python/vyos/authutils.py +++ b/python/vyos/utils/auth.py @@ -15,7 +15,7 @@ import re -from vyos.util import cmd +from vyos.utils.process import cmd def make_password_hash(password): diff --git a/python/vyos/utils/boot.py b/python/vyos/utils/boot.py new file mode 100644 index 000000000..3aecbec64 --- /dev/null +++ b/python/vyos/utils/boot.py @@ -0,0 +1,35 @@ +# Copyright 2023 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + +import os + +def boot_configuration_complete() -> bool: + """ Check if the boot config loader has completed + """ + from vyos.defaults import config_status + if os.path.isfile(config_status): + return True + return False + +def boot_configuration_success() -> bool: + from vyos.defaults import config_status + try: + with open(config_status) as f: + res = f.read().strip() + except FileNotFoundError: + return False + if int(res) == 0: + return True + return False diff --git a/python/vyos/utils/commit.py b/python/vyos/utils/commit.py new file mode 100644 index 000000000..105aed8c2 --- /dev/null +++ b/python/vyos/utils/commit.py @@ -0,0 +1,60 @@ +# Copyright 2023 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + +def commit_in_progress(): + """ Not to be used in normal op mode scripts! """ + + # The CStore backend locks the config by opening a file + # The file is not removed after commit, so just checking + # if it exists is insufficient, we need to know if it's open by anyone + + # There are two ways to check if any other process keeps a file open. + # The first one is to try opening it and see if the OS objects. + # That's faster but prone to race conditions and can be intrusive. + # The other one is to actually check if any process keeps it open. + # It's non-intrusive but needs root permissions, else you can't check + # processes of other users. + # + # Since this will be used in scripts that modify the config outside of the CLI + # framework, those knowingly have root permissions. + # For everything else, we add a safeguard. + from psutil import process_iter + from psutil import NoSuchProcess + from getpass import getuser + from vyos.defaults import commit_lock + + if getuser() != 'root': + raise OSError('This functions needs to be run as root to return correct results!') + + for proc in process_iter(): + try: + files = proc.open_files() + if files: + for f in files: + if f.path == commit_lock: + return True + except NoSuchProcess as err: + # Process died before we could examine it + pass + # Default case + return False + + +def wait_for_commit_lock(): + """ Not to be used in normal op mode scripts! """ + from time import sleep + # Very synchronous approach to multiprocessing + while commit_in_progress(): + sleep(1) diff --git a/python/vyos/utils/convert.py b/python/vyos/utils/convert.py index 975c67e0a..ec2333ef0 100644 --- a/python/vyos/utils/convert.py +++ b/python/vyos/utils/convert.py @@ -143,3 +143,33 @@ def mac_to_eui64(mac, prefix=None): return str(net[euil]) except: # pylint: disable=bare-except return + +def convert_data(data): + """Convert multiple types of data to types usable in CLI + + Args: + data (str | bytes | list | OrderedDict): input data + + Returns: + str | list | dict: converted data + """ + from base64 import b64encode + from collections import OrderedDict + + if isinstance(data, str): + return data + if isinstance(data, bytes): + try: + return data.decode() + except UnicodeDecodeError: + return b64encode(data).decode() + if isinstance(data, list): + list_tmp = [] + for item in data: + list_tmp.append(convert_data(item)) + return list_tmp + if isinstance(data, OrderedDict): + dict_tmp = {} + for key, value in data.items(): + dict_tmp[key] = convert_data(value) + return dict_tmp diff --git a/python/vyos/utils/dict.py b/python/vyos/utils/dict.py index 28d32bb8d..9484eacdd 100644 --- a/python/vyos/utils/dict.py +++ b/python/vyos/utils/dict.py @@ -13,7 +13,6 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library. If not, see <http://www.gnu.org/licenses/>. - def colon_separated_to_dict(data_string, uniquekeys=False): """ Converts a string containing newline-separated entries of colon-separated key-value pairs into a dict. @@ -87,7 +86,7 @@ def mangle_dict_keys(data, regex, replacement, abs_path=None, no_tag_node_value_ if abs_path is None: abs_path = [] - new_dict = {} + new_dict = type(data)() for k in data.keys(): if no_tag_node_value_mangle and is_tag_value(abs_path + [k]): @@ -270,3 +269,39 @@ def check_mutually_exclusive_options(d, keys, required=False): if required and (len(present_keys) < 1): raise ValueError(f"At least one of the following options is required: {orig_keys}") + +class FixedDict(dict): + """ + FixedDict: A dictionnary not allowing new keys to be created after initialisation. + + >>> f = FixedDict(**{'count':1}) + >>> f['count'] = 2 + >>> f['king'] = 3 + File "...", line ..., in __setitem__ + raise ConfigError(f'Option "{k}" has no defined default') + """ + + from vyos import ConfigError + + def __init__(self, **options): + self._allowed = options.keys() + super().__init__(**options) + + def __setitem__(self, k, v): + """ + __setitem__ is a builtin which is called by python when setting dict values: + >>> d = dict() + >>> d['key'] = 'value' + >>> d + {'key': 'value'} + + is syntaxic sugar for + + >>> d = dict() + >>> d.__setitem__('key','value') + >>> d + {'key': 'value'} + """ + if k not in self._allowed: + raise ConfigError(f'Option "{k}" has no defined default') + super().__setitem__(k, v) diff --git a/python/vyos/utils/file.py b/python/vyos/utils/file.py index 2560a35be..667a2464b 100644 --- a/python/vyos/utils/file.py +++ b/python/vyos/utils/file.py @@ -14,7 +14,19 @@ # License along with this library. If not, see <http://www.gnu.org/licenses/>. import os +from vyos.utils.permission import chown +def makedir(path, user=None, group=None): + if os.path.exists(path): + return + os.makedirs(path, mode=0o755) + chown(path, user, group) + +def file_is_persistent(path): + import re + location = r'^(/config|/opt/vyatta/etc/config)' + absolute = os.path.abspath(os.path.dirname(path)) + return re.match(location,absolute) def read_file(fname, defaultonfailure=None): """ diff --git a/python/vyos/utils/kernel.py b/python/vyos/utils/kernel.py new file mode 100644 index 000000000..0eb113174 --- /dev/null +++ b/python/vyos/utils/kernel.py @@ -0,0 +1,27 @@ +# Copyright 2023 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + +import os + +def check_kmod(k_mod): + """ Common utility function to load required kernel modules on demand """ + from vyos import ConfigError + from vyos.utils.process import call + if isinstance(k_mod, str): + k_mod = k_mod.split() + for module in k_mod: + if not os.path.exists(f'/sys/module/{module}'): + if call(f'modprobe {module}') != 0: + raise ConfigError(f'Loading Kernel module {module} failed') diff --git a/python/vyos/utils/list.py b/python/vyos/utils/list.py new file mode 100644 index 000000000..63ef720ab --- /dev/null +++ b/python/vyos/utils/list.py @@ -0,0 +1,20 @@ +# Copyright 2023 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + +def is_list_equal(first: list, second: list) -> bool: + """ Check if 2 lists are equal and list not empty """ + if len(first) != len(second) or len(first) == 0: + return False + return sorted(first) == sorted(second) diff --git a/python/vyos/utils/misc.py b/python/vyos/utils/misc.py new file mode 100644 index 000000000..d82655914 --- /dev/null +++ b/python/vyos/utils/misc.py @@ -0,0 +1,66 @@ +# Copyright 2023 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + +def begin(*args): + """ + Evaluate arguments in order and return the result of the *last* argument. + For combining multiple expressions in one statement. Useful for lambdas. + """ + return args[-1] + +def begin0(*args): + """ + Evaluate arguments in order and return the result of the *first* argument. + For combining multiple expressions in one statement. Useful for lambdas. + """ + return args[0] + +def install_into_config(conf, config_paths, override_prompt=True): + # Allows op-mode scripts to install values if called from an active config session + # config_paths: dict of config paths + # override_prompt: if True, user will be prompted before existing nodes are overwritten + if not config_paths: + return None + + from vyos.config import Config + from vyos.utils.io import ask_yes_no + from vyos.utils.process import cmd + if not Config().in_session(): + print('You are not in configure mode, commands to install manually from configure mode:') + for path in config_paths: + print(f'set {path}') + return None + + count = 0 + failed = [] + + for path in config_paths: + if override_prompt and conf.exists(path) and not conf.is_multi(path): + if not ask_yes_no(f'Config node "{node}" already exists. Do you want to overwrite it?'): + continue + + try: + cmd(f'/opt/vyatta/sbin/my_set {path}') + count += 1 + except: + failed.append(path) + + if failed: + print(f'Failed to install {len(failed)} value(s). Commands to manually install:') + for path in failed: + print(f'set {path}') + + if count > 0: + print(f'{count} value(s) installed. Use "compare" to see the pending changes, and "commit" to apply.') diff --git a/python/vyos/utils/network.py b/python/vyos/utils/network.py index 72b7ca6da..3786caf26 100644 --- a/python/vyos/utils/network.py +++ b/python/vyos/utils/network.py @@ -15,7 +15,6 @@ import os - def get_protocol_by_name(protocol_name): """Get protocol number by protocol name @@ -28,3 +27,187 @@ def get_protocol_by_name(protocol_name): return protocol_number except socket.error: return protocol_name + +def interface_exists_in_netns(interface_name, netns): + from vyos.utils.process import rc_cmd + rc, out = rc_cmd(f'ip netns exec {netns} ip link show dev {interface_name}') + if rc == 0: + return True + return False + +def get_interface_vrf(interface): + """ Returns VRF of given interface """ + from vyos.utils.dict import dict_search + from vyos.utils.network import get_interface_config + tmp = get_interface_config(interface) + if dict_search('linkinfo.info_slave_kind', tmp) == 'vrf': + return tmp['master'] + return 'default' + +def get_interface_config(interface): + """ Returns the used encapsulation protocol for given interface. + If interface does not exist, None is returned. + """ + if not os.path.exists(f'/sys/class/net/{interface}'): + return None + from json import loads + from vyos.utils.process import cmd + tmp = loads(cmd(f'ip --detail --json link show dev {interface}'))[0] + return tmp + +def get_interface_address(interface): + """ Returns the used encapsulation protocol for given interface. + If interface does not exist, None is returned. + """ + if not os.path.exists(f'/sys/class/net/{interface}'): + return None + from json import loads + from vyos.utils.process import cmd + tmp = loads(cmd(f'ip --detail --json addr show dev {interface}'))[0] + return tmp + +def get_interface_namespace(iface): + """ + Returns wich netns the interface belongs to + """ + from json import loads + from vyos.utils.process import cmd + # Check if netns exist + tmp = loads(cmd(f'ip --json netns ls')) + if len(tmp) == 0: + return None + + for ns in tmp: + netns = f'{ns["name"]}' + # Search interface in each netns + data = loads(cmd(f'ip netns exec {netns} ip --json link show')) + for tmp in data: + if iface == tmp["ifname"]: + return netns + + +def is_wwan_connected(interface): + """ Determine if a given WWAN interface, e.g. wwan0 is connected to the + carrier network or not """ + import json + from vyos.utils.process import cmd + + if not interface.startswith('wwan'): + raise ValueError(f'Specified interface "{interface}" is not a WWAN interface') + + # ModemManager is required for connection(s) - if service is not running, + # there won't be any connection at all! + if not is_systemd_service_active('ModemManager.service'): + return False + + modem = interface.lstrip('wwan') + + tmp = cmd(f'mmcli --modem {modem} --output-json') + tmp = json.loads(tmp) + + # return True/False if interface is in connected state + return dict_search('modem.generic.state', tmp) == 'connected' + +def get_bridge_fdb(interface): + """ Returns the forwarding database entries for a given interface """ + if not os.path.exists(f'/sys/class/net/{interface}'): + return None + from json import loads + from vyos.utils.process import cmd + tmp = loads(cmd(f'bridge -j fdb show dev {interface}')) + return tmp + +def get_all_vrfs(): + """ Return a dictionary of all system wide known VRF instances """ + from json import loads + from vyos.utils.process import cmd + tmp = loads(cmd('ip --json vrf list')) + # Result is of type [{"name":"red","table":1000},{"name":"blue","table":2000}] + # so we will re-arrange it to a more nicer representation: + # {'red': {'table': 1000}, 'blue': {'table': 2000}} + data = {} + for entry in tmp: + name = entry.pop('name') + data[name] = entry + return data + +def mac2eui64(mac, prefix=None): + """ + Convert a MAC address to a EUI64 address or, with prefix provided, a full + IPv6 address. + Thankfully copied from https://gist.github.com/wido/f5e32576bb57b5cc6f934e177a37a0d3 + """ + import re + from ipaddress import ip_network + # http://tools.ietf.org/html/rfc4291#section-2.5.1 + eui64 = re.sub(r'[.:-]', '', mac).lower() + eui64 = eui64[0:6] + 'fffe' + eui64[6:] + eui64 = hex(int(eui64[0:2], 16) ^ 2)[2:].zfill(2) + eui64[2:] + + if prefix is None: + return ':'.join(re.findall(r'.{4}', eui64)) + else: + try: + net = ip_network(prefix, strict=False) + euil = int('0x{0}'.format(eui64), 16) + return str(net[euil]) + except: # pylint: disable=bare-except + return + +def check_port_availability(ipaddress, port, protocol): + """ + Check if port is available and not used by any service + Return False if a port is busy or IP address does not exists + Should be used carefully for services that can start listening + dynamically, because IP address may be dynamic too + """ + from socketserver import TCPServer, UDPServer + from ipaddress import ip_address + + # verify arguments + try: + ipaddress = ip_address(ipaddress).compressed + except: + raise ValueError(f'The {ipaddress} is not a valid IPv4 or IPv6 address') + if port not in range(1, 65536): + raise ValueError(f'The port number {port} is not in the 1-65535 range') + if protocol not in ['tcp', 'udp']: + raise ValueError(f'The protocol {protocol} is not supported. Only tcp and udp are allowed') + + # check port availability + try: + if protocol == 'tcp': + server = TCPServer((ipaddress, port), None, bind_and_activate=True) + if protocol == 'udp': + server = UDPServer((ipaddress, port), None, bind_and_activate=True) + server.server_close() + except Exception as e: + # errno.h: + #define EADDRINUSE 98 /* Address already in use */ + if e.errno == 98: + return False + + return True + +def is_listen_port_bind_service(port: int, service: str) -> bool: + """Check if listen port bound to expected program name + :param port: Bind port + :param service: Program name + :return: bool + + Example: + % is_listen_port_bind_service(443, 'nginx') + True + % is_listen_port_bind_service(443, 'ocserv-main') + False + """ + from psutil import net_connections as connections + from psutil import Process as process + for connection in connections(): + addr = connection.laddr + pid = connection.pid + pid_name = process(pid).name() + pid_port = addr.port + if service == pid_name and port == pid_port: + return True + return False diff --git a/python/vyos/utils/permission.py b/python/vyos/utils/permission.py new file mode 100644 index 000000000..d938b494f --- /dev/null +++ b/python/vyos/utils/permission.py @@ -0,0 +1,78 @@ +# Copyright 2023 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + +import os + +def chown(path, user, group): + """ change file/directory owner """ + from pwd import getpwnam + from grp import getgrnam + + if user is None or group is None: + return False + + # path may also be an open file descriptor + if not isinstance(path, int) and not os.path.exists(path): + return False + + uid = getpwnam(user).pw_uid + gid = getgrnam(group).gr_gid + os.chown(path, uid, gid) + return True + +def chmod(path, bitmask): + # path may also be an open file descriptor + if not isinstance(path, int) and not os.path.exists(path): + return + if bitmask is None: + return + os.chmod(path, bitmask) + +def chmod_600(path): + """ make file only read/writable by owner """ + from stat import S_IRUSR, S_IWUSR + + bitmask = S_IRUSR | S_IWUSR + chmod(path, bitmask) + +def chmod_750(path): + """ make file/directory only executable to user and group """ + from stat import S_IRUSR, S_IWUSR, S_IXUSR, S_IRGRP, S_IXGRP + + bitmask = S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IXGRP + chmod(path, bitmask) + +def chmod_755(path): + """ make file executable by all """ + from stat import S_IRUSR, S_IWUSR, S_IXUSR, S_IRGRP, S_IXGRP, S_IROTH, S_IXOTH + + bitmask = S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IXGRP | \ + S_IROTH | S_IXOTH + chmod(path, bitmask) + +def is_admin() -> bool: + """Look if current user is in sudo group""" + from getpass import getuser + from grp import getgrnam + current_user = getuser() + (_, _, _, admin_group_members) = getgrnam('sudo') + return current_user in admin_group_members + +def get_cfg_group_id(): + from grp import getgrnam + from vyos.defaults import cfg_group + + group_data = getgrnam(cfg_group) + return group_data.gr_gid diff --git a/python/vyos/utils/process.py b/python/vyos/utils/process.py new file mode 100644 index 000000000..911547995 --- /dev/null +++ b/python/vyos/utils/process.py @@ -0,0 +1,232 @@ +# Copyright 2023 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + +import os + +from subprocess import Popen +from subprocess import PIPE +from subprocess import STDOUT +from subprocess import DEVNULL + +def popen(command, flag='', shell=None, input=None, timeout=None, env=None, + stdout=PIPE, stderr=PIPE, decode='utf-8'): + """ + popen is a wrapper helper aound subprocess.Popen + with it default setting it will return a tuple (out, err) + out: the output of the program run + err: the error code returned by the program + + it can be affected by the following flags: + shell: do not try to auto-detect if a shell is required + for example if a pipe (|) or redirection (>, >>) is used + input: data to sent to the child process via STDIN + the data should be bytes but string will be converted + timeout: time after which the command will be considered to have failed + env: mapping that defines the environment variables for the new process + stdout: define how the output of the program should be handled + - PIPE (default), sends stdout to the output + - DEVNULL, discard the output + stderr: define how the output of the program should be handled + - None (default), send/merge the data to/with stderr + - PIPE, popen will append it to output + - STDOUT, send the data to be merged with stdout + - DEVNULL, discard the output + decode: specify the expected text encoding (utf-8, ascii, ...) + the default is explicitely utf-8 which is python's own default + + usage: + get both stdout and stderr: popen('command', stdout=PIPE, stderr=STDOUT) + discard stdout and get stderr: popen('command', stdout=DEVNUL, stderr=PIPE) + """ + + # airbag must be left as an import in the function as otherwise we have a + # a circual import dependency + from vyos import debug + from vyos import airbag + + # log if the flag is set, otherwise log if command is set + if not debug.enabled(flag): + flag = 'command' + + cmd_msg = f"cmd '{command}'" + debug.message(cmd_msg, flag) + + use_shell = shell + stdin = None + if shell is None: + use_shell = False + if ' ' in command: + use_shell = True + if env: + use_shell = True + + if input: + stdin = PIPE + input = input.encode() if type(input) is str else input + + p = Popen(command, stdin=stdin, stdout=stdout, stderr=stderr, + env=env, shell=use_shell) + + pipe = p.communicate(input, timeout) + + pipe_out = b'' + if stdout == PIPE: + pipe_out = pipe[0] + + pipe_err = b'' + if stderr == PIPE: + pipe_err = pipe[1] + + str_out = pipe_out.decode(decode).replace('\r\n', '\n').strip() + str_err = pipe_err.decode(decode).replace('\r\n', '\n').strip() + + out_msg = f"returned (out):\n{str_out}" + if str_out: + debug.message(out_msg, flag) + + if str_err: + from sys import stderr + err_msg = f"returned (err):\n{str_err}" + # this message will also be send to syslog via airbag + debug.message(err_msg, flag, destination=stderr) + + # should something go wrong, report this too via airbag + airbag.noteworthy(cmd_msg) + airbag.noteworthy(out_msg) + airbag.noteworthy(err_msg) + + return str_out, p.returncode + + +def run(command, flag='', shell=None, input=None, timeout=None, env=None, + stdout=DEVNULL, stderr=PIPE, decode='utf-8'): + """ + A wrapper around popen, which discard the stdout and + will return the error code of a command + """ + _, code = popen( + command, flag, + stdout=stdout, stderr=stderr, + input=input, timeout=timeout, + env=env, shell=shell, + decode=decode, + ) + return code + + +def cmd(command, flag='', shell=None, input=None, timeout=None, env=None, + stdout=PIPE, stderr=PIPE, decode='utf-8', raising=None, message='', + expect=[0]): + """ + A wrapper around popen, which returns the stdout and + will raise the error code of a command + + raising: specify which call should be used when raising + the class should only require a string as parameter + (default is OSError) with the error code + expect: a list of error codes to consider as normal + """ + decoded, code = popen( + command, flag, + stdout=stdout, stderr=stderr, + input=input, timeout=timeout, + env=env, shell=shell, + decode=decode, + ) + if code not in expect: + feedback = message + '\n' if message else '' + feedback += f'failed to run command: {command}\n' + feedback += f'returned: {decoded}\n' + feedback += f'exit code: {code}' + if raising is None: + # error code can be recovered with .errno + raise OSError(code, feedback) + else: + raise raising(feedback) + return decoded + + +def rc_cmd(command, flag='', shell=None, input=None, timeout=None, env=None, + stdout=PIPE, stderr=STDOUT, decode='utf-8'): + """ + A wrapper around popen, which returns the return code + of a command and stdout + + % rc_cmd('uname') + (0, 'Linux') + % rc_cmd('ip link show dev eth99') + (1, 'Device "eth99" does not exist.') + """ + out, code = popen( + command, flag, + stdout=stdout, stderr=stderr, + input=input, timeout=timeout, + env=env, shell=shell, + decode=decode, + ) + return code, out + +def call(command, flag='', shell=None, input=None, timeout=None, env=None, + stdout=PIPE, stderr=PIPE, decode='utf-8'): + """ + A wrapper around popen, which print the stdout and + will return the error code of a command + """ + out, code = popen( + command, flag, + stdout=stdout, stderr=stderr, + input=input, timeout=timeout, + env=env, shell=shell, + decode=decode, + ) + if out: + print(out) + return code + +def process_running(pid_file): + """ Checks if a process with PID in pid_file is running """ + from psutil import pid_exists + if not os.path.isfile(pid_file): + return False + with open(pid_file, 'r') as f: + pid = f.read().strip() + return pid_exists(int(pid)) + +def process_named_running(name, cmdline: str=None): + """ Checks if process with given name is running and returns its PID. + If Process is not running, return None + """ + from psutil import process_iter + for p in process_iter(['name', 'pid', 'cmdline']): + if cmdline: + if p.info['name'] == name and cmdline in p.info['cmdline']: + return p.info['pid'] + elif p.info['name'] == name: + return p.info['pid'] + return None + +def is_systemd_service_active(service): + """ Test is a specified systemd service is activated. + Returns True if service is active, false otherwise. + Copied from: https://unix.stackexchange.com/a/435317 """ + tmp = cmd(f'systemctl show --value -p ActiveState {service}') + return bool((tmp == 'active')) + +def is_systemd_service_running(service): + """ Test is a specified systemd service is actually running. + Returns True if service is running, false otherwise. + Copied from: https://unix.stackexchange.com/a/435317 """ + tmp = cmd(f'systemctl show --value -p SubState {service}') + return bool((tmp == 'running')) diff --git a/python/vyos/utils/system.py b/python/vyos/utils/system.py new file mode 100644 index 000000000..5d41c0c05 --- /dev/null +++ b/python/vyos/utils/system.py @@ -0,0 +1,107 @@ +# Copyright 2023 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + +import os +from subprocess import run + +def sysctl_read(name: str) -> str: + """Read and return current value of sysctl() option + + Args: + name (str): sysctl key name + + Returns: + str: sysctl key value + """ + tmp = run(['sysctl', '-nb', name], capture_output=True) + return tmp.stdout.decode() + +def sysctl_write(name: str, value: str | int) -> bool: + """Change value via sysctl() + + Args: + name (str): sysctl key name + value (str | int): sysctl key value + + Returns: + bool: True if changed, False otherwise + """ + # convert other types to string before comparison + if not isinstance(value, str): + value = str(value) + # do not change anything if a value is already configured + if sysctl_read(name) == value: + return True + # return False if sysctl call failed + if run(['sysctl', '-wq', f'{name}={value}']).returncode != 0: + return False + # compare old and new values + # sysctl may apply value, but its actual value will be + # different from requested + if sysctl_read(name) == value: + return True + # False in other cases + return False + +def sysctl_apply(sysctl_dict: dict[str, str], revert: bool = True) -> bool: + """Apply sysctl values. + + Args: + sysctl_dict (dict[str, str]): dictionary with sysctl keys with values + revert (bool, optional): Revert to original values if new were not + applied. Defaults to True. + + Returns: + bool: True if all params configured properly, False in other cases + """ + # get current values + sysctl_original: dict[str, str] = {} + for key_name in sysctl_dict.keys(): + sysctl_original[key_name] = sysctl_read(key_name) + # apply new values and revert in case one of them was not applied + for key_name, value in sysctl_dict.items(): + if not sysctl_write(key_name, value): + if revert: + sysctl_apply(sysctl_original, revert=False) + return False + # everything applied + return True + +def get_half_cpus(): + """ return 1/2 of the numbers of available CPUs """ + cpu = os.cpu_count() + if cpu > 1: + cpu /= 2 + return int(cpu) + +def find_device_file(device): + """ Recurively search /dev for the given device file and return its full path. + If no device file was found 'None' is returned """ + from fnmatch import fnmatch + + for root, dirs, files in os.walk('/dev'): + for basename in files: + if fnmatch(basename, device): + return os.path.join(root, basename) + + return None + +def load_as_module(name: str, path: str): + import importlib.util + + spec = importlib.util.spec_from_file_location(name, path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod diff --git a/python/vyos/validate.py b/python/vyos/validate.py index e5d8c6043..567f4c972 100644 --- a/python/vyos/validate.py +++ b/python/vyos/validate.py @@ -100,8 +100,9 @@ def is_intf_addr_assigned(intf, address) -> bool: def is_addr_assigned(ip_address, vrf=None) -> bool: """ Verify if the given IPv4/IPv6 address is assigned to any interface """ from netifaces import interfaces - from vyos.util import get_interface_config - from vyos.util import dict_search + from vyos.utils.network import get_interface_config + from vyos.utils.dict import dict_search + for interface in interfaces(): # Check if interface belongs to the requested VRF, if this is not the # case there is no need to proceed with this data set - continue loop @@ -218,7 +219,7 @@ def assert_mtu(mtu, ifname): assert_number(mtu) import json - from vyos.util import cmd + from vyos.utils.process import cmd out = cmd(f'ip -j -d link show dev {ifname}') # [{"ifindex":2,"ifname":"eth0","flags":["BROADCAST","MULTICAST","UP","LOWER_UP"],"mtu":1500,"qdisc":"pfifo_fast","operstate":"UP","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"08:00:27:d9:5b:04","broadcast":"ff:ff:ff:ff:ff:ff","promiscuity":0,"min_mtu":46,"max_mtu":16110,"inet6_addr_gen_mode":"none","num_tx_queues":1,"num_rx_queues":1,"gso_max_size":65536,"gso_max_segs":65535}] parsed = json.loads(out)[0] diff --git a/python/vyos/version.py b/python/vyos/version.py index fb706ad44..1c5651c83 100644 --- a/python/vyos/version.py +++ b/python/vyos/version.py @@ -1,4 +1,4 @@ -# Copyright 2017-2020 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2017-2023 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -34,11 +34,11 @@ import json import requests import vyos.defaults -from vyos.util import read_file -from vyos.util import read_json -from vyos.util import popen -from vyos.util import run -from vyos.util import DEVNULL +from vyos.utils.file import read_file +from vyos.utils.file import read_json +from vyos.utils.process import popen +from vyos.utils.process import run +from vyos.utils.process import DEVNULL version_file = os.path.join(vyos.defaults.directories['data'], 'version.json') diff --git a/python/vyos/xml_ref/__init__.py b/python/vyos/xml_ref/__init__.py index 62d3680a1..ad2130dca 100644 --- a/python/vyos/xml_ref/__init__.py +++ b/python/vyos/xml_ref/__init__.py @@ -48,6 +48,9 @@ def is_leaf(path: list) -> bool: def cli_defined(path: list, node: str, non_local=False) -> bool: return load_reference().cli_defined(path, node, non_local=non_local) +def from_source(d: dict, path: list) -> bool: + return load_reference().from_source(d, path) + def component_version() -> dict: return load_reference().component_version() diff --git a/python/vyos/xml_ref/definition.py b/python/vyos/xml_ref/definition.py index 33a49ca69..d95d580e2 100644 --- a/python/vyos/xml_ref/definition.py +++ b/python/vyos/xml_ref/definition.py @@ -13,7 +13,12 @@ # You should have received a copy of the GNU Lesser General Public License # along with this library. If not, see <http://www.gnu.org/licenses/>. -from typing import Optional, Union, Any +from typing import Optional, Union, Any, TYPE_CHECKING + +# https://peps.python.org/pep-0484/#forward-references +# for type 'ConfigDict' +if TYPE_CHECKING: + from vyos.config import ConfigDict class Xml: def __init__(self): @@ -123,9 +128,6 @@ class Xml: return d def multi_to_list(self, rpath: list, conf: dict) -> dict: - if rpath and rpath[-1] in list(conf): - raise ValueError('rpath should be disjoint from conf keys') - res: Any = {} for k in list(conf): @@ -210,19 +212,42 @@ class Xml: return False return True + def _set_source_recursive(self, o: Union[dict, str, list], b: bool): + d = {} + if not isinstance(o, dict): + d = {'_source': b} + else: + for k, v in o.items(): + d[k] = self._set_source_recursive(v, b) + d |= {'_source': b} + return d + # use local copy of function in module configdict, to avoid circular # import + # + # extend dict_merge to keep track of keys only in source def _dict_merge(self, source, destination): from copy import deepcopy - tmp = deepcopy(destination) + dest = deepcopy(destination) + from_source = {} for key, value in source.items(): - if key not in tmp: - tmp[key] = value + if key not in dest: + dest[key] = value + from_source[key] = self._set_source_recursive(value, True) elif isinstance(source[key], dict): - tmp[key] = self._dict_merge(source[key], tmp[key]) + dest[key], f = self._dict_merge(source[key], dest[key]) + f |= {'_source': False} + from_source[key] = f + + return dest, from_source - return tmp + def from_source(self, d: dict, path: list) -> bool: + for key in path: + d = d[key] if key in d else {} + if not d or not isinstance(d, dict): + return False + return d.get('_source', False) def _relative_defaults(self, rpath: list, conf: dict, recursive=False) -> dict: res: dict = {} @@ -246,13 +271,14 @@ class Xml: if not conf: return self.get_defaults(path, get_first_key=get_first_key, recursive=recursive) - if path and path[-1] in list(conf): - conf = conf[path[-1]] - conf = {} if not isinstance(conf, dict) else conf - if not self._well_defined(path, conf): - print('path to config dict does not define full config paths') - return {} + # adjust for possible overlap: + if path and path[-1] in list(conf): + conf = conf[path[-1]] + conf = {} if not isinstance(conf, dict) else conf + if not self._well_defined(path, conf): + print('path to config dict does not define full config paths') + return {} res = self._relative_defaults(path, conf, recursive=recursive) @@ -264,13 +290,16 @@ class Xml: return res - def merge_defaults(self, path: list, conf: dict, get_first_key=False, - recursive=False) -> dict: + def merge_defaults(self, path: list, conf: Union[dict, 'ConfigDict'], + get_first_key=False, recursive=False) -> dict: """Return config dict with defaults non-destructively merged This merges non-recursive defaults relative to the config dict. """ d = self.relative_defaults(path, conf, get_first_key=get_first_key, recursive=recursive) - d = self._dict_merge(d, conf) + d, f = self._dict_merge(d, conf) + d = type(conf)(d) + if hasattr(d, '_from_defaults'): + setattr(d, '_from_defaults', f) return d diff --git a/scripts/update-configd-include-file b/scripts/update-configd-include-file index 6615e21ff..ca23408f0 100755 --- a/scripts/update-configd-include-file +++ b/scripts/update-configd-include-file @@ -31,7 +31,7 @@ from inspect import signature, getsource from vyos.defaults import directories from vyos.version import get_version -from vyos.util import cmd +from vyos.utils.process import cmd # Defaults diff --git a/smoketest/scripts/cli/base_accel_ppp_test.py b/smoketest/scripts/cli/base_accel_ppp_test.py index 471bdaffb..989028f64 100644 --- a/smoketest/scripts/cli/base_accel_ppp_test.py +++ b/smoketest/scripts/cli/base_accel_ppp_test.py @@ -1,4 +1,4 @@ -# Copyright (C) 2020-2022 VyOS maintainers and contributors +# Copyright (C) 2020-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -21,9 +21,9 @@ from configparser import ConfigParser from vyos.configsession import ConfigSession from vyos.configsession import ConfigSessionError from vyos.template import is_ipv4 -from vyos.util import cmd -from vyos.util import get_half_cpus -from vyos.util import process_named_running +from vyos.utils.system import get_half_cpus +from vyos.utils.process import process_named_running +from vyos.utils.process import cmd class BasicAccelPPPTest: class TestCase(VyOSUnitTestSHIM.TestCase): diff --git a/smoketest/scripts/cli/base_interfaces_test.py b/smoketest/scripts/cli/base_interfaces_test.py index 348741715..b4afac0e2 100644 --- a/smoketest/scripts/cli/base_interfaces_test.py +++ b/smoketest/scripts/cli/base_interfaces_test.py @@ -28,11 +28,11 @@ from vyos.configsession import ConfigSession from vyos.configsession import ConfigSessionError from vyos.ifconfig import Interface from vyos.ifconfig import Section -from vyos.util import read_file -from vyos.util import cmd -from vyos.util import dict_search -from vyos.util import process_named_running -from vyos.util import get_interface_config +from vyos.utils.file import read_file +from vyos.utils.dict import dict_search +from vyos.utils.process import process_named_running +from vyos.utils.network import get_interface_config +from vyos.utils.process import cmd from vyos.validate import is_intf_addr_assigned from vyos.validate import is_ipv6_link_local from vyos.xml_ref import cli_defined diff --git a/smoketest/scripts/cli/base_vyostest_shim.py b/smoketest/scripts/cli/base_vyostest_shim.py index 9415d6f44..f694f539d 100644 --- a/smoketest/scripts/cli/base_vyostest_shim.py +++ b/smoketest/scripts/cli/base_vyostest_shim.py @@ -1,4 +1,4 @@ -# Copyright (C) 2021 VyOS maintainers and contributors +# Copyright (C) 2021-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -23,8 +23,8 @@ from vyos.configsession import ConfigSession from vyos.configsession import ConfigSessionError from vyos import ConfigError from vyos.defaults import commit_lock -from vyos.util import cmd -from vyos.util import run +from vyos.utils.process import cmd +from vyos.utils.process import run save_config = '/tmp/vyos-smoketest-save' diff --git a/smoketest/scripts/cli/test_configd_init.py b/smoketest/scripts/cli/test_configd_init.py index 5dec89963..245c03824 100755 --- a/smoketest/scripts/cli/test_configd_init.py +++ b/smoketest/scripts/cli/test_configd_init.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2021 VyOS maintainers and contributors +# Copyright (C) 2021-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -17,7 +17,8 @@ import unittest from time import sleep -from vyos.util import cmd, is_systemd_service_running +from vyos.utils.process import is_systemd_service_running +from vyos.utils.process import cmd class TestConfigdInit(unittest.TestCase): def setUp(self): diff --git a/smoketest/scripts/cli/test_container.py b/smoketest/scripts/cli/test_container.py index 902156ee6..b43c05fae 100755 --- a/smoketest/scripts/cli/test_container.py +++ b/smoketest/scripts/cli/test_container.py @@ -21,9 +21,9 @@ import json from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError -from vyos.util import cmd -from vyos.util import process_named_running -from vyos.util import read_file +from vyos.utils.process import cmd +from vyos.utils.process import process_named_running +from vyos.utils.file import read_file base_path = ['container'] cont_image = 'busybox:stable' # busybox is included in vyos-build diff --git a/smoketest/scripts/cli/test_firewall.py b/smoketest/scripts/cli/test_firewall.py index 99d3b3ca1..0c56c2c93 100755 --- a/smoketest/scripts/cli/test_firewall.py +++ b/smoketest/scripts/cli/test_firewall.py @@ -22,8 +22,8 @@ from time import sleep from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError -from vyos.util import cmd -from vyos.util import run +from vyos.utils.process import cmd +from vyos.utils.process import run sysfs_config = { 'all_ping': {'sysfs': '/proc/sys/net/ipv4/icmp_echo_ignore_all', 'default': '0', 'test_value': 'disable'}, diff --git a/smoketest/scripts/cli/test_ha_virtual_server.py b/smoketest/scripts/cli/test_ha_virtual_server.py index e3a91283e..51ccfa4df 100755 --- a/smoketest/scripts/cli/test_ha_virtual_server.py +++ b/smoketest/scripts/cli/test_ha_virtual_server.py @@ -20,9 +20,9 @@ from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError from vyos.ifconfig.vrrp import VRRP -from vyos.util import cmd -from vyos.util import process_named_running -from vyos.util import read_file +from vyos.utils.process import cmd +from vyos.utils.process import process_named_running +from vyos.utils.file import read_file from vyos.template import inc_ip PROCESS_NAME = 'keepalived' @@ -47,6 +47,7 @@ class TestHAVirtualServer(VyOSUnitTestSHIM.TestCase): delay = '10' method = 'nat' persistence_timeout = '600' + vs = 'serv-one' vip = '203.0.113.111' vport = '2222' rservers = ['192.0.2.21', '192.0.2.22', '192.0.2.23'] @@ -56,21 +57,23 @@ class TestHAVirtualServer(VyOSUnitTestSHIM.TestCase): vserver_base = base_path + ['virtual-server'] - self.cli_set(vserver_base + [vip, 'algorithm', algo]) - self.cli_set(vserver_base + [vip, 'delay-loop', delay]) - self.cli_set(vserver_base + [vip, 'forward-method', method]) - self.cli_set(vserver_base + [vip, 'persistence-timeout', persistence_timeout]) - self.cli_set(vserver_base + [vip, 'port', vport]) - self.cli_set(vserver_base + [vip, 'protocol', proto]) + self.cli_set(vserver_base + [vs, 'address', vip]) + self.cli_set(vserver_base + [vs, 'algorithm', algo]) + self.cli_set(vserver_base + [vs, 'delay-loop', delay]) + self.cli_set(vserver_base + [vs, 'forward-method', method]) + self.cli_set(vserver_base + [vs, 'persistence-timeout', persistence_timeout]) + self.cli_set(vserver_base + [vs, 'port', vport]) + self.cli_set(vserver_base + [vs, 'protocol', proto]) for rs in rservers: - self.cli_set(vserver_base + [vip, 'real-server', rs, 'connection-timeout', connection_timeout]) - self.cli_set(vserver_base + [vip, 'real-server', rs, 'port', rport]) + self.cli_set(vserver_base + [vs, 'real-server', rs, 'connection-timeout', connection_timeout]) + self.cli_set(vserver_base + [vs, 'real-server', rs, 'port', rport]) # commit changes self.cli_commit() config = read_file(KEEPALIVED_CONF) + self.assertIn(f'virtual_server {vip} {vport}', config) self.assertIn(f'delay_loop {delay}', config) self.assertIn(f'lb_algo lc', config) self.assertIn(f'lb_kind {method.upper()}', config) @@ -86,6 +89,7 @@ class TestHAVirtualServer(VyOSUnitTestSHIM.TestCase): delay = '15' method = 'nat' persistence_timeout = '300' + vs = 'serv-two' vip = '203.0.113.222' vport = '22322' rservers = ['192.0.2.11', '192.0.2.12'] @@ -107,15 +111,16 @@ class TestHAVirtualServer(VyOSUnitTestSHIM.TestCase): self.cli_set(vrrp_base + [group, 'vrid', vrid]) # Virtual-server config - self.cli_set(vserver_base + [vip, 'algorithm', algo]) - self.cli_set(vserver_base + [vip, 'delay-loop', delay]) - self.cli_set(vserver_base + [vip, 'forward-method', method]) - self.cli_set(vserver_base + [vip, 'persistence-timeout', persistence_timeout]) - self.cli_set(vserver_base + [vip, 'port', vport]) - self.cli_set(vserver_base + [vip, 'protocol', proto]) + self.cli_set(vserver_base + [vs, 'address', vip]) + self.cli_set(vserver_base + [vs, 'algorithm', algo]) + self.cli_set(vserver_base + [vs, 'delay-loop', delay]) + self.cli_set(vserver_base + [vs, 'forward-method', method]) + self.cli_set(vserver_base + [vs, 'persistence-timeout', persistence_timeout]) + self.cli_set(vserver_base + [vs, 'port', vport]) + self.cli_set(vserver_base + [vs, 'protocol', proto]) for rs in rservers: - self.cli_set(vserver_base + [vip, 'real-server', rs, 'connection-timeout', connection_timeout]) - self.cli_set(vserver_base + [vip, 'real-server', rs, 'port', rport]) + self.cli_set(vserver_base + [vs, 'real-server', rs, 'connection-timeout', connection_timeout]) + self.cli_set(vserver_base + [vs, 'real-server', rs, 'port', rport]) # commit changes self.cli_commit() @@ -131,6 +136,7 @@ class TestHAVirtualServer(VyOSUnitTestSHIM.TestCase): self.assertIn(f'preempt_delay 0', config) # default value # Keepalived virtual-server + self.assertIn(f'virtual_server {vip} {vport}', config) self.assertIn(f'delay_loop {delay}', config) self.assertIn(f'lb_algo lc', config) self.assertIn(f'lb_kind {method.upper()}', config) diff --git a/smoketest/scripts/cli/test_ha_vrrp.py b/smoketest/scripts/cli/test_ha_vrrp.py index 03bdbdd61..98259d830 100755 --- a/smoketest/scripts/cli/test_ha_vrrp.py +++ b/smoketest/scripts/cli/test_ha_vrrp.py @@ -20,9 +20,9 @@ from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError from vyos.ifconfig.vrrp import VRRP -from vyos.util import cmd -from vyos.util import process_named_running -from vyos.util import read_file +from vyos.utils.process import cmd +from vyos.utils.process import process_named_running +from vyos.utils.file import read_file from vyos.template import inc_ip PROCESS_NAME = 'keepalived' diff --git a/smoketest/scripts/cli/test_interfaces_bonding.py b/smoketest/scripts/cli/test_interfaces_bonding.py index 2e09173a7..d8e6bde5c 100755 --- a/smoketest/scripts/cli/test_interfaces_bonding.py +++ b/smoketest/scripts/cli/test_interfaces_bonding.py @@ -22,8 +22,8 @@ from base_interfaces_test import BasicInterfaceTest from vyos.ifconfig import Section from vyos.ifconfig.interface import Interface from vyos.configsession import ConfigSessionError -from vyos.util import get_interface_config -from vyos.util import read_file +from vyos.utils.network import get_interface_config +from vyos.utils.file import read_file class BondingInterfaceTest(BasicInterfaceTest.TestCase): @classmethod @@ -243,4 +243,4 @@ class BondingInterfaceTest(BasicInterfaceTest.TestCase): self.assertIn(member, slaves) if __name__ == '__main__': - unittest.main(verbosity=2, failfast=True) + unittest.main(verbosity=2) diff --git a/smoketest/scripts/cli/test_interfaces_bridge.py b/smoketest/scripts/cli/test_interfaces_bridge.py index 85b45c8fa..674b0535a 100755 --- a/smoketest/scripts/cli/test_interfaces_bridge.py +++ b/smoketest/scripts/cli/test_interfaces_bridge.py @@ -24,9 +24,9 @@ from glob import glob from netifaces import interfaces from vyos.ifconfig import Section -from vyos.util import cmd -from vyos.util import read_file -from vyos.util import get_interface_config +from vyos.utils.process import cmd +from vyos.utils.file import read_file +from vyos.utils.network import get_interface_config from vyos.validate import is_intf_addr_assigned class BridgeInterfaceTest(BasicInterfaceTest.TestCase): diff --git a/smoketest/scripts/cli/test_interfaces_ethernet.py b/smoketest/scripts/cli/test_interfaces_ethernet.py index a1bc8481d..eec3ddbe8 100755 --- a/smoketest/scripts/cli/test_interfaces_ethernet.py +++ b/smoketest/scripts/cli/test_interfaces_ethernet.py @@ -28,9 +28,9 @@ from vyos.configsession import ConfigSessionError from vyos.ifconfig import Section from vyos.pki import CERT_BEGIN from vyos.template import is_ipv6 -from vyos.util import cmd -from vyos.util import process_named_running -from vyos.util import read_file +from vyos.utils.process import cmd +from vyos.utils.process import process_named_running +from vyos.utils.file import read_file from vyos.validate import is_ipv6_link_local server_ca_root_cert_data = """ diff --git a/smoketest/scripts/cli/test_interfaces_geneve.py b/smoketest/scripts/cli/test_interfaces_geneve.py index b2efb0349..5f8fae91e 100755 --- a/smoketest/scripts/cli/test_interfaces_geneve.py +++ b/smoketest/scripts/cli/test_interfaces_geneve.py @@ -17,7 +17,7 @@ import unittest from vyos.ifconfig import Interface -from vyos.util import get_interface_config +from vyos.utils.network import get_interface_config from base_interfaces_test import BasicInterfaceTest diff --git a/smoketest/scripts/cli/test_interfaces_input.py b/smoketest/scripts/cli/test_interfaces_input.py index b4cae4695..3ddf86000 100755 --- a/smoketest/scripts/cli/test_interfaces_input.py +++ b/smoketest/scripts/cli/test_interfaces_input.py @@ -16,7 +16,7 @@ import unittest -from vyos.util import read_file +from vyos.utils.file import read_file from vyos.ifconfig import Interface from base_vyostest_shim import VyOSUnitTestSHIM diff --git a/smoketest/scripts/cli/test_interfaces_l2tpv3.py b/smoketest/scripts/cli/test_interfaces_l2tpv3.py index b1d5b7c19..af3d49f75 100755 --- a/smoketest/scripts/cli/test_interfaces_l2tpv3.py +++ b/smoketest/scripts/cli/test_interfaces_l2tpv3.py @@ -19,7 +19,7 @@ import json import unittest from base_interfaces_test import BasicInterfaceTest -from vyos.util import cmd +from vyos.utils.process import cmd class L2TPv3InterfaceTest(BasicInterfaceTest.TestCase): @classmethod diff --git a/smoketest/scripts/cli/test_interfaces_macsec.py b/smoketest/scripts/cli/test_interfaces_macsec.py index 433336f07..b32a6f524 100755 --- a/smoketest/scripts/cli/test_interfaces_macsec.py +++ b/smoketest/scripts/cli/test_interfaces_macsec.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2020-2022 VyOS maintainers and contributors +# Copyright (C) 2020-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -23,10 +23,10 @@ from netifaces import interfaces from vyos.configsession import ConfigSessionError from vyos.ifconfig import Section -from vyos.util import cmd -from vyos.util import read_file -from vyos.util import get_interface_config -from vyos.util import process_named_running +from vyos.utils.process import cmd +from vyos.utils.file import read_file +from vyos.utils.network import get_interface_config +from vyos.utils.process import process_named_running PROCESS_NAME = 'wpa_supplicant' @@ -209,5 +209,4 @@ class MACsecInterfaceTest(BasicInterfaceTest.TestCase): self.assertTrue(process_named_running(PROCESS_NAME)) if __name__ == '__main__': - unittest.main(verbosity=2, failfast=True) - + unittest.main(verbosity=2) diff --git a/smoketest/scripts/cli/test_interfaces_netns.py b/smoketest/scripts/cli/test_interfaces_netns.py index dffa7e0ac..b8bebb221 100755 --- a/smoketest/scripts/cli/test_interfaces_netns.py +++ b/smoketest/scripts/cli/test_interfaces_netns.py @@ -23,7 +23,7 @@ from vyos.configsession import ConfigSession from vyos.configsession import ConfigSessionError from vyos.ifconfig import Interface from vyos.ifconfig import Section -from vyos.util import cmd +from vyos.utils.process import cmd base_path = ['netns'] namespaces = ['mgmt', 'front', 'back', 'ams-ix'] diff --git a/smoketest/scripts/cli/test_interfaces_openvpn.py b/smoketest/scripts/cli/test_interfaces_openvpn.py index c80c7cf80..5c54f58a3 100755 --- a/smoketest/scripts/cli/test_interfaces_openvpn.py +++ b/smoketest/scripts/cli/test_interfaces_openvpn.py @@ -24,9 +24,9 @@ from netifaces import interfaces from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError -from vyos.util import cmd -from vyos.util import process_named_running -from vyos.util import read_file +from vyos.utils.process import cmd +from vyos.utils.process import process_named_running +from vyos.utils.file import read_file from vyos.template import address_from_cidr from vyos.template import dec_ip from vyos.template import inc_ip diff --git a/smoketest/scripts/cli/test_interfaces_tunnel.py b/smoketest/scripts/cli/test_interfaces_tunnel.py index 9dce2b52c..2a7a519fd 100755 --- a/smoketest/scripts/cli/test_interfaces_tunnel.py +++ b/smoketest/scripts/cli/test_interfaces_tunnel.py @@ -19,7 +19,7 @@ import unittest from base_interfaces_test import BasicInterfaceTest from vyos.configsession import ConfigSessionError -from vyos.util import get_interface_config +from vyos.utils.network import get_interface_config from vyos.template import inc_ip remote_ip4 = '192.0.2.100' diff --git a/smoketest/scripts/cli/test_interfaces_virtual_ethernet.py b/smoketest/scripts/cli/test_interfaces_virtual_ethernet.py index 4710930a0..7874589ca 100755 --- a/smoketest/scripts/cli/test_interfaces_virtual_ethernet.py +++ b/smoketest/scripts/cli/test_interfaces_virtual_ethernet.py @@ -19,7 +19,7 @@ import unittest from netifaces import interfaces from vyos.ifconfig import Section -from vyos.util import process_named_running +from vyos.utils.process import process_named_running from base_interfaces_test import BasicInterfaceTest class VEthInterfaceTest(BasicInterfaceTest.TestCase): diff --git a/smoketest/scripts/cli/test_interfaces_vxlan.py b/smoketest/scripts/cli/test_interfaces_vxlan.py index 60a9d1966..f6b203de4 100755 --- a/smoketest/scripts/cli/test_interfaces_vxlan.py +++ b/smoketest/scripts/cli/test_interfaces_vxlan.py @@ -18,8 +18,8 @@ import unittest from vyos.configsession import ConfigSessionError from vyos.ifconfig import Interface -from vyos.util import get_bridge_fdb -from vyos.util import get_interface_config +from vyos.utils.network import get_bridge_fdb +from vyos.utils.network import get_interface_config from vyos.template import is_ipv6 from base_interfaces_test import BasicInterfaceTest diff --git a/smoketest/scripts/cli/test_interfaces_wireless.py b/smoketest/scripts/cli/test_interfaces_wireless.py index 85432cf04..875ca9dc6 100755 --- a/smoketest/scripts/cli/test_interfaces_wireless.py +++ b/smoketest/scripts/cli/test_interfaces_wireless.py @@ -22,9 +22,9 @@ from base_interfaces_test import BasicInterfaceTest from glob import glob from vyos.configsession import ConfigSessionError -from vyos.util import process_named_running -from vyos.util import check_kmod -from vyos.util import read_file +from vyos.utils.process import process_named_running +from vyos.utils.kernel import check_kmod +from vyos.utils.file import read_file def get_config_value(interface, key): tmp = read_file(f'/run/hostapd/{interface}.conf') diff --git a/smoketest/scripts/cli/test_load_balancing_reverse_proxy.py b/smoketest/scripts/cli/test_load_balancing_reverse_proxy.py index 23a681321..a33fd5c18 100755 --- a/smoketest/scripts/cli/test_load_balancing_reverse_proxy.py +++ b/smoketest/scripts/cli/test_load_balancing_reverse_proxy.py @@ -19,8 +19,8 @@ import unittest from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError -from vyos.util import process_named_running -from vyos.util import read_file +from vyos.utils.process import process_named_running +from vyos.utils.file import read_file PROCESS_NAME = 'haproxy' HAPROXY_CONF = '/run/haproxy/haproxy.cfg' diff --git a/smoketest/scripts/cli/test_load_balancing_wan.py b/smoketest/scripts/cli/test_load_balancing_wan.py index 8df3471f7..9b2cb0fac 100755 --- a/smoketest/scripts/cli/test_load_balancing_wan.py +++ b/smoketest/scripts/cli/test_load_balancing_wan.py @@ -21,8 +21,8 @@ import time from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError from vyos.ifconfig import Section -from vyos.util import call -from vyos.util import cmd +from vyos.utils.process import call +from vyos.utils.process import cmd base_path = ['load-balancing'] diff --git a/smoketest/scripts/cli/test_nat.py b/smoketest/scripts/cli/test_nat.py index 1f2b777a8..28d566eba 100755 --- a/smoketest/scripts/cli/test_nat.py +++ b/smoketest/scripts/cli/test_nat.py @@ -21,8 +21,8 @@ import unittest from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError -from vyos.util import cmd -from vyos.util import dict_search +from vyos.utils.process import cmd +from vyos.utils.dict import dict_search base_path = ['nat'] src_path = base_path + ['source'] @@ -231,5 +231,26 @@ class TestNAT(VyOSUnitTestSHIM.TestCase): self.verify_nftables(nftables_search, 'ip vyos_static_nat') + def test_dnat_redirect(self): + dst_addr_1 = '10.0.1.1' + dest_port = '5122' + protocol = 'tcp' + redirected_port = '22' + ifname = 'eth0' + + self.cli_set(dst_path + ['rule', '10', 'destination', 'address', dst_addr_1]) + self.cli_set(dst_path + ['rule', '10', 'destination', 'port', dest_port]) + self.cli_set(dst_path + ['rule', '10', 'protocol', protocol]) + self.cli_set(dst_path + ['rule', '10', 'inbound-interface', ifname]) + self.cli_set(dst_path + ['rule', '10', 'translation', 'redirect', 'port', redirected_port]) + + self.cli_commit() + + nftables_search = [ + [f'iifname "{ifname}"', f'ip daddr {dst_addr_1}', f'{protocol} dport {dest_port}', f'redirect to :{redirected_port}'] + ] + + self.verify_nftables(nftables_search, 'ip vyos_nat') + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/smoketest/scripts/cli/test_nat66.py b/smoketest/scripts/cli/test_nat66.py index 50806b3e8..e062f28a6 100755 --- a/smoketest/scripts/cli/test_nat66.py +++ b/smoketest/scripts/cli/test_nat66.py @@ -22,8 +22,8 @@ import unittest from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError -from vyos.util import cmd -from vyos.util import dict_search +from vyos.utils.process import cmd +from vyos.utils.dict import dict_search base_path = ['nat66'] src_path = base_path + ['source'] diff --git a/smoketest/scripts/cli/test_policy.py b/smoketest/scripts/cli/test_policy.py index f35cdaa4c..354f791bd 100755 --- a/smoketest/scripts/cli/test_policy.py +++ b/smoketest/scripts/cli/test_policy.py @@ -19,7 +19,7 @@ import unittest from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError -from vyos.util import cmd +from vyos.utils.process import cmd base_path = ['policy'] diff --git a/smoketest/scripts/cli/test_policy_route.py b/smoketest/scripts/cli/test_policy_route.py index c83e633b2..d9b64544a 100755 --- a/smoketest/scripts/cli/test_policy_route.py +++ b/smoketest/scripts/cli/test_policy_route.py @@ -18,7 +18,7 @@ import unittest from base_vyostest_shim import VyOSUnitTestSHIM -from vyos.util import cmd +from vyos.utils.process import cmd mark = '100' conn_mark = '555' diff --git a/smoketest/scripts/cli/test_protocols_bfd.py b/smoketest/scripts/cli/test_protocols_bfd.py index fdc254a05..451565664 100755 --- a/smoketest/scripts/cli/test_protocols_bfd.py +++ b/smoketest/scripts/cli/test_protocols_bfd.py @@ -18,7 +18,7 @@ import unittest from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError -from vyos.util import process_named_running +from vyos.utils.process import process_named_running PROCESS_NAME = 'bfdd' base_path = ['protocols', 'bfd'] diff --git a/smoketest/scripts/cli/test_protocols_bgp.py b/smoketest/scripts/cli/test_protocols_bgp.py index 2fd5d0c9b..5b247a413 100755 --- a/smoketest/scripts/cli/test_protocols_bgp.py +++ b/smoketest/scripts/cli/test_protocols_bgp.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2021-2022 VyOS maintainers and contributors +# Copyright (C) 2021-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -18,9 +18,10 @@ import unittest from base_vyostest_shim import VyOSUnitTestSHIM +from vyos.ifconfig import Section from vyos.configsession import ConfigSessionError from vyos.template import is_ipv6 -from vyos.util import process_named_running +from vyos.utils.process import process_named_running PROCESS_NAME = 'bgpd' ASN = '64512' @@ -164,7 +165,6 @@ peer_group_config = { 'local_role_strict': '', }, } - class TestProtocolsBGP(VyOSUnitTestSHIM.TestCase): @classmethod def setUpClass(cls): @@ -201,12 +201,8 @@ class TestProtocolsBGP(VyOSUnitTestSHIM.TestCase): # Check for running process self.assertTrue(process_named_running(PROCESS_NAME)) - def create_bgp_instances_for_import_test(self): table = '1000' - self.cli_set(base_path + ['system-as', ASN]) - # testing only one AFI is sufficient as it's generic code - self.cli_set(import_vrf_base + [import_vrf, 'table', table]) self.cli_set(import_vrf_base + [import_vrf, 'protocols', 'bgp', 'system-as', ASN]) @@ -286,7 +282,6 @@ class TestProtocolsBGP(VyOSUnitTestSHIM.TestCase): if 'disable_conn_chk' in peer_config: self.assertIn(f' neighbor {peer} disable-connected-check', frrconfig) - def test_bgp_01_simple(self): router_id = '127.0.0.1' local_pref = '500' @@ -374,7 +369,6 @@ class TestProtocolsBGP(VyOSUnitTestSHIM.TestCase): self.assertIn(f' maximum-paths {max_path_v6}', afiv6_config) self.assertIn(f' maximum-paths ibgp {max_path_v6ibgp}', afiv6_config) - def test_bgp_02_neighbors(self): # Test out individual neighbor configuration items, not all of them are # also available to a peer-group! @@ -569,7 +563,6 @@ class TestProtocolsBGP(VyOSUnitTestSHIM.TestCase): if 'peer_group' in peer_config: self.cli_set(base_path + ['neighbor', peer, 'peer-group', peer_config['peer_group']]) - # commit changes self.cli_commit() @@ -585,7 +578,6 @@ class TestProtocolsBGP(VyOSUnitTestSHIM.TestCase): if 'peer_group' in peer_config: self.assertIn(f' neighbor {peer} peer-group {peer_config["peer_group"]}', frrconfig) - def test_bgp_04_afi_ipv4(self): networks = { '10.0.0.0/8' : { @@ -633,7 +625,6 @@ class TestProtocolsBGP(VyOSUnitTestSHIM.TestCase): if 'summary_only' in network_config: self.assertIn(f' aggregate-address {network} summary-only', frrconfig) - def test_bgp_05_afi_ipv6(self): networks = { '2001:db8:100::/48' : { @@ -680,7 +671,6 @@ class TestProtocolsBGP(VyOSUnitTestSHIM.TestCase): if 'as_set' in network_config: self.assertIn(f' aggregate-address {network} summary-only', frrconfig) - def test_bgp_06_listen_range(self): # Implemented via T1875 limit = '64' @@ -791,7 +781,6 @@ class TestProtocolsBGP(VyOSUnitTestSHIM.TestCase): self.assertIn(f'{family}', frrconfig) self.assertIn(f'local-install {flowspec_int}', frrconfig) - def test_bgp_10_vrf_simple(self): router_id = '127.0.0.3' vrfs = ['red', 'green', 'blue'] @@ -801,9 +790,7 @@ class TestProtocolsBGP(VyOSUnitTestSHIM.TestCase): # templates and Jinja2 FRR template. table = '1000' - self.cli_set(base_path + ['system-as', ASN]) # testing only one AFI is sufficient as it's generic code - for vrf in vrfs: vrf_base = ['vrf', 'name', vrf] self.cli_set(vrf_base + ['table', table]) @@ -834,7 +821,6 @@ class TestProtocolsBGP(VyOSUnitTestSHIM.TestCase): confed_id = str(int(ASN) + 1) confed_asns = '10 20 30 40' - self.cli_set(base_path + ['system-as', ASN]) self.cli_set(base_path + ['parameters', 'router-id', router_id]) self.cli_set(base_path + ['parameters', 'confederation', 'identifier', confed_id]) for asn in confed_asns.split(): @@ -850,12 +836,10 @@ class TestProtocolsBGP(VyOSUnitTestSHIM.TestCase): self.assertIn(f' bgp confederation identifier {confed_id}', frrconfig) self.assertIn(f' bgp confederation peers {confed_asns}', frrconfig) - def test_bgp_12_v6_link_local(self): remote_asn = str(int(ASN) + 10) interface = 'eth0' - self.cli_set(base_path + ['system-as', ASN]) self.cli_set(base_path + ['neighbor', interface, 'address-family', 'ipv6-unicast']) self.cli_set(base_path + ['neighbor', interface, 'interface', 'v6only', 'remote-as', remote_asn]) @@ -870,7 +854,6 @@ class TestProtocolsBGP(VyOSUnitTestSHIM.TestCase): self.assertIn(f' neighbor {interface} activate', frrconfig) self.assertIn(f' exit-address-family', frrconfig) - def test_bgp_13_vpn(self): remote_asn = str(int(ASN) + 150) neighbor = '192.0.2.55' @@ -880,7 +863,6 @@ class TestProtocolsBGP(VyOSUnitTestSHIM.TestCase): rt_export = f'{neighbor}:1002 1.2.3.4:567' rt_import = f'{neighbor}:1003 500:100' - self.cli_set(base_path + ['system-as', ASN]) # testing only one AFI is sufficient as it's generic code for afi in ['ipv4-unicast', 'ipv6-unicast']: self.cli_set(base_path + ['address-family', afi, 'export', 'vpn']) @@ -919,7 +901,6 @@ class TestProtocolsBGP(VyOSUnitTestSHIM.TestCase): peer_group = 'bar' interface = 'eth0' - self.cli_set(base_path + ['system-as', ASN]) self.cli_set(base_path + ['neighbor', neighbor, 'remote-as', remote_asn]) self.cli_set(base_path + ['neighbor', neighbor, 'peer-group', peer_group]) self.cli_set(base_path + ['peer-group', peer_group, 'remote-as', remote_asn]) @@ -959,7 +940,6 @@ class TestProtocolsBGP(VyOSUnitTestSHIM.TestCase): remote_asn = '500' local_asn = '400' - self.cli_set(base_path + ['system-as', ASN]) self.cli_set(base_path + ['neighbor', neighbor, 'remote-as', ASN]) self.cli_set(base_path + ['neighbor', neighbor, 'local-as', local_asn]) @@ -1072,5 +1052,32 @@ class TestProtocolsBGP(VyOSUnitTestSHIM.TestCase): with self.assertRaises(ConfigSessionError): self.cli_commit() + def test_bgp_22_interface_mpls_forwarding(self): + interfaces = Section.interfaces('ethernet', vlan=False) + for interface in interfaces: + self.cli_set(base_path + ['interface', interface, 'mpls', 'forwarding']) + + self.cli_commit() + + for interface in interfaces: + frrconfig = self.getFRRconfig(f'interface {interface}') + self.assertIn(f'interface {interface}', frrconfig) + self.assertIn(f' mpls bgp forwarding', frrconfig) + + def test_bgp_23_vrf_interface_mpls_forwarding(self): + self.create_bgp_instances_for_import_test() + interfaces = Section.interfaces('ethernet', vlan=False) + for interface in interfaces: + self.cli_set(['interfaces', 'ethernet', interface, 'vrf', import_vrf]) + self.cli_set(import_vrf_base + [import_vrf] + base_path + ['interface', interface, 'mpls', 'forwarding']) + + self.cli_commit() + + for interface in interfaces: + frrconfig = self.getFRRconfig(f'interface {interface}') + self.assertIn(f'interface {interface}', frrconfig) + self.assertIn(f' mpls bgp forwarding', frrconfig) + self.cli_delete(['interfaces', 'ethernet', interface, 'vrf']) + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/smoketest/scripts/cli/test_protocols_igmp-proxy.py b/smoketest/scripts/cli/test_protocols_igmp-proxy.py index 079b5bee5..a75003b12 100755 --- a/smoketest/scripts/cli/test_protocols_igmp-proxy.py +++ b/smoketest/scripts/cli/test_protocols_igmp-proxy.py @@ -19,8 +19,8 @@ import unittest from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError -from vyos.util import read_file -from vyos.util import process_named_running +from vyos.utils.file import read_file +from vyos.utils.process import process_named_running PROCESS_NAME = 'igmpproxy' IGMP_PROXY_CONF = '/etc/igmpproxy.conf' diff --git a/smoketest/scripts/cli/test_protocols_isis.py b/smoketest/scripts/cli/test_protocols_isis.py index f1a030e77..511a5eb8b 100755 --- a/smoketest/scripts/cli/test_protocols_isis.py +++ b/smoketest/scripts/cli/test_protocols_isis.py @@ -19,7 +19,7 @@ import unittest from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError from vyos.ifconfig import Section -from vyos.util import process_named_running +from vyos.utils.process import process_named_running PROCESS_NAME = 'isisd' base_path = ['protocols', 'isis'] diff --git a/smoketest/scripts/cli/test_protocols_mpls.py b/smoketest/scripts/cli/test_protocols_mpls.py index 76e6ca35a..06f21c6e1 100755 --- a/smoketest/scripts/cli/test_protocols_mpls.py +++ b/smoketest/scripts/cli/test_protocols_mpls.py @@ -19,7 +19,7 @@ import unittest from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError from vyos.ifconfig import Section -from vyos.util import process_named_running +from vyos.utils.process import process_named_running PROCESS_NAME = 'ldpd' base_path = ['protocols', 'mpls', 'ldp'] diff --git a/smoketest/scripts/cli/test_protocols_nhrp.py b/smoketest/scripts/cli/test_protocols_nhrp.py index 7dbe836f7..45ef539f6 100755 --- a/smoketest/scripts/cli/test_protocols_nhrp.py +++ b/smoketest/scripts/cli/test_protocols_nhrp.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2021 VyOS maintainers and contributors +# Copyright (C) 2021-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -19,7 +19,9 @@ import unittest from base_vyostest_shim import VyOSUnitTestSHIM from vyos.firewall import find_nftables_rule -from vyos.util import call, process_named_running, read_file +from vyos.utils.process import process_named_running +from vyos.utils.file import read_file +from vyos.utils.process import call tunnel_path = ['interfaces', 'tunnel'] nhrp_path = ['protocols', 'nhrp'] diff --git a/smoketest/scripts/cli/test_protocols_ospf.py b/smoketest/scripts/cli/test_protocols_ospf.py index e4907596e..80befbfd6 100755 --- a/smoketest/scripts/cli/test_protocols_ospf.py +++ b/smoketest/scripts/cli/test_protocols_ospf.py @@ -20,7 +20,7 @@ from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError from vyos.ifconfig import Section -from vyos.util import process_named_running +from vyos.utils.process import process_named_running PROCESS_NAME = 'ospfd' base_path = ['protocols', 'ospf'] @@ -479,5 +479,32 @@ class TestProtocolsOSPF(VyOSUnitTestSHIM.TestCase): self.assertIn(f' ip ospf dead-interval 40', config) self.assertIn(f' no ip ospf mpls ldp-sync', config) + def test_ospf_16_graceful_restart(self): + period = '300' + supported_grace_time = '400' + router_ids = ['192.0.2.1', '192.0.2.2'] + + self.cli_set(base_path + ['capability', 'opaque']) + self.cli_set(base_path + ['graceful-restart', 'grace-period', period]) + self.cli_set(base_path + ['graceful-restart', 'helper', 'planned-only']) + self.cli_set(base_path + ['graceful-restart', 'helper', 'no-strict-lsa-checking']) + self.cli_set(base_path + ['graceful-restart', 'helper', 'supported-grace-time', supported_grace_time]) + for router_id in router_ids: + self.cli_set(base_path + ['graceful-restart', 'helper', 'enable', 'router-id', router_id]) + + # commit changes + self.cli_commit() + + # Verify FRR ospfd configuration + frrconfig = self.getFRRconfig('router ospf') + self.assertIn(f'router ospf', frrconfig) + self.assertIn(f' capability opaque', frrconfig) + self.assertIn(f' graceful-restart grace-period {period}', frrconfig) + self.assertIn(f' graceful-restart helper planned-only', frrconfig) + self.assertIn(f' no graceful-restart helper strict-lsa-checking', frrconfig) + self.assertIn(f' graceful-restart helper supported-grace-time {supported_grace_time}', frrconfig) + for router_id in router_ids: + self.assertIn(f' graceful-restart helper enable {router_id}', frrconfig) + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/smoketest/scripts/cli/test_protocols_ospfv3.py b/smoketest/scripts/cli/test_protocols_ospfv3.py index fa80ad555..64dfa18db 100755 --- a/smoketest/scripts/cli/test_protocols_ospfv3.py +++ b/smoketest/scripts/cli/test_protocols_ospfv3.py @@ -20,7 +20,7 @@ from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError from vyos.ifconfig import Section -from vyos.util import process_named_running +from vyos.utils.process import process_named_running PROCESS_NAME = 'ospf6d' base_path = ['protocols', 'ospfv3'] @@ -281,5 +281,31 @@ class TestProtocolsOSPFv3(VyOSUnitTestSHIM.TestCase): self.cli_delete(['vrf', 'name', vrf]) self.cli_delete(['interfaces', 'ethernet', vrf_iface, 'vrf']) + + def test_ospfv3_09_graceful_restart(self): + period = '300' + supported_grace_time = '400' + router_ids = ['192.0.2.1', '192.0.2.2'] + + self.cli_set(base_path + ['graceful-restart', 'grace-period', period]) + self.cli_set(base_path + ['graceful-restart', 'helper', 'planned-only']) + self.cli_set(base_path + ['graceful-restart', 'helper', 'lsa-check-disable']) + self.cli_set(base_path + ['graceful-restart', 'helper', 'supported-grace-time', supported_grace_time]) + for router_id in router_ids: + self.cli_set(base_path + ['graceful-restart', 'helper', 'enable', 'router-id', router_id]) + + # commit changes + self.cli_commit() + + # Verify FRR ospfd configuration + frrconfig = self.getFRRconfig('router ospf6') + self.assertIn(f'router ospf6', frrconfig) + self.assertIn(f' graceful-restart grace-period {period}', frrconfig) + self.assertIn(f' graceful-restart helper planned-only', frrconfig) + self.assertIn(f' graceful-restart helper lsa-check-disable', frrconfig) + self.assertIn(f' graceful-restart helper supported-grace-time {supported_grace_time}', frrconfig) + for router_id in router_ids: + self.assertIn(f' graceful-restart helper enable {router_id}', frrconfig) + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/smoketest/scripts/cli/test_protocols_rip.py b/smoketest/scripts/cli/test_protocols_rip.py index 11385adb5..925499fc8 100755 --- a/smoketest/scripts/cli/test_protocols_rip.py +++ b/smoketest/scripts/cli/test_protocols_rip.py @@ -19,7 +19,7 @@ import unittest from base_vyostest_shim import VyOSUnitTestSHIM from vyos.ifconfig import Section -from vyos.util import process_named_running +from vyos.utils.process import process_named_running PROCESS_NAME = 'ripd' acl_in = '198' diff --git a/smoketest/scripts/cli/test_protocols_ripng.py b/smoketest/scripts/cli/test_protocols_ripng.py index 53336a533..0a8ce7eef 100755 --- a/smoketest/scripts/cli/test_protocols_ripng.py +++ b/smoketest/scripts/cli/test_protocols_ripng.py @@ -19,7 +19,7 @@ import unittest from base_vyostest_shim import VyOSUnitTestSHIM from vyos.ifconfig import Section -from vyos.util import process_named_running +from vyos.utils.process import process_named_running PROCESS_NAME = 'ripngd' acl_in = '198' diff --git a/smoketest/scripts/cli/test_protocols_rpki.py b/smoketest/scripts/cli/test_protocols_rpki.py index e5e45565b..f4aedcbc3 100755 --- a/smoketest/scripts/cli/test_protocols_rpki.py +++ b/smoketest/scripts/cli/test_protocols_rpki.py @@ -20,8 +20,8 @@ import unittest from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError -from vyos.util import cmd -from vyos.util import process_named_running +from vyos.utils.process import cmd +from vyos.utils.process import process_named_running base_path = ['protocols', 'rpki'] PROCESS_NAME = 'bgpd' diff --git a/smoketest/scripts/cli/test_protocols_static.py b/smoketest/scripts/cli/test_protocols_static.py index 275f1a1df..abf1080ab 100755 --- a/smoketest/scripts/cli/test_protocols_static.py +++ b/smoketest/scripts/cli/test_protocols_static.py @@ -20,7 +20,7 @@ from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError from vyos.template import is_ipv6 -from vyos.util import get_interface_config +from vyos.utils.network import get_interface_config base_path = ['protocols', 'static'] vrf_path = ['protocols', 'vrf'] diff --git a/smoketest/scripts/cli/test_protocols_static_arp.py b/smoketest/scripts/cli/test_protocols_static_arp.py index b61d8f854..7f8047249 100755 --- a/smoketest/scripts/cli/test_protocols_static_arp.py +++ b/smoketest/scripts/cli/test_protocols_static_arp.py @@ -19,7 +19,7 @@ import unittest from base_vyostest_shim import VyOSUnitTestSHIM -from vyos.util import cmd +from vyos.utils.process import cmd base_path = ['protocols', 'static', 'arp'] interface = 'eth0' diff --git a/smoketest/scripts/cli/test_qos.py b/smoketest/scripts/cli/test_qos.py index 0092473d6..3743be788 100755 --- a/smoketest/scripts/cli/test_qos.py +++ b/smoketest/scripts/cli/test_qos.py @@ -22,7 +22,7 @@ from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError from vyos.ifconfig import Section -from vyos.util import cmd +from vyos.utils.process import cmd base_path = ['qos'] @@ -544,4 +544,4 @@ class TestQoS(VyOSUnitTestSHIM.TestCase): self.assertEqual(f'{dport:x}', filter['options']['match']['value']) if __name__ == '__main__': - unittest.main(verbosity=2, failfast=True) + unittest.main(verbosity=2) diff --git a/smoketest/scripts/cli/test_service_dhcp-relay.py b/smoketest/scripts/cli/test_service_dhcp-relay.py index 92f87c06c..59c4b59a9 100755 --- a/smoketest/scripts/cli/test_service_dhcp-relay.py +++ b/smoketest/scripts/cli/test_service_dhcp-relay.py @@ -20,8 +20,8 @@ from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError from vyos.ifconfig import Section -from vyos.util import process_named_running -from vyos.util import read_file +from vyos.utils.process import process_named_running +from vyos.utils.file import read_file PROCESS_NAME = 'dhcrelay' RELAY_CONF = '/run/dhcp-relay/dhcrelay.conf' diff --git a/smoketest/scripts/cli/test_service_dhcp-server.py b/smoketest/scripts/cli/test_service_dhcp-server.py index 64ba100a2..093e43494 100755 --- a/smoketest/scripts/cli/test_service_dhcp-server.py +++ b/smoketest/scripts/cli/test_service_dhcp-server.py @@ -19,8 +19,8 @@ import unittest from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError -from vyos.util import process_named_running -from vyos.util import read_file +from vyos.utils.process import process_named_running +from vyos.utils.file import read_file from vyos.template import address_from_cidr from vyos.template import inc_ip from vyos.template import dec_ip diff --git a/smoketest/scripts/cli/test_service_dhcpv6-relay.py b/smoketest/scripts/cli/test_service_dhcpv6-relay.py index 8bb58d296..4487f4b0f 100755 --- a/smoketest/scripts/cli/test_service_dhcpv6-relay.py +++ b/smoketest/scripts/cli/test_service_dhcpv6-relay.py @@ -21,8 +21,8 @@ from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError from vyos.ifconfig import Section from vyos.template import address_from_cidr -from vyos.util import process_named_running -from vyos.util import read_file +from vyos.utils.process import process_named_running +from vyos.utils.file import read_file PROCESS_NAME = 'dhcrelay' RELAY_CONF = '/run/dhcp-relay/dhcrelay6.conf' diff --git a/smoketest/scripts/cli/test_service_dhcpv6-server.py b/smoketest/scripts/cli/test_service_dhcpv6-server.py index f83453323..4d9dabc3f 100755 --- a/smoketest/scripts/cli/test_service_dhcpv6-server.py +++ b/smoketest/scripts/cli/test_service_dhcpv6-server.py @@ -20,8 +20,8 @@ from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError from vyos.template import inc_ip -from vyos.util import process_named_running -from vyos.util import read_file +from vyos.utils.process import process_named_running +from vyos.utils.file import read_file PROCESS_NAME = 'dhcpd' DHCPD_CONF = '/run/dhcp-server/dhcpdv6.conf' diff --git a/smoketest/scripts/cli/test_service_dns_dynamic.py b/smoketest/scripts/cli/test_service_dns_dynamic.py index 11d411cb4..f6c42e8c9 100755 --- a/smoketest/scripts/cli/test_service_dns_dynamic.py +++ b/smoketest/scripts/cli/test_service_dns_dynamic.py @@ -22,8 +22,8 @@ import tempfile from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError -from vyos.util import cmd -from vyos.util import process_running +from vyos.utils.process import cmd +from vyos.utils.process import process_running DDCLIENT_CONF = '/run/ddclient/ddclient.conf' DDCLIENT_PID = '/run/ddclient/ddclient.pid' diff --git a/smoketest/scripts/cli/test_service_dns_forwarding.py b/smoketest/scripts/cli/test_service_dns_forwarding.py index 88492e348..bc50a4ffe 100755 --- a/smoketest/scripts/cli/test_service_dns_forwarding.py +++ b/smoketest/scripts/cli/test_service_dns_forwarding.py @@ -21,8 +21,8 @@ from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError from vyos.template import bracketize_ipv6 -from vyos.util import read_file -from vyos.util import process_named_running +from vyos.utils.file import read_file +from vyos.utils.process import process_named_running CONFIG_FILE = '/run/powerdns/recursor.conf' FORWARD_FILE = '/run/powerdns/recursor.forward-zones.conf' @@ -256,4 +256,4 @@ class TestServicePowerDNS(VyOSUnitTestSHIM.TestCase): self.assertEqual(tmp, port) if __name__ == '__main__': - unittest.main(verbosity=2, failfast=True) + unittest.main(verbosity=2) diff --git a/smoketest/scripts/cli/test_service_https.py b/smoketest/scripts/cli/test_service_https.py index 1adf1f5cf..1ae5c104c 100755 --- a/smoketest/scripts/cli/test_service_https.py +++ b/smoketest/scripts/cli/test_service_https.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2019-2022 VyOS maintainers and contributors +# Copyright (C) 2019-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -21,8 +21,8 @@ from urllib3.exceptions import InsecureRequestWarning from base_vyostest_shim import VyOSUnitTestSHIM from base_vyostest_shim import ignore_warning -from vyos.util import read_file -from vyos.util import run +from vyos.utils.file import read_file +from vyos.utils.process import run base_path = ['service', 'https'] pki_base = ['pki'] diff --git a/smoketest/scripts/cli/test_service_ids.py b/smoketest/scripts/cli/test_service_ids.py index dcf2bcefe..91b056eea 100755 --- a/smoketest/scripts/cli/test_service_ids.py +++ b/smoketest/scripts/cli/test_service_ids.py @@ -20,8 +20,8 @@ import unittest from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError -from vyos.util import process_named_running -from vyos.util import read_file +from vyos.utils.process import process_named_running +from vyos.utils.file import read_file PROCESS_NAME = 'fastnetmon' FASTNETMON_CONF = '/run/fastnetmon/fastnetmon.conf' diff --git a/smoketest/scripts/cli/test_service_ipoe-server.py b/smoketest/scripts/cli/test_service_ipoe-server.py index 8a141b8f0..4dd3e761c 100755 --- a/smoketest/scripts/cli/test_service_ipoe-server.py +++ b/smoketest/scripts/cli/test_service_ipoe-server.py @@ -19,7 +19,7 @@ import unittest from base_accel_ppp_test import BasicAccelPPPTest from vyos.configsession import ConfigSessionError -from vyos.util import cmd +from vyos.utils.process import cmd from configparser import ConfigParser diff --git a/smoketest/scripts/cli/test_service_lldp.py b/smoketest/scripts/cli/test_service_lldp.py index 439c96c33..ee26844ab 100755 --- a/smoketest/scripts/cli/test_service_lldp.py +++ b/smoketest/scripts/cli/test_service_lldp.py @@ -22,9 +22,9 @@ from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError from vyos.ifconfig import Section -from vyos.util import cmd -from vyos.util import process_named_running -from vyos.util import read_file +from vyos.utils.process import cmd +from vyos.utils.process import process_named_running +from vyos.utils.file import read_file from vyos.version import get_version_data PROCESS_NAME = 'lldpd' diff --git a/smoketest/scripts/cli/test_service_mdns-repeater.py b/smoketest/scripts/cli/test_service_mdns-repeater.py index 880c13043..9a9839025 100755 --- a/smoketest/scripts/cli/test_service_mdns-repeater.py +++ b/smoketest/scripts/cli/test_service_mdns-repeater.py @@ -19,7 +19,7 @@ import unittest from base_vyostest_shim import VyOSUnitTestSHIM from configparser import ConfigParser -from vyos.util import process_named_running +from vyos.utils.process import process_named_running base_path = ['service', 'mdns', 'repeater'] intf_base = ['interfaces', 'dummy'] diff --git a/smoketest/scripts/cli/test_service_monitoring_telegraf.py b/smoketest/scripts/cli/test_service_monitoring_telegraf.py index ed486c3b9..f3355b735 100755 --- a/smoketest/scripts/cli/test_service_monitoring_telegraf.py +++ b/smoketest/scripts/cli/test_service_monitoring_telegraf.py @@ -20,8 +20,8 @@ from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSession from vyos.configsession import ConfigSessionError -from vyos.util import process_named_running -from vyos.util import read_file +from vyos.utils.process import process_named_running +from vyos.utils.file import read_file PROCESS_NAME = 'telegraf' TELEGRAF_CONF = '/run/telegraf/telegraf.conf' diff --git a/smoketest/scripts/cli/test_service_ntp.py b/smoketest/scripts/cli/test_service_ntp.py index 47e012913..5e385d5ad 100755 --- a/smoketest/scripts/cli/test_service_ntp.py +++ b/smoketest/scripts/cli/test_service_ntp.py @@ -19,8 +19,8 @@ import unittest from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError -from vyos.util import cmd -from vyos.util import process_named_running +from vyos.utils.process import cmd +from vyos.utils.process import process_named_running PROCESS_NAME = 'chronyd' NTP_CONF = '/run/chrony/chrony.conf' diff --git a/smoketest/scripts/cli/test_service_pppoe-server.py b/smoketest/scripts/cli/test_service_pppoe-server.py index bb6a1c1cd..963784f0a 100755 --- a/smoketest/scripts/cli/test_service_pppoe-server.py +++ b/smoketest/scripts/cli/test_service_pppoe-server.py @@ -19,7 +19,7 @@ import unittest from base_accel_ppp_test import BasicAccelPPPTest from configparser import ConfigParser -from vyos.util import read_file +from vyos.utils.file import read_file from vyos.template import range_to_regex local_if = ['interfaces', 'dummy', 'dum667'] diff --git a/smoketest/scripts/cli/test_service_router-advert.py b/smoketest/scripts/cli/test_service_router-advert.py index 0169b7934..5fc2019fd 100755 --- a/smoketest/scripts/cli/test_service_router-advert.py +++ b/smoketest/scripts/cli/test_service_router-advert.py @@ -20,8 +20,8 @@ import unittest from vyos.configsession import ConfigSessionError from base_vyostest_shim import VyOSUnitTestSHIM -from vyos.util import read_file -from vyos.util import process_named_running +from vyos.utils.file import read_file +from vyos.utils.process import process_named_running PROCESS_NAME = 'radvd' RADVD_CONF = '/run/radvd/radvd.conf' diff --git a/smoketest/scripts/cli/test_service_salt.py b/smoketest/scripts/cli/test_service_salt.py index 00a4f2020..48a588b72 100755 --- a/smoketest/scripts/cli/test_service_salt.py +++ b/smoketest/scripts/cli/test_service_salt.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2022 VyOS maintainers and contributors +# Copyright (C) 2022-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -19,9 +19,9 @@ import unittest from socket import gethostname from base_vyostest_shim import VyOSUnitTestSHIM -from vyos.util import process_named_running -from vyos.util import read_file -from vyos.util import cmd +from vyos.utils.process import process_named_running +from vyos.utils.file import read_file +from vyos.utils.process import cmd PROCESS_NAME = 'salt-minion' SALT_CONF = '/etc/salt/minion' diff --git a/smoketest/scripts/cli/test_service_snmp.py b/smoketest/scripts/cli/test_service_snmp.py index b18b9e7a1..52a72ec4f 100755 --- a/smoketest/scripts/cli/test_service_snmp.py +++ b/smoketest/scripts/cli/test_service_snmp.py @@ -22,10 +22,10 @@ from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError from vyos.template import is_ipv4 from vyos.template import address_from_cidr -from vyos.util import call -from vyos.util import DEVNULL -from vyos.util import read_file -from vyos.util import process_named_running +from vyos.utils.process import call +from vyos.utils.process import DEVNULL +from vyos.utils.file import read_file +from vyos.utils.process import process_named_running from vyos.version import get_version_data PROCESS_NAME = 'snmpd' diff --git a/smoketest/scripts/cli/test_service_ssh.py b/smoketest/scripts/cli/test_service_ssh.py index e03907dd8..947d7d568 100755 --- a/smoketest/scripts/cli/test_service_ssh.py +++ b/smoketest/scripts/cli/test_service_ssh.py @@ -24,10 +24,10 @@ from pwd import getpwall from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError -from vyos.util import cmd -from vyos.util import is_systemd_service_running -from vyos.util import process_named_running -from vyos.util import read_file +from vyos.utils.process import cmd +from vyos.utils.process import is_systemd_service_running +from vyos.utils.process import process_named_running +from vyos.utils.file import read_file PROCESS_NAME = 'sshd' SSHD_CONF = '/run/sshd/sshd_config' diff --git a/smoketest/scripts/cli/test_service_tftp-server.py b/smoketest/scripts/cli/test_service_tftp-server.py index 99d81e203..d60794980 100755 --- a/smoketest/scripts/cli/test_service_tftp-server.py +++ b/smoketest/scripts/cli/test_service_tftp-server.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2019-2020 VyOS maintainers and contributors +# Copyright (C) 2019-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -20,9 +20,9 @@ from psutil import process_iter from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError -from vyos.util import cmd -from vyos.util import read_file -from vyos.util import process_named_running +from vyos.utils.process import cmd +from vyos.utils.file import read_file +from vyos.utils.process import process_named_running from vyos.template import is_ipv6 PROCESS_NAME = 'in.tftpd' diff --git a/smoketest/scripts/cli/test_service_upnp.py b/smoketest/scripts/cli/test_service_upnp.py index e4df88c1e..c3fb0ec9d 100755 --- a/smoketest/scripts/cli/test_service_upnp.py +++ b/smoketest/scripts/cli/test_service_upnp.py @@ -22,8 +22,8 @@ from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSession from vyos.configsession import ConfigSessionError from vyos.template import ip_from_cidr -from vyos.util import read_file -from vyos.util import process_named_running +from vyos.utils.file import read_file +from vyos.utils.process import process_named_running UPNP_CONF = '/run/upnp/miniupnp.conf' DAEMON = 'miniupnpd' diff --git a/smoketest/scripts/cli/test_service_webproxy.py b/smoketest/scripts/cli/test_service_webproxy.py index fb9b46a06..2b3f6d21c 100755 --- a/smoketest/scripts/cli/test_service_webproxy.py +++ b/smoketest/scripts/cli/test_service_webproxy.py @@ -19,9 +19,9 @@ import unittest from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError -from vyos.util import cmd -from vyos.util import process_named_running -from vyos.util import read_file +from vyos.utils.process import cmd +from vyos.utils.process import process_named_running +from vyos.utils.file import read_file PROCESS_NAME = 'squid' PROXY_CONF = '/etc/squid/squid.conf' diff --git a/smoketest/scripts/cli/test_system_conntrack.py b/smoketest/scripts/cli/test_system_conntrack.py index fd16146b1..2a89aa98b 100755 --- a/smoketest/scripts/cli/test_system_conntrack.py +++ b/smoketest/scripts/cli/test_system_conntrack.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2021 VyOS maintainers and contributors +# Copyright (C) 2021-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -21,8 +21,8 @@ import unittest from base_vyostest_shim import VyOSUnitTestSHIM from vyos.firewall import find_nftables_rule -from vyos.util import cmd -from vyos.util import read_file +from vyos.utils.process import cmd +from vyos.utils.file import read_file base_path = ['system', 'conntrack'] diff --git a/smoketest/scripts/cli/test_system_flow-accounting.py b/smoketest/scripts/cli/test_system_flow-accounting.py index df60b9613..d55ea616e 100755 --- a/smoketest/scripts/cli/test_system_flow-accounting.py +++ b/smoketest/scripts/cli/test_system_flow-accounting.py @@ -22,9 +22,9 @@ from vyos.configsession import ConfigSessionError from vyos.ifconfig import Section from vyos.template import bracketize_ipv6 from vyos.template import is_ipv6 -from vyos.util import cmd -from vyos.util import process_named_running -from vyos.util import read_file +from vyos.utils.process import cmd +from vyos.utils.process import process_named_running +from vyos.utils.file import read_file PROCESS_NAME = 'uacctd' base_path = ['system', 'flow-accounting'] diff --git a/smoketest/scripts/cli/test_system_frr.py b/smoketest/scripts/cli/test_system_frr.py index 331133ed4..3eb0cd0ab 100755 --- a/smoketest/scripts/cli/test_system_frr.py +++ b/smoketest/scripts/cli/test_system_frr.py @@ -17,7 +17,7 @@ import re import unittest from base_vyostest_shim import VyOSUnitTestSHIM -from vyos.util import read_file +from vyos.utils.file import read_file config_file = '/etc/frr/daemons' base_path = ['system', 'frr'] @@ -143,4 +143,4 @@ class TestSystemFRR(VyOSUnitTestSHIM.TestCase): if __name__ == '__main__': - unittest.main(verbosity=2, failfast=True) + unittest.main(verbosity=2) diff --git a/smoketest/scripts/cli/test_system_ip.py b/smoketest/scripts/cli/test_system_ip.py index e7f7e3345..bd2531084 100755 --- a/smoketest/scripts/cli/test_system_ip.py +++ b/smoketest/scripts/cli/test_system_ip.py @@ -18,7 +18,7 @@ import unittest from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError -from vyos.util import read_file +from vyos.utils.file import read_file base_path = ['system', 'ip'] diff --git a/smoketest/scripts/cli/test_system_ipv6.py b/smoketest/scripts/cli/test_system_ipv6.py index e91b924fc..b540be9ff 100755 --- a/smoketest/scripts/cli/test_system_ipv6.py +++ b/smoketest/scripts/cli/test_system_ipv6.py @@ -20,8 +20,8 @@ from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError from vyos.template import is_ipv4 -from vyos.util import read_file -from vyos.util import get_interface_config +from vyos.utils.file import read_file +from vyos.utils.network import get_interface_config from vyos.validate import is_intf_addr_assigned base_path = ['system', 'ipv6'] diff --git a/smoketest/scripts/cli/test_system_lcd.py b/smoketest/scripts/cli/test_system_lcd.py index 831fba979..fc440ca8a 100755 --- a/smoketest/scripts/cli/test_system_lcd.py +++ b/smoketest/scripts/cli/test_system_lcd.py @@ -19,7 +19,7 @@ import unittest from base_vyostest_shim import VyOSUnitTestSHIM from configparser import ConfigParser -from vyos.util import process_named_running +from vyos.utils.process import process_named_running config_file = '/run/LCDd/LCDd.conf' base_path = ['system', 'lcd'] diff --git a/smoketest/scripts/cli/test_system_login.py b/smoketest/scripts/cli/test_system_login.py index 8a4f5fdd1..195b127a4 100755 --- a/smoketest/scripts/cli/test_system_login.py +++ b/smoketest/scripts/cli/test_system_login.py @@ -26,8 +26,8 @@ from pwd import getpwall from time import sleep from vyos.configsession import ConfigSessionError -from vyos.util import cmd -from vyos.util import read_file +from vyos.utils.process import cmd +from vyos.utils.file import read_file from vyos.template import inc_ip base_path = ['system', 'login'] diff --git a/smoketest/scripts/cli/test_system_logs.py b/smoketest/scripts/cli/test_system_logs.py index 92fa9c3d9..17cce5ca1 100755 --- a/smoketest/scripts/cli/test_system_logs.py +++ b/smoketest/scripts/cli/test_system_logs.py @@ -17,7 +17,7 @@ import re import unittest from base_vyostest_shim import VyOSUnitTestSHIM -from vyos.util import read_file +from vyos.utils.file import read_file # path to logrotate configs logrotate_atop_file = '/etc/logrotate.d/vyos-atop' diff --git a/smoketest/scripts/cli/test_system_nameserver.py b/smoketest/scripts/cli/test_system_nameserver.py index 58c84988e..4979a7c72 100755 --- a/smoketest/scripts/cli/test_system_nameserver.py +++ b/smoketest/scripts/cli/test_system_nameserver.py @@ -21,7 +21,7 @@ from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError -from vyos.util import read_file +from vyos.utils.file import read_file RESOLV_CONF = '/etc/resolv.conf' diff --git a/smoketest/scripts/cli/test_system_sflow.py b/smoketest/scripts/cli/test_system_sflow.py index 1aec050a4..63262db69 100755 --- a/smoketest/scripts/cli/test_system_sflow.py +++ b/smoketest/scripts/cli/test_system_sflow.py @@ -20,9 +20,9 @@ from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError from vyos.ifconfig import Section -from vyos.util import cmd -from vyos.util import process_named_running -from vyos.util import read_file +from vyos.utils.process import cmd +from vyos.utils.process import process_named_running +from vyos.utils.file import read_file PROCESS_NAME = 'hsflowd' base_path = ['system', 'sflow'] diff --git a/smoketest/scripts/cli/test_vpn_ipsec.py b/smoketest/scripts/cli/test_vpn_ipsec.py index acb41e410..01b0406bf 100755 --- a/smoketest/scripts/cli/test_vpn_ipsec.py +++ b/smoketest/scripts/cli/test_vpn_ipsec.py @@ -18,9 +18,9 @@ import os import unittest from base_vyostest_shim import VyOSUnitTestSHIM -from vyos.util import call -from vyos.util import process_named_running -from vyos.util import read_file +from vyos.utils.process import call +from vyos.utils.process import process_named_running +from vyos.utils.file import read_file ethernet_path = ['interfaces', 'ethernet'] tunnel_path = ['interfaces', 'tunnel'] diff --git a/smoketest/scripts/cli/test_vpn_openconnect.py b/smoketest/scripts/cli/test_vpn_openconnect.py index ec8ecacb9..04abeb1aa 100755 --- a/smoketest/scripts/cli/test_vpn_openconnect.py +++ b/smoketest/scripts/cli/test_vpn_openconnect.py @@ -19,8 +19,8 @@ import unittest from base_vyostest_shim import VyOSUnitTestSHIM from vyos.template import ip_from_cidr -from vyos.util import process_named_running -from vyos.util import read_file +from vyos.utils.process import process_named_running +from vyos.utils.file import read_file OCSERV_CONF = '/run/ocserv/ocserv.conf' base_path = ['vpn', 'openconnect'] diff --git a/smoketest/scripts/cli/test_vpn_sstp.py b/smoketest/scripts/cli/test_vpn_sstp.py index 434e3aa05..232eafcf2 100755 --- a/smoketest/scripts/cli/test_vpn_sstp.py +++ b/smoketest/scripts/cli/test_vpn_sstp.py @@ -17,7 +17,7 @@ import unittest from base_accel_ppp_test import BasicAccelPPPTest -from vyos.util import read_file +from vyos.utils.file import read_file pki_path = ['pki'] diff --git a/smoketest/scripts/cli/test_vrf.py b/smoketest/scripts/cli/test_vrf.py index 926616727..932f7b4f1 100755 --- a/smoketest/scripts/cli/test_vrf.py +++ b/smoketest/scripts/cli/test_vrf.py @@ -26,9 +26,9 @@ from vyos.configsession import ConfigSessionError from vyos.ifconfig import Interface from vyos.ifconfig import Section from vyos.template import is_ipv4 -from vyos.util import cmd -from vyos.util import read_file -from vyos.util import get_interface_config +from vyos.utils.process import cmd +from vyos.utils.file import read_file +from vyos.utils.network import get_interface_config from vyos.validate import is_intf_addr_assigned base_path = ['vrf'] diff --git a/smoketest/scripts/system/test_kernel_options.py b/smoketest/scripts/system/test_kernel_options.py index fe2a1c48a..a39ae50bc 100755 --- a/smoketest/scripts/system/test_kernel_options.py +++ b/smoketest/scripts/system/test_kernel_options.py @@ -20,14 +20,13 @@ import os import platform import unittest -from vyos.util import call -from vyos.util import read_file +from vyos.utils.process import call +from vyos.utils.file import read_file kernel = platform.release() config = read_file(f'/boot/config-{kernel}') CONFIG = '/proc/config.gz' - class TestKernelModules(unittest.TestCase): """ VyOS makes use of a lot of Kernel drivers, modules and features. The required modules which are essential for VyOS should be tested that they are diff --git a/smoketest/scripts/system/test_module_load.py b/smoketest/scripts/system/test_module_load.py index bd30c57ec..9d94f01e6 100755 --- a/smoketest/scripts/system/test_module_load.py +++ b/smoketest/scripts/system/test_module_load.py @@ -15,7 +15,7 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. import unittest -from vyos.util import cmd +from vyos.utils.process import cmd modules = { "intel": ["e1000", "e1000e", "igb", "ixgb", "ixgbe", "ixgbevf", "i40e", @@ -23,7 +23,8 @@ modules = { "intel_qat": ["qat_200xx", "qat_200xxvf", "qat_c3xxx", "qat_c3xxxvf", "qat_c62x", "qat_c62xvf", "qat_d15xx", "qat_d15xxvf", "qat_dh895xcc", "qat_dh895xccvf"], - "accel_ppp": ["ipoe", "vlan_mon"] + "accel_ppp": ["ipoe", "vlan_mon"], + "openvpn": ["ovpn-dco-v2"] } class TestKernelModules(unittest.TestCase): diff --git a/src/completion/list_dumpable_interfaces.py b/src/completion/list_dumpable_interfaces.py index 67bf6206b..f9748352f 100755 --- a/src/completion/list_dumpable_interfaces.py +++ b/src/completion/list_dumpable_interfaces.py @@ -4,7 +4,7 @@ import re -from vyos.util import cmd +from vyos.utils.process import cmd if __name__ == '__main__': out = cmd('tcpdump -D').split('\n') diff --git a/src/completion/list_ipoe.py b/src/completion/list_ipoe.py index c386b46a2..5a8f4b0c5 100755 --- a/src/completion/list_ipoe.py +++ b/src/completion/list_ipoe.py @@ -1,7 +1,21 @@ #!/usr/bin/env python3 +# Copyright 2020-2023 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. import argparse -from vyos.util import popen +from vyos.utils.process import popen if __name__ == '__main__': parser = argparse.ArgumentParser() diff --git a/src/completion/list_ipsec_profile_tunnels.py b/src/completion/list_ipsec_profile_tunnels.py index df6c52f6d..4a917dbd6 100644 --- a/src/completion/list_ipsec_profile_tunnels.py +++ b/src/completion/list_ipsec_profile_tunnels.py @@ -19,7 +19,7 @@ import sys import argparse from vyos.config import Config -from vyos.util import dict_search +from vyos.utils.dict import dict_search def get_tunnels_from_ipsecprofile(profile): config = Config() diff --git a/src/completion/list_openconnect_users.py b/src/completion/list_openconnect_users.py index a266fd893..db2f4b4da 100755 --- a/src/completion/list_openconnect_users.py +++ b/src/completion/list_openconnect_users.py @@ -15,7 +15,7 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. from vyos.config import Config -from vyos.util import dict_search +from vyos.utils.dict import dict_search def get_user_from_ocserv(): config = Config() diff --git a/src/completion/list_openvpn_users.py b/src/completion/list_openvpn_users.py index c472dbeab..b2b0149fc 100755 --- a/src/completion/list_openvpn_users.py +++ b/src/completion/list_openvpn_users.py @@ -19,7 +19,7 @@ import sys import argparse from vyos.config import Config -from vyos.util import dict_search +from vyos.utils.dict import dict_search def get_user_from_interface(interface): config = Config() diff --git a/src/conf_mode/arp.py b/src/conf_mode/arp.py index 7dc5206e0..b141f1141 100755 --- a/src/conf_mode/arp.py +++ b/src/conf_mode/arp.py @@ -18,7 +18,7 @@ from sys import exit from vyos.config import Config from vyos.configdict import node_changed -from vyos.util import call +from vyos.utils.process import call from vyos import ConfigError from vyos import airbag airbag.enable() diff --git a/src/conf_mode/bcast_relay.py b/src/conf_mode/bcast_relay.py index 7b93a31c0..ced5d212e 100755 --- a/src/conf_mode/bcast_relay.py +++ b/src/conf_mode/bcast_relay.py @@ -23,7 +23,7 @@ from sys import exit from vyos.config import Config from vyos.configverify import verify_interface_exists from vyos.template import render -from vyos.util import call +from vyos.utils.process import call from vyos.validate import is_afi_configured from vyos import ConfigError from vyos import airbag diff --git a/src/conf_mode/conntrack.py b/src/conf_mode/conntrack.py index 82289526f..2a77540f7 100755 --- a/src/conf_mode/conntrack.py +++ b/src/conf_mode/conntrack.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2021 VyOS maintainers and contributors +# Copyright (C) 2021-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -23,10 +23,10 @@ from vyos.config import Config from vyos.configdict import dict_merge from vyos.firewall import find_nftables_rule from vyos.firewall import remove_nftables_rule -from vyos.util import cmd -from vyos.util import run -from vyos.util import process_named_running -from vyos.util import dict_search +from vyos.utils.process import process_named_running +from vyos.utils.dict import dict_search +from vyos.utils.process import cmd +from vyos.utils.process import run from vyos.template import render from vyos.xml import defaults from vyos import ConfigError diff --git a/src/conf_mode/conntrack_sync.py b/src/conf_mode/conntrack_sync.py index c4b2bb488..a83c2274d 100755 --- a/src/conf_mode/conntrack_sync.py +++ b/src/conf_mode/conntrack_sync.py @@ -20,11 +20,11 @@ from sys import exit from vyos.config import Config from vyos.configdict import dict_merge from vyos.configverify import verify_interface_exists -from vyos.util import call -from vyos.util import dict_search -from vyos.util import process_named_running -from vyos.util import read_file -from vyos.util import run +from vyos.utils.dict import dict_search +from vyos.utils.process import process_named_running +from vyos.utils.file import read_file +from vyos.utils.process import call +from vyos.utils.process import run from vyos.template import render from vyos.template import get_ipv4 from vyos.validate import is_addr_assigned diff --git a/src/conf_mode/container.py b/src/conf_mode/container.py index 6198bb65f..3378aac63 100755 --- a/src/conf_mode/container.py +++ b/src/conf_mode/container.py @@ -28,11 +28,11 @@ from vyos.configdict import node_changed from vyos.configdict import is_node_changed from vyos.configverify import verify_vrf from vyos.ifconfig import Interface -from vyos.util import call -from vyos.util import cmd -from vyos.util import run -from vyos.util import rc_cmd -from vyos.util import write_file +from vyos.utils.file import write_file +from vyos.utils.process import call +from vyos.utils.process import cmd +from vyos.utils.process import run +from vyos.utils.process import rc_cmd from vyos.template import inc_ip from vyos.template import is_ipv4 from vyos.template import is_ipv6 diff --git a/src/conf_mode/dhcp_relay.py b/src/conf_mode/dhcp_relay.py index 7e702a446..fd39bd9fe 100755 --- a/src/conf_mode/dhcp_relay.py +++ b/src/conf_mode/dhcp_relay.py @@ -23,8 +23,8 @@ from vyos.config import Config from vyos.configdict import dict_merge from vyos.template import render from vyos.base import Warning -from vyos.util import call -from vyos.util import dict_search +from vyos.utils.process import call +from vyos.utils.dict import dict_search from vyos.xml import defaults from vyos import ConfigError from vyos import airbag @@ -51,7 +51,7 @@ def get_config(config=None): def verify(relay): # bail out early - looks like removal from running config - if not relay: + if not relay or 'disable' in relay: return None if 'lo' in (dict_search('interface', relay) or []): @@ -78,7 +78,7 @@ def verify(relay): def generate(relay): # bail out early - looks like removal from running config - if not relay: + if not relay or 'disable' in relay: return None render(config_file, 'dhcp-relay/dhcrelay.conf.j2', relay) @@ -87,7 +87,7 @@ def generate(relay): def apply(relay): # bail out early - looks like removal from running config service_name = 'isc-dhcp-relay.service' - if not relay: + if not relay or 'disable' in relay: call(f'systemctl stop {service_name}') if os.path.exists(config_file): os.unlink(config_file) diff --git a/src/conf_mode/dhcp_server.py b/src/conf_mode/dhcp_server.py index 2b2af252d..c29270367 100755 --- a/src/conf_mode/dhcp_server.py +++ b/src/conf_mode/dhcp_server.py @@ -25,9 +25,9 @@ from sys import exit from vyos.config import Config from vyos.configdict import dict_merge from vyos.template import render -from vyos.util import call -from vyos.util import dict_search -from vyos.util import run +from vyos.utils.dict import dict_search +from vyos.utils.process import call +from vyos.utils.process import run from vyos.validate import is_subnet_connected from vyos.validate import is_addr_assigned from vyos.xml import defaults diff --git a/src/conf_mode/dhcpv6_relay.py b/src/conf_mode/dhcpv6_relay.py index c1bd51f62..0e7da6f89 100755 --- a/src/conf_mode/dhcpv6_relay.py +++ b/src/conf_mode/dhcpv6_relay.py @@ -22,8 +22,9 @@ from vyos.config import Config from vyos.configdict import dict_merge from vyos.ifconfig import Interface from vyos.template import render -from vyos.util import call -from vyos.util import dict_search +from vyos.template import is_ipv6 +from vyos.utils.process import call +from vyos.utils.dict import dict_search from vyos.validate import is_ipv6_link_local from vyos.xml import defaults from vyos import ConfigError @@ -51,7 +52,7 @@ def get_config(config=None): def verify(relay): # bail out early - looks like removal from running config - if not relay: + if not relay or 'disable' in relay: return None if 'upstream_interface' not in relay: @@ -69,7 +70,7 @@ def verify(relay): for interface in relay['listen_interface']: has_global = False for addr in Interface(interface).get_addr(): - if not is_ipv6_link_local(addr): + if is_ipv6(addr) and not is_ipv6_link_local(addr): has_global = True if not has_global: raise ConfigError(f'Interface {interface} does not have global '\ @@ -79,7 +80,7 @@ def verify(relay): def generate(relay): # bail out early - looks like removal from running config - if not relay: + if not relay or 'disable' in relay: return None render(config_file, 'dhcp-relay/dhcrelay6.conf.j2', relay) @@ -88,7 +89,7 @@ def generate(relay): def apply(relay): # bail out early - looks like removal from running config service_name = 'isc-dhcp-relay6.service' - if not relay: + if not relay or 'disable' in relay: # DHCPv6 relay support is removed in the commit call(f'systemctl stop {service_name}') if os.path.exists(config_file): diff --git a/src/conf_mode/dhcpv6_server.py b/src/conf_mode/dhcpv6_server.py index 078ff327c..f89ad5b9c 100755 --- a/src/conf_mode/dhcpv6_server.py +++ b/src/conf_mode/dhcpv6_server.py @@ -23,8 +23,8 @@ from sys import exit from vyos.config import Config from vyos.template import render from vyos.template import is_ipv6 -from vyos.util import call -from vyos.util import dict_search +from vyos.utils.process import call +from vyos.utils.dict import dict_search from vyos.validate import is_subnet_connected from vyos import ConfigError from vyos import airbag diff --git a/src/conf_mode/dns_dynamic.py b/src/conf_mode/dns_dynamic.py index 67134e681..d78eb70bc 100755 --- a/src/conf_mode/dns_dynamic.py +++ b/src/conf_mode/dns_dynamic.py @@ -21,7 +21,7 @@ from sys import exit from vyos.config import Config from vyos.configdict import dict_merge from vyos.template import render -from vyos.util import call +from vyos.utils.process import call from vyos.xml import defaults from vyos import ConfigError from vyos import airbag diff --git a/src/conf_mode/dns_forwarding.py b/src/conf_mode/dns_forwarding.py index 0d86c6a52..2d98bffe3 100755 --- a/src/conf_mode/dns_forwarding.py +++ b/src/conf_mode/dns_forwarding.py @@ -25,9 +25,9 @@ from vyos.configdict import dict_merge from vyos.hostsd_client import Client as hostsd_client from vyos.template import render from vyos.template import bracketize_ipv6 -from vyos.util import call -from vyos.util import chown -from vyos.util import dict_search +from vyos.utils.process import call +from vyos.utils.permission import chown +from vyos.utils.dict import dict_search from vyos.xml import defaults from vyos import ConfigError diff --git a/src/conf_mode/firewall.py b/src/conf_mode/firewall.py index 190587980..07166d457 100755 --- a/src/conf_mode/firewall.py +++ b/src/conf_mode/firewall.py @@ -31,12 +31,12 @@ from vyos.configdep import set_dependents, call_dependents from vyos.firewall import fqdn_config_parse from vyos.firewall import geoip_update from vyos.template import render -from vyos.util import call -from vyos.util import cmd -from vyos.util import dict_search_args -from vyos.util import dict_search_recursive -from vyos.util import process_named_running -from vyos.util import rc_cmd +from vyos.utils.process import call +from vyos.utils.process import cmd +from vyos.utils.dict import dict_search_args +from vyos.utils.dict import dict_search_recursive +from vyos.utils.process import process_named_running +from vyos.utils.process import rc_cmd from vyos.xml import defaults from vyos import ConfigError from vyos import airbag diff --git a/src/conf_mode/flow_accounting_conf.py b/src/conf_mode/flow_accounting_conf.py index c36d52e05..bfe906c87 100755 --- a/src/conf_mode/flow_accounting_conf.py +++ b/src/conf_mode/flow_accounting_conf.py @@ -26,8 +26,8 @@ from vyos.configdict import dict_merge from vyos.configverify import verify_vrf from vyos.ifconfig import Section from vyos.template import render -from vyos.util import call -from vyos.util import cmd +from vyos.utils.process import call +from vyos.utils.process import cmd from vyos.validate import is_addr_assigned from vyos.xml import defaults from vyos import ConfigError diff --git a/src/conf_mode/high-availability.py b/src/conf_mode/high-availability.py index e18b426b1..0cbd4c49c 100755 --- a/src/conf_mode/high-availability.py +++ b/src/conf_mode/high-availability.py @@ -28,8 +28,8 @@ from vyos.ifconfig.vrrp import VRRP from vyos.template import render from vyos.template import is_ipv4 from vyos.template import is_ipv6 -from vyos.util import call -from vyos.util import dict_search +from vyos.utils.process import call +from vyos.utils.dict import dict_search from vyos.xml import defaults from vyos import ConfigError from vyos import airbag @@ -175,6 +175,11 @@ def verify(ha): # Virtual-server if 'virtual_server' in ha: for vs, vs_config in ha['virtual_server'].items(): + + if 'address' not in vs_config and 'fwmark' not in vs_config: + raise ConfigError('Either address or fwmark is required ' + f'but not set for virtual-server "{vs}"') + if 'port' not in vs_config and 'fwmark' not in vs_config: raise ConfigError(f'Port or fwmark is required but not set for virtual-server "{vs}"') if 'port' in vs_config and 'fwmark' in vs_config: diff --git a/src/conf_mode/host_name.py b/src/conf_mode/host_name.py index 93f244f42..36d1f6493 100755 --- a/src/conf_mode/host_name.py +++ b/src/conf_mode/host_name.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2018-2021 VyOS maintainers and contributors +# Copyright (C) 2018-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -18,16 +18,15 @@ import re import sys import copy -import vyos.util import vyos.hostsd_client from vyos.base import Warning from vyos.config import Config from vyos.ifconfig import Section from vyos.template import is_ip -from vyos.util import cmd -from vyos.util import call -from vyos.util import process_named_running +from vyos.utils.process import cmd +from vyos.utils.process import call +from vyos.utils.process import process_named_running from vyos import ConfigError from vyos import airbag airbag.enable() diff --git a/src/conf_mode/http-api.py b/src/conf_mode/http-api.py index 7e801eb26..7bdf448a3 100755 --- a/src/conf_mode/http-api.py +++ b/src/conf_mode/http-api.py @@ -27,8 +27,8 @@ from vyos.config import Config from vyos.configdict import dict_merge from vyos.configdep import set_dependents, call_dependents from vyos.template import render -from vyos.util import cmd -from vyos.util import call +from vyos.utils.process import cmd +from vyos.utils.process import call from vyos.xml import defaults from vyos import ConfigError from vyos import airbag diff --git a/src/conf_mode/https.py b/src/conf_mode/https.py index b0c38e8d3..010490c7e 100755 --- a/src/conf_mode/https.py +++ b/src/conf_mode/https.py @@ -28,10 +28,10 @@ from vyos import ConfigError from vyos.pki import wrap_certificate from vyos.pki import wrap_private_key from vyos.template import render -from vyos.util import call -from vyos.util import check_port_availability -from vyos.util import is_listen_port_bind_service -from vyos.util import write_file +from vyos.utils.process import call +from vyos.utils.network import check_port_availability +from vyos.utils.network import is_listen_port_bind_service +from vyos.utils.file import write_file from vyos import airbag airbag.enable() diff --git a/src/conf_mode/igmp_proxy.py b/src/conf_mode/igmp_proxy.py index de6a51c64..4ec2f1835 100755 --- a/src/conf_mode/igmp_proxy.py +++ b/src/conf_mode/igmp_proxy.py @@ -23,8 +23,8 @@ from vyos.base import Warning from vyos.config import Config from vyos.configdict import dict_merge from vyos.template import render -from vyos.util import call -from vyos.util import dict_search +from vyos.utils.process import call +from vyos.utils.dict import dict_search from vyos.xml import defaults from vyos import ConfigError from vyos import airbag diff --git a/src/conf_mode/intel_qat.py b/src/conf_mode/intel_qat.py index dd04a002d..e4b248675 100755 --- a/src/conf_mode/intel_qat.py +++ b/src/conf_mode/intel_qat.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2019-2020 VyOS maintainers and contributors +# Copyright (C) 2019-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -20,7 +20,8 @@ import re from sys import exit from vyos.config import Config -from vyos.util import popen, run +from vyos.utils.process import popen +from vyos.utils.process import run from vyos import ConfigError from vyos import airbag airbag.enable() diff --git a/src/conf_mode/interfaces-bonding.py b/src/conf_mode/interfaces-bonding.py index 9936620c8..c2a569fa9 100755 --- a/src/conf_mode/interfaces-bonding.py +++ b/src/conf_mode/interfaces-bonding.py @@ -35,7 +35,7 @@ from vyos.configverify import verify_vlan_config from vyos.configverify import verify_vrf from vyos.ifconfig import BondIf from vyos.ifconfig import Section -from vyos.util import dict_search +from vyos.utils.dict import dict_search from vyos.validate import has_address_configured from vyos.validate import has_vrf_configured from vyos import ConfigError diff --git a/src/conf_mode/interfaces-bridge.py b/src/conf_mode/interfaces-bridge.py index 4da3b097f..087ead20a 100755 --- a/src/conf_mode/interfaces-bridge.py +++ b/src/conf_mode/interfaces-bridge.py @@ -34,8 +34,8 @@ from vyos.validate import has_address_configured from vyos.validate import has_vrf_configured from vyos.xml import defaults -from vyos.util import cmd -from vyos.util import dict_search +from vyos.utils.process import cmd +from vyos.utils.dict import dict_search from vyos import ConfigError from vyos import airbag diff --git a/src/conf_mode/interfaces-ethernet.py b/src/conf_mode/interfaces-ethernet.py index 9d2ea2eeb..b015bba88 100755 --- a/src/conf_mode/interfaces-ethernet.py +++ b/src/conf_mode/interfaces-ethernet.py @@ -40,9 +40,9 @@ from vyos.pki import encode_certificate from vyos.pki import load_certificate from vyos.pki import wrap_private_key from vyos.template import render -from vyos.util import call -from vyos.util import dict_search -from vyos.util import write_file +from vyos.utils.process import call +from vyos.utils.dict import dict_search +from vyos.utils.file import write_file from vyos import ConfigError from vyos import airbag airbag.enable() diff --git a/src/conf_mode/interfaces-l2tpv3.py b/src/conf_mode/interfaces-l2tpv3.py index ca321e01d..6efeac302 100755 --- a/src/conf_mode/interfaces-l2tpv3.py +++ b/src/conf_mode/interfaces-l2tpv3.py @@ -28,7 +28,7 @@ from vyos.configverify import verify_mtu_ipv6 from vyos.configverify import verify_mirror_redirect from vyos.configverify import verify_bond_bridge_member from vyos.ifconfig import L2TPv3If -from vyos.util import check_kmod +from vyos.utils.kernel import check_kmod from vyos.validate import is_addr_assigned from vyos import ConfigError from vyos import airbag diff --git a/src/conf_mode/interfaces-macsec.py b/src/conf_mode/interfaces-macsec.py index 649ea8d50..3f86e2638 100755 --- a/src/conf_mode/interfaces-macsec.py +++ b/src/conf_mode/interfaces-macsec.py @@ -33,9 +33,9 @@ from vyos.configverify import verify_bond_bridge_member from vyos.ifconfig import MACsecIf from vyos.ifconfig import Interface from vyos.template import render -from vyos.util import call -from vyos.util import dict_search -from vyos.util import is_systemd_service_running +from vyos.utils.process import call +from vyos.utils.dict import dict_search +from vyos.utils.process import is_systemd_service_running from vyos import ConfigError from vyos import airbag airbag.enable() diff --git a/src/conf_mode/interfaces-openvpn.py b/src/conf_mode/interfaces-openvpn.py index 6f227b0d1..607a19385 100755 --- a/src/conf_mode/interfaces-openvpn.py +++ b/src/conf_mode/interfaces-openvpn.py @@ -50,15 +50,15 @@ from vyos.pki import wrap_private_key from vyos.template import render from vyos.template import is_ipv4 from vyos.template import is_ipv6 -from vyos.util import call -from vyos.util import chown -from vyos.util import cmd -from vyos.util import dict_search -from vyos.util import dict_search_args -from vyos.util import is_list_equal -from vyos.util import makedir -from vyos.util import read_file -from vyos.util import write_file +from vyos.utils.dict import dict_search +from vyos.utils.dict import dict_search_args +from vyos.utils.list import is_list_equal +from vyos.utils.file import makedir +from vyos.utils.file import read_file +from vyos.utils.file import write_file +from vyos.utils.process import call +from vyos.utils.permission import chown +from vyos.utils.process import cmd from vyos.validate import is_addr_assigned from vyos import ConfigError diff --git a/src/conf_mode/interfaces-pppoe.py b/src/conf_mode/interfaces-pppoe.py index 5f0b76f90..fca91253c 100755 --- a/src/conf_mode/interfaces-pppoe.py +++ b/src/conf_mode/interfaces-pppoe.py @@ -32,8 +32,8 @@ from vyos.configverify import verify_mtu_ipv6 from vyos.configverify import verify_mirror_redirect from vyos.ifconfig import PPPoEIf from vyos.template import render -from vyos.util import call -from vyos.util import is_systemd_service_running +from vyos.utils.process import call +from vyos.utils.process import is_systemd_service_running from vyos import ConfigError from vyos import airbag airbag.enable() diff --git a/src/conf_mode/interfaces-sstpc.py b/src/conf_mode/interfaces-sstpc.py index b5cc4cf4e..b588910dc 100755 --- a/src/conf_mode/interfaces-sstpc.py +++ b/src/conf_mode/interfaces-sstpc.py @@ -27,10 +27,10 @@ from vyos.pki import encode_certificate from vyos.pki import find_chain from vyos.pki import load_certificate from vyos.template import render -from vyos.util import call -from vyos.util import dict_search -from vyos.util import is_systemd_service_running -from vyos.util import write_file +from vyos.utils.process import call +from vyos.utils.dict import dict_search +from vyos.utils.process import is_systemd_service_running +from vyos.utils.file import write_file from vyos import ConfigError from vyos import airbag airbag.enable() diff --git a/src/conf_mode/interfaces-tunnel.py b/src/conf_mode/interfaces-tunnel.py index 0a3726e94..6a075970e 100755 --- a/src/conf_mode/interfaces-tunnel.py +++ b/src/conf_mode/interfaces-tunnel.py @@ -33,8 +33,8 @@ from vyos.configverify import verify_bond_bridge_member from vyos.ifconfig import Interface from vyos.ifconfig import Section from vyos.ifconfig import TunnelIf -from vyos.util import get_interface_config -from vyos.util import dict_search +from vyos.utils.network import get_interface_config +from vyos.utils.dict import dict_search from vyos import ConfigError from vyos import airbag airbag.enable() diff --git a/src/conf_mode/interfaces-vti.py b/src/conf_mode/interfaces-vti.py index f4b0436af..9871810ae 100755 --- a/src/conf_mode/interfaces-vti.py +++ b/src/conf_mode/interfaces-vti.py @@ -21,7 +21,7 @@ from vyos.config import Config from vyos.configdict import get_interface_dict from vyos.configverify import verify_mirror_redirect from vyos.ifconfig import VTIIf -from vyos.util import dict_search +from vyos.utils.dict import dict_search from vyos import ConfigError from vyos import airbag airbag.enable() diff --git a/src/conf_mode/interfaces-wireguard.py b/src/conf_mode/interfaces-wireguard.py index 762bad94f..a02baba82 100755 --- a/src/conf_mode/interfaces-wireguard.py +++ b/src/conf_mode/interfaces-wireguard.py @@ -27,8 +27,8 @@ from vyos.configverify import verify_mtu_ipv6 from vyos.configverify import verify_mirror_redirect from vyos.configverify import verify_bond_bridge_member from vyos.ifconfig import WireGuardIf -from vyos.util import check_kmod -from vyos.util import check_port_availability +from vyos.utils.kernel import check_kmod +from vyos.utils.network import check_port_availability from vyos import ConfigError from vyos import airbag airbag.enable() diff --git a/src/conf_mode/interfaces-wireless.py b/src/conf_mode/interfaces-wireless.py index dd798b5a2..42326bea0 100755 --- a/src/conf_mode/interfaces-wireless.py +++ b/src/conf_mode/interfaces-wireless.py @@ -33,8 +33,8 @@ from vyos.configverify import verify_vrf from vyos.configverify import verify_bond_bridge_member from vyos.ifconfig import WiFiIf from vyos.template import render -from vyos.util import call -from vyos.util import dict_search +from vyos.utils.process import call +from vyos.utils.dict import dict_search from vyos import ConfigError from vyos import airbag airbag.enable() diff --git a/src/conf_mode/interfaces-wwan.py b/src/conf_mode/interfaces-wwan.py index 9ca495476..6658ca86a 100755 --- a/src/conf_mode/interfaces-wwan.py +++ b/src/conf_mode/interfaces-wwan.py @@ -27,12 +27,12 @@ from vyos.configverify import verify_interface_exists from vyos.configverify import verify_mirror_redirect from vyos.configverify import verify_vrf from vyos.ifconfig import WWANIf -from vyos.util import cmd -from vyos.util import call -from vyos.util import dict_search -from vyos.util import DEVNULL -from vyos.util import is_systemd_service_active -from vyos.util import write_file +from vyos.utils.dict import dict_search +from vyos.utils.process import cmd +from vyos.utils.process import call +from vyos.utils.process import DEVNULL +from vyos.utils.process import is_systemd_service_active +from vyos.utils.file import write_file from vyos import ConfigError from vyos import airbag airbag.enable() diff --git a/src/conf_mode/le_cert.py b/src/conf_mode/le_cert.py index 6e169a3d5..06c7e7b72 100755 --- a/src/conf_mode/le_cert.py +++ b/src/conf_mode/le_cert.py @@ -20,9 +20,9 @@ import os import vyos.defaults from vyos.config import Config from vyos import ConfigError -from vyos.util import cmd -from vyos.util import call -from vyos.util import is_systemd_service_running +from vyos.utils.process import cmd +from vyos.utils.process import call +from vyos.utils.process import is_systemd_service_running from vyos import airbag airbag.enable() diff --git a/src/conf_mode/lldp.py b/src/conf_mode/lldp.py index c703c1fe0..0e5fc29d3 100755 --- a/src/conf_mode/lldp.py +++ b/src/conf_mode/lldp.py @@ -24,8 +24,8 @@ from vyos.configdict import dict_merge from vyos.validate import is_addr_assigned from vyos.validate import is_loopback_addr from vyos.version import get_version_data -from vyos.util import call -from vyos.util import dict_search +from vyos.utils.process import call +from vyos.utils.dict import dict_search from vyos.xml import defaults from vyos.template import render from vyos import ConfigError diff --git a/src/conf_mode/load-balancing-haproxy.py b/src/conf_mode/load-balancing-haproxy.py index b29fdffc7..2fb0edf8e 100755 --- a/src/conf_mode/load-balancing-haproxy.py +++ b/src/conf_mode/load-balancing-haproxy.py @@ -21,9 +21,9 @@ from shutil import rmtree from vyos.config import Config from vyos.configdict import dict_merge -from vyos.util import call -from vyos.util import check_port_availability -from vyos.util import is_listen_port_bind_service +from vyos.utils.process import call +from vyos.utils.network import check_port_availability +from vyos.utils.network import is_listen_port_bind_service from vyos.pki import wrap_certificate from vyos.pki import wrap_private_key from vyos.template import render diff --git a/src/conf_mode/load-balancing-wan.py b/src/conf_mode/load-balancing-wan.py index 7086aaf8b..3533a5a04 100755 --- a/src/conf_mode/load-balancing-wan.py +++ b/src/conf_mode/load-balancing-wan.py @@ -22,7 +22,7 @@ from shutil import rmtree from vyos.base import Warning from vyos.config import Config from vyos.configdict import dict_merge -from vyos.util import cmd +from vyos.utils.process import cmd from vyos.template import render from vyos.xml import defaults from vyos import ConfigError diff --git a/src/conf_mode/nat.py b/src/conf_mode/nat.py index 9f8221514..e19b12937 100755 --- a/src/conf_mode/nat.py +++ b/src/conf_mode/nat.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2020-2022 VyOS maintainers and contributors +# Copyright (C) 2020-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -28,11 +28,11 @@ from vyos.config import Config from vyos.configdict import dict_merge from vyos.template import render from vyos.template import is_ip_network -from vyos.util import cmd -from vyos.util import run -from vyos.util import check_kmod -from vyos.util import dict_search -from vyos.util import dict_search_args +from vyos.utils.kernel import check_kmod +from vyos.utils.dict import dict_search +from vyos.utils.dict import dict_search_args +from vyos.utils.process import cmd +from vyos.utils.process import run from vyos.validate import is_addr_assigned from vyos.xml import defaults from vyos import ConfigError @@ -72,6 +72,7 @@ def verify_rule(config, err_msg, groups_dict): """ Common verify steps used for both source and destination NAT """ if (dict_search('translation.port', config) != None or + dict_search('translation.redirect.port', config) != None or dict_search('destination.port', config) != None or dict_search('source.port', config)): @@ -221,7 +222,7 @@ def verify(nat): elif config['inbound_interface'] not in 'any' and config['inbound_interface'] not in interfaces(): Warning(f'rule "{rule}" interface "{config["inbound_interface"]}" does not exist on this system') - if not dict_search('translation.address', config) and not dict_search('translation.port', config): + if not dict_search('translation.address', config) and not dict_search('translation.port', config) and not dict_search('translation.redirect.port', config): if 'exclude' not in config: raise ConfigError(f'{err_msg} translation requires address and/or port') diff --git a/src/conf_mode/nat66.py b/src/conf_mode/nat66.py index d8f913b0c..25f625b84 100755 --- a/src/conf_mode/nat66.py +++ b/src/conf_mode/nat66.py @@ -25,9 +25,9 @@ from vyos.base import Warning from vyos.config import Config from vyos.configdict import dict_merge from vyos.template import render -from vyos.util import cmd -from vyos.util import check_kmod -from vyos.util import dict_search +from vyos.utils.process import cmd +from vyos.utils.kernel import check_kmod +from vyos.utils.dict import dict_search from vyos.template import is_ipv6 from vyos.xml import defaults from vyos import ConfigError diff --git a/src/conf_mode/netns.py b/src/conf_mode/netns.py index 20129ce65..95ab83dbc 100755 --- a/src/conf_mode/netns.py +++ b/src/conf_mode/netns.py @@ -22,9 +22,9 @@ from tempfile import NamedTemporaryFile from vyos.config import Config from vyos.configdict import node_changed from vyos.ifconfig import Interface -from vyos.util import call -from vyos.util import dict_search -from vyos.util import get_interface_config +from vyos.utils.process import call +from vyos.utils.dict import dict_search +from vyos.utils.network import get_interface_config from vyos import ConfigError from vyos import airbag airbag.enable() diff --git a/src/conf_mode/ntp.py b/src/conf_mode/ntp.py index 95766c44c..1cc23a7df 100755 --- a/src/conf_mode/ntp.py +++ b/src/conf_mode/ntp.py @@ -20,9 +20,9 @@ from vyos.config import Config from vyos.configdict import is_node_changed from vyos.configverify import verify_vrf from vyos.configverify import verify_interface_exists -from vyos.util import call -from vyos.util import chmod_750 -from vyos.util import get_interface_config +from vyos.utils.process import call +from vyos.utils.permission import chmod_750 +from vyos.utils.network import get_interface_config from vyos.template import render from vyos.template import is_ipv4 from vyos import ConfigError diff --git a/src/conf_mode/pki.py b/src/conf_mode/pki.py index 54de467ca..eb8cb3940 100755 --- a/src/conf_mode/pki.py +++ b/src/conf_mode/pki.py @@ -26,8 +26,8 @@ from vyos.pki import load_public_key from vyos.pki import load_private_key from vyos.pki import load_crl from vyos.pki import load_dh_parameters -from vyos.util import dict_search_args -from vyos.util import dict_search_recursive +from vyos.utils.dict import dict_search_args +from vyos.utils.dict import dict_search_recursive from vyos.xml import defaults from vyos import ConfigError from vyos import airbag diff --git a/src/conf_mode/policy-local-route.py b/src/conf_mode/policy-local-route.py index 3f834f55c..79526f82a 100755 --- a/src/conf_mode/policy-local-route.py +++ b/src/conf_mode/policy-local-route.py @@ -24,7 +24,7 @@ from vyos.configdict import dict_merge from vyos.configdict import node_changed from vyos.configdict import leaf_node_changed from vyos.template import render -from vyos.util import call +from vyos.utils.process import call from vyos import ConfigError from vyos import airbag airbag.enable() diff --git a/src/conf_mode/policy-route.py b/src/conf_mode/policy-route.py index 40a32efb3..adad012de 100755 --- a/src/conf_mode/policy-route.py +++ b/src/conf_mode/policy-route.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2021 VyOS maintainers and contributors +# Copyright (C) 2021-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -22,9 +22,9 @@ from sys import exit from vyos.base import Warning from vyos.config import Config from vyos.template import render -from vyos.util import cmd -from vyos.util import dict_search_args -from vyos.util import run +from vyos.utils.dict import dict_search_args +from vyos.utils.process import cmd +from vyos.utils.process import run from vyos import ConfigError from vyos import airbag airbag.enable() diff --git a/src/conf_mode/policy.py b/src/conf_mode/policy.py index 331194fec..4df893ebf 100755 --- a/src/conf_mode/policy.py +++ b/src/conf_mode/policy.py @@ -19,7 +19,7 @@ from sys import exit from vyos.config import Config from vyos.configdict import dict_merge from vyos.template import render_to_string -from vyos.util import dict_search +from vyos.utils.dict import dict_search from vyos import ConfigError from vyos import frr from vyos import airbag diff --git a/src/conf_mode/protocols_babel.py b/src/conf_mode/protocols_babel.py index 20821c7f2..f5ac56f65 100755 --- a/src/conf_mode/protocols_babel.py +++ b/src/conf_mode/protocols_babel.py @@ -24,7 +24,7 @@ from vyos.configdict import node_changed from vyos.configverify import verify_common_route_maps from vyos.configverify import verify_access_list from vyos.configverify import verify_prefix_list -from vyos.util import dict_search +from vyos.utils.dict import dict_search from vyos.xml import defaults from vyos.template import render_to_string from vyos import ConfigError diff --git a/src/conf_mode/protocols_bgp.py b/src/conf_mode/protocols_bgp.py index b23584bdb..cec025fea 100755 --- a/src/conf_mode/protocols_bgp.py +++ b/src/conf_mode/protocols_bgp.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2020-2022 VyOS maintainers and contributors +# Copyright (C) 2020-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -20,13 +20,15 @@ from sys import argv from vyos.base import Warning from vyos.config import Config from vyos.configdict import dict_merge +from vyos.configdict import node_changed from vyos.configverify import verify_prefix_list from vyos.configverify import verify_route_map from vyos.configverify import verify_vrf from vyos.template import is_ip from vyos.template import is_interface from vyos.template import render_to_string -from vyos.util import dict_search +from vyos.utils.dict import dict_search +from vyos.utils.network import get_interface_vrf from vyos.validate import is_addr_assigned from vyos import ConfigError from vyos import frr @@ -55,6 +57,12 @@ def get_config(config=None): get_first_key=True, no_tag_node_value_mangle=True) + # Remove per interface MPLS configuration - get a list if changed + # nodes under the interface tagNode + interfaces_removed = node_changed(conf, base + ['interface']) + if interfaces_removed: + bgp['interface_removed'] = list(interfaces_removed) + # Assign the name of our VRF context. This MUST be done before the return # statement below, else on deletion we will delete the default instance # instead of the VRF instance. @@ -195,14 +203,21 @@ def verify_remote_as(peer_config, bgp_config): return None def verify_afi(peer_config, bgp_config): + # If address_family configured under neighboor if 'address_family' in peer_config: return True + # If address_family configured under peer-group + # if neighbor interface configured + peer_group_name = '' + if dict_search('interface.peer_group', peer_config): + peer_group_name = peer_config['interface']['peer_group'] + # if neighbor IP configured. if 'peer_group' in peer_config: peer_group_name = peer_config['peer_group'] + if peer_group_name: tmp = dict_search(f'peer_group.{peer_group_name}.address_family', bgp_config) if tmp: return True - return False def verify(bgp): @@ -231,6 +246,18 @@ def verify(bgp): if 'system_as' not in bgp: raise ConfigError('BGP system-as number must be defined!') + # Verify vrf on interface and bgp section + if 'interface' in bgp: + for interface in bgp['interface']: + error_msg = f'Interface "{interface}" belongs to different VRF instance' + tmp = get_interface_vrf(interface) + if 'vrf' in bgp: + if bgp['vrf'] != tmp: + vrf = bgp['vrf'] + raise ConfigError(f'{error_msg} "{vrf}"!') + elif tmp != 'default': + raise ConfigError(f'{error_msg} "{tmp}"!') + # Common verification for both peer-group and neighbor statements for neighbor in ['neighbor', 'peer_group']: # bail out early if there is no neighbor or peer-group statement @@ -520,6 +547,14 @@ def apply(bgp): vrf = ' vrf ' + bgp['vrf'] frr_cfg.load_configuration(bgp_daemon) + + # Remove interface specific config + for key in ['interface', 'interface_removed']: + if key not in bgp: + continue + for interface in bgp[key]: + frr_cfg.modify_section(f'^interface {interface}', stop_pattern='^exit', remove_stop_mark=True) + frr_cfg.modify_section(f'^router bgp \d+{vrf}', stop_pattern='^exit', remove_stop_mark=True) if 'frr_bgpd_config' in bgp: frr_cfg.add_before(frr.default_add_before, bgp['frr_bgpd_config']) diff --git a/src/conf_mode/protocols_failover.py b/src/conf_mode/protocols_failover.py index 85e984afe..faf56d741 100755 --- a/src/conf_mode/protocols_failover.py +++ b/src/conf_mode/protocols_failover.py @@ -21,7 +21,7 @@ from pathlib import Path from vyos.config import Config from vyos.configdict import dict_merge from vyos.template import render -from vyos.util import call +from vyos.utils.process import call from vyos.xml import defaults from vyos import ConfigError from vyos import airbag diff --git a/src/conf_mode/protocols_igmp.py b/src/conf_mode/protocols_igmp.py index 65cc2beba..f6097e282 100755 --- a/src/conf_mode/protocols_igmp.py +++ b/src/conf_mode/protocols_igmp.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2020 VyOS maintainers and contributors +# Copyright (C) 2020-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -21,7 +21,8 @@ from sys import exit from vyos import ConfigError from vyos.config import Config -from vyos.util import call, process_named_running +from vyos.utils.process import process_named_running +from vyos.utils.process import call from vyos.template import render from signal import SIGTERM diff --git a/src/conf_mode/protocols_isis.py b/src/conf_mode/protocols_isis.py index ecca87db0..4c637a99f 100755 --- a/src/conf_mode/protocols_isis.py +++ b/src/conf_mode/protocols_isis.py @@ -25,8 +25,8 @@ from vyos.configdict import node_changed from vyos.configverify import verify_common_route_maps from vyos.configverify import verify_interface_exists from vyos.ifconfig import Interface -from vyos.util import dict_search -from vyos.util import get_interface_config +from vyos.utils.dict import dict_search +from vyos.utils.network import get_interface_config from vyos.template import render_to_string from vyos.xml import defaults from vyos import ConfigError diff --git a/src/conf_mode/protocols_mpls.py b/src/conf_mode/protocols_mpls.py index 73af6595b..177a43444 100755 --- a/src/conf_mode/protocols_mpls.py +++ b/src/conf_mode/protocols_mpls.py @@ -21,9 +21,9 @@ from sys import exit from glob import glob from vyos.config import Config from vyos.template import render_to_string -from vyos.util import dict_search -from vyos.util import read_file -from vyos.util import sysctl_write +from vyos.utils.dict import dict_search +from vyos.utils.file import read_file +from vyos.utils.system import sysctl_write from vyos.configverify import verify_interface_exists from vyos import ConfigError from vyos import frr diff --git a/src/conf_mode/protocols_nhrp.py b/src/conf_mode/protocols_nhrp.py index d28ced4fd..5ec0bc9e5 100755 --- a/src/conf_mode/protocols_nhrp.py +++ b/src/conf_mode/protocols_nhrp.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2021 VyOS maintainers and contributors +# Copyright (C) 2021-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -19,8 +19,8 @@ import os from vyos.config import Config from vyos.configdict import node_changed from vyos.template import render -from vyos.util import process_named_running -from vyos.util import run +from vyos.utils.process import process_named_running +from vyos.utils.process import run from vyos import ConfigError from vyos import airbag airbag.enable() diff --git a/src/conf_mode/protocols_ospf.py b/src/conf_mode/protocols_ospf.py index 460c9f1a4..f2075d25b 100755 --- a/src/conf_mode/protocols_ospf.py +++ b/src/conf_mode/protocols_ospf.py @@ -27,8 +27,8 @@ from vyos.configverify import verify_route_map from vyos.configverify import verify_interface_exists from vyos.configverify import verify_access_list from vyos.template import render_to_string -from vyos.util import dict_search -from vyos.util import get_interface_config +from vyos.utils.dict import dict_search +from vyos.utils.network import get_interface_config from vyos.xml import defaults from vyos import ConfigError from vyos import frr @@ -88,6 +88,8 @@ def get_config(config=None): del default_values['area']['area_type']['nssa'] if 'mpls_te' not in ospf: del default_values['mpls_te'] + if 'graceful_restart' not in ospf: + del default_values['graceful_restart'] for protocol in ['babel', 'bgp', 'connected', 'isis', 'kernel', 'rip', 'static', 'table']: # table is a tagNode thus we need to clean out all occurances for the diff --git a/src/conf_mode/protocols_ospfv3.py b/src/conf_mode/protocols_ospfv3.py index cb21bd83c..fbea51f56 100755 --- a/src/conf_mode/protocols_ospfv3.py +++ b/src/conf_mode/protocols_ospfv3.py @@ -27,8 +27,8 @@ from vyos.configverify import verify_route_map from vyos.configverify import verify_interface_exists from vyos.template import render_to_string from vyos.ifconfig import Interface -from vyos.util import dict_search -from vyos.util import get_interface_config +from vyos.utils.dict import dict_search +from vyos.utils.network import get_interface_config from vyos.xml import defaults from vyos import ConfigError from vyos import frr @@ -83,6 +83,8 @@ def get_config(config=None): # need to check this first and probably drop that key. if dict_search('default_information.originate', ospfv3) is None: del default_values['default_information'] + if 'graceful_restart' not in ospfv3: + del default_values['graceful_restart'] # XXX: T2665: we currently have no nice way for defaults under tag nodes, # clean them out and add them manually :( diff --git a/src/conf_mode/protocols_pim.py b/src/conf_mode/protocols_pim.py index 78df9b6f8..0aaa0d2c6 100755 --- a/src/conf_mode/protocols_pim.py +++ b/src/conf_mode/protocols_pim.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2020 VyOS maintainers and contributors +# Copyright (C) 2020-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -21,7 +21,8 @@ from sys import exit from vyos.config import Config from vyos import ConfigError -from vyos.util import call, process_named_running +from vyos.utils.process import process_named_running +from vyos.utils.process import call from vyos.template import render from signal import SIGTERM diff --git a/src/conf_mode/protocols_rip.py b/src/conf_mode/protocols_rip.py index c78d90396..5661dc377 100755 --- a/src/conf_mode/protocols_rip.py +++ b/src/conf_mode/protocols_rip.py @@ -24,7 +24,7 @@ from vyos.configdict import node_changed from vyos.configverify import verify_common_route_maps from vyos.configverify import verify_access_list from vyos.configverify import verify_prefix_list -from vyos.util import dict_search +from vyos.utils.dict import dict_search from vyos.xml import defaults from vyos.template import render_to_string from vyos import ConfigError diff --git a/src/conf_mode/protocols_ripng.py b/src/conf_mode/protocols_ripng.py index 21ff710b3..e3c904e33 100755 --- a/src/conf_mode/protocols_ripng.py +++ b/src/conf_mode/protocols_ripng.py @@ -23,7 +23,7 @@ from vyos.configdict import dict_merge from vyos.configverify import verify_common_route_maps from vyos.configverify import verify_access_list from vyos.configverify import verify_prefix_list -from vyos.util import dict_search +from vyos.utils.dict import dict_search from vyos.xml import defaults from vyos.template import render_to_string from vyos import ConfigError diff --git a/src/conf_mode/protocols_rpki.py b/src/conf_mode/protocols_rpki.py index 62ea9c878..035b7db05 100755 --- a/src/conf_mode/protocols_rpki.py +++ b/src/conf_mode/protocols_rpki.py @@ -21,7 +21,7 @@ from sys import exit from vyos.config import Config from vyos.configdict import dict_merge from vyos.template import render_to_string -from vyos.util import dict_search +from vyos.utils.dict import dict_search from vyos.xml import defaults from vyos import ConfigError from vyos import frr diff --git a/src/conf_mode/protocols_static.py b/src/conf_mode/protocols_static.py index 7b6150696..5def8d645 100755 --- a/src/conf_mode/protocols_static.py +++ b/src/conf_mode/protocols_static.py @@ -47,7 +47,7 @@ def get_config(config=None): base_path = ['protocols', 'static'] # eqivalent of the C foo ? 'a' : 'b' statement base = vrf and ['vrf', 'name', vrf, 'protocols', 'static'] or base_path - static = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True) + static = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True, no_tag_node_value_mangle=True) # Assign the name of our VRF context if vrf: static['vrf'] = vrf diff --git a/src/conf_mode/protocols_static_multicast.py b/src/conf_mode/protocols_static_multicast.py index 6afdf31f3..7f6ae3680 100755 --- a/src/conf_mode/protocols_static_multicast.py +++ b/src/conf_mode/protocols_static_multicast.py @@ -21,7 +21,7 @@ from sys import exit from vyos import ConfigError from vyos.config import Config -from vyos.util import call +from vyos.utils.process import call from vyos.template import render from vyos import airbag diff --git a/src/conf_mode/qos.py b/src/conf_mode/qos.py index 1be2c283f..53e9ff50d 100755 --- a/src/conf_mode/qos.py +++ b/src/conf_mode/qos.py @@ -36,8 +36,8 @@ from vyos.qos import RateLimiter from vyos.qos import RoundRobin from vyos.qos import TrafficShaper from vyos.qos import TrafficShaperHFSC -from vyos.util import call -from vyos.util import dict_search_recursive +from vyos.utils.process import call +from vyos.utils.dict import dict_search_recursive from vyos.xml import defaults from vyos import ConfigError from vyos import airbag diff --git a/src/conf_mode/salt-minion.py b/src/conf_mode/salt-minion.py index 00b889a11..3ff7880b2 100755 --- a/src/conf_mode/salt-minion.py +++ b/src/conf_mode/salt-minion.py @@ -25,8 +25,8 @@ from vyos.config import Config from vyos.configdict import dict_merge from vyos.configverify import verify_interface_exists from vyos.template import render -from vyos.util import call -from vyos.util import chown +from vyos.utils.process import call +from vyos.utils.permission import chown from vyos.xml import defaults from vyos import ConfigError diff --git a/src/conf_mode/service_config_sync.py b/src/conf_mode/service_config_sync.py new file mode 100755 index 000000000..5cde735a1 --- /dev/null +++ b/src/conf_mode/service_config_sync.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2023 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import os +import json +from pathlib import Path + +from vyos.config import Config +from vyos.configdict import dict_merge +from vyos.xml import defaults +from vyos import ConfigError +from vyos import airbag + +airbag.enable() + + +service_conf = Path(f'/run/config_sync_conf.conf') +post_commit_dir = '/run/scripts/commit/post-hooks.d' +post_commit_file_src = '/usr/libexec/vyos/vyos_config_sync.py' +post_commit_file = f'{post_commit_dir}/vyos_config_sync' + + +def get_config(config=None): + if config: + conf = config + else: + conf = Config() + + base = ['service', 'config-sync'] + if not conf.exists(base): + return None + config = conf.get_config_dict(base, + get_first_key=True, + no_tag_node_value_mangle=True) + + default_values = defaults(base) + config = dict_merge(default_values, config) + + return config + + +def verify(config): + # bail out early - looks like removal from running config + if not config: + return None + + if 'mode' not in config: + raise ConfigError(f'config-sync mode is mandatory!') + + for option in ['secondary', 'section']: + if option not in config: + raise ConfigError(f"config-sync '{option}' is not configured!") + + if 'address' not in config['secondary']: + raise ConfigError(f'secondary address is mandatory!') + if 'key' not in config['secondary']: + raise ConfigError(f'secondary key is mandatory!') + + +def generate(config): + if not config: + + if os.path.exists(post_commit_file): + os.unlink(post_commit_file) + + if service_conf.exists(): + service_conf.unlink() + + return None + + # Write configuration file + conf_json = json.dumps(config, indent=4) + service_conf.write_text(conf_json) + + # Create post commit dir + if not os.path.isdir(post_commit_dir): + os.makedirs(post_commit_dir) + + # Symlink from helpers to post-commit + if not os.path.exists(post_commit_file): + os.symlink(post_commit_file_src, post_commit_file) + + return None + + +def apply(config): + return None + + +if __name__ == '__main__': + try: + c = get_config() + verify(c) + generate(c) + apply(c) + except ConfigError as e: + print(e) + exit(1) diff --git a/src/conf_mode/service_console-server.py b/src/conf_mode/service_console-server.py index 60eff6543..7eb41ea87 100755 --- a/src/conf_mode/service_console-server.py +++ b/src/conf_mode/service_console-server.py @@ -22,7 +22,7 @@ from psutil import process_iter from vyos.config import Config from vyos.configdict import dict_merge from vyos.template import render -from vyos.util import call +from vyos.utils.process import call from vyos.xml import defaults from vyos import ConfigError diff --git a/src/conf_mode/service_event_handler.py b/src/conf_mode/service_event_handler.py index 5440d1056..5028ef52f 100755 --- a/src/conf_mode/service_event_handler.py +++ b/src/conf_mode/service_event_handler.py @@ -18,7 +18,8 @@ import json from pathlib import Path from vyos.config import Config -from vyos.util import call, dict_search +from vyos.utils.dict import dict_search +from vyos.utils.process import call from vyos import ConfigError from vyos import airbag diff --git a/src/conf_mode/service_ids_fastnetmon.py b/src/conf_mode/service_ids_fastnetmon.py index c58f8db9a..2e678cf0b 100755 --- a/src/conf_mode/service_ids_fastnetmon.py +++ b/src/conf_mode/service_ids_fastnetmon.py @@ -21,7 +21,7 @@ from sys import exit from vyos.config import Config from vyos.configdict import dict_merge from vyos.template import render -from vyos.util import call +from vyos.utils.process import call from vyos.xml import defaults from vyos import ConfigError from vyos import airbag diff --git a/src/conf_mode/service_ipoe-server.py b/src/conf_mode/service_ipoe-server.py index 95c72df47..b70e32373 100755 --- a/src/conf_mode/service_ipoe-server.py +++ b/src/conf_mode/service_ipoe-server.py @@ -24,8 +24,8 @@ from vyos.configdict import get_accel_dict from vyos.configverify import verify_accel_ppp_base_service from vyos.configverify import verify_interface_exists from vyos.template import render -from vyos.util import call -from vyos.util import dict_search +from vyos.utils.process import call +from vyos.utils.dict import dict_search from vyos import ConfigError from vyos import airbag airbag.enable() diff --git a/src/conf_mode/service_mdns-repeater.py b/src/conf_mode/service_mdns-repeater.py index 2383a53fb..a2c90b537 100755 --- a/src/conf_mode/service_mdns-repeater.py +++ b/src/conf_mode/service_mdns-repeater.py @@ -23,7 +23,7 @@ from netifaces import ifaddresses, interfaces, AF_INET from vyos.config import Config from vyos.ifconfig.vrrp import VRRP from vyos.template import render -from vyos.util import call +from vyos.utils.process import call from vyos import ConfigError from vyos import airbag airbag.enable() diff --git a/src/conf_mode/service_monitoring_telegraf.py b/src/conf_mode/service_monitoring_telegraf.py index 47510ce80..0269bedd9 100755 --- a/src/conf_mode/service_monitoring_telegraf.py +++ b/src/conf_mode/service_monitoring_telegraf.py @@ -27,9 +27,9 @@ from vyos.configdict import is_node_changed from vyos.configverify import verify_vrf from vyos.ifconfig import Section from vyos.template import render -from vyos.util import call -from vyos.util import chown -from vyos.util import cmd +from vyos.utils.process import call +from vyos.utils.permission import chown +from vyos.utils.process import cmd from vyos.xml import defaults from vyos import ConfigError from vyos import airbag diff --git a/src/conf_mode/service_pppoe-server.py b/src/conf_mode/service_pppoe-server.py index adeefaa37..aace267a7 100755 --- a/src/conf_mode/service_pppoe-server.py +++ b/src/conf_mode/service_pppoe-server.py @@ -24,8 +24,8 @@ from vyos.configdict import is_node_changed from vyos.configverify import verify_accel_ppp_base_service from vyos.configverify import verify_interface_exists from vyos.template import render -from vyos.util import call -from vyos.util import dict_search +from vyos.utils.process import call +from vyos.utils.dict import dict_search from vyos import ConfigError from vyos import airbag airbag.enable() diff --git a/src/conf_mode/service_router-advert.py b/src/conf_mode/service_router-advert.py index 1dd973d67..fe33c43ea 100755 --- a/src/conf_mode/service_router-advert.py +++ b/src/conf_mode/service_router-advert.py @@ -21,7 +21,7 @@ from vyos.base import Warning from vyos.config import Config from vyos.configdict import dict_merge from vyos.template import render -from vyos.util import call +from vyos.utils.process import call from vyos.xml import defaults from vyos import ConfigError from vyos import airbag diff --git a/src/conf_mode/service_sla.py b/src/conf_mode/service_sla.py index b1e22f37b..54b72e029 100755 --- a/src/conf_mode/service_sla.py +++ b/src/conf_mode/service_sla.py @@ -21,7 +21,7 @@ from sys import exit from vyos.config import Config from vyos.configdict import dict_merge from vyos.template import render -from vyos.util import call +from vyos.utils.process import call from vyos.xml import defaults from vyos import ConfigError from vyos import airbag diff --git a/src/conf_mode/service_upnp.py b/src/conf_mode/service_upnp.py index c798fd515..b37d502c2 100755 --- a/src/conf_mode/service_upnp.py +++ b/src/conf_mode/service_upnp.py @@ -24,7 +24,7 @@ from ipaddress import IPv6Network from vyos.config import Config from vyos.configdict import dict_merge -from vyos.util import call +from vyos.utils.process import call from vyos.template import render from vyos.template import is_ipv4 from vyos.template import is_ipv6 diff --git a/src/conf_mode/service_webproxy.py b/src/conf_mode/service_webproxy.py index 658e496a6..bbdb756bd 100755 --- a/src/conf_mode/service_webproxy.py +++ b/src/conf_mode/service_webproxy.py @@ -22,10 +22,10 @@ from sys import exit from vyos.config import Config from vyos.configdict import dict_merge from vyos.template import render -from vyos.util import call -from vyos.util import chmod_755 -from vyos.util import dict_search -from vyos.util import write_file +from vyos.utils.process import call +from vyos.utils.permission import chmod_755 +from vyos.utils.dict import dict_search +from vyos.utils.file import write_file from vyos.validate import is_addr_assigned from vyos.xml import defaults from vyos.base import Warning diff --git a/src/conf_mode/snmp.py b/src/conf_mode/snmp.py index 9b7c04eb0..0f0d97ac3 100755 --- a/src/conf_mode/snmp.py +++ b/src/conf_mode/snmp.py @@ -26,9 +26,9 @@ from vyos.snmpv3_hashgen import plaintext_to_md5 from vyos.snmpv3_hashgen import plaintext_to_sha1 from vyos.snmpv3_hashgen import random from vyos.template import render -from vyos.util import call -from vyos.util import chmod_755 -from vyos.util import dict_search +from vyos.utils.process import call +from vyos.utils.permission import chmod_755 +from vyos.utils.dict import dict_search from vyos.validate import is_addr_assigned from vyos.version import get_version_data from vyos.xml import defaults @@ -161,8 +161,12 @@ def verify(snmp): for address in snmp['listen_address']: # We only wan't to configure addresses that exist on the system. # Hint the user if they don't exist - if not is_addr_assigned(address): - Warning(f'SNMP listen address "{address}" not configured!') + if 'vrf' in snmp: + vrf_name = snmp['vrf'] + if not is_addr_assigned(address, vrf_name) and address not in ['::1','127.0.0.1']: + raise ConfigError(f'SNMP listen address "{address}" not configured in vrf "{vrf_name}"!') + elif not is_addr_assigned(address): + raise ConfigError(f'SNMP listen address "{address}" not configured in default vrf!') if 'trap_target' in snmp: for trap, trap_config in snmp['trap_target'].items(): diff --git a/src/conf_mode/ssh.py b/src/conf_mode/ssh.py index 8de0617af..3b63fcb7d 100755 --- a/src/conf_mode/ssh.py +++ b/src/conf_mode/ssh.py @@ -24,7 +24,7 @@ from vyos.config import Config from vyos.configdict import dict_merge from vyos.configdict import is_node_changed from vyos.configverify import verify_vrf -from vyos.util import call +from vyos.utils.process import call from vyos.template import render from vyos.xml import defaults from vyos import ConfigError diff --git a/src/conf_mode/system-ip.py b/src/conf_mode/system-ip.py index 95865c690..cca996e4f 100755 --- a/src/conf_mode/system-ip.py +++ b/src/conf_mode/system-ip.py @@ -20,10 +20,10 @@ from vyos.config import Config from vyos.configdict import dict_merge from vyos.configverify import verify_route_map from vyos.template import render_to_string -from vyos.util import call -from vyos.util import dict_search -from vyos.util import sysctl_write -from vyos.util import write_file +from vyos.utils.process import call +from vyos.utils.dict import dict_search +from vyos.utils.file import write_file +from vyos.utils.system import sysctl_write from vyos.xml import defaults from vyos import ConfigError from vyos import frr diff --git a/src/conf_mode/system-ipv6.py b/src/conf_mode/system-ipv6.py index b6d3a79c3..22210c27a 100755 --- a/src/conf_mode/system-ipv6.py +++ b/src/conf_mode/system-ipv6.py @@ -21,9 +21,9 @@ from vyos.config import Config from vyos.configdict import dict_merge from vyos.configverify import verify_route_map from vyos.template import render_to_string -from vyos.util import dict_search -from vyos.util import sysctl_write -from vyos.util import write_file +from vyos.utils.dict import dict_search +from vyos.utils.system import sysctl_write +from vyos.utils.file import write_file from vyos.xml import defaults from vyos import ConfigError from vyos import frr diff --git a/src/conf_mode/system-login-banner.py b/src/conf_mode/system-login-banner.py index a521c9834..65fa04417 100755 --- a/src/conf_mode/system-login-banner.py +++ b/src/conf_mode/system-login-banner.py @@ -18,7 +18,7 @@ from sys import exit from copy import deepcopy from vyos.config import Config -from vyos.util import write_file +from vyos.utils.file import write_file from vyos import ConfigError from vyos import airbag airbag.enable() diff --git a/src/conf_mode/system-login.py b/src/conf_mode/system-login.py index 24766a5b5..afd75913e 100755 --- a/src/conf_mode/system-login.py +++ b/src/conf_mode/system-login.py @@ -29,12 +29,12 @@ from vyos.configverify import verify_vrf from vyos.defaults import directories from vyos.template import render from vyos.template import is_ipv4 -from vyos.util import cmd -from vyos.util import call -from vyos.util import rc_cmd -from vyos.util import run -from vyos.util import DEVNULL -from vyos.util import dict_search +from vyos.utils.dict import dict_search +from vyos.utils.process import cmd +from vyos.utils.process import call +from vyos.utils.process import rc_cmd +from vyos.utils.process import run +from vyos.utils.process import DEVNULL from vyos.xml import defaults from vyos import ConfigError from vyos import airbag @@ -389,7 +389,7 @@ def apply(login): # command until user is removed - userdel might return 8 as # SSH sessions are not all yet properly cleaned away, thus we # simply re-run the command until the account wen't away - while run(f'userdel --remove {user}', stderr=DEVNULL): + while run(f'userdel {user}', stderr=DEVNULL): sleep(0.250) except Exception as e: diff --git a/src/conf_mode/system-logs.py b/src/conf_mode/system-logs.py index c71938a79..12145d641 100755 --- a/src/conf_mode/system-logs.py +++ b/src/conf_mode/system-logs.py @@ -22,7 +22,7 @@ from vyos.config import Config from vyos.configdict import dict_merge from vyos.logger import syslog from vyos.template import render -from vyos.util import dict_search +from vyos.utils.dict import dict_search from vyos.xml import defaults airbag.enable() diff --git a/src/conf_mode/system-option.py b/src/conf_mode/system-option.py index e6c7a0ed2..5172b492e 100755 --- a/src/conf_mode/system-option.py +++ b/src/conf_mode/system-option.py @@ -24,8 +24,8 @@ from vyos.config import Config from vyos.configdict import dict_merge from vyos.configverify import verify_source_interface from vyos.template import render -from vyos.util import cmd -from vyos.util import is_systemd_service_running +from vyos.utils.process import cmd +from vyos.utils.process import is_systemd_service_running from vyos.validate import is_addr_assigned from vyos.validate import is_intf_addr_assigned from vyos.xml import defaults diff --git a/src/conf_mode/system-syslog.py b/src/conf_mode/system-syslog.py index cf34bad2e..19c87bcee 100755 --- a/src/conf_mode/system-syslog.py +++ b/src/conf_mode/system-syslog.py @@ -22,7 +22,7 @@ from vyos.config import Config from vyos.configdict import dict_merge from vyos.configdict import is_node_changed from vyos.configverify import verify_vrf -from vyos.util import call +from vyos.utils.process import call from vyos.template import render from vyos.xml import defaults from vyos import ConfigError @@ -46,6 +46,7 @@ def get_config(config=None): get_first_key=True, no_tag_node_value_mangle=True) syslog.update({ 'logrotate' : logrotate_conf }) + tmp = is_node_changed(conf, base + ['vrf']) if tmp: syslog.update({'restart_required': {}}) @@ -70,35 +71,22 @@ def get_config(config=None): syslog['console']['facility'][facility]) # XXX: add defaults for "host" tree - if 'host' in syslog: - default_values_host = defaults(base + ['host']) + for syslog_type in ['host', 'user', 'file']: + # Bail out early if there is nothing to do + if syslog_type not in syslog: + continue + + default_values_host = defaults(base + [syslog_type]) if 'facility' in default_values_host: del default_values_host['facility'] - default_values_facility = defaults(base + ['host', 'facility']) - - for host, host_config in syslog['host'].items(): - syslog['host'][host] = dict_merge(default_values_host, syslog['host'][host]) - if 'facility' in host_config: - for facility in host_config['facility']: - syslog['host'][host]['facility'][facility] = dict_merge(default_values_facility, - syslog['host'][host]['facility'][facility]) - - # XXX: add defaults for "user" tree - if 'user' in syslog: - default_values = defaults(base + ['user', 'facility']) - for user, user_config in syslog['user'].items(): - if 'facility' in user_config: - for facility in user_config['facility']: - syslog['user'][user]['facility'][facility] = dict_merge(default_values, - syslog['user'][user]['facility'][facility]) - - # XXX: add defaults for "file" tree - if 'file' in syslog: - default_values = defaults(base + ['file']) - for file, file_config in syslog['file'].items(): - for facility in file_config['facility']: - syslog['file'][file]['facility'][facility] = dict_merge(default_values, - syslog['file'][file]['facility'][facility]) + + for tmp, tmp_config in syslog[syslog_type].items(): + syslog[syslog_type][tmp] = dict_merge(default_values_host, syslog[syslog_type][tmp]) + if 'facility' in tmp_config: + default_values_facility = defaults(base + [syslog_type, 'facility']) + for facility in tmp_config['facility']: + syslog[syslog_type][tmp]['facility'][facility] = dict_merge(default_values_facility, + syslog[syslog_type][tmp]['facility'][facility]) return syslog diff --git a/src/conf_mode/system-timezone.py b/src/conf_mode/system-timezone.py index 3d98ba774..cd3d4b229 100755 --- a/src/conf_mode/system-timezone.py +++ b/src/conf_mode/system-timezone.py @@ -20,7 +20,7 @@ import os from copy import deepcopy from vyos.config import Config from vyos import ConfigError -from vyos.util import call +from vyos.utils.process import call from vyos import airbag airbag.enable() diff --git a/src/conf_mode/system_console.py b/src/conf_mode/system_console.py index e922edc4e..87d587959 100755 --- a/src/conf_mode/system_console.py +++ b/src/conf_mode/system_console.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2020 VyOS maintainers and contributors +# Copyright (C) 2020-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -20,9 +20,9 @@ from pathlib import Path from vyos.config import Config from vyos.configdict import dict_merge -from vyos.util import call -from vyos.util import read_file -from vyos.util import write_file +from vyos.utils.process import call +from vyos.utils.file import read_file +from vyos.utils.file import write_file from vyos.template import render from vyos.xml import defaults from vyos import ConfigError diff --git a/src/conf_mode/system_frr.py b/src/conf_mode/system_frr.py index 1af0055f6..fb252238a 100755 --- a/src/conf_mode/system_frr.py +++ b/src/conf_mode/system_frr.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2021 VyOS maintainers and contributors +# Copyright (C) 2021-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -22,7 +22,9 @@ from vyos import airbag from vyos.config import Config from vyos.logger import syslog from vyos.template import render_to_string -from vyos.util import read_file, write_file, run +from vyos.utils.file import read_file +from vyos.utils.file import write_file +from vyos.utils.process import run airbag.enable() # path to daemons config and config status files diff --git a/src/conf_mode/system_lcd.py b/src/conf_mode/system_lcd.py index 3341dd738..eb88224d1 100755 --- a/src/conf_mode/system_lcd.py +++ b/src/conf_mode/system_lcd.py @@ -19,8 +19,8 @@ import os from sys import exit from vyos.config import Config -from vyos.util import call -from vyos.util import find_device_file +from vyos.utils.process import call +from vyos.utils.system import find_device_file from vyos.template import render from vyos import ConfigError from vyos import airbag diff --git a/src/conf_mode/system_sflow.py b/src/conf_mode/system_sflow.py index a0c3fca7f..9e3d41100 100755 --- a/src/conf_mode/system_sflow.py +++ b/src/conf_mode/system_sflow.py @@ -21,7 +21,7 @@ from sys import exit from vyos.config import Config from vyos.configdict import dict_merge from vyos.template import render -from vyos.util import call +from vyos.utils.process import call from vyos.validate import is_addr_assigned from vyos.xml import defaults from vyos import ConfigError diff --git a/src/conf_mode/system_sysctl.py b/src/conf_mode/system_sysctl.py index 2e0004ffa..f6b02023d 100755 --- a/src/conf_mode/system_sysctl.py +++ b/src/conf_mode/system_sysctl.py @@ -20,7 +20,7 @@ from sys import exit from vyos.config import Config from vyos.template import render -from vyos.util import cmd +from vyos.utils.process import cmd from vyos import ConfigError from vyos import airbag airbag.enable() diff --git a/src/conf_mode/system_update_check.py b/src/conf_mode/system_update_check.py index 08ecfcb81..8d641a97d 100755 --- a/src/conf_mode/system_update_check.py +++ b/src/conf_mode/system_update_check.py @@ -22,7 +22,7 @@ from pathlib import Path from sys import exit from vyos.config import Config -from vyos.util import call +from vyos.utils.process import call from vyos import ConfigError from vyos import airbag airbag.enable() diff --git a/src/conf_mode/tftp_server.py b/src/conf_mode/tftp_server.py index c5daccb7f..2735772dc 100755 --- a/src/conf_mode/tftp_server.py +++ b/src/conf_mode/tftp_server.py @@ -28,8 +28,8 @@ from vyos.configdict import dict_merge from vyos.configverify import verify_vrf from vyos.template import render from vyos.template import is_ipv4 -from vyos.util import call -from vyos.util import chmod_755 +from vyos.utils.process import call +from vyos.utils.permission import chmod_755 from vyos.validate import is_addr_assigned from vyos.xml import defaults from vyos import ConfigError diff --git a/src/conf_mode/vpn_ipsec.py b/src/conf_mode/vpn_ipsec.py index b82d90e4d..b0825d0ee 100755 --- a/src/conf_mode/vpn_ipsec.py +++ b/src/conf_mode/vpn_ipsec.py @@ -40,10 +40,10 @@ from vyos.template import is_ipv4 from vyos.template import is_ipv6 from vyos.template import render from vyos.validate import is_ipv6_link_local -from vyos.util import call -from vyos.util import dict_search -from vyos.util import dict_search_args -from vyos.util import run +from vyos.utils.dict import dict_search +from vyos.utils.dict import dict_search_args +from vyos.utils.process import call +from vyos.utils.process import run from vyos.xml import defaults from vyos import ConfigError from vyos import airbag diff --git a/src/conf_mode/vpn_l2tp.py b/src/conf_mode/vpn_l2tp.py index ffac3b023..6232ce64a 100755 --- a/src/conf_mode/vpn_l2tp.py +++ b/src/conf_mode/vpn_l2tp.py @@ -26,10 +26,10 @@ from ipaddress import ip_network from vyos.config import Config from vyos.template import is_ipv4 from vyos.template import render -from vyos.util import call -from vyos.util import get_half_cpus -from vyos.util import check_port_availability -from vyos.util import is_listen_port_bind_service +from vyos.utils.process import call +from vyos.utils.system import get_half_cpus +from vyos.utils.network import check_port_availability +from vyos.utils.network import is_listen_port_bind_service from vyos import ConfigError from vyos import airbag diff --git a/src/conf_mode/vpn_openconnect.py b/src/conf_mode/vpn_openconnect.py index 3d5dc12a4..e82862fa3 100755 --- a/src/conf_mode/vpn_openconnect.py +++ b/src/conf_mode/vpn_openconnect.py @@ -23,11 +23,11 @@ from vyos.configdict import dict_merge from vyos.pki import wrap_certificate from vyos.pki import wrap_private_key from vyos.template import render -from vyos.util import call -from vyos.util import check_port_availability -from vyos.util import is_systemd_service_running -from vyos.util import is_listen_port_bind_service -from vyos.util import dict_search +from vyos.utils.process import call +from vyos.utils.network import check_port_availability +from vyos.utils.process import is_systemd_service_running +from vyos.utils.network import is_listen_port_bind_service +from vyos.utils.dict import dict_search from vyos.xml import defaults from vyos import ConfigError from passlib.hash import sha512_crypt diff --git a/src/conf_mode/vpn_pptp.py b/src/conf_mode/vpn_pptp.py index b9d18110a..d542f57fe 100755 --- a/src/conf_mode/vpn_pptp.py +++ b/src/conf_mode/vpn_pptp.py @@ -23,7 +23,8 @@ from sys import exit from vyos.config import Config from vyos.template import render -from vyos.util import call, get_half_cpus +from vyos.utils.system import get_half_cpus +from vyos.utils.process import call from vyos import ConfigError from vyos import airbag diff --git a/src/conf_mode/vpn_sstp.py b/src/conf_mode/vpn_sstp.py index 2949ab290..e98d8385b 100755 --- a/src/conf_mode/vpn_sstp.py +++ b/src/conf_mode/vpn_sstp.py @@ -25,11 +25,11 @@ from vyos.configverify import verify_accel_ppp_base_service from vyos.pki import wrap_certificate from vyos.pki import wrap_private_key from vyos.template import render -from vyos.util import call -from vyos.util import check_port_availability -from vyos.util import dict_search -from vyos.util import is_listen_port_bind_service -from vyos.util import write_file +from vyos.utils.process import call +from vyos.utils.network import check_port_availability +from vyos.utils.dict import dict_search +from vyos.utils.network import is_listen_port_bind_service +from vyos.utils.file import write_file from vyos import ConfigError from vyos import airbag airbag.enable() diff --git a/src/conf_mode/vpp.py b/src/conf_mode/vpp.py index dc13f4e60..80ce1e8e3 100755 --- a/src/conf_mode/vpp.py +++ b/src/conf_mode/vpp.py @@ -15,7 +15,7 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. import os -import psutil +from psutil import virtual_memory from pathlib import Path from re import search as re_search, MULTILINE as re_M @@ -25,7 +25,11 @@ from vyos.configdep import set_dependents, call_dependents from vyos.configdict import dict_merge from vyos.configdict import node_changed from vyos.ifconfig import Section -from vyos.util import call, rc_cmd, boot_configuration_complete +from vyos.utils.boot import boot_configuration_complete +from vyos.utils.process import call +from vyos.utils.process import rc_cmd +from vyos.utils.system import sysctl_read +from vyos.utils.system import sysctl_apply from vyos.template import render from vyos.xml import defaults @@ -39,10 +43,10 @@ airbag.enable() service_name = 'vpp' service_conf = Path(f'/run/vpp/{service_name}.conf') systemd_override = '/run/systemd/system/vpp.service.d/10-override.conf' -sysctl_vpp = '/etc/sysctl.d/80-vpp.conf' -# Min memory 6GB (2GB reserved for vpp) -MIN_TOTAL_MEMORY = 6 +# Free memory required for VPP +# 2 GB for hugepages + 1 GB for other services +MIN_AVAILABLE_MEMORY: int = 3 * 1024**3 def _get_pci_address_by_interface(iface) -> str: @@ -64,7 +68,6 @@ def _get_pci_address_by_interface(iface) -> str: raise ConfigError(f'Cannot find PCI address for interface {iface}') - def get_config(config=None): if config: conf = config @@ -131,32 +134,45 @@ def verify(config): return None if 'interface' not in config: - raise ConfigError(f'"interface" is required but not set!') + raise ConfigError('"interface" is required but not set!') if 'cpu' in config: - if 'corelist_workers' in config['cpu'] and 'main_core' not in config['cpu']: - raise ConfigError(f'"cpu main-core" is required but not set!') + if 'corelist_workers' in config['cpu'] and 'main_core' not in config[ + 'cpu']: + raise ConfigError('"cpu main-core" is required but not set!') - memory = psutil.virtual_memory() - memory_total = round(memory.total / (1024 ** 3), 2) - if memory_total < MIN_TOTAL_MEMORY: + memory_available: int = virtual_memory().available + if memory_available < MIN_AVAILABLE_MEMORY: raise ConfigError( - f'Not enough installed memory {memory_total}GB! ' - f'The minimum required memory is {MIN_TOTAL_MEMORY}GB.' - ) + 'Not enough free memory to start VPP:\n' + f'available: {round(memory_available / 1024**3, 1)}GB\n' + f'required: {round(MIN_AVAILABLE_MEMORY / 1024**3, 1)}GB') def generate(config): if not config or (len(config) == 1 and 'removed_ifaces' in config): # Remove old config and return service_conf.unlink(missing_ok=True) - if os.path.isfile(sysctl_vpp): - os.unlink(sysctl_vpp) return None render(service_conf, 'vpp/startup.conf.j2', config) render(systemd_override, 'vpp/override.conf.j2', config) - render(sysctl_vpp, 'vpp/sysctl.conf.j2', config) + + # apply default sysctl values from + # https://github.com/FDio/vpp/blob/v23.06/src/vpp/conf/80-vpp.conf + sysctl_config: dict[str, str] = { + 'vm.nr_hugepages': '1024', + 'vm.max_map_count': '3096', + 'vm.hugetlb_shm_group': '0', + 'kernel.shmmax': '2147483648' + } + # we do not want to reduce `kernel.shmmax` + kernel_shmnax_current: str = sysctl_read('kernel.shmmax') + if int(kernel_shmnax_current) > int(sysctl_config['kernel.shmmax']): + sysctl_config['kernel.shmmax'] = kernel_shmnax_current + + if not sysctl_apply(sysctl_config): + raise ConfigError('Cannot configure sysctl parameters for VPP') return None @@ -168,8 +184,6 @@ def apply(config): call('systemctl daemon-reload') call(f'systemctl restart {service_name}.service') - call(f'sysctl -qp {sysctl_vpp}') - # Initialize interfaces removed from VPP for iface in config.get('removed_ifaces', []): host_control = HostControl() diff --git a/src/conf_mode/vrf.py b/src/conf_mode/vrf.py index 0b983293e..be867b208 100755 --- a/src/conf_mode/vrf.py +++ b/src/conf_mode/vrf.py @@ -26,13 +26,13 @@ from vyos.configverify import verify_route_map from vyos.ifconfig import Interface from vyos.template import render from vyos.template import render_to_string -from vyos.util import call -from vyos.util import cmd -from vyos.util import dict_search -from vyos.util import get_interface_config -from vyos.util import popen -from vyos.util import run -from vyos.util import sysctl_write +from vyos.utils.dict import dict_search +from vyos.utils.network import get_interface_config +from vyos.utils.process import call +from vyos.utils.process import cmd +from vyos.utils.process import popen +from vyos.utils.process import run +from vyos.utils.system import sysctl_write from vyos import ConfigError from vyos import frr from vyos import airbag diff --git a/src/conf_mode/vrf_vni.py b/src/conf_mode/vrf_vni.py index 9f33536e5..23b341079 100644 --- a/src/conf_mode/vrf_vni.py +++ b/src/conf_mode/vrf_vni.py @@ -19,7 +19,7 @@ from sys import exit from vyos.config import Config from vyos.template import render_to_string -from vyos.util import dict_search +from vyos.utils.dict import dict_search from vyos import ConfigError from vyos import frr from vyos import airbag diff --git a/src/etc/dhcp/dhclient-exit-hooks.d/99-ipsec-dhclient-hook b/src/etc/dhcp/dhclient-exit-hooks.d/99-ipsec-dhclient-hook index 1f1926e17..c7a92fe26 100755 --- a/src/etc/dhcp/dhclient-exit-hooks.d/99-ipsec-dhclient-hook +++ b/src/etc/dhcp/dhclient-exit-hooks.d/99-ipsec-dhclient-hook @@ -15,7 +15,7 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. if [ "$reason" == "REBOOT" ] || [ "$reason" == "EXPIRE" ]; then - exit 0 + return 0 fi DHCP_HOOK_IFLIST="/tmp/ipsec_dhcp_waiting" @@ -24,22 +24,22 @@ if [ -f $DHCP_HOOK_IFLIST ] && [ "$reason" == "BOUND" ]; then if grep -qw $interface $DHCP_HOOK_IFLIST; then sudo rm $DHCP_HOOK_IFLIST sudo /usr/libexec/vyos/conf_mode/vpn_ipsec.py - exit 0 + return 0 fi fi if [ "$old_ip_address" == "$new_ip_address" ] && [ "$reason" == "BOUND" ]; then - exit 0 + return 0 fi python3 - <<PYEND import os import re -from vyos.util import call -from vyos.util import cmd -from vyos.util import read_file -from vyos.util import write_file +from vyos.utils.process import call +from vyos.utils.process import cmd +from vyos.utils.file import read_file +from vyos.utils.file import write_file SWANCTL_CONF="/etc/swanctl/swanctl.conf" @@ -83,4 +83,4 @@ if __name__ == '__main__': call('sudo swanctl -q') exit(0) -PYEND
\ No newline at end of file +PYEND diff --git a/src/etc/ipsec.d/vti-up-down b/src/etc/ipsec.d/vti-up-down index 1ffb32955..9eb6fac48 100755 --- a/src/etc/ipsec.d/vti-up-down +++ b/src/etc/ipsec.d/vti-up-down @@ -25,9 +25,9 @@ from syslog import LOG_PID from syslog import LOG_INFO from vyos.configquery import ConfigTreeQuery -from vyos.util import call -from vyos.util import get_interface_config -from vyos.util import get_interface_address +from vyos.utils.process import call +from vyos.utils.network import get_interface_config +from vyos.utils.network import get_interface_address if __name__ == '__main__': verb = os.getenv('PLUTO_VERB') diff --git a/src/etc/opennhrp/opennhrp-script.py b/src/etc/opennhrp/opennhrp-script.py index 688c7af2a..f6f6d075c 100755 --- a/src/etc/opennhrp/opennhrp-script.py +++ b/src/etc/opennhrp/opennhrp-script.py @@ -23,8 +23,8 @@ from json import loads from pathlib import Path from vyos.logger import getLogger -from vyos.util import cmd -from vyos.util import process_named_running +from vyos.utils.process import cmd +from vyos.utils.process import process_named_running NHRP_CONFIG: str = '/run/opennhrp/opennhrp.conf' diff --git a/src/etc/systemd/system-generators/vyos-generator b/src/etc/systemd/system-generators/vyos-generator new file mode 100755 index 000000000..34faab6a2 --- /dev/null +++ b/src/etc/systemd/system-generators/vyos-generator @@ -0,0 +1,94 @@ +#!/bin/sh +set -f + +LOG="" +DEBUG_LEVEL=1 +LOG_D="/run/vyos-router" +ENABLE="enabled" +DISABLE="disabled" +FOUND="found" +NOTFOUND="notfound" +RUN_ENABLED_FILE="$LOG_D/$ENABLE" +VYOS_SYSTEM_TARGET="/lib/systemd/system/vyos.target" +VYOS_TARGET_NAME="vyos.target" + +debug() { + local lvl="$1" + shift + [ "$lvl" -gt "$DEBUG_LEVEL" ] && return + if [ -z "$LOG" ]; then + local log="$LOG_D/${0##*/}.log" + { [ -d "$LOG_D" ] || mkdir -p "$LOG_D"; } && + { : > "$log"; } >/dev/null 2>&1 && LOG="$log" || + LOG="/dev/kmsg" + fi + echo "$@" >> "$LOG" +} + +default() { + _RET="$ENABLE" +} + +main() { + local normal_d="$1" early_d="$2" late_d="$3" + local target_name="multi-user.target" gen_d="$early_d" + local link_path="$gen_d/${target_name}.wants/${VYOS_TARGET_NAME}" + local ds="$NOTFOUND" + + debug 1 "$0 normal=$normal_d early=$early_d late=$late_d" + debug 2 "$0 $*" + + local search result="error" ret="" + for search in default; do + if $search; then + debug 1 "$search found $_RET" + [ "$_RET" = "$ENABLE" -o "$_RET" = "$DISABLE" ] && + result=$_RET && break + else + ret=$? + debug 0 "search $search returned $ret" + fi + done + + # enable AND ds=found == enable + # enable AND ds=notfound == disable + # disable || <any> == disabled + if [ "$result" = "$ENABLE" ]; then + if [ -e "$link_path" ]; then + debug 1 "already enabled: no change needed" + else + [ -d "${link_path%/*}" ] || mkdir -p "${link_path%/*}" || + debug 0 "failed to make dir $link_path" + if ln -snf "$VYOS_SYSTEM_TARGET" "$link_path"; then + debug 1 "enabled via $link_path -> $VYOS_SYSTEM_TARGET" + else + ret=$? + debug 0 "[$ret] enable failed:" \ + "ln $VYOS_SYSTEM_TARGET $link_path" + fi + fi + : > "$RUN_ENABLED_FILE" + elif [ "$result" = "$DISABLE" ]; then + if [ -f "$link_path" ]; then + if rm -f "$link_path"; then + debug 1 "disabled. removed existing $link_path" + else + ret=$? + debug 0 "[$ret] disable failed, remove $link_path" + fi + else + debug 1 "already disabled: no change needed [no $link_path]" + fi + if [ -e "$RUN_ENABLED_FILE" ]; then + rm -f "$RUN_ENABLED_FILE" + fi + else + debug 0 "unexpected result '$result' 'ds=$ds'" + ret=3 + fi + return $ret +} + +main "$@" + +# vi: ts=4 expandtab diff --git a/src/etc/systemd/system/getty@.service.d/aftervyos.conf b/src/etc/systemd/system/getty@.service.d/aftervyos.conf new file mode 100644 index 000000000..c5753900e --- /dev/null +++ b/src/etc/systemd/system/getty@.service.d/aftervyos.conf @@ -0,0 +1,3 @@ +[Service] +ExecStartPre=-/usr/libexec/vyos/init/vyos-config +StandardOutput=journal+console diff --git a/src/etc/systemd/system/serial-getty@.service.d/aftervyos.conf b/src/etc/systemd/system/serial-getty@.service.d/aftervyos.conf new file mode 100644 index 000000000..8ba42778d --- /dev/null +++ b/src/etc/systemd/system/serial-getty@.service.d/aftervyos.conf @@ -0,0 +1,3 @@ +[Service] +ExecStartPre=-/usr/libexec/vyos/init/vyos-config SERIAL +StandardOutput=journal+console diff --git a/src/etc/telegraf/custom_scripts/show_firewall_input_filter.py b/src/etc/telegraf/custom_scripts/show_firewall_input_filter.py index d7eca5894..bb7515a90 100755 --- a/src/etc/telegraf/custom_scripts/show_firewall_input_filter.py +++ b/src/etc/telegraf/custom_scripts/show_firewall_input_filter.py @@ -4,7 +4,7 @@ import json import re import time -from vyos.util import cmd +from vyos.utils.process import cmd def get_nft_filter_chains(): diff --git a/src/etc/telegraf/custom_scripts/vyos_services_input_filter.py b/src/etc/telegraf/custom_scripts/vyos_services_input_filter.py index df4eed131..00f2f184c 100755 --- a/src/etc/telegraf/custom_scripts/vyos_services_input_filter.py +++ b/src/etc/telegraf/custom_scripts/vyos_services_input_filter.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2021 VyOS maintainers and contributors +# Copyright (C) 2021-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -17,7 +17,8 @@ import time from vyos.configquery import ConfigTreeQuery -from vyos.util import is_systemd_service_running, process_named_running +from vyos.utils.process import is_systemd_service_running +from vyos.utils.process import process_named_running # Availible services and prouceses # 1 - service diff --git a/src/etc/vmware-tools/scripts/resume-vm-default.d/ether-resume.py b/src/etc/vmware-tools/scripts/resume-vm-default.d/ether-resume.py index 4e7fb117c..7da57bca8 100755 --- a/src/etc/vmware-tools/scripts/resume-vm-default.d/ether-resume.py +++ b/src/etc/vmware-tools/scripts/resume-vm-default.d/ether-resume.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2018-2021 VyOS maintainers and contributors +# Copyright (C) 2018-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -17,9 +17,9 @@ import sys import syslog -from vyos.config import Config from vyos import ConfigError -from vyos.util import run +from vyos.config import Config +from vyos.utils.process import run def get_config(): c = Config() diff --git a/src/helpers/run-config-migration.py b/src/helpers/run-config-migration.py index cc7166c22..ce647ad0a 100755 --- a/src/helpers/run-config-migration.py +++ b/src/helpers/run-config-migration.py @@ -20,7 +20,7 @@ import sys import argparse import datetime -from vyos.util import cmd +from vyos.utils.process import cmd from vyos.migrator import Migrator, VirtualMigrator def main(): diff --git a/src/helpers/vyos-boot-config-loader.py b/src/helpers/vyos-boot-config-loader.py index b9cc87bfa..01b06526d 100755 --- a/src/helpers/vyos-boot-config-loader.py +++ b/src/helpers/vyos-boot-config-loader.py @@ -26,7 +26,7 @@ from datetime import datetime from vyos.defaults import directories, config_status from vyos.configsession import ConfigSession, ConfigSessionError from vyos.configtree import ConfigTree -from vyos.util import cmd +from vyos.utils.process import cmd STATUS_FILE = config_status TRACE_FILE = '/tmp/boot-config-trace' diff --git a/src/helpers/vyos-check-wwan.py b/src/helpers/vyos-check-wwan.py index 2ff9a574f..334f08dd3 100755 --- a/src/helpers/vyos-check-wwan.py +++ b/src/helpers/vyos-check-wwan.py @@ -17,7 +17,7 @@ from vyos.configquery import VbashOpRun from vyos.configquery import ConfigTreeQuery -from vyos.util import is_wwan_connected +from vyos.utils.network import is_wwan_connected conf = ConfigTreeQuery() dict = conf.get_config_dict(['interfaces', 'wwan'], key_mangling=('-', '_'), diff --git a/src/helpers/vyos-domain-resolver.py b/src/helpers/vyos-domain-resolver.py index e31d9238e..2036ca72e 100755 --- a/src/helpers/vyos-domain-resolver.py +++ b/src/helpers/vyos-domain-resolver.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2022 VyOS maintainers and contributors +# Copyright (C) 2022-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -22,10 +22,10 @@ from vyos.configdict import dict_merge from vyos.configquery import ConfigTreeQuery from vyos.firewall import fqdn_config_parse from vyos.firewall import fqdn_resolve -from vyos.util import cmd -from vyos.util import commit_in_progress -from vyos.util import dict_search_args -from vyos.util import run +from vyos.utils.commit import commit_in_progress +from vyos.utils.dict import dict_search_args +from vyos.utils.process import cmd +from vyos.utils.process import run from vyos.xml import defaults base = ['firewall'] diff --git a/src/helpers/vyos-failover.py b/src/helpers/vyos-failover.py index ce4cf8fa4..cc7610370 100755 --- a/src/helpers/vyos-failover.py +++ b/src/helpers/vyos-failover.py @@ -20,7 +20,7 @@ import subprocess import socket import time -from vyos.util import rc_cmd +from vyos.utils.process import rc_cmd from pathlib import Path from systemd import journal diff --git a/src/helpers/vyos-interface-rescan.py b/src/helpers/vyos-interface-rescan.py index 1ac1810e0..012357259 100755 --- a/src/helpers/vyos-interface-rescan.py +++ b/src/helpers/vyos-interface-rescan.py @@ -24,7 +24,7 @@ import netaddr from vyos.configtree import ConfigTree from vyos.defaults import directories -from vyos.util import get_cfg_group_id +from vyos.utils.permission import get_cfg_group_id debug = False diff --git a/src/helpers/vyos-merge-config.py b/src/helpers/vyos-merge-config.py index 14df2734b..8997705fe 100755 --- a/src/helpers/vyos-merge-config.py +++ b/src/helpers/vyos-merge-config.py @@ -1,6 +1,6 @@ #!/usr/bin/python3 -# Copyright 2019 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2019-2023 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -20,11 +20,12 @@ import os import tempfile import vyos.defaults import vyos.remote + from vyos.config import Config from vyos.configtree import ConfigTree from vyos.migrator import Migrator, VirtualMigrator -from vyos.util import cmd, DEVNULL - +from vyos.utils.process import cmd +from vyos.utils.process import DEVNULL if (len(sys.argv) < 2): print("Need config file name to merge.") diff --git a/src/helpers/vyos-sudo.py b/src/helpers/vyos-sudo.py index 3e4c196d9..75dd7f29d 100755 --- a/src/helpers/vyos-sudo.py +++ b/src/helpers/vyos-sudo.py @@ -18,7 +18,7 @@ import os import sys -from vyos.util import is_admin +from vyos.utils.permission import is_admin if __name__ == '__main__': diff --git a/src/helpers/vyos_config_sync.py b/src/helpers/vyos_config_sync.py new file mode 100755 index 000000000..7cfa8fe88 --- /dev/null +++ b/src/helpers/vyos_config_sync.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2023 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +# + +import os +import json +import requests +import urllib3 +import logging +from typing import Optional, List, Union, Dict, Any + +from vyos.config import Config +from vyos.template import bracketize_ipv6 + + +CONFIG_FILE = '/run/config_sync_conf.conf' + +# Logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) +logger.name = os.path.basename(__file__) + +# API +API_HEADERS = {'Content-Type': 'application/json'} + + +def post_request(url: str, + data: str, + headers: Dict[str, str]) -> requests.Response: + """Sends a POST request to the specified URL + + Args: + url (str): The URL to send the POST request to. + data (Dict[str, Any]): The data to send with the POST request. + headers (Dict[str, str]): The headers to include with the POST request. + + Returns: + requests.Response: The response object representing the server's response to the request + """ + + response = requests.post(url, + data=data, + headers=headers, + verify=False, + timeout=timeout) + return response + + +def retrieve_config(section: str = None) -> Optional[Dict[str, Any]]: + """Retrieves the configuration from the local server. + + Args: + section: str: The section of the configuration to retrieve. Default is None. + + Returns: + Optional[Dict[str, Any]]: The retrieved configuration as a dictionary, or None if an error occurred. + """ + if section is None: + section = [] + else: + section = section.split() + + conf = Config() + config = conf.get_config_dict(section, get_first_key=True) + if config: + return config + return None + + +def set_remote_config( + address: str, + key: str, + op: str, + path: str = None, + section: Optional[str] = None) -> Optional[Dict[str, Any]]: + """Loads the VyOS configuration in JSON format to a remote host. + + Args: + address (str): The address of the remote host. + key (str): The key to use for loading the configuration. + path (Optional[str]): The path of the configuration. Default is None. + section (Optional[str]): The section of the configuration to load. Default is None. + + Returns: + Optional[Dict[str, Any]]: The response from the remote host as a dictionary, or None if an error occurred. + """ + + if path is None: + path = [] + else: + path = path.split() + headers = {'Content-Type': 'application/json'} + + # Disable the InsecureRequestWarning + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + url = f'https://{address}/configure-section' + data = json.dumps({ + 'op': mode, + 'path': path, + 'section': section, + 'key': key + }) + + try: + config = post_request(url, data, headers) + return config.json() + except requests.exceptions.RequestException as e: + print(f"An error occurred: {e}") + logger.error(f"An error occurred: {e}") + return None + + +def is_section_revised(section: str) -> bool: + from vyos.config_mgmt import is_node_revised + return is_node_revised([section]) + + +def config_sync(secondary_address: str, + secondary_key: str, + sections: List[str], + mode: str): + """Retrieve a config section from primary router in JSON format and send it to + secondary router + """ + # Config sync only if sections changed + if not any(map(is_section_revised, sections)): + return + + logger.info( + f"Config synchronization: Mode={mode}, Secondary={secondary_address}" + ) + + # Sync sections ("nat", "firewall", etc) + for section in sections: + config_json = retrieve_config(section=section) + # Check if config path deesn't exist, for example "set nat" + # we set empty value for config_json data + # As we cannot send to the remote host section "nat None" config + if not config_json: + config_json = "" + logger.debug( + f"Retrieved config for section '{section}': {config_json}") + set_config = set_remote_config(address=secondary_address, + key=secondary_key, + op=mode, + path=section, + section=config_json) + logger.debug(f"Set config for section '{section}': {set_config}") + + +if __name__ == '__main__': + # Read configuration from file + if not os.path.exists(CONFIG_FILE): + logger.error(f"Post-commit: No config file '{CONFIG_FILE}' exists") + exit(0) + + with open(CONFIG_FILE, 'r') as f: + config_data = f.read() + + config = json.loads(config_data) + + mode = config.get('mode') + secondary_address = config.get('secondary', {}).get('address') + secondary_address = bracketize_ipv6(secondary_address) + secondary_key = config.get('secondary', {}).get('key') + sections = config.get('section') + timeout = int(config.get('secondary', {}).get('timeout')) + + if not all([ + mode, secondary_address, secondary_key, sections + ]): + logger.error( + "Missing required configuration data for config synchronization.") + exit(0) + + config_sync(secondary_address, secondary_key, + sections, mode) diff --git a/src/helpers/vyos_net_name b/src/helpers/vyos_net_name index 1798e92db..8c0992414 100755 --- a/src/helpers/vyos_net_name +++ b/src/helpers/vyos_net_name @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2021 VyOS maintainers and contributors +# Copyright (C) 2021-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -13,8 +13,6 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -# -# import os import re @@ -26,7 +24,8 @@ from sys import argv from vyos.configtree import ConfigTree from vyos.defaults import directories -from vyos.util import cmd, boot_configuration_complete +from vyos.utils.process import cmd +from vyos.utils.boot import boot_configuration_complete from vyos.migrator import VirtualMigrator vyos_udev_dir = directories['vyos_udev_dir'] diff --git a/src/init/vyos-config b/src/init/vyos-config new file mode 100755 index 000000000..356427024 --- /dev/null +++ b/src/init/vyos-config @@ -0,0 +1,16 @@ +#!/bin/bash + +while [ ! -f /tmp/vyos-config-status ] +do + sleep 1 +done + +status=$(cat /tmp/vyos-config-status) + +if [ -z "$1" ]; then + if [ $status -ne 0 ]; then + echo "Configuration error" + else + echo "Configuration success" + fi +fi diff --git a/src/init/vyos-router b/src/init/vyos-router new file mode 100755 index 000000000..7b752b84b --- /dev/null +++ b/src/init/vyos-router @@ -0,0 +1,421 @@ +#!/bin/bash +# Copyright (C) 2021 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +. /lib/lsb/init-functions + +: ${vyatta_env:=/etc/default/vyatta} +source $vyatta_env + +declare progname=${0##*/} +declare action=$1; shift + +declare -x BOOTFILE=$vyatta_sysconfdir/config/config.boot + +# If vyos-config= boot option is present, use that file instead +for x in $(cat /proc/cmdline); do + [[ $x = vyos-config=* ]] || continue + VYOS_CONFIG="${x#vyos-config=}" +done + +if [ ! -z "$VYOS_CONFIG" ]; then + if [ -r "$VYOS_CONFIG" ]; then + echo "Config selected manually: $VYOS_CONFIG" + declare -x BOOTFILE="$VYOS_CONFIG" + else + echo "WARNING: Could not read selected config file, using default!" + fi +fi + +declare -a subinit +declare -a all_subinits=( firewall ) + +if [ $# -gt 0 ] ; then + for s in $@ ; do + [ -x ${vyatta_sbindir}/${s}.init ] && subinit[${#subinit}]=$s + done +else + for s in ${all_subinits[@]} ; do + [ -x ${vyatta_sbindir}/${s}.init ] && subinit[${#subinit}]=$s + done +fi + +GROUP=vyattacfg + +# easy way to make empty file without any command +empty() +{ + >$1 +} + +# check if bootup of this portion is disabled +disabled () { + grep -q -w no-vyos-$1 /proc/cmdline +} + +# if necessary, provide initial config +init_bootfile () { + if [ ! -r $BOOTFILE ] ; then + if [ -f $vyatta_sysconfdir/config.boot.default ]; then + cp $vyatta_sysconfdir/config.boot.default $BOOTFILE + else + $vyos_libexec_dir/system-versions-foot.py > $BOOTFILE + fi + chgrp ${GROUP} $BOOTFILE + chmod 660 $BOOTFILE + fi +} + +# if necessary, migrate initial config +migrate_bootfile () +{ + if [ -x $vyos_libexec_dir/run-config-migration.py ]; then + log_progress_msg migrate + sg ${GROUP} -c "$vyos_libexec_dir/run-config-migration.py $BOOTFILE" + fi +} + +# load the initial config +load_bootfile () +{ + log_progress_msg configure + ( + if [ -f /etc/default/vyatta-load-boot ]; then + # build-specific environment for boot-time config loading + source /etc/default/vyatta-load-boot + fi + if [ -x $vyos_libexec_dir/vyos-boot-config-loader.py ]; then + sg ${GROUP} -c "$vyos_libexec_dir/vyos-boot-config-loader.py $BOOTFILE" + fi + ) +} + +# execute the pre-config script +run_preconfig_script () +{ + if [ -x $vyatta_sysconfdir/config/scripts/vyos-preconfig-bootup.script ]; then + $vyatta_sysconfdir/config/scripts/vyos-preconfig-bootup.script + fi +} + +# execute the post-config scripts +run_postconfig_scripts () +{ + if [ -x $vyatta_sysconfdir/config/scripts/vyatta-postconfig-bootup.script ]; then + $vyatta_sysconfdir/config/scripts/vyatta-postconfig-bootup.script + fi + if [ -x $vyatta_sysconfdir/config/scripts/vyos-postconfig-bootup.script ]; then + $vyatta_sysconfdir/config/scripts/vyos-postconfig-bootup.script + fi +} + +run_postupgrade_script () +{ + if [ -f $vyatta_sysconfdir/config/.upgraded ]; then + # Run the system script + /usr/libexec/vyos/system/post-upgrade + + # Run user scripts + if [ -d $vyatta_sysconfdir/config/scripts/post-upgrade.d ]; then + run-parts $vyatta_sysconfdir/config/scripts/post-upgrade.d + fi + rm -f $vyatta_sysconfdir/config/.upgraded + fi +} + +# +# On image booted machines, we need to mount /boot from the image-specific +# boot directory so that kernel package installation will put the +# files in the right place. We also have to mount /boot/grub from the +# system-wide grub directory so that tools that edit the grub.cfg +# file will find it in the expected location. +# +bind_mount_boot () +{ + persist_path=$(/opt/vyatta/sbin/vyos-persistpath) + if [ $? == 0 ]; then + if [ -e $persist_path/boot ]; then + image_name=$(cat /proc/cmdline | sed -e s+^.*vyos-union=/boot/++ | sed -e 's/ .*$//') + + if [ -n "$image_name" ]; then + mount --bind $persist_path/boot/$image_name /boot + if [ $? -ne 0 ]; then + echo "Couldn't bind mount /boot" + fi + + if [ ! -d /boot/grub ]; then + mkdir /boot/grub + fi + + mount --bind $persist_path/boot/grub /boot/grub + if [ $? -ne 0 ]; then + echo "Couldn't bind mount /boot/grub" + fi + fi + fi + fi +} + +clear_or_override_config_files () +{ + for conf in snmp/snmpd.conf snmp/snmptrapd.conf snmp/snmp.conf \ + keepalived/keepalived.conf cron.d/vyos-crontab \ + ipvsadm.rules default/ipvsadm resolv.conf + do + if [ -s /etc/$conf ] ; then + empty /etc/$conf + chmod 0644 /etc/$conf + fi + done +} + +update_interface_config () +{ + if [ -d /run/udev/vyos ]; then + $vyos_libexec_dir/vyos-interface-rescan.py $BOOTFILE + fi +} + +cleanup_post_commit_hooks () { + # Remove links from the post-commit hooks directory. + # note that this approach only supports hooks that are "configured", + # i.e., it does not support hooks that need to always be present. + cpostdir=$(cli-shell-api getPostCommitHookDir) + # exclude commits hooks from vyatta-cfg + excluded="10vyatta-log-commit.pl 99vyos-user-postcommit-hooks" + if [ -d "$cpostdir" ]; then + for f in $cpostdir/*; do + if [[ ! $excluded =~ $(basename $f) ]]; then + rm -f $cpostdir/$(basename $f) + fi + done + fi +} + +# These are all the default security setting which are later +# overridden when configuration is read. These are the values the +# system defaults. +security_reset () +{ + # restore PAM back to virgin state (no radius/tacacs services) + pam-auth-update --package --remove radius + rm -f /etc/pam_radius_auth.conf + pam-auth-update --package --remove tacplus + rm -f /etc/tacplus_nss.conf /etc/tacplus_servers + + # Certain configuration files are re-generated by the configuration + # subsystem and must reside under /etc and can not easily be moved to /run. + # So on every boot we simply delete any remaining files and let the CLI + # regenearte them. + + # PPPoE + rm -f /etc/ppp/peers/pppoe* /etc/ppp/peers/wlm* + + # IPSec + rm -rf /etc/ipsec.conf /etc/ipsec.secrets + find /etc/swanctl -type f | xargs rm -f + + # limit cleanup + rm -f /etc/security/limits.d/10-vyos.conf + + # iproute2 cleanup + rm -f /etc/iproute2/rt_tables.d/vyos-*.conf + + # Container + rm -f /etc/containers/storage.conf /etc/containers/registries.conf /etc/containers/containers.conf + # Clean all networks and re-create them from our CLI + rm -f /etc/containers/networks/* + + # System Options (SSH/cURL) + rm -f /etc/ssh/ssh_config.d/*vyos*.conf + rm -f /etc/curlrc +} + +# XXX: T3885 - generate persistend DHCPv6 DUID (Type4 - UUID based) +gen_duid () +{ + DUID_FILE="/var/lib/dhcpv6/dhcp6c_duid" + UUID_FILE="/sys/class/dmi/id/product_uuid" + UUID_FILE_ALT="/sys/class/dmi/id/product_serial" + if [ ! -f ${UUID_FILE} ] && [ ! -f ${UUID_FILE_ALT} ]; then + return 1 + fi + + # DUID is based on the BIOS/EFI UUID. We omit additional - characters + if [ -f ${UUID_FILE} ]; then + UUID=$(cat ${UUID_FILE} | tr -d -) + fi + if [ -z ${UUID} ]; then + UUID=$(uuidgen --sha1 --namespace @dns --name $(cat ${UUID_FILE_ALT}) | tr -d -) + fi + # Add DUID type4 (UUID) information + DUID_TYPE="0004" + + # The length-information (as per RFC6355 UUID is 128 bits long) is in big-endian + # format - beware when porting to ARM64. The length field consists out of the + # UUID (128 bit + 16 bits DUID type) resulting in hex 12. + DUID_LEN="0012" + if [ "$(echo -n I | od -to2 | head -n1 | cut -f2 -d" " | cut -c6 )" -eq 1 ]; then + # true on little-endian (x86) systems + DUID_LEN="1200" + fi + + for i in $(echo -n ${DUID_LEN}${DUID_TYPE}${UUID} | sed 's/../& /g'); do + echo -ne "\x$i" + done > ${DUID_FILE} +} + +start () +{ + # reset and clean config files + security_reset || log_failure_msg "security reset failed" + + # some legacy directories migrated over from old rl-system.init + mkdir -p /var/run/vyatta /var/log/vyatta + chgrp vyattacfg /var/run/vyatta /var/log/vyatta + chmod 775 /var/run/vyatta /var/log/vyatta + + log_daemon_msg "Waiting for NICs to settle down" + # On boot time udev migth take a long time to reorder nic's, this will ensure that + # all udev activity is completed and all nics presented at boot-time will have their + # final name before continuing with vyos-router initialization. + SECONDS=0 + udevadm settle + STATUS=$? + log_progress_msg "settled in ${SECONDS}sec." + log_end_msg ${STATUS} + + # mountpoint for bpf maps required by xdp + mount -t bpf none /sys/fs/bpf + + # Clear out Debian APT source config file + empty /etc/apt/sources.list + + # Generate DHCPv6 DUID + gen_duid || log_failure_msg "could not generate DUID" + + # Mount a temporary filesystem for container networks. + # Configuration should be loaded from VyOS cli. + cni_dir="/etc/cni/net.d" + [ ! -d ${cni_dir} ] && mkdir -p ${cni_dir} + mount -t tmpfs none ${cni_dir} + + # Init firewall + nfct helper add rpc inet tcp + nfct helper add rpc inet udp + nfct helper add tns inet tcp + nft -f /usr/share/vyos/vyos-firewall-init.conf || log_failure_msg "could not initiate firewall rules" + + rm -f /etc/hostname + ${vyos_conf_scripts_dir}/host_name.py || log_failure_msg "could not reset host-name" + systemctl start frr.service + + # As VyOS does not execute commands that are not present in the CLI we call + # the script by hand to have a single source for the login banner and MOTD + ${vyos_conf_scripts_dir}/system_console.py || log_failure_msg "could not reset serial console" + ${vyos_conf_scripts_dir}/system-login.py || log_failure_msg "could not reset system login" + ${vyos_conf_scripts_dir}/system-login-banner.py || log_failure_msg "could not reset motd and issue files" + ${vyos_conf_scripts_dir}/system-option.py || log_failure_msg "could not reset system option files" + ${vyos_conf_scripts_dir}/conntrack.py || log_failure_msg "could not reset conntrack subsystem" + ${vyos_conf_scripts_dir}/container.py || log_failure_msg "could not reset container subsystem" + + clear_or_override_config_files || log_failure_msg "could not reset config files" + + # enable some debugging before loading the configuration + if grep -q vyos-debug /proc/cmdline; then + log_action_begin_msg "Enable runtime debugging options" + touch /tmp/vyos.container.debug + touch /tmp/vyos.ifconfig.debug + touch /tmp/vyos.frr.debug + touch /tmp/vyos.container.debug + fi + + log_action_begin_msg "Mounting VyOS Config" + # ensure the vyatta_configdir supports a large number of inodes since + # the config hierarchy is often inode-bound (instead of size). + # impose a minimum and then scale up dynamically with the actual size + # of the system memory. + local tmem=$(sed -n 's/^MemTotal: \+\([0-9]\+\) kB$/\1/p' /proc/meminfo) + local tpages + local tmpfs_opts="nosuid,nodev,mode=775,nr_inodes=0" #automatically allocate inodes + mount -o $tmpfs_opts -t tmpfs none ${vyatta_configdir} \ + && chgrp ${GROUP} ${vyatta_configdir} + log_action_end_msg $? + + disabled bootfile || init_bootfile + + cleanup_post_commit_hooks + + log_daemon_msg "Starting VyOS router" + disabled migrate || migrate_bootfile + + run_preconfig_script + + run_postupgrade_script + + update_interface_config + + for s in ${subinit[@]} ; do + if ! disabled $s; then + log_progress_msg $s + if ! ${vyatta_sbindir}/${s}.init start + then log_failure_msg + exit 1 + fi + fi + done + + bind_mount_boot + + disabled configure || load_bootfile + log_end_msg $? + + telinit q + chmod g-w,o-w / + + run_postconfig_scripts +} + +stop() +{ + local -i status=0 + log_daemon_msg "Stopping VyOS router" + for ((i=${#sub_inits[@]} - 1; i >= 0; i--)) ; do + s=${subinit[$i]} + log_progress_msg $s + ${vyatta_sbindir}/${s}.init stop + let status\|=$? + done + log_end_msg $status + log_action_begin_msg "Un-mounting VyOS Config" + umount ${vyatta_configdir} + log_action_end_msg $? + + systemctl stop frr.service +} + +case "$action" in + start) start ;; + stop) stop ;; + restart|force-reload) stop && start ;; + *) log_failure_msg "usage: $progname [ start|stop|restart ] [ subinit ... ]" ; + false ;; +esac + +exit $? + +# Local Variables: +# mode: shell-script +# sh-indentation: 4 +# End: diff --git a/src/migration-scripts/container/0-to-1 b/src/migration-scripts/container/0-to-1 index d0461389b..86f89ee04 100755 --- a/src/migration-scripts/container/0-to-1 +++ b/src/migration-scripts/container/0-to-1 @@ -21,7 +21,7 @@ import shutil import sys from vyos.configtree import ConfigTree -from vyos.util import call +from vyos.utils.process import call if (len(sys.argv) < 1): print("Must specify file name!") @@ -39,12 +39,12 @@ config = ConfigTree(config_file) if config.exists(base): for container in config.list_nodes(base): # Stop any given container first - call(f'systemctl stop vyos-container-{container}.service') + call(f'sudo systemctl stop vyos-container-{container}.service') # Export container image for later re-import to new filesystem. We store # the backup on a real disk as a tmpfs (like /tmp) could probably lack # memory if a host has too many containers stored. image_name = config.return_value(base + [container, 'image']) - call(f'podman image save --quiet --output /root/{container}.tar --format oci-archive {image_name}') + call(f'sudo podman image save --quiet --output /root/{container}.tar --format oci-archive {image_name}') # No need to adjust the strage driver online (this is only used for testing and # debugging on a live system) - it is already overlay2 when the migration script @@ -66,10 +66,10 @@ if config.exists(base): # Export container image for later re-import to new filesystem image_name = config.return_value(base + [container, 'image']) image_path = f'/root/{container}.tar' - call(f'podman image load --quiet --input {image_path}') + call(f'sudo podman image load --quiet --input {image_path}') # Start any given container first - call(f'systemctl start vyos-container-{container}.service') + call(f'sudo systemctl start vyos-container-{container}.service') # Delete temporary container image if os.path.exists(image_path): diff --git a/src/migration-scripts/interfaces/24-to-25 b/src/migration-scripts/interfaces/24-to-25 index 4095f2a3e..f3a1dc464 100755 --- a/src/migration-scripts/interfaces/24-to-25 +++ b/src/migration-scripts/interfaces/24-to-25 @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2021 VyOS maintainers and contributors +# Copyright (C) 2021-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -19,6 +19,7 @@ import os import sys + from vyos.configtree import ConfigTree from vyos.pki import CERT_BEGIN from vyos.pki import load_certificate @@ -29,7 +30,7 @@ from vyos.pki import encode_certificate from vyos.pki import encode_dh_parameters from vyos.pki import encode_private_key from vyos.pki import verify_crl -from vyos.util import run +from vyos.utils.process import run def wrapped_pem_to_config_value(pem): out = [] @@ -241,7 +242,7 @@ if config.exists(base): config.set(pki_base + ['certificate', pki_name, 'private', 'key'], value=wrapped_pem_to_config_value(key_pem)) else: print(f'Failed to migrate private key on openvpn interface {interface}') - + config.delete(x509_base + ['key-file']) if config.exists(x509_base + ['dh-file']): @@ -276,7 +277,7 @@ base = ['interfaces', 'wireguard'] if config.exists(base): for interface in config.list_nodes(base): private_key_path = base + [interface, 'private-key'] - + key_file = 'default' if config.exists(private_key_path): key_file = config.return_value(private_key_path) @@ -375,7 +376,7 @@ if config.exists(base): config.set(pki_base + ['certificate', pki_name, 'private', 'key'], value=wrapped_pem_to_config_value(key_pem)) else: print(f'Failed to migrate private key on eapol config for interface {interface}') - + config.delete(x509_base + ['key-file']) try: diff --git a/src/migration-scripts/interfaces/7-to-8 b/src/migration-scripts/interfaces/7-to-8 index a4051301f..9845098a7 100755 --- a/src/migration-scripts/interfaces/7-to-8 +++ b/src/migration-scripts/interfaces/7-to-8 @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2020 VyOS maintainers and contributors +# Copyright (C) 2020-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -21,7 +21,8 @@ import os from sys import exit, argv from vyos.configtree import ConfigTree -from vyos.util import chown, chmod_750 +from vyos.utils.permission import chown +from vyos.utils.permission import chmod_750 def migrate_default_keys(): kdir = r'/config/auth/wireguard' diff --git a/src/migration-scripts/ipsec/6-to-7 b/src/migration-scripts/ipsec/6-to-7 index 788a87095..649a18cb3 100755 --- a/src/migration-scripts/ipsec/6-to-7 +++ b/src/migration-scripts/ipsec/6-to-7 @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2021 VyOS maintainers and contributors +# Copyright (C) 2021-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -27,7 +27,7 @@ from vyos.pki import load_crl from vyos.pki import load_private_key from vyos.pki import encode_certificate from vyos.pki import encode_private_key -from vyos.util import run +from vyos.utils.process import run if (len(argv) < 1): print("Must specify file name!") @@ -127,7 +127,7 @@ if config.exists(ipsec_site_base): config.set(pki_base + ['ca', pki_name, 'crl'], value=wrapped_pem_to_config_value(crl_pem)) else: print(f'Failed to migrate CRL on peer "{peer}"') - + config.delete(peer_x509_base + ['crl-file']) if config.exists(peer_x509_base + ['key', 'file']): @@ -157,7 +157,7 @@ if config.exists(ipsec_site_base): config.set(peer_x509_base + ['private-key-passphrase'], value=key_passphrase) else: print(f'Failed to migrate private key on peer "{peer}"') - + config.delete(peer_x509_base + ['key']) if changes_made: diff --git a/src/migration-scripts/l2tp/3-to-4 b/src/migration-scripts/l2tp/3-to-4 index 18eabadec..ee6079864 100755 --- a/src/migration-scripts/l2tp/3-to-4 +++ b/src/migration-scripts/l2tp/3-to-4 @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2021 VyOS maintainers and contributors +# Copyright (C) 2021-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -27,7 +27,7 @@ from vyos.pki import load_crl from vyos.pki import load_private_key from vyos.pki import encode_certificate from vyos.pki import encode_private_key -from vyos.util import run +from vyos.utils.process import run if (len(argv) < 1): print("Must specify file name!") @@ -156,7 +156,7 @@ if config.exists(x509_base + ['server-key-file']): config.set(x509_base + ['private-key-passphrase'], value=key_passphrase) else: print(f'Failed to migrate private key on l2tp remote-access config') - + config.delete(x509_base + ['server-key-file']) if config.exists(x509_base + ['server-key-password']): config.delete(x509_base + ['server-key-password']) diff --git a/src/migration-scripts/openconnect/0-to-1 b/src/migration-scripts/openconnect/0-to-1 index 83cd09143..b26023a5b 100755 --- a/src/migration-scripts/openconnect/0-to-1 +++ b/src/migration-scripts/openconnect/0-to-1 @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2021 VyOS maintainers and contributors +# Copyright (C) 2021-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -26,7 +26,7 @@ from vyos.pki import load_crl from vyos.pki import load_private_key from vyos.pki import encode_certificate from vyos.pki import encode_private_key -from vyos.util import run +from vyos.utils.process import run if (len(argv) < 1): print("Must specify file name!") @@ -125,7 +125,7 @@ if config.exists(x509_base + ['key-file']): config.set(pki_base + ['certificate', pki_name, 'private', 'key'], value=wrapped_pem_to_config_value(key_pem)) else: print(f'Failed to migrate private key on openconnect config') - + config.delete(x509_base + ['key-file']) try: diff --git a/src/migration-scripts/policy/3-to-4 b/src/migration-scripts/policy/3-to-4 index bae30cffc..49e0b4c38 100755 --- a/src/migration-scripts/policy/3-to-4 +++ b/src/migration-scripts/policy/3-to-4 @@ -51,7 +51,7 @@ def community_migrate(config: ConfigTree, rule: list[str]) -> bool: :rtype: bool """ community_list = list((config.return_value(rule)).split(" ")) - config.delete(rule) + if 'none' in community_list: config.set(rule + ['none']) return False @@ -61,8 +61,10 @@ def community_migrate(config: ConfigTree, rule: list[str]) -> bool: community_action = 'add' community_list.remove('additive') for community in community_list: - config.set(rule + [community_action], value=community, - replace=False) + if len(community): + config.set(rule + [community_action], value=community, + replace=False) + config.delete(rule) if community_action == 'replace': return False else: diff --git a/src/migration-scripts/qos/1-to-2 b/src/migration-scripts/qos/1-to-2 index 14d3a6e0a..a689bacc5 100755 --- a/src/migration-scripts/qos/1-to-2 +++ b/src/migration-scripts/qos/1-to-2 @@ -18,7 +18,7 @@ from sys import argv,exit from vyos.base import Warning from vyos.configtree import ConfigTree -from vyos.util import read_file +from vyos.utils.file import read_file def bandwidth_percent_to_val(interface, percent) -> int: speed = read_file(f'/sys/class/net/{interface}/speed') diff --git a/src/migration-scripts/sstp/3-to-4 b/src/migration-scripts/sstp/3-to-4 index 0568f043f..ea814fdc5 100755 --- a/src/migration-scripts/sstp/3-to-4 +++ b/src/migration-scripts/sstp/3-to-4 @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2021 VyOS maintainers and contributors +# Copyright (C) 2021-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -26,7 +26,7 @@ from vyos.pki import load_crl from vyos.pki import load_private_key from vyos.pki import encode_certificate from vyos.pki import encode_private_key -from vyos.util import run +from vyos.utils.process import run if (len(argv) < 1): print("Must specify file name!") @@ -125,7 +125,7 @@ if config.exists(x509_base + ['key-file']): config.set(pki_base + ['certificate', pki_name, 'private', 'key'], value=wrapped_pem_to_config_value(key_pem)) else: print(f'Failed to migrate private key on sstp config') - + config.delete(x509_base + ['key-file']) try: diff --git a/src/migration-scripts/system/13-to-14 b/src/migration-scripts/system/13-to-14 index 5b068f4fc..1751f1865 100755 --- a/src/migration-scripts/system/13-to-14 +++ b/src/migration-scripts/system/13-to-14 @@ -12,7 +12,7 @@ import re import sys from vyos.configtree import ConfigTree -from vyos.util import cmd +from vyos.utils.process import cmd if (len(sys.argv) < 1): diff --git a/src/migration-scripts/vrrp/3-to-4 b/src/migration-scripts/vrrp/3-to-4 new file mode 100755 index 000000000..b0a6975c2 --- /dev/null +++ b/src/migration-scripts/vrrp/3-to-4 @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2023 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +from sys import argv +from vyos.configtree import ConfigTree + +if (len(argv) < 1): + print('Must specify file name!') + exit(1) + +file_name = argv[1] + +with open(file_name, 'r') as f: + config_file = f.read() + +base = ['high-availability', 'virtual-server'] +config = ConfigTree(config_file) + +if not config.exists(base): + # Nothing to do + exit(0) + +if config.exists(base): + for vs in config.list_nodes(base): + vs_base = base + [vs] + + # If the fwmark is used, the address is not required + if not config.exists(vs_base + ['fwmark']): + # add option: 'virtual-server <tag> address x.x.x.x' + config.set(vs_base + ['address'], value=vs) + + +try: + with open(file_name, 'w') as f: + f.write(config.to_string()) +except OSError as e: + print(f'Failed to save the modified config: {e}') + exit(1) diff --git a/src/op_mode/accelppp.py b/src/op_mode/accelppp.py index 00de45fc8..67ce786d0 100755 --- a/src/op_mode/accelppp.py +++ b/src/op_mode/accelppp.py @@ -21,7 +21,7 @@ import vyos.accel_ppp import vyos.opmode from vyos.configquery import ConfigTreeQuery -from vyos.util import rc_cmd +from vyos.utils.process import rc_cmd accel_dict = { diff --git a/src/op_mode/bgp.py b/src/op_mode/bgp.py index af9ea788b..096113cb4 100755 --- a/src/op_mode/bgp.py +++ b/src/op_mode/bgp.py @@ -81,7 +81,7 @@ ArgFamily = typing.Literal['inet', 'inet6', 'l2vpn'] ArgFamilyModifier = typing.Literal['unicast', 'labeled_unicast', 'multicast', 'vpn', 'flowspec'] def show_summary(raw: bool): - from vyos.util import cmd + from vyos.utils.process import cmd if raw: from json import loads @@ -96,7 +96,7 @@ def show_summary(raw: bool): return output def show_neighbors(raw: bool): - from vyos.util import cmd + from vyos.utils.process import cmd from vyos.utils.dict import dict_to_list if raw: @@ -129,7 +129,7 @@ def show(raw: bool, frr_command = frr_command_template.render(kwargs) frr_command = re.sub(r'\s+', ' ', frr_command) - from vyos.util import cmd + from vyos.utils.process import cmd output = cmd(f"vtysh -c '{frr_command}'") if raw: diff --git a/src/op_mode/bridge.py b/src/op_mode/bridge.py index d6098c158..1834b9cc9 100755 --- a/src/op_mode/bridge.py +++ b/src/op_mode/bridge.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2022 VyOS maintainers and contributors +# Copyright (C) 2022-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -22,8 +22,10 @@ import typing from sys import exit from tabulate import tabulate -from vyos.util import cmd, rc_cmd -from vyos.util import dict_search +from vyos.utils.process import cmd +from vyos.utils.process import rc_cmd +from vyos.utils.process import call +from vyos.utils.dict import dict_search import vyos.opmode @@ -128,7 +130,8 @@ def _get_formatted_output_vlan(data): if vlan_entry.get('vlanEnd'): vlan_end = vlan_entry.get('vlanEnd') vlan = f'{vlan}-{vlan_end}' - flags = ', '.join(vlan_entry.get('flags')).lower() + flags_raw = vlan_entry.get('flags') + flags = ', '.join(flags_raw if isinstance(flags_raw,list) else "").lower() data_entries.append([interface, vlan, flags]) headers = ["Interface", "Vlan", "Flags"] @@ -163,6 +166,23 @@ def _get_formatted_output_mdb(data): output = tabulate(data_entries, headers) return output +def _get_bridge_detail(iface): + """Get interface detail statistics""" + return call(f'vtysh -c "show interface {iface}"') + +def _get_bridge_detail_nexthop_group(iface): + """Get interface detail nexthop_group statistics""" + return call(f'vtysh -c "show interface {iface} nexthop-group"') + +def _get_bridge_detail_nexthop_group_raw(iface): + out = cmd(f'vtysh -c "show interface {iface} nexthop-group"') + return out + +def _get_bridge_detail_raw(iface): + """Get interface detail json statistics""" + data = cmd(f'vtysh -c "show interface {iface} json"') + data_dict = json.loads(data) + return data_dict def show(raw: bool): bridge_data = _get_raw_data_summary() @@ -195,6 +215,17 @@ def show_mdb(raw: bool, interface: str): else: return _get_formatted_output_mdb(mdb_data) +def show_detail(raw: bool, nexthop_group: typing.Optional[bool], interface: str): + if raw: + if nexthop_group: + return _get_bridge_detail_nexthop_group_raw(interface) + else: + return _get_bridge_detail_raw(interface) + else: + if nexthop_group: + return _get_bridge_detail_nexthop_group(interface) + else: + return _get_bridge_detail(interface) if __name__ == '__main__': try: diff --git a/src/op_mode/clear_conntrack.py b/src/op_mode/clear_conntrack.py index 423694187..fec7cf144 100755 --- a/src/op_mode/clear_conntrack.py +++ b/src/op_mode/clear_conntrack.py @@ -16,8 +16,9 @@ import sys -from vyos.util import ask_yes_no -from vyos.util import cmd, DEVNULL +from vyos.utils.io import ask_yes_no +from vyos.utils.process import cmd +from vyos.utils.process import DEVNULL if not ask_yes_no("This will clear all currently tracked and expected connections. Continue?"): sys.exit(1) diff --git a/src/op_mode/clear_dhcp_lease.py b/src/op_mode/clear_dhcp_lease.py index 250dbcce1..f372d3af0 100755 --- a/src/op_mode/clear_dhcp_lease.py +++ b/src/op_mode/clear_dhcp_lease.py @@ -7,9 +7,9 @@ from isc_dhcp_leases import Lease from isc_dhcp_leases import IscDhcpLeases from vyos.configquery import ConfigTreeQuery -from vyos.util import ask_yes_no -from vyos.util import call -from vyos.util import commit_in_progress +from vyos.utils.io import ask_yes_no +from vyos.utils.process import call +from vyos.utils.commit import commit_in_progress config = ConfigTreeQuery() diff --git a/src/op_mode/connect_disconnect.py b/src/op_mode/connect_disconnect.py index d39e88bf3..89f929be7 100755 --- a/src/op_mode/connect_disconnect.py +++ b/src/op_mode/connect_disconnect.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2020-2021 VyOS maintainers and contributors +# Copyright (C) 2020-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -19,10 +19,10 @@ import argparse from psutil import process_iter -from vyos.util import call -from vyos.util import commit_in_progress -from vyos.util import DEVNULL -from vyos.util import is_wwan_connected +from vyos.utils.process import call +from vyos.utils.commit import commit_in_progress +from vyos.utils.network import is_wwan_connected +from vyos.utils.process import DEVNULL def check_ppp_interface(interface): if not os.path.isfile(f'/etc/ppp/peers/{interface}'): diff --git a/src/op_mode/conntrack.py b/src/op_mode/conntrack.py index ea7c4c208..cf8adf795 100755 --- a/src/op_mode/conntrack.py +++ b/src/op_mode/conntrack.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2022 VyOS maintainers and contributors +# Copyright (C) 2022-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -19,8 +19,8 @@ import typing import xmltodict from tabulate import tabulate -from vyos.util import cmd -from vyos.util import run +from vyos.utils.process import cmd +from vyos.utils.process import run import vyos.opmode diff --git a/src/op_mode/conntrack_sync.py b/src/op_mode/conntrack_sync.py index c3345a936..a38688e45 100755 --- a/src/op_mode/conntrack_sync.py +++ b/src/op_mode/conntrack_sync.py @@ -24,10 +24,10 @@ import vyos.opmode from argparse import ArgumentParser from vyos.configquery import CliShellApiConfigQuery from vyos.configquery import ConfigTreeQuery -from vyos.util import call -from vyos.util import commit_in_progress -from vyos.util import cmd -from vyos.util import run +from vyos.utils.commit import commit_in_progress +from vyos.utils.process import call +from vyos.utils.process import cmd +from vyos.utils.process import run from vyos.template import render_to_string conntrackd_bin = '/usr/sbin/conntrackd' diff --git a/src/op_mode/container.py b/src/op_mode/container.py index 7f726f076..5a022d0c0 100755 --- a/src/op_mode/container.py +++ b/src/op_mode/container.py @@ -19,7 +19,7 @@ import sys from sys import exit -from vyos.util import cmd +from vyos.utils.process import cmd import vyos.opmode @@ -36,14 +36,14 @@ def _get_raw_data(command: str) -> list: return data def add_image(name: str): - from vyos.util import rc_cmd + from vyos.utils.process import rc_cmd rc, output = rc_cmd(f'podman image pull {name}') if rc != 0: raise vyos.opmode.InternalError(output) def delete_image(name: str): - from vyos.util import rc_cmd + from vyos.utils.process import rc_cmd rc, output = rc_cmd(f'podman image rm --force {name}') if rc != 0: @@ -77,7 +77,7 @@ def show_network(raw: bool): def restart(name: str): - from vyos.util import rc_cmd + from vyos.utils.process import rc_cmd rc, output = rc_cmd(f'systemctl restart vyos-container-{name}.service') if rc != 0: diff --git a/src/op_mode/dhcp.py b/src/op_mode/dhcp.py index fe7f252ba..3e51e990b 100755 --- a/src/op_mode/dhcp.py +++ b/src/op_mode/dhcp.py @@ -27,9 +27,9 @@ import vyos.opmode from vyos.base import Warning from vyos.configquery import ConfigTreeQuery -from vyos.util import cmd -from vyos.util import dict_search -from vyos.util import is_systemd_service_running +from vyos.utils.process import cmd +from vyos.utils.dict import dict_search +from vyos.utils.process import is_systemd_service_running config = ConfigTreeQuery() lease_valid_states = ['all', 'active', 'free', 'expired', 'released', 'abandoned', 'reset', 'backup'] diff --git a/src/op_mode/dns.py b/src/op_mode/dns.py index f8863c530..2168aef89 100755 --- a/src/op_mode/dns.py +++ b/src/op_mode/dns.py @@ -20,7 +20,7 @@ import sys from tabulate import tabulate from vyos.configquery import ConfigTreeQuery -from vyos.util import cmd +from vyos.utils.process import cmd import vyos.opmode diff --git a/src/op_mode/dns_dynamic.py b/src/op_mode/dns_dynamic.py index 76ca5249b..12aa5494a 100755 --- a/src/op_mode/dns_dynamic.py +++ b/src/op_mode/dns_dynamic.py @@ -22,7 +22,7 @@ from tabulate import tabulate from vyos.config import Config from vyos.template import is_ipv4, is_ipv6 -from vyos.util import call +from vyos.utils.process import call cache_file = r'/run/ddclient/ddclient.cache' diff --git a/src/op_mode/dns_forwarding_reset.py b/src/op_mode/dns_forwarding_reset.py index bfc640a26..55e20918f 100755 --- a/src/op_mode/dns_forwarding_reset.py +++ b/src/op_mode/dns_forwarding_reset.py @@ -25,7 +25,7 @@ import argparse from sys import exit from vyos.config import Config -from vyos.util import call +from vyos.utils.process import call PDNS_CMD='/usr/bin/rec_control --socket-dir=/run/powerdns' diff --git a/src/op_mode/dns_forwarding_statistics.py b/src/op_mode/dns_forwarding_statistics.py index d79b6c024..32b5c76a7 100755 --- a/src/op_mode/dns_forwarding_statistics.py +++ b/src/op_mode/dns_forwarding_statistics.py @@ -4,7 +4,7 @@ import jinja2 from sys import exit from vyos.config import Config -from vyos.util import cmd +from vyos.utils.process import cmd PDNS_CMD='/usr/bin/rec_control --socket-dir=/run/powerdns' diff --git a/src/op_mode/firewall.py b/src/op_mode/firewall.py index 46bda5f7e..8260bbb77 100755 --- a/src/op_mode/firewall.py +++ b/src/op_mode/firewall.py @@ -21,8 +21,8 @@ import re import tabulate from vyos.config import Config -from vyos.util import cmd -from vyos.util import dict_search_args +from vyos.utils.process import cmd +from vyos.utils.dict import dict_search_args def get_firewall_interfaces(firewall, name=None, ipv6=False): directions = ['in', 'out', 'local'] diff --git a/src/op_mode/flow_accounting_op.py b/src/op_mode/flow_accounting_op.py index 514143cd7..497ccafdf 100755 --- a/src/op_mode/flow_accounting_op.py +++ b/src/op_mode/flow_accounting_op.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2018 VyOS maintainers and contributors +# Copyright (C) 2018-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -13,18 +13,18 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -# import sys import argparse import re import ipaddress import os.path + from tabulate import tabulate from json import loads -from vyos.util import cmd -from vyos.util import commit_in_progress -from vyos.util import run +from vyos.utils.commit import commit_in_progress +from vyos.utils.process import cmd +from vyos.utils.process import run from vyos.logger import syslog # some default values diff --git a/src/op_mode/format_disk.py b/src/op_mode/format_disk.py index b3ba44e87..31ceb196a 100755 --- a/src/op_mode/format_disk.py +++ b/src/op_mode/format_disk.py @@ -20,10 +20,10 @@ import re from datetime import datetime -from vyos.util import ask_yes_no -from vyos.util import call -from vyos.util import cmd -from vyos.util import DEVNULL +from vyos.utils.io import ask_yes_no +from vyos.utils.process import call +from vyos.utils.process import cmd +from vyos.utils.process import DEVNULL def list_disks(): disks = set() diff --git a/src/op_mode/generate_interfaces_debug_archive.py b/src/op_mode/generate_interfaces_debug_archive.py index f5767080a..3059aad23 100755 --- a/src/op_mode/generate_interfaces_debug_archive.py +++ b/src/op_mode/generate_interfaces_debug_archive.py @@ -20,7 +20,7 @@ from shutil import rmtree from socket import gethostname from sys import exit from tarfile import open as tar_open -from vyos.util import rc_cmd +from vyos.utils.process import rc_cmd import os # define a list of commands that needs to be executed diff --git a/src/op_mode/generate_ipsec_debug_archive.py b/src/op_mode/generate_ipsec_debug_archive.py index 1422559a8..60195d48b 100755 --- a/src/op_mode/generate_ipsec_debug_archive.py +++ b/src/op_mode/generate_ipsec_debug_archive.py @@ -20,7 +20,7 @@ from shutil import rmtree from socket import gethostname from sys import exit from tarfile import open as tar_open -from vyos.util import rc_cmd +from vyos.utils.process import rc_cmd # define a list of commands that needs to be executed CMD_LIST: list[str] = [ diff --git a/src/op_mode/generate_openconnect_otp_key.py b/src/op_mode/generate_openconnect_otp_key.py index 363bcf3ea..99b67d261 100755 --- a/src/op_mode/generate_openconnect_otp_key.py +++ b/src/op_mode/generate_openconnect_otp_key.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2022 VyOS maintainers and contributors +# Copyright (C) 2022-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -17,7 +17,7 @@ import argparse import os -from vyos.util import popen +from vyos.utils.process import popen from secrets import token_hex from base64 import b32encode diff --git a/src/op_mode/generate_ovpn_client_file.py b/src/op_mode/generate_ovpn_client_file.py index 0628e6135..cec370a07 100755 --- a/src/op_mode/generate_ovpn_client_file.py +++ b/src/op_mode/generate_ovpn_client_file.py @@ -22,7 +22,7 @@ from textwrap import fill from vyos.configquery import ConfigTreeQuery from vyos.ifconfig import Section -from vyos.util import cmd +from vyos.utils.process import cmd client_config = """ diff --git a/src/op_mode/generate_ssh_server_key.py b/src/op_mode/generate_ssh_server_key.py index 43e94048d..d6063c43c 100755 --- a/src/op_mode/generate_ssh_server_key.py +++ b/src/op_mode/generate_ssh_server_key.py @@ -15,9 +15,9 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. from sys import exit -from vyos.util import ask_yes_no -from vyos.util import cmd -from vyos.util import commit_in_progress +from vyos.utils.io import ask_yes_no +from vyos.utils.process import cmd +from vyos.utils.commit import commit_in_progress if not ask_yes_no('Do you really want to remove the existing SSH host keys?'): exit(0) diff --git a/src/op_mode/generate_system_login_user.py b/src/op_mode/generate_system_login_user.py index 8f8827b1b..1b328eae0 100755 --- a/src/op_mode/generate_system_login_user.py +++ b/src/op_mode/generate_system_login_user.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2022 VyOS maintainers and contributors +# Copyright (C) 2022-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -17,7 +17,7 @@ import argparse import os -from vyos.util import popen +from vyos.utils.process import popen from secrets import token_hex from base64 import b32encode diff --git a/src/op_mode/igmp-proxy.py b/src/op_mode/igmp-proxy.py index 0086c9aa6..709e25915 100755 --- a/src/op_mode/igmp-proxy.py +++ b/src/op_mode/igmp-proxy.py @@ -28,16 +28,14 @@ import tabulate import vyos.config import vyos.opmode -from vyos.util import bytes_to_human, print_error +from vyos.utils.convert import bytes_to_human +from vyos.utils.io import print_error +from vyos.utils.process import process_named_running def _is_configured(): """Check if IGMP proxy is configured""" return vyos.config.Config().exists_effective('protocols igmp-proxy') -def _is_running(): - """Check if IGMP proxy is currently running""" - return not vyos.util.run('ps -C igmpproxy') - def _kernel_to_ip(addr): """ Convert any given address from Linux kernel to a proper, IPv4 address @@ -84,7 +82,7 @@ def show_interface(raw: bool): if not _is_configured(): print_error('IGMP proxy is not configured.') sys.exit(0) -if not _is_running(): +if not process_named_running('igmpproxy'): print_error('IGMP proxy is not running.') sys.exit(0) diff --git a/src/op_mode/ikev2_profile_generator.py b/src/op_mode/ikev2_profile_generator.py index a22f04c45..5454cc0ce 100755 --- a/src/op_mode/ikev2_profile_generator.py +++ b/src/op_mode/ikev2_profile_generator.py @@ -24,7 +24,7 @@ from cryptography.x509.oid import NameOID from vyos.configquery import ConfigTreeQuery from vyos.pki import load_certificate from vyos.template import render_to_string -from vyos.util import ask_input +from vyos.utils.io import ask_input # Apple profiles only support one IKE/ESP encryption cipher and hash, whereas # VyOS comes with a multitude of different proposals for a connection. diff --git a/src/op_mode/interfaces.py b/src/op_mode/interfaces.py index f38b95a71..782e178c6 100755 --- a/src/op_mode/interfaces.py +++ b/src/op_mode/interfaces.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2022 VyOS maintainers and contributors +# Copyright (C) 2022-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -28,7 +28,9 @@ import vyos.opmode from vyos.ifconfig import Section from vyos.ifconfig import Interface from vyos.ifconfig import VRRP -from vyos.util import cmd, rc_cmd, call +from vyos.utils.process import cmd +from vyos.utils.process import rc_cmd +from vyos.utils.process import call def catch_broken_pipe(func): def wrapped(*args, **kwargs): @@ -384,7 +386,7 @@ def _format_show_counters(data: list): rx_errors = entry.get('rx_over_errors') tx_errors = entry.get('tx_carrier_errors') data_entries.append([interface, rx_packets, rx_bytes, tx_packets, tx_bytes, rx_dropped, tx_dropped, rx_errors, tx_errors]) - + headers = ['Interface', 'Rx Packets', 'Rx Bytes', 'Tx Packets', 'Tx Bytes', 'Rx Dropped', 'Tx Dropped', 'Rx Errors', 'Tx Errors'] output = tabulate(data_entries, headers, numalign="left") print (output) diff --git a/src/op_mode/ipoe-control.py b/src/op_mode/ipoe-control.py index 7111498b2..0f33beca7 100755 --- a/src/op_mode/ipoe-control.py +++ b/src/op_mode/ipoe-control.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2020 VyOS maintainers and contributors +# Copyright (C) 2020-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -18,7 +18,8 @@ import sys import argparse from vyos.config import Config -from vyos.util import popen, run +from vyos.utils.process import popen +from vyos.utils.process import run cmd_dict = { 'cmd_base' : '/usr/bin/accel-cmd -p 2002 ', diff --git a/src/op_mode/ipsec.py b/src/op_mode/ipsec.py index 823bd039d..57d3cfed9 100755 --- a/src/op_mode/ipsec.py +++ b/src/op_mode/ipsec.py @@ -21,9 +21,9 @@ from hurry import filesize from re import split as re_split from tabulate import tabulate -from vyos.util import convert_data -from vyos.util import seconds_to_human -from vyos.util import cmd +from vyos.utils.convert import convert_data +from vyos.utils.convert import seconds_to_human +from vyos.utils.process import cmd from vyos.configquery import ConfigTreeQuery import vyos.opmode diff --git a/src/op_mode/lldp.py b/src/op_mode/lldp.py index 1a1b94783..c287b8fa6 100755 --- a/src/op_mode/lldp.py +++ b/src/op_mode/lldp.py @@ -22,8 +22,8 @@ import typing from tabulate import tabulate from vyos.configquery import ConfigTreeQuery -from vyos.util import cmd -from vyos.util import dict_search +from vyos.utils.process import cmd +from vyos.utils.dict import dict_search import vyos.opmode unconf_message = 'LLDP is not configured' diff --git a/src/op_mode/log.py b/src/op_mode/log.py index b0abd6191..797ba5a88 100755 --- a/src/op_mode/log.py +++ b/src/op_mode/log.py @@ -21,7 +21,7 @@ import typing from jinja2 import Template -from vyos.util import rc_cmd +from vyos.utils.process import rc_cmd import vyos.opmode diff --git a/src/op_mode/memory.py b/src/op_mode/memory.py index 7666de646..eb530035b 100755 --- a/src/op_mode/memory.py +++ b/src/op_mode/memory.py @@ -54,7 +54,7 @@ def _get_raw_data(): return mem_data def _get_formatted_output(mem): - from vyos.util import bytes_to_human + from vyos.utils.convert import bytes_to_human # For human-readable outputs, we convert bytes to more convenient units # (100M, 1.3G...) diff --git a/src/op_mode/nat.py b/src/op_mode/nat.py index c92795745..71a40c0e1 100755 --- a/src/op_mode/nat.py +++ b/src/op_mode/nat.py @@ -25,8 +25,8 @@ from tabulate import tabulate import vyos.opmode from vyos.configquery import ConfigTreeQuery -from vyos.util import cmd -from vyos.util import dict_search +from vyos.utils.process import cmd +from vyos.utils.dict import dict_search base = 'nat' unconf_message = 'NAT is not configured' diff --git a/src/op_mode/neighbor.py b/src/op_mode/neighbor.py index b329ea280..1edeb0045 100755 --- a/src/op_mode/neighbor.py +++ b/src/op_mode/neighbor.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2022 VyOS maintainers and contributors +# Copyright (C) 2022-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -41,7 +41,7 @@ def interface_exists(interface): def get_raw_data(family, interface=None, state=None): from json import loads - from vyos.util import cmd + from vyos.utils.process import cmd if interface: if not interface_exists(interface): @@ -102,7 +102,7 @@ def show(raw: bool, family: ArgFamily, interface: typing.Optional[str], return format_neighbors(data, interface) def reset(family: ArgFamily, interface: typing.Optional[str], address: typing.Optional[str]): - from vyos.util import run + from vyos.utils.process import run if address and interface: raise ValueError("interface and address parameters are mutually exclusive") @@ -114,7 +114,6 @@ def reset(family: ArgFamily, interface: typing.Optional[str], address: typing.Op # Flush an entire neighbor table run(f"""ip --family {family} neighbor flush""") - if __name__ == '__main__': try: res = vyos.opmode.run(sys.modules[__name__]) diff --git a/src/op_mode/nhrp.py b/src/op_mode/nhrp.py index 5ff91a59c..e66f33079 100755 --- a/src/op_mode/nhrp.py +++ b/src/op_mode/nhrp.py @@ -18,9 +18,9 @@ import sys import tabulate import vyos.opmode -from vyos.util import cmd -from vyos.util import process_named_running -from vyos.util import colon_separated_to_dict +from vyos.utils.process import cmd +from vyos.utils.process import process_named_running +from vyos.utils.dict import colon_separated_to_dict def _get_formatted_output(output_dict: dict) -> str: diff --git a/src/op_mode/openconnect-control.py b/src/op_mode/openconnect-control.py index 20c50e779..b70d4fa16 100755 --- a/src/op_mode/openconnect-control.py +++ b/src/op_mode/openconnect-control.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2020 VyOS maintainers and contributors +# Copyright (C) 2020-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -18,12 +18,13 @@ import sys import argparse import json -from vyos.config import Config -from vyos.util import popen -from vyos.util import run -from vyos.util import DEVNULL from tabulate import tabulate +from vyos.config import Config +from vyos.utils.process import popen +from vyos.utils.process import run +from vyos.utils.process import DEVNULL + occtl = '/usr/bin/occtl' occtl_socket = '/run/ocserv/occtl.socket' diff --git a/src/op_mode/openconnect.py b/src/op_mode/openconnect.py index b21890728..cfa0678a7 100755 --- a/src/op_mode/openconnect.py +++ b/src/op_mode/openconnect.py @@ -19,7 +19,7 @@ import json from tabulate import tabulate from vyos.configquery import ConfigTreeQuery -from vyos.util import rc_cmd +from vyos.utils.process import rc_cmd import vyos.opmode diff --git a/src/op_mode/openvpn.py b/src/op_mode/openvpn.py index d9ae965c5..fd9d2db92 100755 --- a/src/op_mode/openvpn.py +++ b/src/op_mode/openvpn.py @@ -23,10 +23,10 @@ import typing from tabulate import tabulate import vyos.opmode -from vyos.util import bytes_to_human -from vyos.util import commit_in_progress -from vyos.util import call -from vyos.util import rc_cmd +from vyos.utils.convert import bytes_to_human +from vyos.utils.commit import commit_in_progress +from vyos.utils.process import call +from vyos.utils.process import rc_cmd from vyos.config import Config ArgMode = typing.Literal['client', 'server', 'site_to_site'] diff --git a/src/op_mode/ping.py b/src/op_mode/ping.py index 610e63cb3..f1d87a118 100755 --- a/src/op_mode/ping.py +++ b/src/op_mode/ping.py @@ -18,7 +18,7 @@ import os import sys import socket import ipaddress -from vyos.util import get_all_vrfs +from vyos.utils.network import get_all_vrfs from vyos.ifconfig import Section @@ -90,6 +90,16 @@ options = { 'type': '<seconds>', 'help': 'Number of seconds to wait between requests' }, + 'ipv4': { + 'ping': '{command} -4', + 'type': 'noarg', + 'help': 'Use IPv4 only' + }, + 'ipv6': { + 'ping': '{command} -6', + 'type': 'noarg', + 'help': 'Use IPv6 only' + }, 'mark': { 'ping': '{command} -m {value}', 'type': '<fwmark>', diff --git a/src/op_mode/pki.py b/src/op_mode/pki.py index b054690b0..4c31291ad 100755 --- a/src/op_mode/pki.py +++ b/src/op_mode/pki.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2021 VyOS maintainers and contributors +# Copyright (C) 2021-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -34,10 +34,11 @@ from vyos.pki import create_dh_parameters from vyos.pki import load_certificate, load_certificate_request, load_private_key from vyos.pki import load_crl, load_dh_parameters, load_public_key from vyos.pki import verify_certificate +from vyos.utils.io import ask_input +from vyos.utils.io import ask_yes_no +from vyos.utils.misc import install_into_config +from vyos.utils.process import cmd from vyos.xml import defaults -from vyos.util import ask_input, ask_yes_no -from vyos.util import cmd -from vyos.util import install_into_config CERT_REQ_END = '-----END CERTIFICATE REQUEST-----' auth_dir = '/config/auth' @@ -190,7 +191,7 @@ def install_ssh_key(name, public_key, private_key, passphrase=None): def install_keypair(name, key_type, private_key=None, public_key=None, passphrase=None, prompt=True): # Show/install conf commands for key-pair - + config_paths = [] if public_key: @@ -840,7 +841,7 @@ def import_openvpn_secret(name, path): install_openvpn_key(name, key_data, key_version) # Show functions -def show_certificate_authority(name=None): +def show_certificate_authority(name=None, pem=False): headers = ['Name', 'Subject', 'Issuer CN', 'Issued', 'Expiry', 'Private Key', 'Parent'] data = [] certs = get_config_ca_certificate() @@ -852,6 +853,11 @@ def show_certificate_authority(name=None): continue cert = load_certificate(cert_dict['certificate']) + + if name and pem: + print(encode_certificate(cert)) + return + parent_ca_name = get_certificate_ca(cert, certs) cert_issuer_cn = cert.issuer.rfc4514_string().split(",")[0] @@ -867,7 +873,7 @@ def show_certificate_authority(name=None): print("Certificate Authorities:") print(tabulate.tabulate(data, headers)) -def show_certificate(name=None): +def show_certificate(name=None, pem=False): headers = ['Name', 'Type', 'Subject CN', 'Issuer CN', 'Issued', 'Expiry', 'Revoked', 'Private Key', 'CA Present'] data = [] certs = get_config_certificate() @@ -885,6 +891,10 @@ def show_certificate(name=None): if not cert: continue + if name and pem: + print(encode_certificate(cert)) + return + ca_name = get_certificate_ca(cert, ca_certs) cert_subject_cn = cert.subject.rfc4514_string().split(",")[0] cert_issuer_cn = cert.issuer.rfc4514_string().split(",")[0] @@ -906,7 +916,7 @@ def show_certificate(name=None): print("Certificates:") print(tabulate.tabulate(data, headers)) -def show_crl(name=None): +def show_crl(name=None, pem=False): headers = ['CA Name', 'Updated', 'Revokes'] data = [] certs = get_config_ca_certificate() @@ -927,9 +937,16 @@ def show_crl(name=None): if not crl: continue + if name and pem: + print(encode_certificate(crl)) + continue + certs = get_revoked_by_serial_numbers([revoked.serial_number for revoked in crl]) data.append([cert_name, crl.last_update, ", ".join(certs)]) + if name and pem: + return + print("Certificate Revocation Lists:") print(tabulate.tabulate(data, headers)) @@ -943,6 +960,7 @@ if __name__ == '__main__': parser.add_argument('--crl', help='Certificate Revocation List', required=False) parser.add_argument('--sign', help='Sign certificate with specified CA', required=False) parser.add_argument('--self-sign', help='Self-sign the certificate', action='store_true') + parser.add_argument('--pem', help='Output using PEM encoding', action='store_true') # SSH parser.add_argument('--ssh', help='SSH Key', required=False) @@ -1032,16 +1050,16 @@ if __name__ == '__main__': if not conf.exists(['pki', 'ca', ca_name]): print(f'CA "{ca_name}" does not exist!') exit(1) - show_certificate_authority(ca_name) + show_certificate_authority(ca_name, args.pem) elif args.certificate: cert_name = None if args.certificate == 'all' else args.certificate if cert_name: if not conf.exists(['pki', 'certificate', cert_name]): print(f'Certificate "{cert_name}" does not exist!') exit(1) - show_certificate(None if args.certificate == 'all' else args.certificate) + show_certificate(None if args.certificate == 'all' else args.certificate, args.pem) elif args.crl: - show_crl(None if args.crl == 'all' else args.crl) + show_crl(None if args.crl == 'all' else args.crl, args.pem) else: show_certificate_authority() show_certificate() diff --git a/src/op_mode/policy_route.py b/src/op_mode/policy_route.py index fae47adec..eff99de7f 100755 --- a/src/op_mode/policy_route.py +++ b/src/op_mode/policy_route.py @@ -19,8 +19,8 @@ import re import tabulate from vyos.config import Config -from vyos.util import cmd -from vyos.util import dict_search_args +from vyos.utils.process import cmd +from vyos.utils.dict import dict_search_args def get_config_policy(conf, name=None, ipv6=False): config_path = ['policy'] diff --git a/src/op_mode/powerctrl.py b/src/op_mode/powerctrl.py index dfacd45c2..3ac5991b4 100755 --- a/src/op_mode/powerctrl.py +++ b/src/op_mode/powerctrl.py @@ -22,7 +22,11 @@ from datetime import datetime, timedelta, time as type_time, date as type_date from sys import exit from time import time -from vyos.util import ask_yes_no, cmd, call, run, STDOUT +from vyos.utils.io import ask_yes_no +from vyos.utils.process import call +from vyos.utils.process import cmd +from vyos.utils.process import run +from vyos.utils.process import STDOUT systemd_sched_file = "/run/systemd/shutdown/scheduled" @@ -104,7 +108,7 @@ def cancel_shutdown(): def check_unsaved_config(): from vyos.config_mgmt import unsaved_commits - from vyos.util import boot_configuration_success + from vyos.utils.boot import boot_configuration_success if unsaved_commits() and boot_configuration_success(): print("Warning: there are unsaved configuration changes!") diff --git a/src/op_mode/ppp-server-ctrl.py b/src/op_mode/ppp-server-ctrl.py index e93963fdd..2bae5b32a 100755 --- a/src/op_mode/ppp-server-ctrl.py +++ b/src/op_mode/ppp-server-ctrl.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2019 VyOS maintainers and contributors +# Copyright (C) 2019-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -18,7 +18,8 @@ import sys import argparse from vyos.config import Config -from vyos.util import popen, DEVNULL +from vyos.utils.process import popen +from vyos.utils.process import DEVNULL cmd_dict = { 'cmd_base' : '/usr/bin/accel-cmd -p {} ', diff --git a/src/op_mode/reset_openvpn.py b/src/op_mode/reset_openvpn.py index efbf65083..cef5299da 100755 --- a/src/op_mode/reset_openvpn.py +++ b/src/op_mode/reset_openvpn.py @@ -16,8 +16,8 @@ import os from sys import argv, exit -from vyos.util import call -from vyos.util import commit_in_progress +from vyos.utils.process import call +from vyos.utils.commit import commit_in_progress if __name__ == '__main__': if (len(argv) < 1): diff --git a/src/op_mode/reset_vpn.py b/src/op_mode/reset_vpn.py index 46195d6cd..61d7c8c81 100755 --- a/src/op_mode/reset_vpn.py +++ b/src/op_mode/reset_vpn.py @@ -13,10 +13,11 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. + import sys import typing -from vyos.util import run +from vyos.utils.process import run import vyos.opmode @@ -29,7 +30,6 @@ cmd_dict = { } } - def reset_conn(protocol: str, username: typing.Optional[str] = None, interface: typing.Optional[str] = None): if protocol in cmd_dict['vpn_types']: diff --git a/src/op_mode/restart_dhcp_relay.py b/src/op_mode/restart_dhcp_relay.py index 9203c009f..3ead97f4c 100755 --- a/src/op_mode/restart_dhcp_relay.py +++ b/src/op_mode/restart_dhcp_relay.py @@ -23,8 +23,8 @@ import argparse import os import vyos.config -from vyos.util import call -from vyos.util import commit_in_progress +from vyos.utils.process import call +from vyos.utils.commit import commit_in_progress parser = argparse.ArgumentParser() diff --git a/src/op_mode/restart_frr.py b/src/op_mode/restart_frr.py index 680d9f8cc..5cce377eb 100755 --- a/src/op_mode/restart_frr.py +++ b/src/op_mode/restart_frr.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2019-2021 VyOS maintainers and contributors +# Copyright (C) 2019-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -23,10 +23,10 @@ from logging.handlers import SysLogHandler from shutil import rmtree from vyos.base import Warning -from vyos.util import call -from vyos.util import ask_yes_no -from vyos.util import process_named_running -from vyos.util import makedir +from vyos.utils.io import ask_yes_no +from vyos.utils.file import makedir +from vyos.utils.process import call +from vyos.utils.process import process_named_running # some default values watchfrr = '/usr/lib/frr/watchfrr.sh' diff --git a/src/op_mode/route.py b/src/op_mode/route.py index d6d6b7d6f..4aa57dbf4 100755 --- a/src/op_mode/route.py +++ b/src/op_mode/route.py @@ -57,7 +57,7 @@ frr_command_template = Template(""" ArgFamily = typing.Literal['inet', 'inet6'] def show_summary(raw: bool, family: ArgFamily, table: typing.Optional[int], vrf: typing.Optional[str]): - from vyos.util import cmd + from vyos.utils.process import cmd if family == 'inet': family_cmd = 'ip' @@ -119,7 +119,7 @@ def show(raw: bool, frr_command = frr_command_template.render(kwargs) frr_command = re.sub(r'\s+', ' ', frr_command) - from vyos.util import cmd + from vyos.utils.process import cmd output = cmd(f"vtysh -c '{frr_command}'") if raw: diff --git a/src/op_mode/sflow.py b/src/op_mode/sflow.py index 88f70d6bd..dca7f44cb 100755 --- a/src/op_mode/sflow.py +++ b/src/op_mode/sflow.py @@ -20,7 +20,7 @@ import sys from tabulate import tabulate from vyos.configquery import ConfigTreeQuery -from vyos.util import cmd +from vyos.utils.process import cmd import vyos.opmode diff --git a/src/op_mode/show-bond.py b/src/op_mode/show-bond.py index edf7847fc..f676e0841 100755 --- a/src/op_mode/show-bond.py +++ b/src/op_mode/show-bond.py @@ -19,7 +19,7 @@ import jinja2 from argparse import ArgumentParser from vyos.ifconfig import Section from vyos.ifconfig import BondIf -from vyos.util import read_file +from vyos.utils.file import read_file from sys import exit diff --git a/src/op_mode/show_acceleration.py b/src/op_mode/show_acceleration.py index 48c31d4d9..1c4831f1d 100755 --- a/src/op_mode/show_acceleration.py +++ b/src/op_mode/show_acceleration.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2019-2022 VyOS maintainers and contributors +# Copyright (C) 2019-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -20,8 +20,8 @@ import re import argparse from vyos.config import Config -from vyos.util import popen -from vyos.util import call +from vyos.utils.process import call +from vyos.utils.process import popen def detect_qat_dev(): output, err = popen('lspci -nn', decode='utf-8') diff --git a/src/op_mode/show_ntp.sh b/src/op_mode/show_ntp.sh index 85f8eda15..4b59b801e 100755 --- a/src/op_mode/show_ntp.sh +++ b/src/op_mode/show_ntp.sh @@ -18,7 +18,7 @@ if ! ps -C chronyd &>/dev/null; then fi PID=$(pgrep chronyd | head -n1) -VRF_NAME=$(ip vrf identify ) +VRF_NAME=$(ip vrf identify ${PID}) if [ ! -z ${VRF_NAME} ]; then VRF_CMD="sudo ip vrf exec ${VRF_NAME}" diff --git a/src/op_mode/show_openconnect_otp.py b/src/op_mode/show_openconnect_otp.py index 88982c50b..415a5f72c 100755 --- a/src/op_mode/show_openconnect_otp.py +++ b/src/op_mode/show_openconnect_otp.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2017, 2022 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2017-2023 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -21,7 +21,7 @@ import os from vyos.config import Config from vyos.xml import defaults from vyos.configdict import dict_merge -from vyos.util import popen +from vyos.utils.process import popen from base64 import b32encode otp_file = '/run/ocserv/users.oath' diff --git a/src/op_mode/show_openvpn_mfa.py b/src/op_mode/show_openvpn_mfa.py index 1ab54600c..100c42154 100755 --- a/src/op_mode/show_openvpn_mfa.py +++ b/src/op_mode/show_openvpn_mfa.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 - -# Copyright 2017, 2021 VyOS maintainers and contributors <maintainers@vyos.io> +# +# Copyright 2017-2023 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -20,7 +20,7 @@ import socket import urllib.parse import argparse -from vyos.util import popen +from vyos.utils.process import popen otp_file = '/config/auth/openvpn/{interface}-otp-secrets' diff --git a/src/op_mode/show_sensors.py b/src/op_mode/show_sensors.py index 6ae477647..5e3084fe9 100755 --- a/src/op_mode/show_sensors.py +++ b/src/op_mode/show_sensors.py @@ -1,9 +1,25 @@ #!/usr/bin/env python3 +# +# Copyright 2017-2023 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. import re import sys -from vyos.util import popen -from vyos.util import DEVNULL +from vyos.utils.process import popen +from vyos.utils.process import DEVNULL + output,retcode = popen("sensors --no-adapter", stderr=DEVNULL) if retcode == 0: print (output) @@ -23,5 +39,3 @@ else: print ("No sensors found") sys.exit(1) - - diff --git a/src/op_mode/show_techsupport_report.py b/src/op_mode/show_techsupport_report.py index 782004144..d27221e54 100644 --- a/src/op_mode/show_techsupport_report.py +++ b/src/op_mode/show_techsupport_report.py @@ -17,7 +17,7 @@ import os from typing import List -from vyos.util import rc_cmd +from vyos.utils.process import rc_cmd from vyos.ifconfig import Section from vyos.ifconfig import Interface diff --git a/src/op_mode/show_virtual_server.py b/src/op_mode/show_virtual_server.py index 377180dec..7880edc97 100755 --- a/src/op_mode/show_virtual_server.py +++ b/src/op_mode/show_virtual_server.py @@ -15,7 +15,7 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. from vyos.configquery import ConfigTreeQuery -from vyos.util import call +from vyos.utils.process import call def is_configured(): """ Check if high-availability virtual-server is configured """ diff --git a/src/op_mode/show_wireless.py b/src/op_mode/show_wireless.py index 19ab6771c..340163057 100755 --- a/src/op_mode/show_wireless.py +++ b/src/op_mode/show_wireless.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2019 VyOS maintainers and contributors +# Copyright (C) 2019-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -21,7 +21,7 @@ from sys import exit from copy import deepcopy from vyos.config import Config -from vyos.util import popen +from vyos.utils.process import popen parser = argparse.ArgumentParser() parser.add_argument("-s", "--scan", help="Scan for Wireless APs on given interface, e.g. 'wlan0'") diff --git a/src/op_mode/show_wwan.py b/src/op_mode/show_wwan.py index eb601a456..bd97bb0e5 100755 --- a/src/op_mode/show_wwan.py +++ b/src/op_mode/show_wwan.py @@ -18,7 +18,7 @@ import argparse from sys import exit from vyos.configquery import ConfigTreeQuery -from vyos.util import cmd +from vyos.utils.process import cmd parser = argparse.ArgumentParser() parser.add_argument("--model", help="Get module model", action="store_true") diff --git a/src/op_mode/snmp.py b/src/op_mode/snmp.py index 5fae67881..43f5d9e0a 100755 --- a/src/op_mode/snmp.py +++ b/src/op_mode/snmp.py @@ -24,7 +24,7 @@ import sys import argparse from vyos.config import Config -from vyos.util import call +from vyos.utils.process import call config_file_daemon = r'/etc/snmp/snmpd.conf' diff --git a/src/op_mode/snmp_ifmib.py b/src/op_mode/snmp_ifmib.py index 2479936bd..c71febac9 100755 --- a/src/op_mode/snmp_ifmib.py +++ b/src/op_mode/snmp_ifmib.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2018 VyOS maintainers and contributors +# Copyright (C) 2018-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -24,7 +24,7 @@ import argparse import netifaces from vyos.config import Config -from vyos.util import popen +from vyos.utils.process import popen parser = argparse.ArgumentParser(description='Retrieve SNMP interfaces information') parser.add_argument('--ifindex', action='store', nargs='?', const='all', help='Show interface index') diff --git a/src/op_mode/storage.py b/src/op_mode/storage.py index d16e271bd..6bc3d3a2d 100755 --- a/src/op_mode/storage.py +++ b/src/op_mode/storage.py @@ -18,7 +18,7 @@ import sys import vyos.opmode -from vyos.util import cmd +from vyos.utils.process import cmd # FIY: As of coreutils from Debian Buster and Bullseye, # the outpt looks like this: @@ -43,7 +43,7 @@ def _get_system_storage(only_persistent=False): def _get_raw_data(): from re import sub as re_sub - from vyos.util import human_to_bytes + from vyos.utils.convert import human_to_bytes out = _get_system_storage(only_persistent=True) lines = out.splitlines() diff --git a/src/op_mode/traceroute.py b/src/op_mode/traceroute.py index 6c7030ea0..2f0edf53a 100755 --- a/src/op_mode/traceroute.py +++ b/src/op_mode/traceroute.py @@ -18,7 +18,7 @@ import os import sys import socket import ipaddress -from vyos.util import get_all_vrfs +from vyos.utils.network import get_all_vrfs from vyos.ifconfig import Section diff --git a/src/op_mode/uptime.py b/src/op_mode/uptime.py index 2ebe6783b..d6adf6f4d 100755 --- a/src/op_mode/uptime.py +++ b/src/op_mode/uptime.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2021-2022 VyOS maintainers and contributors +# Copyright (C) 2021-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -20,7 +20,7 @@ import vyos.opmode def _get_uptime_seconds(): from re import search - from vyos.util import read_file + from vyos.utils.file import read_file data = read_file("/proc/uptime") seconds = search("([0-9\.]+)\s", data).group(1) @@ -29,7 +29,7 @@ def _get_uptime_seconds(): def _get_load_averages(): from re import search - from vyos.util import cmd + from vyos.utils.process import cmd from vyos.cpu import get_core_count data = cmd("uptime") @@ -45,7 +45,7 @@ def _get_load_averages(): return res def _get_raw_data(): - from vyos.util import seconds_to_human + from vyos.utils.convert import seconds_to_human res = {} res["uptime_seconds"] = _get_uptime_seconds() diff --git a/src/op_mode/vpn_ike_sa.py b/src/op_mode/vpn_ike_sa.py index 4b44c5c15..069c12069 100755 --- a/src/op_mode/vpn_ike_sa.py +++ b/src/op_mode/vpn_ike_sa.py @@ -19,7 +19,7 @@ import re import sys import vici -from vyos.util import process_named_running +from vyos.utils.process import process_named_running ike_sa_peer_prefix = """\ Peer ID / IP Local ID / IP @@ -39,8 +39,6 @@ def ike_sa(peer, nat): peers = [] for conn in sas: for name, sa in conn.items(): - if peer and not name.startswith('peer_' + peer): - continue if name.startswith('peer_') and name in peers: continue if nat and 'nat-local' not in sa: @@ -70,7 +68,7 @@ if __name__ == '__main__': args = parser.parse_args() - if not process_named_running('charon'): + if not process_named_running('charon-systemd'): print("IPsec Process NOT Running") sys.exit(0) diff --git a/src/op_mode/vpn_ipsec.py b/src/op_mode/vpn_ipsec.py index b81d1693e..ef89e605f 100755 --- a/src/op_mode/vpn_ipsec.py +++ b/src/op_mode/vpn_ipsec.py @@ -17,7 +17,7 @@ import re import argparse -from vyos.util import call +from vyos.utils.process import call SWANCTL_CONF = '/etc/swanctl/swanctl.conf' diff --git a/src/op_mode/vrf.py b/src/op_mode/vrf.py index a9a416761..1f0bbbaeb 100755 --- a/src/op_mode/vrf.py +++ b/src/op_mode/vrf.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2022 VyOS maintainers and contributors +# Copyright (C) 2022-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -20,7 +20,7 @@ import sys import typing from tabulate import tabulate -from vyos.util import cmd +from vyos.utils.process import cmd import vyos.opmode diff --git a/src/op_mode/vrrp.py b/src/op_mode/vrrp.py index dab146d28..b3ab55cc3 100755 --- a/src/op_mode/vrrp.py +++ b/src/op_mode/vrrp.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2018-2022 VyOS maintainers and contributors +# Copyright (C) 2018-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -13,7 +13,6 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -# import sys import time @@ -21,12 +20,10 @@ import argparse import json import tabulate -import vyos.util - from vyos.configquery import ConfigTreeQuery from vyos.ifconfig.vrrp import VRRP -from vyos.ifconfig.vrrp import VRRPError, VRRPNoData - +from vyos.ifconfig.vrrp import VRRPError +from vyos.ifconfig.vrrp import VRRPNoData parser = argparse.ArgumentParser() group = parser.add_mutually_exclusive_group() @@ -44,7 +41,7 @@ def is_configured(): return True # Exit early if VRRP is dead or not configured -if is_configured() == False: +if is_configured() == False: print('VRRP not configured!') exit(0) if not VRRP.is_running(): diff --git a/src/op_mode/wireguard_client.py b/src/op_mode/wireguard_client.py index 76c1ff7d1..04d8ce28c 100755 --- a/src/op_mode/wireguard_client.py +++ b/src/op_mode/wireguard_client.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2021 VyOS maintainers and contributors +# Copyright (C) 2021-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -23,8 +23,8 @@ from ipaddress import ip_interface from vyos.ifconfig import Section from vyos.template import is_ipv4 from vyos.template import is_ipv6 -from vyos.util import cmd -from vyos.util import popen +from vyos.utils.process import cmd +from vyos.utils.process import popen if os.geteuid() != 0: exit("You need to have root privileges to run this script.\nPlease try again, this time using 'sudo'. Exiting.") diff --git a/src/op_mode/zone.py b/src/op_mode/zone.py index f326215b1..17ce90396 100755 --- a/src/op_mode/zone.py +++ b/src/op_mode/zone.py @@ -19,8 +19,8 @@ import vyos.opmode import tabulate from vyos.configquery import ConfigTreeQuery -from vyos.util import dict_search_args -from vyos.util import dict_search +from vyos.utils.dict import dict_search_args +from vyos.utils.dict import dict_search def get_config_zone(conf, name=None): diff --git a/src/services/api/graphql/generate/schema_from_op_mode.py b/src/services/api/graphql/generate/schema_from_op_mode.py index 229ccf90f..ab7cb691f 100755 --- a/src/services/api/graphql/generate/schema_from_op_mode.py +++ b/src/services/api/graphql/generate/schema_from_op_mode.py @@ -27,7 +27,7 @@ from jinja2 import Template from vyos.defaults import directories from vyos.opmode import _is_op_mode_function_name as is_op_mode_function_name from vyos.opmode import _get_literal_values as get_literal_values -from vyos.util import load_as_module +from vyos.utils.system import load_as_module if __package__ is None or __package__ == '': sys.path.append(os.path.join(directories['services'], 'api')) from graphql.libs.op_mode import is_show_function_name diff --git a/src/services/api/graphql/libs/op_mode.py b/src/services/api/graphql/libs/op_mode.py index e91d8bd0f..5022f7d4e 100644 --- a/src/services/api/graphql/libs/op_mode.py +++ b/src/services/api/graphql/libs/op_mode.py @@ -20,7 +20,7 @@ from typing import Union, Tuple, Optional from humps import decamelize from vyos.defaults import directories -from vyos.util import load_as_module +from vyos.utils.system import load_as_module from vyos.opmode import _normalize_field_names from vyos.opmode import _is_literal_type, _get_literal_values diff --git a/src/services/vyos-configd b/src/services/vyos-configd index 48c9135e2..355182b26 100755 --- a/src/services/vyos-configd +++ b/src/services/vyos-configd @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2020 VyOS maintainers and contributors +# Copyright (C) 2020-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -13,8 +13,6 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -# -# import os import sys @@ -28,8 +26,9 @@ import zmq from contextlib import contextmanager from vyos.defaults import directories -from vyos.util import boot_configuration_complete -from vyos.configsource import ConfigSourceString, ConfigSourceError +from vyos.utils.boot import boot_configuration_complete +from vyos.configsource import ConfigSourceString +from vyos.configsource import ConfigSourceError from vyos.config import Config from vyos import ConfigError diff --git a/src/services/vyos-hostsd b/src/services/vyos-hostsd index 894f9e24d..e34a4b740 100755 --- a/src/services/vyos-hostsd +++ b/src/services/vyos-hostsd @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2019-2020 VyOS maintainers and contributors +# Copyright (C) 2019-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -241,9 +241,14 @@ import traceback import re import logging import zmq + from voluptuous import Schema, MultipleInvalid, Required, Any from collections import OrderedDict -from vyos.util import popen, chown, chmod_755, makedir, process_named_running +from vyos.utils.file import makedir +from vyos.utils.permission import chown +from vyos.utils.permission import chmod_755 +from vyos.utils.process import popen +from vyos.utils.process import process_named_running from vyos.template import render debug = True diff --git a/src/system/keepalived-fifo.py b/src/system/keepalived-fifo.py index 864ee8419..5e19bdbad 100755 --- a/src/system/keepalived-fifo.py +++ b/src/system/keepalived-fifo.py @@ -28,9 +28,9 @@ from logging.handlers import SysLogHandler from vyos.ifconfig.vrrp import VRRP from vyos.configquery import ConfigTreeQuery -from vyos.util import cmd -from vyos.util import dict_search -from vyos.util import commit_in_progress +from vyos.utils.process import cmd +from vyos.utils.dict import dict_search +from vyos.utils.commit import commit_in_progress # configure logging logger = logging.getLogger(__name__) diff --git a/src/system/vyos-event-handler.py b/src/system/vyos-event-handler.py index 1c85380bc..74112ec91 100755 --- a/src/system/vyos-event-handler.py +++ b/src/system/vyos-event-handler.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2022 VyOS maintainers and contributors +# Copyright (C) 2022-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -18,6 +18,7 @@ import argparse import json import re import select + from copy import deepcopy from os import getpid, environ from pathlib import Path @@ -25,13 +26,13 @@ from signal import signal, SIGTERM, SIGINT from sys import exit from systemd import journal -from vyos.util import run, dict_search +from vyos.utils.dict import dict_search +from vyos.utils.process import run # Identify this script my_pid = getpid() my_name = Path(__file__).stem - # handle termination signal def handle_signal(signal_type, frame): if signal_type == SIGTERM: diff --git a/src/system/vyos-system-update-check.py b/src/system/vyos-system-update-check.py index c9597721b..c874f1e2c 100755 --- a/src/system/vyos-system-update-check.py +++ b/src/system/vyos-system-update-check.py @@ -22,7 +22,7 @@ from pathlib import Path from sys import exit from time import sleep -from vyos.util import call +from vyos.utils.process import call import vyos.version diff --git a/src/systemd/vyos-router.service b/src/systemd/vyos-router.service new file mode 100644 index 000000000..6f683cebb --- /dev/null +++ b/src/systemd/vyos-router.service @@ -0,0 +1,19 @@ +[Unit] +Description=VyOS Router +After=systemd-journald-dev-log.socket time-sync.target local-fs.target cloud-config.service +Requires=frr.service +Conflicts=shutdown.target +Before=systemd-user-sessions.service + +[Service] +Type=simple +Restart=no +TimeoutSec=20min +KillMode=process +RemainAfterExit=yes +ExecStart=/usr/libexec/vyos/init/vyos-router start +ExecStop=/usr/libexec/vyos/init/vyos-router stop +StandardOutput=journal+console + +[Install] +WantedBy=vyos.target diff --git a/src/systemd/vyos.target b/src/systemd/vyos.target new file mode 100644 index 000000000..47c91c1cc --- /dev/null +++ b/src/systemd/vyos.target @@ -0,0 +1,3 @@ +[Unit] +Description=VyOS target +After=multi-user.target diff --git a/src/tests/test_configverify.py b/src/tests/test_configverify.py index 6fb43ece2..15ccdf13d 100644 --- a/src/tests/test_configverify.py +++ b/src/tests/test_configverify.py @@ -16,7 +16,7 @@ from unittest import TestCase from vyos.configverify import verify_diffie_hellman_length -from vyos.util import cmd +from vyos.utils.process import cmd dh_file = '/tmp/dh.pem' diff --git a/src/tests/test_dict_search.py b/src/tests/test_dict_search.py index 1028437b2..2435d89c7 100644 --- a/src/tests/test_dict_search.py +++ b/src/tests/test_dict_search.py @@ -15,8 +15,8 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. from unittest import TestCase -from vyos.util import dict_search -from vyos.util import dict_search_recursive +from vyos.utils.dict import dict_search +from vyos.utils.dict import dict_search_recursive data = { 'string': 'fooo', diff --git a/src/tests/test_find_device_file.py b/src/tests/test_find_device_file.py index 43c80dc76..f18043d65 100755 --- a/src/tests/test_find_device_file.py +++ b/src/tests/test_find_device_file.py @@ -15,7 +15,7 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. from unittest import TestCase -from vyos.util import find_device_file +from vyos.utils.system import find_device_file class TestDeviceFile(TestCase): """ used to find USB devices on target """ diff --git a/src/tests/test_util.py b/src/tests/test_util.py index 473052bef..27ee648d4 100644 --- a/src/tests/test_util.py +++ b/src/tests/test_util.py @@ -15,14 +15,15 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. from unittest import TestCase -from vyos.util import * class TestVyOSUtil(TestCase): - def test_key_mangline(self): + def test_key_mangling(self): + from vyos.utils.dict import mangle_dict_keys data = {"foo-bar": {"baz-quux": None}} expected_data = {"foo_bar": {"baz_quux": None}} new_data = mangle_dict_keys(data, '-', '_') self.assertEqual(new_data, expected_data) def test_sysctl_read(self): + from vyos.utils.system import sysctl_read self.assertEqual(sysctl_read('net.ipv4.conf.lo.forwarding'), '1') diff --git a/src/validators/port-multi b/src/validators/port-multi index bd6f0ef60..ed6ff6849 100755 --- a/src/validators/port-multi +++ b/src/validators/port-multi @@ -4,7 +4,7 @@ from sys import argv from sys import exit import re -from vyos.util import read_file +from vyos.utils.file import read_file services_file = '/etc/services' diff --git a/src/validators/port-range b/src/validators/port-range index 5468000a7..526c639ad 100755 --- a/src/validators/port-range +++ b/src/validators/port-range @@ -3,7 +3,7 @@ import sys import re -from vyos.util import read_file +from vyos.utils.file import read_file services_file = '/etc/services' diff --git a/src/validators/script b/src/validators/script index 4ffdeb2a0..eb176d23b 100755 --- a/src/validators/script +++ b/src/validators/script @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2018-2021 VyOS maintainers and contributors +# Copyright (C) 2018-2023 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -19,7 +19,7 @@ import os import sys import shlex -import vyos.util +from vyos.utils.file import file_is_persistent if __name__ == '__main__': if len(sys.argv) < 2: @@ -35,7 +35,7 @@ if __name__ == '__main__': sys.exit(f'File {script} is not an executable file') # File outside the config dir is just a warning - if not vyos.util.file_is_persistent(script): + if not file_is_persistent(script): sys.exit(0)( f'Warning: file {script} is outside the "/config" directory\n' 'It will not be automatically migrated to a new image on system update' diff --git a/src/validators/timezone b/src/validators/timezone index 107571181..e55af8d2a 100755 --- a/src/validators/timezone +++ b/src/validators/timezone @@ -17,7 +17,7 @@ import argparse import sys -from vyos.util import cmd +from vyos.utils.process import cmd if __name__ == '__main__': |