diff options
68 files changed, 1552 insertions, 624 deletions
diff --git a/data/op-mode-standardized.json b/data/op-mode-standardized.json index 03b85b50f..ec950765d 100644 --- a/data/op-mode-standardized.json +++ b/data/op-mode-standardized.json @@ -1,4 +1,5 @@ [ +"bgp.py", "bridge.py", "conntrack.py", "container.py", diff --git a/data/templates/container/systemd-unit.j2 b/data/templates/container/systemd-unit.j2 new file mode 100644 index 000000000..fa48384ab --- /dev/null +++ b/data/templates/container/systemd-unit.j2 @@ -0,0 +1,17 @@ +### Autogenerated by container.py ### +[Unit] +Description=VyOS Container {{ name }} + +[Service] +Environment=PODMAN_SYSTEMD_UNIT=%n +Restart=on-failure +ExecStartPre=/bin/rm -f %t/%n.pid %t/%n.cid +ExecStart=/usr/bin/podman run \ + --conmon-pidfile %t/%n.pid --cidfile %t/%n.cid --cgroups=no-conmon \ + {{ run_args }} +ExecStop=/usr/bin/podman stop --ignore --cidfile %t/%n.cid -t 5 +ExecStopPost=/usr/bin/podman rm --ignore -f --cidfile %t/%n.cid +ExecStopPost=/bin/rm -f %t/%n.cid +PIDFile=%t/%n.pid +KillMode=none +Type=forking diff --git a/data/templates/firewall/nftables-nat.j2 b/data/templates/firewall/nftables-nat.j2 index 55fe6024b..c5c0a2c86 100644 --- a/data/templates/firewall/nftables-nat.j2 +++ b/data/templates/firewall/nftables-nat.j2 @@ -24,6 +24,7 @@ add rule ip raw NAT_CONNTRACK counter accept {% if first_install is not vyos_defined %} delete table ip vyos_nat {% endif %} +{% if deleted is not vyos_defined %} table ip vyos_nat { # # Destination NAT rules build up here @@ -31,11 +32,11 @@ table ip vyos_nat { chain PREROUTING { type nat hook prerouting priority -100; policy accept; counter jump VYOS_PRE_DNAT_HOOK -{% if destination.rule is vyos_defined %} -{% for rule, config in destination.rule.items() if config.disable is not vyos_defined %} +{% if destination.rule is vyos_defined %} +{% for rule, config in destination.rule.items() if config.disable is not vyos_defined %} {{ config | nat_rule(rule, 'destination') }} -{% endfor %} -{% endif %} +{% endfor %} +{% endif %} } # @@ -44,11 +45,11 @@ table ip vyos_nat { chain POSTROUTING { type nat hook postrouting priority 100; policy accept; counter jump VYOS_PRE_SNAT_HOOK -{% if source.rule is vyos_defined %} -{% for rule, config in source.rule.items() if config.disable is not vyos_defined %} +{% if source.rule is vyos_defined %} +{% for rule, config in source.rule.items() if config.disable is not vyos_defined %} {{ config | nat_rule(rule, 'source') }} -{% endfor %} -{% endif %} +{% endfor %} +{% endif %} } chain VYOS_PRE_DNAT_HOOK { @@ -59,3 +60,4 @@ table ip vyos_nat { return } } +{% endif %} diff --git a/data/templates/firewall/nftables-static-nat.j2 b/data/templates/firewall/nftables-static-nat.j2 index 790c33ce9..e5e3da867 100644 --- a/data/templates/firewall/nftables-static-nat.j2 +++ b/data/templates/firewall/nftables-static-nat.j2 @@ -3,6 +3,7 @@ {% if first_install is not vyos_defined %} delete table ip vyos_static_nat {% endif %} +{% if deleted is not vyos_defined %} table ip vyos_static_nat { # # Destination NAT rules build up here @@ -10,11 +11,11 @@ table ip vyos_static_nat { chain PREROUTING { type nat hook prerouting priority -100; policy accept; -{% if static.rule is vyos_defined %} -{% for rule, config in static.rule.items() if config.disable is not vyos_defined %} +{% if static.rule is vyos_defined %} +{% for rule, config in static.rule.items() if config.disable is not vyos_defined %} {{ config | nat_static_rule(rule, 'destination') }} -{% endfor %} -{% endif %} +{% endfor %} +{% endif %} } # @@ -22,10 +23,11 @@ table ip vyos_static_nat { # chain POSTROUTING { type nat hook postrouting priority 100; policy accept; -{% if static.rule is vyos_defined %} -{% for rule, config in static.rule.items() if config.disable is not vyos_defined %} +{% if static.rule is vyos_defined %} +{% for rule, config in static.rule.items() if config.disable is not vyos_defined %} {{ config | nat_static_rule(rule, 'source') }} -{% endfor %} -{% endif %} +{% endfor %} +{% endif %} } } +{% endif %} diff --git a/data/templates/ipsec/charon/eap-radius.conf.j2 b/data/templates/ipsec/charon/eap-radius.conf.j2 index 8495011fe..364377473 100644 --- a/data/templates/ipsec/charon/eap-radius.conf.j2 +++ b/data/templates/ipsec/charon/eap-radius.conf.j2 @@ -49,8 +49,10 @@ eap-radius { # Base to use for calculating exponential back off. # retransmit_base = 1.4 +{% if remote_access.radius.timeout is vyos_defined %} # Timeout in seconds before sending first retransmit. - # retransmit_timeout = 2.0 + retransmit_timeout = {{ remote_access.radius.timeout | float }} +{% endif %} # Number of times to retransmit a packet before giving up. # retransmit_tries = 4 diff --git a/data/templates/ssh/sshd_config.j2 b/data/templates/ssh/sshd_config.j2 index 5bbfdeb88..93735020c 100644 --- a/data/templates/ssh/sshd_config.j2 +++ b/data/templates/ssh/sshd_config.j2 @@ -62,6 +62,11 @@ ListenAddress {{ address }} Ciphers {{ ciphers | join(',') }} {% endif %} +{% if hostkey_algorithm is vyos_defined %} +# Specifies the available Host Key signature algorithms +HostKeyAlgorithms {{ hostkey_algorithm | join(',') }} +{% endif %} + {% if mac is vyos_defined %} # Specifies the available MAC (message authentication code) algorithms MACs {{ mac | join(',') }} diff --git a/debian/control b/debian/control index 0ed8f85c4..cf766a825 100644 --- a/debian/control +++ b/debian/control @@ -131,6 +131,7 @@ Depends: python3-netifaces, python3-paramiko, python3-psutil, + python3-pyhumps, python3-pystache, python3-pyudev, python3-six, @@ -154,6 +155,7 @@ Depends: ssl-cert, strongswan (>= 5.9), strongswan-swanctl (>= 5.9), + stunnel4, sudo, systemd, telegraf (>= 1.20), diff --git a/interface-definitions/https.xml.in b/interface-definitions/https.xml.in index d096c4ff1..6adb07598 100644 --- a/interface-definitions/https.xml.in +++ b/interface-definitions/https.xml.in @@ -107,7 +107,7 @@ <valueless/> </properties> </leafNode> - <node name="gql"> + <node name="graphql"> <properties> <help>GraphQL support</help> </properties> @@ -118,6 +118,59 @@ <valueless/> </properties> </leafNode> + <node name="authentication"> + <properties> + <help>GraphQL authentication</help> + </properties> + <children> + <leafNode name="type"> + <properties> + <help>Authentication type</help> + <completionHelp> + <list>key token</list> + </completionHelp> + <valueHelp> + <format>key</format> + <description>Use API keys</description> + </valueHelp> + <valueHelp> + <format>token</format> + <description>Use JWT token</description> + </valueHelp> + <constraint> + <regex>(key|token)</regex> + </constraint> + </properties> + <defaultValue>key</defaultValue> + </leafNode> + <leafNode name="expiration"> + <properties> + <help>Token time to expire in seconds</help> + <valueHelp> + <format>u32:60-31536000</format> + <description>Token lifetime in seconds</description> + </valueHelp> + <constraint> + <validator name="numeric" argument="--range 60-31536000"/> + </constraint> + </properties> + <defaultValue>3600</defaultValue> + </leafNode> + <leafNode name="secret-length"> + <properties> + <help>Length of shared secret in bytes</help> + <valueHelp> + <format>u32:16-65535</format> + <description>Byte length of generated shared secret</description> + </valueHelp> + <constraint> + <validator name="numeric" argument="--range 16-65535"/> + </constraint> + </properties> + <defaultValue>32</defaultValue> + </leafNode> + </children> + </node> </children> </node> <node name="cors"> diff --git a/interface-definitions/include/radius-timeout.xml.i b/interface-definitions/include/radius-timeout.xml.i new file mode 100644 index 000000000..22bb6d312 --- /dev/null +++ b/interface-definitions/include/radius-timeout.xml.i @@ -0,0 +1,16 @@ +<!-- include start from radius-timeout.xml.i --> +<leafNode name="timeout"> + <properties> + <help>Session timeout</help> + <valueHelp> + <format>u32:1-240</format> + <description>Session timeout in seconds (default: 2)</description> + </valueHelp> + <constraint> + <validator name="numeric" argument="--range 1-240"/> + </constraint> + <constraintErrorMessage>Timeout must be between 1 and 240 seconds</constraintErrorMessage> + </properties> + <defaultValue>2</defaultValue> +</leafNode> +<!-- include end --> diff --git a/interface-definitions/include/static/static-route.xml.i b/interface-definitions/include/static/static-route.xml.i index 2de5dc58f..04ee999c7 100644 --- a/interface-definitions/include/static/static-route.xml.i +++ b/interface-definitions/include/static/static-route.xml.i @@ -14,6 +14,7 @@ #include <include/static/static-route-blackhole.xml.i> #include <include/static/static-route-reject.xml.i> #include <include/dhcp-interface.xml.i> + #include <include/generic-description.xml.i> <tagNode name="interface"> <properties> <help>Next-hop IPv4 router interface</help> diff --git a/interface-definitions/include/static/static-route6.xml.i b/interface-definitions/include/static/static-route6.xml.i index 35feef41c..6131ac7fe 100644 --- a/interface-definitions/include/static/static-route6.xml.i +++ b/interface-definitions/include/static/static-route6.xml.i @@ -13,6 +13,7 @@ <children> #include <include/static/static-route-blackhole.xml.i> #include <include/static/static-route-reject.xml.i> + #include <include/generic-description.xml.i> <tagNode name="interface"> <properties> <help>IPv6 gateway interface name</help> diff --git a/interface-definitions/include/version/https-version.xml.i b/interface-definitions/include/version/https-version.xml.i index 586083649..111076974 100644 --- a/interface-definitions/include/version/https-version.xml.i +++ b/interface-definitions/include/version/https-version.xml.i @@ -1,3 +1,3 @@ <!-- include start from include/version/https-version.xml.i --> -<syntaxVersion component='https' version='3'></syntaxVersion> +<syntaxVersion component='https' version='4'></syntaxVersion> <!-- include end --> diff --git a/interface-definitions/snmp.xml.in b/interface-definitions/snmp.xml.in index b4f72589e..7ec60b2e7 100644 --- a/interface-definitions/snmp.xml.in +++ b/interface-definitions/snmp.xml.in @@ -13,9 +13,9 @@ <properties> <help>Community name</help> <constraint> - <regex>[a-zA-Z0-9\-_]{1,100}</regex> + <regex>[a-zA-Z0-9\-_!@*#]{1,100}</regex> </constraint> - <constraintErrorMessage>Community string is limited to alphanumerical characters only with a total lenght of 100</constraintErrorMessage> + <constraintErrorMessage>Community string is limited to alphanumerical characters, !, @, * and # with a total lenght of 100</constraintErrorMessage> </properties> <children> <leafNode name="authorization"> diff --git a/interface-definitions/ssh.xml.in b/interface-definitions/ssh.xml.in index f3c731fe5..2bcce2cf0 100644 --- a/interface-definitions/ssh.xml.in +++ b/interface-definitions/ssh.xml.in @@ -133,6 +133,19 @@ </leafNode> </children> </node> + <leafNode name="hostkey-algorithm"> + <properties> + <help>Allowed host key signature algorithms</help> + <completionHelp> + <!-- generated by ssh -Q HostKeyAlgorithms | tr '\n' ' ' as this will not change dynamically --> + <list>ssh-ed25519 ssh-ed25519-cert-v01@openssh.com sk-ssh-ed25519@openssh.com sk-ssh-ed25519-cert-v01@openssh.com ssh-rsa rsa-sha2-256 rsa-sha2-512 ssh-dss ecdsa-sha2-nistp256 ecdsa-sha2-nistp384 ecdsa-sha2-nistp521 sk-ecdsa-sha2-nistp256@openssh.com webauthn-sk-ecdsa-sha2-nistp256@openssh.com ssh-rsa-cert-v01@openssh.com rsa-sha2-256-cert-v01@openssh.com rsa-sha2-512-cert-v01@openssh.com ssh-dss-cert-v01@openssh.com ecdsa-sha2-nistp256-cert-v01@openssh.com ecdsa-sha2-nistp384-cert-v01@openssh.com ecdsa-sha2-nistp521-cert-v01@openssh.com sk-ecdsa-sha2-nistp256-cert-v01@openssh.com</list> + </completionHelp> + <multi/> + <constraint> + <regex>(ssh-ed25519|ssh-ed25519-cert-v01@openssh.com|sk-ssh-ed25519@openssh.com|sk-ssh-ed25519-cert-v01@openssh.com|ssh-rsa|rsa-sha2-256|rsa-sha2-512|ssh-dss|ecdsa-sha2-nistp256|ecdsa-sha2-nistp384|ecdsa-sha2-nistp521|sk-ecdsa-sha2-nistp256@openssh.com|webauthn-sk-ecdsa-sha2-nistp256@openssh.com|ssh-rsa-cert-v01@openssh.com|rsa-sha2-256-cert-v01@openssh.com|rsa-sha2-512-cert-v01@openssh.com|ssh-dss-cert-v01@openssh.com|ecdsa-sha2-nistp256-cert-v01@openssh.com|ecdsa-sha2-nistp384-cert-v01@openssh.com|ecdsa-sha2-nistp521-cert-v01@openssh.com|sk-ecdsa-sha2-nistp256-cert-v01@openssh.com)</regex> + </constraint> + </properties> + </leafNode> <leafNode name="key-exchange"> <properties> <help>Allowed key exchange (KEX) algorithms</help> diff --git a/interface-definitions/system-login.xml.in b/interface-definitions/system-login.xml.in index def42544a..027d3f587 100644 --- a/interface-definitions/system-login.xml.in +++ b/interface-definitions/system-login.xml.in @@ -127,32 +127,44 @@ </leafNode> <leafNode name="type"> <properties> - <help>Public key type</help> + <help>SSH public key type</help> <completionHelp> - <list>ssh-dss ssh-rsa ecdsa-sha2-nistp256 ecdsa-sha2-nistp384 ecdsa-sha2-nistp521 ssh-ed25519</list> + <list>ssh-dss ssh-rsa ecdsa-sha2-nistp256 ecdsa-sha2-nistp384 ecdsa-sha2-nistp521 ssh-ed25519 ecdsa-sk ed25519-sk</list> </completionHelp> <valueHelp> <format>ssh-dss</format> - <description/> + <description>Digital Signature Algorithm (DSA) key support</description> </valueHelp> <valueHelp> <format>ssh-rsa</format> - <description/> + <description>Key pair based on RSA algorithm</description> </valueHelp> <valueHelp> <format>ecdsa-sha2-nistp256</format> - <description/> + <description>Elliptic Curve DSA with NIST P-256 curve</description> </valueHelp> <valueHelp> <format>ecdsa-sha2-nistp384</format> - <description/> + <description>Elliptic Curve DSA with NIST P-384 curve</description> + </valueHelp> + <valueHelp> + <format>ecdsa-sha2-nistp521</format> + <description>Elliptic Curve DSA with NIST P-521 curve</description> </valueHelp> <valueHelp> <format>ssh-ed25519</format> - <description/> + <description>Edwards-curve DSA with elliptic curve 25519</description> + </valueHelp> + <valueHelp> + <format>ecdsa-sk</format> + <description>Elliptic Curve DSA security key</description> + </valueHelp> + <valueHelp> + <format>ed25519-sk</format> + <description>Elliptic curve 25519 security key</description> </valueHelp> <constraint> - <regex>(ssh-dss|ssh-rsa|ecdsa-sha2-nistp256|ecdsa-sha2-nistp384|ecdsa-sha2-nistp521|ssh-ed25519)</regex> + <regex>(ssh-dss|ssh-rsa|ecdsa-sha2-nistp256|ecdsa-sha2-nistp384|ecdsa-sha2-nistp521|ssh-ed25519|ecdsa-sk|ed25519-sk)</regex> </constraint> </properties> </leafNode> diff --git a/interface-definitions/vpn-ipsec.xml.in b/interface-definitions/vpn-ipsec.xml.in index 4776c53dc..64966b540 100644 --- a/interface-definitions/vpn-ipsec.xml.in +++ b/interface-definitions/vpn-ipsec.xml.in @@ -888,6 +888,7 @@ <node name="radius"> <children> #include <include/radius-nas-identifier.xml.i> + #include <include/radius-timeout.xml.i> <tagNode name="server"> <children> #include <include/accel-ppp/radius-additions-disable-accounting.xml.i> diff --git a/interface-definitions/vpn-openconnect.xml.in b/interface-definitions/vpn-openconnect.xml.in index 3b3a83bd4..8b60f2e6e 100644 --- a/interface-definitions/vpn-openconnect.xml.in +++ b/interface-definitions/vpn-openconnect.xml.in @@ -140,20 +140,7 @@ #include <include/radius-server-ipv4.xml.i> <node name="radius"> <children> - <leafNode name="timeout"> - <properties> - <help>Session timeout</help> - <valueHelp> - <format>u32:1-240</format> - <description>Session timeout in seconds (default: 2)</description> - </valueHelp> - <constraint> - <validator name="numeric" argument="--range 1-240"/> - </constraint> - <constraintErrorMessage>Timeout must be between 1 and 240 seconds</constraintErrorMessage> - </properties> - <defaultValue>2</defaultValue> - </leafNode> + #include <include/radius-timeout.xml.i> <leafNode name="groupconfig"> <properties> <help>If the groupconfig option is set, then config-per-user will be overriden, and all configuration will be read from RADIUS.</help> diff --git a/op-mode-definitions/nat.xml.in b/op-mode-definitions/nat.xml.in index ce0544390..50abb1555 100644 --- a/op-mode-definitions/nat.xml.in +++ b/op-mode-definitions/nat.xml.in @@ -64,7 +64,7 @@ <properties> <help>Show statistics for configured destination NAT rules</help> </properties> - <command>${vyos_op_scripts_dir}/show_nat_statistics.py --destination</command> + <command>${vyos_op_scripts_dir}/nat.py show_statistics --direction destination --family inet</command> </node> <node name="translations"> <properties> diff --git a/python/vyos/component_version.py b/python/vyos/component_version.py new file mode 100644 index 000000000..a4e318d08 --- /dev/null +++ b/python/vyos/component_version.py @@ -0,0 +1,192 @@ +# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + +""" +Functions for reading/writing component versions. + +The config file version string has the following form: + +VyOS 1.3/1.4: + +// Warning: Do not remove the following line. +// vyos-config-version: "broadcast-relay@1:cluster@1:config-management@1:conntrack@3:conntrack-sync@2:dhcp-relay@2:dhcp-server@6:dhcpv6-server@1:dns-forwarding@3:firewall@5:https@2:interfaces@22:ipoe-server@1:ipsec@5:isis@1:l2tp@3:lldp@1:mdns@1:nat@5:ntp@1:pppoe-server@5:pptp@2:qos@1:quagga@8:rpki@1:salt@1:snmp@2:ssh@2:sstp@3:system@21:vrrp@2:vyos-accel-ppp@2:wanloadbalance@3:webproxy@2:zone-policy@1" +// Release version: 1.3.0 + +VyOS 1.2: + +/* Warning: Do not remove the following line. */ +/* === vyatta-config-version: "broadcast-relay@1:cluster@1:config-management@1:conntrack-sync@1:conntrack@1:dhcp-relay@2:dhcp-server@5:dns-forwarding@1:firewall@5:ipsec@5:l2tp@1:mdns@1:nat@4:ntp@1:pppoe-server@2:pptp@1:qos@1:quagga@7:snmp@1:ssh@1:system@10:vrrp@2:wanloadbalance@3:webgui@1:webproxy@2:zone-policy@1" === */ +/* Release version: 1.2.8 */ + +""" + +import os +import re +import sys +import fileinput + +from vyos.xml import component_version +from vyos.version import get_version +from vyos.defaults import directories + +DEFAULT_CONFIG_PATH = os.path.join(directories['config'], 'config.boot') + +def from_string(string_line, vintage='vyos'): + """ + Get component version dictionary from string. + Return empty dictionary if string contains no config information + or raise error if component version string malformed. + """ + version_dict = {} + + if vintage == 'vyos': + if re.match(r'// vyos-config-version:.+', string_line): + if not re.match(r'// vyos-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s*', string_line): + raise ValueError(f"malformed configuration string: {string_line}") + + for pair in re.findall(r'([\w,-]+)@(\d+)', string_line): + version_dict[pair[0]] = int(pair[1]) + + elif vintage == 'vyatta': + if re.match(r'/\* === vyatta-config-version:.+=== \*/$', string_line): + if not re.match(r'/\* === vyatta-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s+=== \*/$', string_line): + raise ValueError(f"malformed configuration string: {string_line}") + + for pair in re.findall(r'([\w,-]+)@(\d+)', string_line): + version_dict[pair[0]] = int(pair[1]) + else: + raise ValueError("Unknown config string vintage") + + return version_dict + +def from_file(config_file_name=DEFAULT_CONFIG_PATH, vintage='vyos'): + """ + Get component version dictionary parsing config file line by line + """ + with open(config_file_name, 'r') as f: + for line_in_config in f: + version_dict = from_string(line_in_config, vintage=vintage) + if version_dict: + return version_dict + + # no version information + return {} + +def from_system(): + """ + Get system component version dict. + """ + return component_version() + +def legacy_from_system(): + """ + Get system component version dict from legacy location. + This is for a transitional sanity check; the directory will eventually + be removed. + """ + system_versions = {} + legacy_dir = directories['current'] + + # To be removed: + if not os.path.isdir(legacy_dir): + return system_versions + + try: + version_info = os.listdir(legacy_dir) + except OSError as err: + sys.exit(repr(err)) + + for info in version_info: + if re.match(r'[\w,-]+@\d+', info): + pair = info.split('@') + system_versions[pair[0]] = int(pair[1]) + + return system_versions + +def format_string(ver: dict) -> str: + """ + Version dict to string. + """ + keys = list(ver) + keys.sort() + l = [] + for k in keys: + v = ver[k] + l.append(f'{k}@{v}') + sep = ':' + return sep.join(l) + +def version_footer(ver: dict, vintage='vyos') -> str: + """ + Version footer as string. + """ + ver_str = format_string(ver) + release = get_version() + if vintage == 'vyos': + ret_str = (f'// Warning: Do not remove the following line.\n' + + f'// vyos-config-version: "{ver_str}"\n' + + f'// Release version: {release}\n') + elif vintage == 'vyatta': + ret_str = (f'/* Warning: Do not remove the following line. */\n' + + f'/* === vyatta-config-version: "{ver_str}" === */\n' + + f'/* Release version: {release} */\n') + else: + raise ValueError("Unknown config string vintage") + + return ret_str + +def system_footer(vintage='vyos') -> str: + """ + System version footer as string. + """ + ver_d = from_system() + return version_footer(ver_d, vintage=vintage) + +def write_version_footer(ver: dict, file_name, vintage='vyos'): + """ + Write version footer to file. + """ + footer = version_footer(ver=ver, vintage=vintage) + if file_name: + with open(file_name, 'a') as f: + f.write(footer) + else: + sys.stdout.write(footer) + +def write_system_footer(file_name, vintage='vyos'): + """ + Write system version footer to file. + """ + ver_d = from_system() + return write_version_footer(ver_d, file_name=file_name, vintage=vintage) + +def remove_footer(file_name): + """ + Remove old version footer. + """ + for line in fileinput.input(file_name, inplace=True): + if re.match(r'/\* Warning:.+ \*/$', line): + continue + if re.match(r'/\* === vyatta-config-version:.+=== \*/$', line): + continue + if re.match(r'/\* Release version:.+ \*/$', line): + continue + if re.match('// vyos-config-version:.+', line): + continue + if re.match('// Warning:.+', line): + continue + if re.match('// Release version:.+', line): + continue + sys.stdout.write(line) diff --git a/python/vyos/component_versions.py b/python/vyos/component_versions.py deleted file mode 100644 index 90b458aae..000000000 --- a/python/vyos/component_versions.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2017 VyOS maintainers and contributors <maintainers@vyos.io> -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see <http://www.gnu.org/licenses/>. - -""" -The version data looks like: - -/* Warning: Do not remove the following line. */ -/* === vyatta-config-version: -"cluster@1:config-management@1:conntrack-sync@1:conntrack@1:dhcp-relay@1:dhcp-server@4:firewall@5:ipsec@4:nat@4:qos@1:quagga@2:system@8:vrrp@1:wanloadbalance@3:webgui@1:webproxy@1:zone-policy@1" -=== */ -/* Release version: 1.2.0-rolling+201806131737 */ -""" - -import re - -def get_component_version(string_line): - """ - Get component version dictionary from string - return empty dictionary if string contains no config information - or raise error if component version string malformed - """ - return_value = {} - if re.match(r'/\* === vyatta-config-version:.+=== \*/$', string_line): - - if not re.match(r'/\* === vyatta-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s+=== \*/$', string_line): - raise ValueError("malformed configuration string: " + str(string_line)) - - for pair in re.findall(r'([\w,-]+)@(\d+)', string_line): - if pair[0] in return_value.keys(): - raise ValueError("duplicate unit name: \"" + str(pair[0]) + "\" in string: \"" + string_line + "\"") - return_value[pair[0]] = int(pair[1]) - - return return_value - - -def get_component_versions_from_file(config_file_name='/opt/vyatta/etc/config/config.boot'): - """ - Get component version dictionary parsing config file line by line - """ - f = open(config_file_name, 'r') - for line_in_config in f: - component_version = get_component_version(line_in_config) - if component_version: - return component_version - raise ValueError("no config string in file:", config_file_name) diff --git a/python/vyos/formatversions.py b/python/vyos/formatversions.py deleted file mode 100644 index 29117a5d3..000000000 --- a/python/vyos/formatversions.py +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright 2019 VyOS maintainers and contributors <maintainers@vyos.io> -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this library. If not, see <http://www.gnu.org/licenses/>. - -import sys -import os -import re -import fileinput - -def read_vyatta_versions(config_file): - config_file_versions = {} - - with open(config_file, 'r') as config_file_handle: - for config_line in config_file_handle: - if re.match(r'/\* === vyatta-config-version:.+=== \*/$', config_line): - if not re.match(r'/\* === vyatta-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s+=== \*/$', config_line): - raise ValueError("malformed configuration string: " - "{}".format(config_line)) - - for pair in re.findall(r'([\w,-]+)@(\d+)', config_line): - config_file_versions[pair[0]] = int(pair[1]) - - - return config_file_versions - -def read_vyos_versions(config_file): - config_file_versions = {} - - with open(config_file, 'r') as config_file_handle: - for config_line in config_file_handle: - if re.match(r'// vyos-config-version:.+', config_line): - if not re.match(r'// vyos-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s*', config_line): - raise ValueError("malformed configuration string: " - "{}".format(config_line)) - - for pair in re.findall(r'([\w,-]+)@(\d+)', config_line): - config_file_versions[pair[0]] = int(pair[1]) - - return config_file_versions - -def remove_versions(config_file): - """ - Remove old version string. - """ - for line in fileinput.input(config_file, inplace=True): - if re.match(r'/\* Warning:.+ \*/$', line): - continue - if re.match(r'/\* === vyatta-config-version:.+=== \*/$', line): - continue - if re.match(r'/\* Release version:.+ \*/$', line): - continue - if re.match('// vyos-config-version:.+', line): - continue - if re.match('// Warning:.+', line): - continue - if re.match('// Release version:.+', line): - continue - sys.stdout.write(line) - -def format_versions_string(config_versions): - cfg_keys = list(config_versions.keys()) - cfg_keys.sort() - - component_version_strings = [] - - for key in cfg_keys: - cfg_vers = config_versions[key] - component_version_strings.append('{}@{}'.format(key, cfg_vers)) - - separator = ":" - component_version_string = separator.join(component_version_strings) - - return component_version_string - -def write_vyatta_versions_foot(config_file, component_version_string, - os_version_string): - if config_file: - with open(config_file, 'a') as config_file_handle: - config_file_handle.write('/* Warning: Do not remove the following line. */\n') - config_file_handle.write('/* === vyatta-config-version: "{}" === */\n'.format(component_version_string)) - config_file_handle.write('/* Release version: {} */\n'.format(os_version_string)) - else: - sys.stdout.write('/* Warning: Do not remove the following line. */\n') - sys.stdout.write('/* === vyatta-config-version: "{}" === */\n'.format(component_version_string)) - sys.stdout.write('/* Release version: {} */\n'.format(os_version_string)) - -def write_vyos_versions_foot(config_file, component_version_string, - os_version_string): - if config_file: - with open(config_file, 'a') as config_file_handle: - config_file_handle.write('// Warning: Do not remove the following line.\n') - config_file_handle.write('// vyos-config-version: "{}"\n'.format(component_version_string)) - config_file_handle.write('// Release version: {}\n'.format(os_version_string)) - else: - sys.stdout.write('// Warning: Do not remove the following line.\n') - sys.stdout.write('// vyos-config-version: "{}"\n'.format(component_version_string)) - sys.stdout.write('// Release version: {}\n'.format(os_version_string)) - diff --git a/python/vyos/migrator.py b/python/vyos/migrator.py index c6e3435ca..45ea8b0eb 100644 --- a/python/vyos/migrator.py +++ b/python/vyos/migrator.py @@ -1,4 +1,4 @@ -# Copyright 2019 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2019-2022 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -17,10 +17,8 @@ import sys import os import json import subprocess -import vyos.version import vyos.defaults -import vyos.systemversions as systemversions -import vyos.formatversions as formatversions +import vyos.component_version as component_version class MigratorError(Exception): pass @@ -42,13 +40,13 @@ class Migrator(object): cfg_file = self._config_file component_versions = {} - cfg_versions = formatversions.read_vyatta_versions(cfg_file) + cfg_versions = component_version.from_file(cfg_file, vintage='vyatta') if cfg_versions: self._config_file_vintage = 'vyatta' component_versions = cfg_versions - cfg_versions = formatversions.read_vyos_versions(cfg_file) + cfg_versions = component_version.from_file(cfg_file, vintage='vyos') if cfg_versions: self._config_file_vintage = 'vyos' @@ -157,19 +155,15 @@ class Migrator(object): """ Write new versions string. """ - versions_string = formatversions.format_versions_string(cfg_versions) - - os_version_string = vyos.version.get_version() - if self._config_file_vintage == 'vyatta': - formatversions.write_vyatta_versions_foot(self._config_file, - versions_string, - os_version_string) + component_version.write_version_footer(cfg_versions, + self._config_file, + vintage='vyatta') if self._config_file_vintage == 'vyos': - formatversions.write_vyos_versions_foot(self._config_file, - versions_string, - os_version_string) + component_version.write_version_footer(cfg_versions, + self._config_file, + vintage='vyos') def save_json_record(self, component_versions: dict): """ @@ -200,7 +194,7 @@ class Migrator(object): # This will force calling all migration scripts: cfg_versions = {} - sys_versions = systemversions.get_system_component_version() + sys_versions = component_version.from_system() # save system component versions in json file for easy reference self.save_json_record(sys_versions) @@ -216,7 +210,7 @@ class Migrator(object): if not self._changed: return - formatversions.remove_versions(cfg_file) + component_version.remove_footer(cfg_file) self.write_config_file_versions(rev_versions) @@ -237,7 +231,7 @@ class VirtualMigrator(Migrator): if not self._changed: return - formatversions.remove_versions(cfg_file) + component_version.remove_footer(cfg_file) self.write_config_file_versions(cfg_versions) diff --git a/python/vyos/opmode.py b/python/vyos/opmode.py index 7e3545c87..727e118a8 100644 --- a/python/vyos/opmode.py +++ b/python/vyos/opmode.py @@ -44,6 +44,13 @@ class PermissionDenied(Error): """ pass +class InternalError(Error): + """ Any situation when VyOS detects that it could not perform + an operation correctly due to logic errors in its own code + or errors in underlying software. + """ + pass + def _is_op_mode_function_name(name): if re.match(r"^(show|clear|reset|restart)", name): @@ -93,6 +100,51 @@ def _get_arg_type(t): else: return t +def _normalize_field_name(name): + # Convert the name to string if it is not + # (in some cases they may be numbers) + name = str(name) + + # Replace all separators with underscores + name = re.sub(r'(\s|[\(\)\[\]\{\}\-\.\,:\"\'\`])+', '_', name) + + # Replace specific characters with textual descriptions + name = re.sub(r'@', '_at_', name) + name = re.sub(r'%', '_percentage_', name) + name = re.sub(r'~', '_tilde_', name) + + # Force all letters to lowercase + name = name.lower() + + # Remove leading and trailing underscores, if any + name = re.sub(r'(^(_+)(?=[^_])|_+$)', '', name) + + # Ensure there are only single underscores + name = re.sub(r'_+', '_', name) + + return name + +def _normalize_dict_field_names(old_dict): + new_dict = {} + + for key in old_dict: + new_key = _normalize_field_name(key) + new_dict[new_key] = _normalize_field_names(old_dict[key]) + + # Sanity check + if len(old_dict) != len(new_dict): + raise InternalError("Dictionary fields do not allow unique normalization") + else: + return new_dict + +def _normalize_field_names(value): + if isinstance(value, dict): + return _normalize_dict_field_names(value) + elif isinstance(value, list): + return list(map(lambda v: _normalize_field_names(v), value)) + else: + return value + def run(module): from argparse import ArgumentParser @@ -148,6 +200,7 @@ def run(module): if not args["raw"]: return res else: + res = _normalize_field_names(res) from json import dumps return dumps(res, indent=4) else: diff --git a/python/vyos/systemversions.py b/python/vyos/systemversions.py deleted file mode 100644 index f2da76d4f..000000000 --- a/python/vyos/systemversions.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2019 VyOS maintainers and contributors <maintainers@vyos.io> -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this library. If not, see <http://www.gnu.org/licenses/>. - -import os -import re -import sys -import vyos.defaults -from vyos.xml import component_version - -# legacy version, reading from the file names in -# /opt/vyatta/etc/config-migrate/current -def get_system_versions(): - """ - Get component versions from running system; critical failure if - unable to read migration directory. - """ - system_versions = {} - - try: - version_info = os.listdir(vyos.defaults.directories['current']) - except OSError as err: - print("OS error: {}".format(err)) - sys.exit(1) - - for info in version_info: - if re.match(r'[\w,-]+@\d+', info): - pair = info.split('@') - system_versions[pair[0]] = int(pair[1]) - - return system_versions - -# read from xml cache -def get_system_component_version(): - return component_version() diff --git a/python/vyos/util.py b/python/vyos/util.py index 461df9a6e..a80584c5a 100644 --- a/python/vyos/util.py +++ b/python/vyos/util.py @@ -574,6 +574,37 @@ def bytes_to_human(bytes, initial_exponent=0): size_string = "{0:.2f} {1}".format(value, suffix) return size_string +def human_to_bytes(value): + """ Converts a data amount with a unit suffix to bytes, like 2K to 2048 """ + + from re import match as re_match + + res = re_match(r'^\s*(\d+(?:\.\d+)?)\s*([a-zA-Z]+)\s*$', value) + + if not res: + raise ValueError(f"'{value}' is not a valid data amount") + else: + amount = float(res.group(1)) + unit = res.group(2).lower() + + if unit == 'b': + res = amount + elif (unit == 'k') or (unit == 'kb'): + res = amount * 1024 + elif (unit == 'm') or (unit == 'mb'): + res = amount * 1024**2 + elif (unit == 'g') or (unit == 'gb'): + res = amount * 1024**3 + elif (unit == 't') or (unit == 'tb'): + res = amount * 1024**4 + else: + raise ValueError(f"Unsupported data unit '{unit}'") + + # There cannot be fractional bytes, so we convert them to integer. + # However, truncating causes problems with conversion back to human unit, + # so we round instead -- that seems to work well enough. + return round(res) + def get_cfg_group_id(): from grp import getgrnam from vyos.defaults import cfg_group @@ -1105,3 +1136,10 @@ def sysctl_write(name, value): call(f'sysctl -wq {name}={value}') return True return False + +# approach follows a discussion in: +# https://stackoverflow.com/questions/1175208/elegant-python-function-to-convert-camelcase-to-snake-case +def camel_to_snake_case(name: str) -> str: + pattern = r'\d+|[A-Z]?[a-z]+|\W|[A-Z]{2,}(?=[A-Z][a-z]|\d|\W|$)' + words = re.findall(pattern, name) + return '_'.join(map(str.lower, words)) diff --git a/scripts/check-pr-title-and-commit-messages.py b/scripts/check-pr-title-and-commit-messages.py index c30c3ef1f..3317745d6 100755 --- a/scripts/check-pr-title-and-commit-messages.py +++ b/scripts/check-pr-title-and-commit-messages.py @@ -7,7 +7,7 @@ import requests from pprint import pprint # Use the same regex for PR title and commit messages for now -title_regex = r'^(([a-zA-Z]+:\s)?)T\d+:\s+[^\s]+.*' +title_regex = r'^(([a-zA-Z.]+:\s)?)T\d+:\s+[^\s]+.*' commit_regex = title_regex def check_pr_title(title): diff --git a/smoketest/scripts/cli/test_component_version.py b/smoketest/scripts/cli/test_component_version.py index 1355c1f94..7b1b12c53 100755 --- a/smoketest/scripts/cli/test_component_version.py +++ b/smoketest/scripts/cli/test_component_version.py @@ -16,7 +16,7 @@ import unittest -from vyos.systemversions import get_system_versions, get_system_component_version +import vyos.component_version as component_version # After T3474, component versions should be updated in the files in # vyos-1x/interface-definitions/include/version/ @@ -24,8 +24,8 @@ from vyos.systemversions import get_system_versions, get_system_component_versio # that in the xml cache. class TestComponentVersion(unittest.TestCase): def setUp(self): - self.legacy_d = get_system_versions() - self.xml_d = get_system_component_version() + self.legacy_d = component_version.legacy_from_system() + self.xml_d = component_version.from_system() self.set_legacy_d = set(self.legacy_d) self.set_xml_d = set(self.xml_d) diff --git a/smoketest/scripts/cli/test_container.py b/smoketest/scripts/cli/test_container.py index cc0cdaec0..b9d308ae1 100644..100755 --- a/smoketest/scripts/cli/test_container.py +++ b/smoketest/scripts/cli/test_container.py @@ -15,6 +15,7 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. import unittest +import glob import json from base_vyostest_shim import VyOSUnitTestSHIM @@ -25,10 +26,13 @@ from vyos.util import process_named_running from vyos.util import read_file base_path = ['container'] -cont_image = 'busybox' +cont_image = 'busybox:stable' # busybox is included in vyos-build prefix = '192.168.205.0/24' net_name = 'NET01' -PROCESS_NAME = 'podman' +PROCESS_NAME = 'conmon' +PROCESS_PIDFILE = '/run/vyos-container-{0}.service.pid' + +busybox_image_path = '/usr/share/vyos/busybox-stable.tar' def cmd_to_json(command): c = cmd(command + ' --format=json') @@ -37,7 +41,31 @@ def cmd_to_json(command): return data -class TesContainer(VyOSUnitTestSHIM.TestCase): +class TestContainer(VyOSUnitTestSHIM.TestCase): + @classmethod + def setUpClass(cls): + super(TestContainer, cls).setUpClass() + + # Load image for smoketest provided in vyos-build + cmd(f'cat {busybox_image_path} | sudo podman load') + + @classmethod + def tearDownClass(cls): + super(TestContainer, cls).tearDownClass() + + # Cleanup podman image + cmd(f'sudo podman image rm -f {cont_image}') + + def tearDown(self): + self.cli_delete(base_path) + self.cli_commit() + + # Ensure no container process remains + self.assertIsNone(process_named_running(PROCESS_NAME)) + + # Ensure systemd units are removed + units = glob.glob('/run/systemd/system/vyos-container-*') + self.assertEqual(units, []) def test_01_basic_container(self): cont_name = 'c1' @@ -53,13 +81,17 @@ class TesContainer(VyOSUnitTestSHIM.TestCase): # commit changes self.cli_commit() + pid = 0 + with open(PROCESS_PIDFILE.format(cont_name), 'r') as f: + pid = int(f.read()) + # Check for running process - self.assertTrue(process_named_running(PROCESS_NAME)) + self.assertEqual(process_named_running(PROCESS_NAME), pid) def test_02_container_network(self): cont_name = 'c2' cont_ip = '192.168.205.25' - self.cli_set(base_path + ['network', net_name, 'ipv4-prefix', prefix]) + self.cli_set(base_path + ['network', net_name, 'prefix', prefix]) self.cli_set(base_path + ['name', cont_name, 'image', cont_image]) self.cli_set(base_path + ['name', cont_name, 'network', net_name, 'address', cont_ip]) @@ -67,7 +99,7 @@ class TesContainer(VyOSUnitTestSHIM.TestCase): self.cli_commit() n = cmd_to_json(f'sudo podman network inspect {net_name}') - json_subnet = n['plugins'][0]['ipam']['ranges'][0][0]['subnet'] + json_subnet = n['subnets'][0]['subnet'] c = cmd_to_json(f'sudo podman container inspect {cont_name}') json_ip = c['NetworkSettings']['Networks'][net_name]['IPAddress'] diff --git a/smoketest/scripts/cli/test_interfaces_wireguard.py b/smoketest/scripts/cli/test_interfaces_wireguard.py index f3e9670f7..14fc8d109 100755 --- a/smoketest/scripts/cli/test_interfaces_wireguard.py +++ b/smoketest/scripts/cli/test_interfaces_wireguard.py @@ -62,10 +62,10 @@ class WireGuardInterfaceTest(VyOSUnitTestSHIM.TestCase): self.assertTrue(os.path.isdir(f'/sys/class/net/{intf}')) - def test_wireguard_add_remove_peer(self): # T2939: Create WireGuard interfaces with associated peers. # Remove one of the configured peers. + # T4774: Test prevention of duplicate peer public keys interface = 'wg0' port = '12345' privkey = '6ISOkASm6VhHOOSz/5iIxw+Q9adq9zA17iMM4X40dlc=' @@ -80,11 +80,17 @@ class WireGuardInterfaceTest(VyOSUnitTestSHIM.TestCase): self.cli_set(base_path + [interface, 'peer', 'PEER01', 'allowed-ips', '10.205.212.10/32']) self.cli_set(base_path + [interface, 'peer', 'PEER01', 'address', '192.0.2.1']) - self.cli_set(base_path + [interface, 'peer', 'PEER02', 'public-key', pubkey_2]) + self.cli_set(base_path + [interface, 'peer', 'PEER02', 'public-key', pubkey_1]) self.cli_set(base_path + [interface, 'peer', 'PEER02', 'port', port]) self.cli_set(base_path + [interface, 'peer', 'PEER02', 'allowed-ips', '10.205.212.11/32']) self.cli_set(base_path + [interface, 'peer', 'PEER02', 'address', '192.0.2.2']) + # Duplicate pubkey_1 + with self.assertRaises(ConfigSessionError): + self.cli_commit() + + self.cli_set(base_path + [interface, 'peer', 'PEER02', 'public-key', pubkey_2]) + # Commit peers self.cli_commit() diff --git a/smoketest/scripts/cli/test_nat.py b/smoketest/scripts/cli/test_nat.py index f824838c0..2ae90fcaf 100755 --- a/smoketest/scripts/cli/test_nat.py +++ b/smoketest/scripts/cli/test_nat.py @@ -16,6 +16,7 @@ import jmespath import json +import os import unittest from base_vyostest_shim import VyOSUnitTestSHIM @@ -28,6 +29,9 @@ src_path = base_path + ['source'] dst_path = base_path + ['destination'] static_path = base_path + ['static'] +nftables_nat_config = '/run/nftables_nat.conf' +nftables_static_nat_conf = '/run/nftables_static-nat-rules.nft' + class TestNAT(VyOSUnitTestSHIM.TestCase): @classmethod def setUpClass(cls): @@ -40,6 +44,8 @@ class TestNAT(VyOSUnitTestSHIM.TestCase): def tearDown(self): self.cli_delete(base_path) self.cli_commit() + self.assertFalse(os.path.exists(nftables_nat_config)) + self.assertFalse(os.path.exists(nftables_static_nat_conf)) def verify_nftables(self, nftables_search, table, inverse=False, args=''): nftables_output = cmd(f'sudo nft {args} list table {table}') diff --git a/smoketest/scripts/cli/test_service_https.py b/smoketest/scripts/cli/test_service_https.py index 72c1d4e43..0f4b1393c 100755 --- a/smoketest/scripts/cli/test_service_https.py +++ b/smoketest/scripts/cli/test_service_https.py @@ -143,10 +143,10 @@ class TestHTTPSService(VyOSUnitTestSHIM.TestCase): # caught by the resolver, and returns success 'False', so one must # check the return value. - self.cli_set(base_path + ['api', 'gql']) + self.cli_set(base_path + ['api', 'graphql']) self.cli_commit() - gql_url = f'https://{address}/graphql' + graphql_url = f'https://{address}/graphql' query_valid_key = f""" {{ @@ -160,7 +160,7 @@ class TestHTTPSService(VyOSUnitTestSHIM.TestCase): }} """ - r = request('POST', gql_url, verify=False, headers=headers, json={'query': query_valid_key}) + r = request('POST', graphql_url, verify=False, headers=headers, json={'query': query_valid_key}) success = r.json()['data']['SystemStatus']['success'] self.assertTrue(success) @@ -176,7 +176,7 @@ class TestHTTPSService(VyOSUnitTestSHIM.TestCase): } """ - r = request('POST', gql_url, verify=False, headers=headers, json={'query': query_invalid_key}) + r = request('POST', graphql_url, verify=False, headers=headers, json={'query': query_invalid_key}) success = r.json()['data']['SystemStatus']['success'] self.assertFalse(success) @@ -192,8 +192,52 @@ class TestHTTPSService(VyOSUnitTestSHIM.TestCase): } """ - r = request('POST', gql_url, verify=False, headers=headers, json={'query': query_no_key}) + r = request('POST', graphql_url, verify=False, headers=headers, json={'query': query_no_key}) self.assertEqual(r.status_code, 400) + # GraphQL token authentication test: request token; pass in header + # of query. + + self.cli_set(base_path + ['api', 'graphql', 'authentication', 'type', 'token']) + self.cli_commit() + + mutation = """ + mutation { + AuthToken (data: {username: "vyos", password: "vyos"}) { + success + errors + data { + result + } + } + } + """ + r = request('POST', graphql_url, verify=False, headers=headers, json={'query': mutation}) + + token = r.json()['data']['AuthToken']['data']['result']['token'] + + headers = {'Authorization': f'Bearer {token}'} + + query = """ + { + ShowVersion (data: {}) { + success + errors + op_mode_error { + name + message + vyos_code + } + data { + result + } + } + } + """ + + r = request('POST', graphql_url, verify=False, headers=headers, json={'query': query}) + success = r.json()['data']['ShowVersion']['success'] + self.assertTrue(success) + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/smoketest/scripts/cli/test_service_ssh.py b/smoketest/scripts/cli/test_service_ssh.py index 0b029dd00..8de98f34f 100755 --- a/smoketest/scripts/cli/test_service_ssh.py +++ b/smoketest/scripts/cli/test_service_ssh.py @@ -262,5 +262,42 @@ class TestServiceSSH(VyOSUnitTestSHIM.TestCase): self.assertFalse(process_named_running(SSHGUARD_PROCESS)) + + # Network Device Collaborative Protection Profile + def test_ssh_ndcpp(self): + ciphers = ['aes128-cbc', 'aes128-ctr', 'aes256-cbc', 'aes256-ctr'] + host_key_algs = ['sk-ssh-ed25519@openssh.com', 'ssh-rsa', 'ssh-ed25519'] + kexes = ['diffie-hellman-group14-sha1', 'ecdh-sha2-nistp256', 'ecdh-sha2-nistp384', 'ecdh-sha2-nistp521'] + macs = ['hmac-sha1', 'hmac-sha2-256', 'hmac-sha2-512'] + rekey_time = '60' + rekey_data = '1024' + + for cipher in ciphers: + self.cli_set(base_path + ['ciphers', cipher]) + for host_key in host_key_algs: + self.cli_set(base_path + ['hostkey-algorithm', host_key]) + for kex in kexes: + self.cli_set(base_path + ['key-exchange', kex]) + for mac in macs: + self.cli_set(base_path + ['mac', mac]) + # Optional rekey parameters + self.cli_set(base_path + ['rekey', 'data', rekey_data]) + self.cli_set(base_path + ['rekey', 'time', rekey_time]) + + # commit changes + self.cli_commit() + + ssh_lines = ['Ciphers aes128-cbc,aes128-ctr,aes256-cbc,aes256-ctr', + 'HostKeyAlgorithms sk-ssh-ed25519@openssh.com,ssh-rsa,ssh-ed25519', + 'MACs hmac-sha1,hmac-sha2-256,hmac-sha2-512', + 'KexAlgorithms diffie-hellman-group14-sha1,ecdh-sha2-nistp256,ecdh-sha2-nistp384,ecdh-sha2-nistp521', + 'RekeyLimit 1024M 60M' + ] + tmp_sshd_conf = read_file(SSHD_CONF) + + for line in ssh_lines: + self.assertIn(line, tmp_sshd_conf) + + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/src/conf_mode/container.py b/src/conf_mode/container.py index ac3dc536b..70d149f0d 100755 --- a/src/conf_mode/container.py +++ b/src/conf_mode/container.py @@ -40,20 +40,7 @@ airbag.enable() config_containers_registry = '/etc/containers/registries.conf' config_containers_storage = '/etc/containers/storage.conf' - -def _run_rerun(container_cmd): - counter = 0 - while True: - if counter >= 10: - break - try: - _cmd(container_cmd) - break - except: - counter = counter +1 - sleep(0.5) - - return None +systemd_unit_path = '/run/systemd/system' def _cmd(command): if os.path.exists('/tmp/vyos.container.debug'): @@ -122,7 +109,7 @@ def verify(container): # of image upgrade and deletion. image = container_config['image'] if run(f'podman image exists {image}') != 0: - Warning(f'Image "{image}" used in contianer "{name}" does not exist '\ + Warning(f'Image "{image}" used in container "{name}" does not exist '\ f'locally. Please use "add container image {image}" to add it '\ f'to the system! Container "{name}" will not be started!') @@ -136,9 +123,6 @@ def verify(container): raise ConfigError(f'Container network "{network_name}" does not exist!') if 'address' in container_config['network'][network_name]: - if 'network' not in container_config: - raise ConfigError(f'Can not use "address" without "network" for container "{name}"!') - address = container_config['network'][network_name]['address'] network = None if is_ipv4(address): @@ -220,6 +204,71 @@ def verify(container): return None +def generate_run_arguments(name, container_config): + image = container_config['image'] + memory = container_config['memory'] + restart = container_config['restart'] + + # Add capability options. Should be in uppercase + cap_add = '' + if 'cap_add' in container_config: + for c in container_config['cap_add']: + c = c.upper() + c = c.replace('-', '_') + cap_add += f' --cap-add={c}' + + # Add a host device to the container /dev/x:/dev/x + device = '' + if 'device' in container_config: + for dev, dev_config in container_config['device'].items(): + source_dev = dev_config['source'] + dest_dev = dev_config['destination'] + device += f' --device={source_dev}:{dest_dev}' + + # Check/set environment options "-e foo=bar" + env_opt = '' + if 'environment' in container_config: + for k, v in container_config['environment'].items(): + env_opt += f" -e \"{k}={v['value']}\"" + + # Publish ports + port = '' + if 'port' in container_config: + protocol = '' + for portmap in container_config['port']: + if 'protocol' in container_config['port'][portmap]: + protocol = container_config['port'][portmap]['protocol'] + protocol = f'/{protocol}' + else: + protocol = '/tcp' + sport = container_config['port'][portmap]['source'] + dport = container_config['port'][portmap]['destination'] + port += f' -p {sport}:{dport}{protocol}' + + # Bind volume + volume = '' + if 'volume' in container_config: + for vol, vol_config in container_config['volume'].items(): + svol = vol_config['source'] + dvol = vol_config['destination'] + volume += f' -v {svol}:{dvol}' + + container_base_cmd = f'--detach --interactive --tty --replace {cap_add} ' \ + f'--memory {memory}m --memory-swap 0 --restart {restart} ' \ + f'--name {name} {device} {port} {volume} {env_opt}' + + if 'allow_host_networks' in container_config: + return f'{container_base_cmd} --net host {image}' + + ip_param = '' + networks = ",".join(container_config['network']) + for network in container_config['network']: + if 'address' in container_config['network'][network]: + address = container_config['network'][network]['address'] + ip_param = f'--ip {address}' + + return f'{container_base_cmd} --net {networks} {ip_param} {image}' + def generate(container): # bail out early - looks like removal from running config if not container: @@ -263,6 +312,15 @@ def generate(container): render(config_containers_registry, 'container/registries.conf.j2', container) render(config_containers_storage, 'container/storage.conf.j2', container) + if 'name' in container: + for name, container_config in container['name'].items(): + if 'disable' in container_config: + continue + + file_path = os.path.join(systemd_unit_path, f'vyos-container-{name}.service') + run_args = generate_run_arguments(name, container_config) + render(file_path, 'container/systemd-unit.j2', {'name': name, 'run_args': run_args}) + return None def apply(container): @@ -270,8 +328,12 @@ def apply(container): # Option "--force" allows to delete containers with any status if 'container_remove' in container: for name in container['container_remove']: - call(f'podman stop --time 3 {name}') - call(f'podman rm --force {name}') + file_path = os.path.join(systemd_unit_path, f'vyos-container-{name}.service') + call(f'systemctl stop vyos-container-{name}.service') + if os.path.exists(file_path): + os.unlink(file_path) + + call('systemctl daemon-reload') # Delete old networks if needed if 'network_remove' in container: @@ -282,6 +344,7 @@ def apply(container): os.unlink(tmp) # Add container + disabled_new = False if 'name' in container: for name, container_config in container['name'].items(): image = container_config['image'] @@ -295,70 +358,17 @@ def apply(container): # check if there is a container by that name running tmp = _cmd('podman ps -a --format "{{.Names}}"') if name in tmp: - _cmd(f'podman stop --time 3 {name}') - _cmd(f'podman rm --force {name}') + file_path = os.path.join(systemd_unit_path, f'vyos-container-{name}.service') + call(f'systemctl stop vyos-container-{name}.service') + if os.path.exists(file_path): + disabled_new = True + os.unlink(file_path) continue - memory = container_config['memory'] - restart = container_config['restart'] - - # Add capability options. Should be in uppercase - cap_add = '' - if 'cap_add' in container_config: - for c in container_config['cap_add']: - c = c.upper() - c = c.replace('-', '_') - cap_add += f' --cap-add={c}' - - # Add a host device to the container /dev/x:/dev/x - device = '' - if 'device' in container_config: - for dev, dev_config in container_config['device'].items(): - source_dev = dev_config['source'] - dest_dev = dev_config['destination'] - device += f' --device={source_dev}:{dest_dev}' - - # Check/set environment options "-e foo=bar" - env_opt = '' - if 'environment' in container_config: - for k, v in container_config['environment'].items(): - env_opt += f" -e \"{k}={v['value']}\"" - - # Publish ports - port = '' - if 'port' in container_config: - protocol = '' - for portmap in container_config['port']: - if 'protocol' in container_config['port'][portmap]: - protocol = container_config['port'][portmap]['protocol'] - protocol = f'/{protocol}' - else: - protocol = '/tcp' - sport = container_config['port'][portmap]['source'] - dport = container_config['port'][portmap]['destination'] - port += f' -p {sport}:{dport}{protocol}' - - # Bind volume - volume = '' - if 'volume' in container_config: - for vol, vol_config in container_config['volume'].items(): - svol = vol_config['source'] - dvol = vol_config['destination'] - volume += f' -v {svol}:{dvol}' - - container_base_cmd = f'podman run --detach --interactive --tty --replace {cap_add} ' \ - f'--memory {memory}m --memory-swap 0 --restart {restart} ' \ - f'--name {name} {device} {port} {volume} {env_opt}' - if 'allow_host_networks' in container_config: - _run_rerun(f'{container_base_cmd} --net host {image}') - else: - for network in container_config['network']: - ipparam = '' - if 'address' in container_config['network'][network]: - address = container_config['network'][network]['address'] - ipparam = f'--ip {address}' + cmd(f'systemctl restart vyos-container-{name}.service') - _run_rerun(f'{container_base_cmd} --net {network} {ipparam} {image}') + if disabled_new: + call('systemctl daemon-reload') return None diff --git a/src/conf_mode/http-api.py b/src/conf_mode/http-api.py index c196e272b..be80613c6 100755 --- a/src/conf_mode/http-api.py +++ b/src/conf_mode/http-api.py @@ -86,7 +86,7 @@ def get_config(config=None): if 'api_keys' in api_dict: keys_added = True - if 'gql' in api_dict: + if 'graphql' in api_dict: api_dict = dict_merge(defaults(base), api_dict) http_api.update(api_dict) diff --git a/src/conf_mode/interfaces-wireguard.py b/src/conf_mode/interfaces-wireguard.py index 8d738f55e..762bad94f 100755 --- a/src/conf_mode/interfaces-wireguard.py +++ b/src/conf_mode/interfaces-wireguard.py @@ -87,6 +87,8 @@ def verify(wireguard): 'cannot be used for the interface!') # run checks on individual configured WireGuard peer + public_keys = [] + for tmp in wireguard['peer']: peer = wireguard['peer'][tmp] @@ -100,6 +102,11 @@ def verify(wireguard): raise ConfigError('Both Wireguard port and address must be defined ' f'for peer "{tmp}" if either one of them is set!') + if peer['public_key'] in public_keys: + raise ConfigError(f'Duplicate public-key defined on peer "{tmp}"') + + public_keys.append(peer['public_key']) + def apply(wireguard): tmp = WireGuardIf(wireguard['ifname']) if 'deleted' in wireguard: diff --git a/src/conf_mode/nat.py b/src/conf_mode/nat.py index 8b1a5a720..978c043e9 100755 --- a/src/conf_mode/nat.py +++ b/src/conf_mode/nat.py @@ -146,6 +146,10 @@ def verify(nat): if config['outbound_interface'] not in 'any' and config['outbound_interface'] not in interfaces(): Warning(f'rule "{rule}" interface "{config["outbound_interface"]}" does not exist on this system') + if not dict_search('translation.address', config) and not dict_search('translation.port', config): + if 'exclude' not in config: + raise ConfigError(f'{err_msg} translation requires address and/or port') + addr = dict_search('translation.address', config) if addr != None and addr != 'masquerade' and not is_ip_network(addr): for ip in addr.split('-'): @@ -166,6 +170,10 @@ def verify(nat): elif config['inbound_interface'] not in 'any' and config['inbound_interface'] not in interfaces(): Warning(f'rule "{rule}" interface "{config["inbound_interface"]}" does not exist on this system') + if not dict_search('translation.address', config) and not dict_search('translation.port', config): + if 'exclude' not in config: + raise ConfigError(f'{err_msg} translation requires address and/or port') + # common rule verification verify_rule(config, err_msg) @@ -204,6 +212,10 @@ def apply(nat): cmd(f'nft -f {nftables_nat_config}') cmd(f'nft -f {nftables_static_nat_conf}') + if not nat or 'deleted' in nat: + os.unlink(nftables_nat_config) + os.unlink(nftables_static_nat_conf) + return None if __name__ == '__main__': diff --git a/src/conf_mode/vpn_ipsec.py b/src/conf_mode/vpn_ipsec.py index 77a425f8b..cfefcfbe8 100755 --- a/src/conf_mode/vpn_ipsec.py +++ b/src/conf_mode/vpn_ipsec.py @@ -117,13 +117,26 @@ def get_config(config=None): ipsec['ike_group'][group]['proposal'][proposal] = dict_merge(default_values, ipsec['ike_group'][group]['proposal'][proposal]) - if 'remote_access' in ipsec and 'connection' in ipsec['remote_access']: + # XXX: T2665: we can not safely rely on the defaults() when there are + # tagNodes in place, it is better to blend in the defaults manually. + if dict_search('remote_access.connection', ipsec): default_values = defaults(base + ['remote-access', 'connection']) for rw in ipsec['remote_access']['connection']: ipsec['remote_access']['connection'][rw] = dict_merge(default_values, ipsec['remote_access']['connection'][rw]) - if 'remote_access' in ipsec and 'radius' in ipsec['remote_access'] and 'server' in ipsec['remote_access']['radius']: + # XXX: T2665: we can not safely rely on the defaults() when there are + # tagNodes in place, it is better to blend in the defaults manually. + if dict_search('remote_access.radius.server', ipsec): + # Fist handle the "base" stuff like RADIUS timeout + default_values = defaults(base + ['remote-access', 'radius']) + if 'server' in default_values: + del default_values['server'] + ipsec['remote_access']['radius'] = dict_merge(default_values, + ipsec['remote_access']['radius']) + + # Take care about individual RADIUS servers implemented as tagNodes - this + # requires special treatment default_values = defaults(base + ['remote-access', 'radius', 'server']) for server in ipsec['remote_access']['radius']['server']: ipsec['remote_access']['radius']['server'][server] = dict_merge(default_values, diff --git a/src/helpers/system-versions-foot.py b/src/helpers/system-versions-foot.py index 2aa687221..9614f0d28 100755 --- a/src/helpers/system-versions-foot.py +++ b/src/helpers/system-versions-foot.py @@ -1,6 +1,6 @@ #!/usr/bin/python3 -# Copyright 2019 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2019, 2022 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -16,24 +16,13 @@ # along with this library. If not, see <http://www.gnu.org/licenses/>. import sys -import vyos.formatversions as formatversions -import vyos.systemversions as systemversions import vyos.defaults -import vyos.version - -sys_versions = systemversions.get_system_component_version() - -component_string = formatversions.format_versions_string(sys_versions) - -os_version_string = vyos.version.get_version() +from vyos.component_version import write_system_footer sys.stdout.write("\n\n") if vyos.defaults.cfg_vintage == 'vyos': - formatversions.write_vyos_versions_foot(None, component_string, - os_version_string) + write_system_footer(None, vintage='vyos') elif vyos.defaults.cfg_vintage == 'vyatta': - formatversions.write_vyatta_versions_foot(None, component_string, - os_version_string) + write_system_footer(None, vintage='vyatta') else: - formatversions.write_vyatta_versions_foot(None, component_string, - os_version_string) + write_system_footer(None, vintage='vyos') diff --git a/src/migration-scripts/https/3-to-4 b/src/migration-scripts/https/3-to-4 new file mode 100755 index 000000000..5ee528b31 --- /dev/null +++ b/src/migration-scripts/https/3-to-4 @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +# T4768 rename node 'gql' to 'graphql'. + +import sys + +from vyos.configtree import ConfigTree + +if (len(sys.argv) < 2): + print("Must specify file name!") + sys.exit(1) + +file_name = sys.argv[1] + +with open(file_name, 'r') as f: + config_file = f.read() + +config = ConfigTree(config_file) + +old_base = ['service', 'https', 'api', 'gql'] +if not config.exists(old_base): + # Nothing to do + sys.exit(0) + +new_base = ['service', 'https', 'api', 'graphql'] +config.set(new_base) + +nodes = config.list_nodes(old_base) +for node in nodes: + config.copy(old_base + [node], new_base + [node]) + +config.delete(old_base) + +try: + with open(file_name, 'w') as f: + f.write(config.to_string()) +except OSError as e: + print("Failed to save the modified config: {}".format(e)) + sys.exit(1) diff --git a/src/op_mode/bgp.py b/src/op_mode/bgp.py new file mode 100755 index 000000000..23001a9d7 --- /dev/null +++ b/src/op_mode/bgp.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +# Purpose: +# Displays bgp neighbors information. +# Used by the "show bgp (vrf <tag>) ipv4|ipv6 neighbors" commands. + +import re +import sys +import typing + +import jmespath +from jinja2 import Template +from humps import decamelize + +from vyos.configquery import ConfigTreeQuery + +import vyos.opmode + + +frr_command_template = Template(""" +{% if family %} + show bgp + {{ 'vrf ' ~ vrf if vrf else '' }} + {{ 'ipv6' if family == 'inet6' else 'ipv4'}} + {{ 'neighbor ' ~ peer if peer else 'summary' }} +{% endif %} + +{% if raw %} + json +{% endif %} +""") + + +def _verify(func): + """Decorator checks if BGP config exists + BGP configuration can be present under vrf <tag> + If we do npt get arg 'peer' then it can be 'bgp summary' + """ + from functools import wraps + + @wraps(func) + def _wrapper(*args, **kwargs): + config = ConfigTreeQuery() + afi = 'ipv6' if kwargs.get('family') == 'inet6' else 'ipv4' + global_vrfs = ['all', 'default'] + peer = kwargs.get('peer') + vrf = kwargs.get('vrf') + unconf_message = f'BGP or neighbor is not configured' + # Add option to check the specific neighbor if we have arg 'peer' + peer_opt = f'neighbor {peer} address-family {afi}-unicast' if peer else '' + vrf_opt = '' + if vrf and vrf not in global_vrfs: + vrf_opt = f'vrf name {vrf}' + # Check if config does not exist + if not config.exists(f'{vrf_opt} protocols bgp {peer_opt}'): + raise vyos.opmode.UnconfiguredSubsystem(unconf_message) + return func(*args, **kwargs) + + return _wrapper + + +@_verify +def show_neighbors(raw: bool, + family: str, + peer: typing.Optional[str], + vrf: typing.Optional[str]): + kwargs = dict(locals()) + frr_command = frr_command_template.render(kwargs) + frr_command = re.sub(r'\s+', ' ', frr_command) + + from vyos.util import cmd + output = cmd(f"vtysh -c '{frr_command}'") + + if raw: + from json import loads + data = loads(output) + # Get list of the peers + peers = jmespath.search('*.peers | [0]', data) + if peers: + # Create new dict, delete old key 'peers' + # add key 'peers' neighbors to the list + list_peers = [] + new_dict = jmespath.search('* | [0]', data) + if 'peers' in new_dict: + new_dict.pop('peers') + + for neighbor, neighbor_options in peers.items(): + neighbor_options['neighbor'] = neighbor + list_peers.append(neighbor_options) + new_dict['peers'] = list_peers + return decamelize(new_dict) + data = jmespath.search('* | [0]', data) + return decamelize(data) + + else: + return output + + +if __name__ == '__main__': + try: + res = vyos.opmode.run(sys.modules[__name__]) + if res: + print(res) + except (ValueError, vyos.opmode.Error) as e: + print(e) + sys.exit(1) diff --git a/src/op_mode/ipsec.py b/src/op_mode/ipsec.py index 7ec35d7bd..aaa0cec5a 100755 --- a/src/op_mode/ipsec.py +++ b/src/op_mode/ipsec.py @@ -43,7 +43,10 @@ def _alphanum_key(key): def _get_vici_sas(): from vici import Session as vici_session - session = vici_session() + try: + session = vici_session() + except Exception: + raise vyos.opmode.UnconfiguredSubsystem("IPsec not initialized") sas = list(session.list_sas()) return sas diff --git a/src/op_mode/log.py b/src/op_mode/log.py new file mode 100755 index 000000000..b0abd6191 --- /dev/null +++ b/src/op_mode/log.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import json +import re +import sys +import typing + +from jinja2 import Template + +from vyos.util import rc_cmd + +import vyos.opmode + +journalctl_command_template = Template(""" +--no-hostname +--quiet + +{% if boot %} + --boot +{% endif %} + +{% if count %} + --lines={{ count }} +{% endif %} + +{% if reverse %} + --reverse +{% endif %} + +{% if since %} + --since={{ since }} +{% endif %} + +{% if unit %} + --unit={{ unit }} +{% endif %} + +{% if utc %} + --utc +{% endif %} + +{% if raw %} +{# By default show 100 only lines for raw option if count does not set #} +{# Protection from parsing the full log by default #} +{% if not boot %} + --lines={{ '' ~ count if count else '100' }} +{% endif %} + --no-pager + --output=json +{% endif %} +""") + + +def show(raw: bool, + boot: typing.Optional[bool], + count: typing.Optional[int], + facility: typing.Optional[str], + reverse: typing.Optional[bool], + utc: typing.Optional[bool], + unit: typing.Optional[str]): + kwargs = dict(locals()) + + journalctl_options = journalctl_command_template.render(kwargs) + journalctl_options = re.sub(r'\s+', ' ', journalctl_options) + rc, output = rc_cmd(f'journalctl {journalctl_options}') + if raw: + # Each 'journalctl --output json' line is a separate JSON object + # So we should return list of dict + return [json.loads(line) for line in output.split('\n')] + return output + + +if __name__ == '__main__': + try: + res = vyos.opmode.run(sys.modules[__name__]) + if res: + print(res) + except (ValueError, vyos.opmode.Error) as e: + print(e) + sys.exit(1) diff --git a/src/op_mode/memory.py b/src/op_mode/memory.py index 178544be4..7666de646 100755 --- a/src/op_mode/memory.py +++ b/src/op_mode/memory.py @@ -20,7 +20,7 @@ import sys import vyos.opmode -def _get_system_memory(): +def _get_raw_data(): from re import search as re_search def find_value(keyword, mem_data): @@ -38,7 +38,7 @@ def _get_system_memory(): used = total - available - res = { + mem_data = { "total": total, "free": available, "used": used, @@ -46,24 +46,21 @@ def _get_system_memory(): "cached": cached } - return res - -def _get_system_memory_human(): - from vyos.util import bytes_to_human - - mem = _get_system_memory() - - for key in mem: + for key in mem_data: # The Linux kernel exposes memory values in kilobytes, # so we need to normalize them - mem[key] = bytes_to_human(mem[key], initial_exponent=10) + mem_data[key] = mem_data[key] * 1024 - return mem - -def _get_raw_data(): - return _get_system_memory_human() + return mem_data def _get_formatted_output(mem): + from vyos.util import bytes_to_human + + # For human-readable outputs, we convert bytes to more convenient units + # (100M, 1.3G...) + for key in mem: + mem[key] = bytes_to_human(mem[key]) + out = "Total: {}\n".format(mem["total"]) out += "Free: {}\n".format(mem["free"]) out += "Used: {}".format(mem["used"]) diff --git a/src/op_mode/nat.py b/src/op_mode/nat.py index 845dbbb2c..f899eb3dc 100755 --- a/src/op_mode/nat.py +++ b/src/op_mode/nat.py @@ -22,12 +22,18 @@ import xmltodict from sys import exit from tabulate import tabulate +from vyos.configquery import ConfigTreeQuery + from vyos.util import cmd from vyos.util import dict_search import vyos.opmode +base = 'nat' +unconf_message = 'NAT is not configured' + + def _get_xml_translation(direction, family): """ Get conntrack XML output --src-nat|--dst-nat @@ -277,6 +283,20 @@ def _get_formatted_translation(dict_data, nat_direction, family): return output +def _verify(func): + """Decorator checks if NAT config exists""" + from functools import wraps + + @wraps(func) + def _wrapper(*args, **kwargs): + config = ConfigTreeQuery() + if not config.exists(base): + raise vyos.opmode.UnconfiguredSubsystem(unconf_message) + return func(*args, **kwargs) + return _wrapper + + +@_verify def show_rules(raw: bool, direction: str, family: str): nat_rules = _get_raw_data_rules(direction, family) if raw: @@ -285,6 +305,7 @@ def show_rules(raw: bool, direction: str, family: str): return _get_formatted_output_rules(nat_rules, direction, family) +@_verify def show_statistics(raw: bool, direction: str, family: str): nat_statistics = _get_raw_data_rules(direction, family) if raw: @@ -293,6 +314,7 @@ def show_statistics(raw: bool, direction: str, family: str): return _get_formatted_output_statistics(nat_statistics, direction) +@_verify def show_translations(raw: bool, direction: str, family: str): family = 'ipv6' if family == 'inet6' else 'ipv4' nat_translation = _get_raw_translation(direction, family) diff --git a/src/op_mode/route.py b/src/op_mode/route.py index e1eee5bbf..d11b00ba0 100755 --- a/src/op_mode/route.py +++ b/src/op_mode/route.py @@ -83,7 +83,12 @@ def show(raw: bool, if raw: from json import loads - return loads(output) + d = loads(output) + collect = [] + for k,_ in d.items(): + for l in d[k]: + collect.append(l) + return collect else: return output diff --git a/src/op_mode/storage.py b/src/op_mode/storage.py index 75964c493..d16e271bd 100755 --- a/src/op_mode/storage.py +++ b/src/op_mode/storage.py @@ -20,6 +20,16 @@ import sys import vyos.opmode from vyos.util import cmd +# FIY: As of coreutils from Debian Buster and Bullseye, +# the outpt looks like this: +# +# $ df -h -t ext4 --output=source,size,used,avail,pcent +# Filesystem Size Used Avail Use% +# /dev/sda1 16G 7.6G 7.3G 51% +# +# Those field names are automatically normalized by vyos.opmode.run, +# so we don't touch them here, +# and only normalize values. def _get_system_storage(only_persistent=False): if not only_persistent: @@ -32,11 +42,19 @@ def _get_system_storage(only_persistent=False): return res def _get_raw_data(): + from re import sub as re_sub + from vyos.util import human_to_bytes + out = _get_system_storage(only_persistent=True) lines = out.splitlines() lists = [l.split() for l in lines] res = {lists[0][i]: lists[1][i] for i in range(len(lists[0]))} + res["Size"] = human_to_bytes(res["Size"]) + res["Used"] = human_to_bytes(res["Used"]) + res["Avail"] = human_to_bytes(res["Avail"]) + res["Use%"] = re_sub(r'%', '', res["Use%"]) + return res def _get_formatted_output(): diff --git a/src/services/api/graphql/__init__.py b/src/services/api/graphql/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/services/api/graphql/__init__.py diff --git a/src/services/api/graphql/bindings.py b/src/services/api/graphql/bindings.py index 0b1260912..aa1ba0eb0 100644 --- a/src/services/api/graphql/bindings.py +++ b/src/services/api/graphql/bindings.py @@ -18,16 +18,26 @@ from . graphql.queries import query from . graphql.mutations import mutation from . graphql.directives import directives_dict from . graphql.errors import op_mode_error -from . utils.schema_from_op_mode import generate_op_mode_definitions +from . graphql.auth_token_mutation import auth_token_mutation +from . generate.schema_from_op_mode import generate_op_mode_definitions +from . generate.schema_from_config_session import generate_config_session_definitions +from . generate.schema_from_composite import generate_composite_definitions +from . libs.token_auth import init_secret +from . import state from ariadne import make_executable_schema, load_schema_from_path, snake_case_fallback_resolvers def generate_schema(): api_schema_dir = vyos.defaults.directories['api_schema'] generate_op_mode_definitions() + generate_config_session_definitions() + generate_composite_definitions() + + if state.settings['app'].state.vyos_auth_type == 'token': + init_secret() type_defs = load_schema_from_path(api_schema_dir) - schema = make_executable_schema(type_defs, query, op_mode_error, mutation, snake_case_fallback_resolvers, directives=directives_dict) + schema = make_executable_schema(type_defs, query, op_mode_error, mutation, auth_token_mutation, snake_case_fallback_resolvers, directives=directives_dict) return schema diff --git a/src/services/api/graphql/utils/composite_function.py b/src/services/api/graphql/generate/composite_function.py index bc9d80fbb..bc9d80fbb 100644 --- a/src/services/api/graphql/utils/composite_function.py +++ b/src/services/api/graphql/generate/composite_function.py diff --git a/src/services/api/graphql/utils/config_session_function.py b/src/services/api/graphql/generate/config_session_function.py index fc0dd7a87..fc0dd7a87 100644 --- a/src/services/api/graphql/utils/config_session_function.py +++ b/src/services/api/graphql/generate/config_session_function.py diff --git a/src/services/api/graphql/utils/schema_from_composite.py b/src/services/api/graphql/generate/schema_from_composite.py index f9983cd98..61a08cb2f 100755 --- a/src/services/api/graphql/utils/schema_from_composite.py +++ b/src/services/api/graphql/generate/schema_from_composite.py @@ -19,28 +19,60 @@ # composite functions comprising several requests. import os +import sys import json from inspect import signature, getmembers, isfunction, isclass, getmro from jinja2 import Template +from vyos.defaults import directories if __package__ is None or __package__ == '': - from util import snake_to_pascal_case, map_type_name + sys.path.append("/usr/libexec/vyos/services/api") + from graphql.libs.op_mode import snake_to_pascal_case, map_type_name + from composite_function import queries, mutations + from vyos.config import Config + from vyos.configdict import dict_merge + from vyos.xml import defaults else: - from . util import snake_to_pascal_case, map_type_name + from .. libs.op_mode import snake_to_pascal_case, map_type_name + from . composite_function import queries, mutations + from .. import state + +SCHEMA_PATH = directories['api_schema'] -# this will be run locally before the build -SCHEMA_PATH = '../graphql/schema' +if __package__ is None or __package__ == '': + # allow running stand-alone + conf = Config() + base = ['service', 'https', 'api'] + graphql_dict = conf.get_config_dict(base, key_mangling=('-', '_'), + no_tag_node_value_mangle=True, + get_first_key=True) + if 'graphql' not in graphql_dict: + exit("graphql is not configured") + + graphql_dict = dict_merge(defaults(base), graphql_dict) + auth_type = graphql_dict['graphql']['authentication']['type'] +else: + auth_type = state.settings['app'].state.vyos_auth_type -schema_data: dict = {'schema_name': '', +schema_data: dict = {'auth_type': auth_type, + 'schema_name': '', 'schema_fields': []} query_template = """ +{%- if auth_type == 'key' %} input {{ schema_name }}Input { key: String! {%- for field_entry in schema_fields %} {{ field_entry }} {%- endfor %} } +{%- elif schema_fields %} +input {{ schema_name }}Input { + {%- for field_entry in schema_fields %} + {{ field_entry }} + {%- endfor %} +} +{%- endif %} type {{ schema_name }} { result: Generic @@ -53,17 +85,29 @@ type {{ schema_name }}Result { } extend type Query { +{%- if auth_type == 'key' or schema_fields %} {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @compositequery +{%- else %} + {{ schema_name }} : {{ schema_name }}Result @compositequery +{%- endif %} } """ mutation_template = """ +{%- if auth_type == 'key' %} input {{ schema_name }}Input { key: String! {%- for field_entry in schema_fields %} {{ field_entry }} {%- endfor %} } +{%- elif schema_fields %} +input {{ schema_name }}Input { + {%- for field_entry in schema_fields %} + {{ field_entry }} + {%- endfor %} +} +{%- endif %} type {{ schema_name }} { result: Generic @@ -76,7 +120,11 @@ type {{ schema_name }}Result { } extend type Mutation { +{%- if auth_type == 'key' or schema_fields %} {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @compositemutation +{%- else %} + {{ schema_name }} : {{ schema_name }}Result @compositemutation +{%- endif %} } """ @@ -100,8 +148,6 @@ def create_schema(func_name: str, func: callable, template: str) -> str: return res def generate_composite_definitions(): - from composite_function import queries, mutations - results = [] for name,func in queries.items(): res = create_schema(name, func, query_template) diff --git a/src/services/api/graphql/utils/schema_from_config_session.py b/src/services/api/graphql/generate/schema_from_config_session.py index ea78aaf88..49bf2440e 100755 --- a/src/services/api/graphql/utils/schema_from_config_session.py +++ b/src/services/api/graphql/generate/schema_from_config_session.py @@ -19,28 +19,60 @@ # (wrappers of) native configsession functions. import os +import sys import json from inspect import signature, getmembers, isfunction, isclass, getmro from jinja2 import Template +from vyos.defaults import directories if __package__ is None or __package__ == '': - from util import snake_to_pascal_case, map_type_name + sys.path.append("/usr/libexec/vyos/services/api") + from graphql.libs.op_mode import snake_to_pascal_case, map_type_name + from config_session_function import queries, mutations + from vyos.config import Config + from vyos.configdict import dict_merge + from vyos.xml import defaults else: - from . util import snake_to_pascal_case, map_type_name + from .. libs.op_mode import snake_to_pascal_case, map_type_name + from . config_session_function import queries, mutations + from .. import state + +SCHEMA_PATH = directories['api_schema'] -# this will be run locally before the build -SCHEMA_PATH = '../graphql/schema' +if __package__ is None or __package__ == '': + # allow running stand-alone + conf = Config() + base = ['service', 'https', 'api'] + graphql_dict = conf.get_config_dict(base, key_mangling=('-', '_'), + no_tag_node_value_mangle=True, + get_first_key=True) + if 'graphql' not in graphql_dict: + exit("graphql is not configured") + + graphql_dict = dict_merge(defaults(base), graphql_dict) + auth_type = graphql_dict['graphql']['authentication']['type'] +else: + auth_type = state.settings['app'].state.vyos_auth_type -schema_data: dict = {'schema_name': '', +schema_data: dict = {'auth_type': auth_type, + 'schema_name': '', 'schema_fields': []} query_template = """ +{%- if auth_type == 'key' %} input {{ schema_name }}Input { key: String! {%- for field_entry in schema_fields %} {{ field_entry }} {%- endfor %} } +{%- elif schema_fields %} +input {{ schema_name }}Input { + {%- for field_entry in schema_fields %} + {{ field_entry }} + {%- endfor %} +} +{%- endif %} type {{ schema_name }} { result: Generic @@ -53,17 +85,29 @@ type {{ schema_name }}Result { } extend type Query { +{%- if auth_type == 'key' or schema_fields %} {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @configsessionquery +{%- else %} + {{ schema_name }} : {{ schema_name }}Result @configsessionquery +{%- endif %} } """ mutation_template = """ +{%- if auth_type == 'key' %} input {{ schema_name }}Input { key: String! {%- for field_entry in schema_fields %} {{ field_entry }} {%- endfor %} } +{%- elif schema_fields %} +input {{ schema_name }}Input { + {%- for field_entry in schema_fields %} + {{ field_entry }} + {%- endfor %} +} +{%- endif %} type {{ schema_name }} { result: Generic @@ -76,7 +120,11 @@ type {{ schema_name }}Result { } extend type Mutation { +{%- if auth_type == 'key' or schema_fields %} {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @configsessionmutation +{%- else %} + {{ schema_name }} : {{ schema_name }}Result @configsessionmutation +{%- endif %} } """ @@ -100,8 +148,6 @@ def create_schema(func_name: str, func: callable, template: str) -> str: return res def generate_config_session_definitions(): - from config_session_function import queries, mutations - results = [] for name,func in queries.items(): res = create_schema(name, func, query_template) diff --git a/src/services/api/graphql/utils/schema_from_op_mode.py b/src/services/api/graphql/generate/schema_from_op_mode.py index 57d63628b..1fd198a37 100755 --- a/src/services/api/graphql/utils/schema_from_op_mode.py +++ b/src/services/api/graphql/generate/schema_from_op_mode.py @@ -19,17 +19,23 @@ # scripts. import os +import sys import json from inspect import signature, getmembers, isfunction, isclass, getmro from jinja2 import Template from vyos.defaults import directories if __package__ is None or __package__ == '': - from util import load_as_module, is_op_mode_function_name, is_show_function_name - from util import snake_to_pascal_case, map_type_name + sys.path.append("/usr/libexec/vyos/services/api") + from graphql.libs.op_mode import load_as_module, is_op_mode_function_name, is_show_function_name + from graphql.libs.op_mode import snake_to_pascal_case, map_type_name + from vyos.config import Config + from vyos.configdict import dict_merge + from vyos.xml import defaults else: - from . util import load_as_module, is_op_mode_function_name, is_show_function_name - from . util import snake_to_pascal_case, map_type_name + from .. libs.op_mode import load_as_module, is_op_mode_function_name, is_show_function_name + from .. libs.op_mode import snake_to_pascal_case, map_type_name + from .. import state OP_MODE_PATH = directories['op_mode'] SCHEMA_PATH = directories['api_schema'] @@ -38,16 +44,40 @@ DATA_DIR = directories['data'] op_mode_include_file = os.path.join(DATA_DIR, 'op-mode-standardized.json') op_mode_error_schema = 'op_mode_error.graphql' -schema_data: dict = {'schema_name': '', +if __package__ is None or __package__ == '': + # allow running stand-alone + conf = Config() + base = ['service', 'https', 'api'] + graphql_dict = conf.get_config_dict(base, key_mangling=('-', '_'), + no_tag_node_value_mangle=True, + get_first_key=True) + if 'graphql' not in graphql_dict: + exit("graphql is not configured") + + graphql_dict = dict_merge(defaults(base), graphql_dict) + auth_type = graphql_dict['graphql']['authentication']['type'] +else: + auth_type = state.settings['app'].state.vyos_auth_type + +schema_data: dict = {'auth_type': auth_type, + 'schema_name': '', 'schema_fields': []} query_template = """ +{%- if auth_type == 'key' %} input {{ schema_name }}Input { key: String! {%- for field_entry in schema_fields %} {{ field_entry }} {%- endfor %} } +{%- elif schema_fields %} +input {{ schema_name }}Input { + {%- for field_entry in schema_fields %} + {{ field_entry }} + {%- endfor %} +} +{%- endif %} type {{ schema_name }} { result: Generic @@ -61,17 +91,29 @@ type {{ schema_name }}Result { } extend type Query { +{%- if auth_type == 'key' or schema_fields %} {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @genopquery +{%- else %} + {{ schema_name }} : {{ schema_name }}Result @genopquery +{%- endif %} } """ mutation_template = """ +{%- if auth_type == 'key' %} input {{ schema_name }}Input { key: String! {%- for field_entry in schema_fields %} {{ field_entry }} {%- endfor %} } +{%- elif schema_fields %} +input {{ schema_name }}Input { + {%- for field_entry in schema_fields %} + {{ field_entry }} + {%- endfor %} +} +{%- endif %} type {{ schema_name }} { result: Generic @@ -85,7 +127,11 @@ type {{ schema_name }}Result { } extend type Mutation { +{%- if auth_type == 'key' or schema_fields %} {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @genopmutation +{%- else %} + {{ schema_name }} : {{ schema_name }}Result @genopquery +{%- endif %} } """ diff --git a/src/services/api/graphql/graphql/auth_token_mutation.py b/src/services/api/graphql/graphql/auth_token_mutation.py new file mode 100644 index 000000000..21ac40094 --- /dev/null +++ b/src/services/api/graphql/graphql/auth_token_mutation.py @@ -0,0 +1,49 @@ +# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see <http://www.gnu.org/licenses/>. + +import jwt +import datetime +from typing import Any, Dict +from ariadne import ObjectType, UnionType +from graphql import GraphQLResolveInfo + +from .. libs.token_auth import generate_token +from .. import state + +auth_token_mutation = ObjectType("Mutation") + +@auth_token_mutation.field('AuthToken') +def auth_token_resolver(obj: Any, info: GraphQLResolveInfo, data: Dict): + # non-nullable fields + user = data['username'] + passwd = data['password'] + + secret = state.settings['secret'] + exp_interval = int(state.settings['app'].state.vyos_token_exp) + expiration = (datetime.datetime.now(tz=datetime.timezone.utc) + + datetime.timedelta(seconds=exp_interval)) + + res = generate_token(user, passwd, secret, expiration) + if res: + data['result'] = res + return { + "success": True, + "data": data + } + + return { + "success": False, + "errors": ['token generation failed'] + } diff --git a/src/services/api/graphql/graphql/mutations.py b/src/services/api/graphql/graphql/mutations.py index 32da0eeb7..2778feb69 100644 --- a/src/services/api/graphql/graphql/mutations.py +++ b/src/services/api/graphql/graphql/mutations.py @@ -20,7 +20,7 @@ from graphql import GraphQLResolveInfo from makefun import with_signature from .. import state -from .. import key_auth +from .. libs import key_auth from api.graphql.session.session import Session from api.graphql.session.errors.op_mode_errors import op_mode_err_msg, op_mode_err_code from vyos.opmode import Error as OpModeError @@ -42,32 +42,54 @@ def make_mutation_resolver(mutation_name, class_name, session_func): func_base_name = convert_camel_case_to_snake(class_name) resolver_name = f'resolve_{func_base_name}' - func_sig = '(obj: Any, info: GraphQLResolveInfo, data: Dict)' + func_sig = '(obj: Any, info: GraphQLResolveInfo, data: Dict = {})' @mutation.field(mutation_name) @convert_kwargs_to_snake_case @with_signature(func_sig, func_name=resolver_name) async def func_impl(*args, **kwargs): try: - if 'data' not in kwargs: - return { - "success": False, - "errors": ['missing data'] - } - - data = kwargs['data'] - key = data['key'] - - auth = key_auth.auth_required(key) - if auth is None: - return { - "success": False, - "errors": ['invalid API key'] - } - - # We are finished with the 'key' entry, and may remove so as to - # pass the rest of data (if any) to function. - del data['key'] + auth_type = state.settings['app'].state.vyos_auth_type + + if auth_type == 'key': + data = kwargs['data'] + key = data['key'] + + auth = key_auth.auth_required(key) + if auth is None: + return { + "success": False, + "errors": ['invalid API key'] + } + + # We are finished with the 'key' entry, and may remove so as to + # pass the rest of data (if any) to function. + del data['key'] + + elif auth_type == 'token': + # there is a subtlety here: with the removal of the key entry, + # some requests will now have empty input, hence no data arg, so + # make it optional in the func_sig. However, it can not be None, + # as the makefun package provides accurate TypeError exceptions; + # hence set it to {}, but now it is a mutable default argument, + # so clear the key 'result', which is added at the end of + # this function. + data = kwargs['data'] + if 'result' in data: + del data['result'] + + info = kwargs['info'] + user = info.context.get('user') + if user is None: + return { + "success": False, + "errors": ['not authenticated'] + } + else: + # AtrributeError will have already been raised if no + # vyos_auth_type; validation and defaultValue ensure it is + # one of the previous cases, so this is never reached. + pass session = state.settings['app'].state.vyos_session diff --git a/src/services/api/graphql/graphql/queries.py b/src/services/api/graphql/graphql/queries.py index 791b0d3e0..9c8a4f064 100644 --- a/src/services/api/graphql/graphql/queries.py +++ b/src/services/api/graphql/graphql/queries.py @@ -20,7 +20,7 @@ from graphql import GraphQLResolveInfo from makefun import with_signature from .. import state -from .. import key_auth +from .. libs import key_auth from api.graphql.session.session import Session from api.graphql.session.errors.op_mode_errors import op_mode_err_msg, op_mode_err_code from vyos.opmode import Error as OpModeError @@ -42,32 +42,54 @@ def make_query_resolver(query_name, class_name, session_func): func_base_name = convert_camel_case_to_snake(class_name) resolver_name = f'resolve_{func_base_name}' - func_sig = '(obj: Any, info: GraphQLResolveInfo, data: Dict)' + func_sig = '(obj: Any, info: GraphQLResolveInfo, data: Dict = {})' @query.field(query_name) @convert_kwargs_to_snake_case @with_signature(func_sig, func_name=resolver_name) async def func_impl(*args, **kwargs): try: - if 'data' not in kwargs: - return { - "success": False, - "errors": ['missing data'] - } - - data = kwargs['data'] - key = data['key'] - - auth = key_auth.auth_required(key) - if auth is None: - return { - "success": False, - "errors": ['invalid API key'] - } - - # We are finished with the 'key' entry, and may remove so as to - # pass the rest of data (if any) to function. - del data['key'] + auth_type = state.settings['app'].state.vyos_auth_type + + if auth_type == 'key': + data = kwargs['data'] + key = data['key'] + + auth = key_auth.auth_required(key) + if auth is None: + return { + "success": False, + "errors": ['invalid API key'] + } + + # We are finished with the 'key' entry, and may remove so as to + # pass the rest of data (if any) to function. + del data['key'] + + elif auth_type == 'token': + # there is a subtlety here: with the removal of the key entry, + # some requests will now have empty input, hence no data arg, so + # make it optional in the func_sig. However, it can not be None, + # as the makefun package provides accurate TypeError exceptions; + # hence set it to {}, but now it is a mutable default argument, + # so clear the key 'result', which is added at the end of + # this function. + data = kwargs['data'] + if 'result' in data: + del data['result'] + + info = kwargs['info'] + user = info.context.get('user') + if user is None: + return { + "success": False, + "errors": ['not authenticated'] + } + else: + # AtrributeError will have already been raised if no + # vyos_auth_type; validation and defaultValue ensure it is + # one of the previous cases, so this is never reached. + pass session = state.settings['app'].state.vyos_session diff --git a/src/services/api/graphql/graphql/schema/auth_token.graphql b/src/services/api/graphql/graphql/schema/auth_token.graphql new file mode 100644 index 000000000..af53a293a --- /dev/null +++ b/src/services/api/graphql/graphql/schema/auth_token.graphql @@ -0,0 +1,19 @@ + +input AuthTokenInput { + username: String! + password: String! +} + +type AuthToken { + result: Generic +} + +type AuthTokenResult { + data: AuthToken + success: Boolean! + errors: [String] +} + +extend type Mutation { + AuthToken(data: AuthTokenInput) : AuthTokenResult +} diff --git a/src/services/api/graphql/graphql/schema/composite.graphql b/src/services/api/graphql/graphql/schema/composite.graphql deleted file mode 100644 index 717fbd89d..000000000 --- a/src/services/api/graphql/graphql/schema/composite.graphql +++ /dev/null @@ -1,18 +0,0 @@ - -input SystemStatusInput { - key: String! -} - -type SystemStatus { - result: Generic -} - -type SystemStatusResult { - data: SystemStatus - success: Boolean! - errors: [String] -} - -extend type Query { - SystemStatus(data: SystemStatusInput) : SystemStatusResult @compositequery -}
\ No newline at end of file diff --git a/src/services/api/graphql/graphql/schema/configsession.graphql b/src/services/api/graphql/graphql/schema/configsession.graphql deleted file mode 100644 index b1deac4b3..000000000 --- a/src/services/api/graphql/graphql/schema/configsession.graphql +++ /dev/null @@ -1,115 +0,0 @@ - -input ShowConfigInput { - key: String! - path: [String!]! - configFormat: String = null -} - -type ShowConfig { - result: Generic -} - -type ShowConfigResult { - data: ShowConfig - success: Boolean! - errors: [String] -} - -extend type Query { - ShowConfig(data: ShowConfigInput) : ShowConfigResult @configsessionquery -} - -input ShowInput { - key: String! - path: [String!]! -} - -type Show { - result: Generic -} - -type ShowResult { - data: Show - success: Boolean! - errors: [String] -} - -extend type Query { - Show(data: ShowInput) : ShowResult @configsessionquery -} - -input SaveConfigFileInput { - key: String! - fileName: String = null -} - -type SaveConfigFile { - result: Generic -} - -type SaveConfigFileResult { - data: SaveConfigFile - success: Boolean! - errors: [String] -} - -extend type Mutation { - SaveConfigFile(data: SaveConfigFileInput) : SaveConfigFileResult @configsessionmutation -} - -input LoadConfigFileInput { - key: String! - fileName: String! -} - -type LoadConfigFile { - result: Generic -} - -type LoadConfigFileResult { - data: LoadConfigFile - success: Boolean! - errors: [String] -} - -extend type Mutation { - LoadConfigFile(data: LoadConfigFileInput) : LoadConfigFileResult @configsessionmutation -} - -input AddSystemImageInput { - key: String! - location: String! -} - -type AddSystemImage { - result: Generic -} - -type AddSystemImageResult { - data: AddSystemImage - success: Boolean! - errors: [String] -} - -extend type Mutation { - AddSystemImage(data: AddSystemImageInput) : AddSystemImageResult @configsessionmutation -} - -input DeleteSystemImageInput { - key: String! - name: String! -} - -type DeleteSystemImage { - result: Generic -} - -type DeleteSystemImageResult { - data: DeleteSystemImage - success: Boolean! - errors: [String] -} - -extend type Mutation { - DeleteSystemImage(data: DeleteSystemImageInput) : DeleteSystemImageResult @configsessionmutation -}
\ No newline at end of file diff --git a/src/services/api/graphql/key_auth.py b/src/services/api/graphql/libs/key_auth.py index f756ed6d8..2db0f7d48 100644 --- a/src/services/api/graphql/key_auth.py +++ b/src/services/api/graphql/libs/key_auth.py @@ -1,5 +1,5 @@ -from . import state +from .. import state def check_auth(key_list, key): if not key_list: diff --git a/src/services/api/graphql/utils/util.py b/src/services/api/graphql/libs/op_mode.py index da2bcdb5b..da2bcdb5b 100644 --- a/src/services/api/graphql/utils/util.py +++ b/src/services/api/graphql/libs/op_mode.py diff --git a/src/services/api/graphql/libs/token_auth.py b/src/services/api/graphql/libs/token_auth.py new file mode 100644 index 000000000..3ecd8b855 --- /dev/null +++ b/src/services/api/graphql/libs/token_auth.py @@ -0,0 +1,68 @@ +import jwt +import uuid +import pam +from secrets import token_hex + +from .. import state + +def _check_passwd_pam(username: str, passwd: str) -> bool: + if pam.authenticate(username, passwd): + return True + return False + +def init_secret(): + length = int(state.settings['app'].state.vyos_secret_len) + secret = token_hex(length) + state.settings['secret'] = secret + +def generate_token(user: str, passwd: str, secret: str, exp: int) -> dict: + if user is None or passwd is None: + return {} + if _check_passwd_pam(user, passwd): + app = state.settings['app'] + try: + users = app.state.vyos_token_users + except AttributeError: + app.state.vyos_token_users = {} + users = app.state.vyos_token_users + user_id = uuid.uuid1().hex + payload_data = {'iss': user, 'sub': user_id, 'exp': exp} + secret = state.settings.get('secret') + if secret is None: + return { + "success": False, + "errors": ['failed secret generation'] + } + token = jwt.encode(payload=payload_data, key=secret, algorithm="HS256") + + users |= {user_id: user} + return {'token': token} + +def get_user_context(request): + context = {} + context['request'] = request + context['user'] = None + if 'Authorization' in request.headers: + auth = request.headers['Authorization'] + scheme, token = auth.split() + if scheme.lower() != 'bearer': + return context + + try: + secret = state.settings.get('secret') + payload = jwt.decode(token, secret, algorithms=["HS256"]) + user_id: str = payload.get('sub') + if user_id is None: + return context + except jwt.PyJWTError: + return context + try: + users = state.settings['app'].state.vyos_token_users + except AttributeError: + return context + + user = users.get(user_id) + if user is not None: + context['user'] = user + + return context diff --git a/src/services/api/graphql/session/composite/system_status.py b/src/services/api/graphql/session/composite/system_status.py index 3c1a3d45b..d809f32e3 100755 --- a/src/services/api/graphql/session/composite/system_status.py +++ b/src/services/api/graphql/session/composite/system_status.py @@ -23,7 +23,7 @@ import importlib.util from vyos.defaults import directories -from api.graphql.utils.util import load_op_mode_as_module +from api.graphql.libs.op_mode import load_op_mode_as_module def get_system_version() -> dict: show_version = load_op_mode_as_module('version.py') diff --git a/src/services/api/graphql/session/session.py b/src/services/api/graphql/session/session.py index f990e63d0..c2c1db1df 100644 --- a/src/services/api/graphql/session/session.py +++ b/src/services/api/graphql/session/session.py @@ -24,7 +24,7 @@ from vyos.defaults import directories from vyos.template import render from vyos.opmode import Error as OpModeError -from api.graphql.utils.util import load_op_mode_as_module, split_compound_op_mode_name +from api.graphql.libs.op_mode import load_op_mode_as_module, split_compound_op_mode_name op_mode_include_file = os.path.join(directories['data'], 'op-mode-standardized.json') diff --git a/src/services/vyos-http-api-server b/src/services/vyos-http-api-server index 4ace981ca..3c390d9dc 100755 --- a/src/services/vyos-http-api-server +++ b/src/services/vyos-http-api-server @@ -647,20 +647,21 @@ def reset_op(data: ResetModel): ### def graphql_init(fast_api_app): - from api.graphql.bindings import generate_schema - + from api.graphql.libs.token_auth import get_user_context api.graphql.state.init() api.graphql.state.settings['app'] = app + # import after initializaion of state + from api.graphql.bindings import generate_schema schema = generate_schema() in_spec = app.state.vyos_introspection if app.state.vyos_origins: origins = app.state.vyos_origins - app.add_route('/graphql', CORSMiddleware(GraphQL(schema, debug=True, introspection=in_spec), allow_origins=origins, allow_methods=("GET", "POST", "OPTIONS"))) + app.add_route('/graphql', CORSMiddleware(GraphQL(schema, context_value=get_user_context, debug=True, introspection=in_spec), allow_origins=origins, allow_methods=("GET", "POST", "OPTIONS"))) else: - app.add_route('/graphql', GraphQL(schema, debug=True, introspection=in_spec)) + app.add_route('/graphql', GraphQL(schema, context_value=get_user_context, debug=True, introspection=in_spec)) ### @@ -688,16 +689,21 @@ if __name__ == '__main__': app.state.vyos_debug = server_config['debug'] app.state.vyos_strict = server_config['strict'] app.state.vyos_origins = server_config.get('cors', {}).get('allow_origin', []) - if 'gql' in server_config: - app.state.vyos_gql = True - if isinstance(server_config['gql'], dict) and 'introspection' in server_config['gql']: - app.state.vyos_introspection = True - else: - app.state.vyos_introspection = False + if 'graphql' in server_config: + app.state.vyos_graphql = True + if isinstance(server_config['graphql'], dict): + if 'introspection' in server_config['graphql']: + app.state.vyos_introspection = True + else: + app.state.vyos_introspection = False + # default value is merged in conf_mode http-api.py, if not set + app.state.vyos_auth_type = server_config['graphql']['authentication']['type'] + app.state.vyos_token_exp = server_config['graphql']['authentication']['expiration'] + app.state.vyos_secret_len = server_config['graphql']['authentication']['secret_length'] else: - app.state.vyos_gql = False + app.state.vyos_graphql = False - if app.state.vyos_gql: + if app.state.vyos_graphql: graphql_init(app) try: diff --git a/src/system/keepalived-fifo.py b/src/system/keepalived-fifo.py index a0fccd1d0..864ee8419 100755 --- a/src/system/keepalived-fifo.py +++ b/src/system/keepalived-fifo.py @@ -67,13 +67,13 @@ class KeepalivedFifo: # For VRRP configuration to be read, the commit must be finished count = 1 while commit_in_progress(): - if ( count <= 40 ): - logger.debug(f'commit in progress try: {count}') + if ( count <= 20 ): + logger.debug(f'Attempt to load keepalived configuration aborted due to a commit in progress (attempt {count}/20)') else: - logger.error(f'commit still in progress after {count} continuing anyway') + logger.error(f'Forced keepalived configuration loading despite a commit in progress ({count} wait time expired, not waiting further)') break count += 1 - time.sleep(0.5) + time.sleep(1) try: base = ['high-availability', 'vrrp'] diff --git a/src/tests/test_op_mode.py b/src/tests/test_op_mode.py new file mode 100644 index 000000000..90963b3c5 --- /dev/null +++ b/src/tests/test_op_mode.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +from unittest import TestCase + +import vyos.opmode + +class TestVyOSOpMode(TestCase): + def test_field_name_normalization(self): + from vyos.opmode import _normalize_field_name + + self.assertEqual(_normalize_field_name(" foo bar "), "foo_bar") + self.assertEqual(_normalize_field_name("foo-bar"), "foo_bar") + self.assertEqual(_normalize_field_name("foo (bar) baz"), "foo_bar_baz") + self.assertEqual(_normalize_field_name("load%"), "load_percentage") + + def test_dict_fields_normalization_non_unique(self): + from vyos.opmode import _normalize_field_names + + # Space and dot are both replaced by an underscore, + # so dicts like this cannor be normalized uniquely + data = {"foo bar": True, "foo.bar": False} + + with self.assertRaises(vyos.opmode.InternalError): + _normalize_field_names(data) + + def test_dict_fields_normalization_simple_dict(self): + from vyos.opmode import _normalize_field_names + + data = {"foo bar": True, "Bar-Baz": False} + self.assertEqual(_normalize_field_names(data), {"foo_bar": True, "bar_baz": False}) + + def test_dict_fields_normalization_nested_dict(self): + from vyos.opmode import _normalize_field_names + + data = {"foo bar": True, "bar-baz": {"baz-quux": {"quux-xyzzy": False}}} + self.assertEqual(_normalize_field_names(data), + {"foo_bar": True, "bar_baz": {"baz_quux": {"quux_xyzzy": False}}}) + + def test_dict_fields_normalization_mixed(self): + from vyos.opmode import _normalize_field_names + + data = [{"foo bar": True, "bar-baz": [{"baz-quux": {"quux-xyzzy": [False]}}]}] + self.assertEqual(_normalize_field_names(data), + [{"foo_bar": True, "bar_baz": [{"baz_quux": {"quux_xyzzy": [False]}}]}]) + + def test_dict_fields_normalization_primitive(self): + from vyos.opmode import _normalize_field_names + + data = [1, False, "foo"] + self.assertEqual(_normalize_field_names(data), [1, False, "foo"]) + diff --git a/src/tests/test_util.py b/src/tests/test_util.py index 8ac9a500a..d8b2b7940 100644 --- a/src/tests/test_util.py +++ b/src/tests/test_util.py @@ -26,3 +26,17 @@ class TestVyOSUtil(TestCase): def test_sysctl_read(self): self.assertEqual(sysctl_read('net.ipv4.conf.lo.forwarding'), '1') + + def test_camel_to_snake_case(self): + self.assertEqual(camel_to_snake_case('ConnectionTimeout'), + 'connection_timeout') + self.assertEqual(camel_to_snake_case('connectionTimeout'), + 'connection_timeout') + self.assertEqual(camel_to_snake_case('TCPConnectionTimeout'), + 'tcp_connection_timeout') + self.assertEqual(camel_to_snake_case('TCPPort'), + 'tcp_port') + self.assertEqual(camel_to_snake_case('UseHTTPProxy'), + 'use_http_proxy') + self.assertEqual(camel_to_snake_case('CustomerID'), + 'customer_id') |