diff options
52 files changed, 1614 insertions, 285 deletions
diff --git a/data/templates/accel-ppp/l2tp.config.tmpl b/data/templates/accel-ppp/l2tp.config.tmpl index 070a966b7..a2a2382fa 100644 --- a/data/templates/accel-ppp/l2tp.config.tmpl +++ b/data/templates/accel-ppp/l2tp.config.tmpl @@ -57,6 +57,9 @@ bind={{ outside_addr }} {% if lns_shared_secret %} secret={{ lns_shared_secret }} {% endif %} +{% if lns_host_name %} +host-name={{ lns_host_name }} +{% endif %} [client-ip-range] 0.0.0.0/0 diff --git a/data/templates/accel-ppp/sstp.config.tmpl b/data/templates/accel-ppp/sstp.config.tmpl index d48e9ab0d..7a40a96aa 100644 --- a/data/templates/accel-ppp/sstp.config.tmpl +++ b/data/templates/accel-ppp/sstp.config.tmpl @@ -52,9 +52,9 @@ verbose=1 check-ip=1 {# MTU #} mtu={{ mtu }} -{% if client_ipv6_pool is defined %} -ipv6=allow -{% endif %} +ipv6={{ 'allow' if ppp_options.ipv6 == "deny" and client_ipv6_pool is defined else ppp_options.ipv6 }} +ipv4={{ ppp_options.ipv4 }} + mppe={{ ppp_options.mppe }} lcp-echo-interval={{ ppp_options.lcp_echo_interval }} lcp-echo-timeout={{ ppp_options.lcp_echo_timeout }} diff --git a/data/templates/https/nginx.default.tmpl b/data/templates/https/nginx.default.tmpl index 26d0b5d73..d25e5193a 100644 --- a/data/templates/https/nginx.default.tmpl +++ b/data/templates/https/nginx.default.tmpl @@ -41,9 +41,11 @@ server { ssl_protocols TLSv1.2 TLSv1.3; # proxy settings for HTTP API, if enabled; 503, if not - location ~ /(retrieve|configure|config-file|image|generate|show) { + location ~ /(retrieve|configure|config-file|image|generate|show|docs|openapi.json|redoc|graphql) { {% if server.api %} proxy_pass http://localhost:{{ server.api.port }}; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; proxy_read_timeout 600; proxy_buffering off; {% else %} diff --git a/data/templates/openvpn/server.conf.tmpl b/data/templates/openvpn/server.conf.tmpl index c96b57fb8..c2b0c2ef9 100644 --- a/data/templates/openvpn/server.conf.tmpl +++ b/data/templates/openvpn/server.conf.tmpl @@ -257,16 +257,3 @@ auth {{ hash }} auth-user-pass {{ auth_user_pass_file }} auth-retry nointeract {% endif %} - -{% if openvpn_option is defined and openvpn_option is not none %} -# -# Custom options added by user (not validated) -# -{% for option in openvpn_option %} -{% for argument in option.split('--') %} -{% if argument is defined and argument != '' %} ---{{ argument }} -{% endif %} -{% endfor %} -{% endfor %} -{% endif %} diff --git a/data/templates/openvpn/service-override.conf.tmpl b/data/templates/openvpn/service-override.conf.tmpl new file mode 100644 index 000000000..069bdbd08 --- /dev/null +++ b/data/templates/openvpn/service-override.conf.tmpl @@ -0,0 +1,20 @@ +[Service] +ExecStart= +ExecStart=/usr/sbin/openvpn --daemon openvpn-%i --config %i.conf --status %i.status 30 --writepid %i.pid +{%- if openvpn_option is defined and openvpn_option is not none %} +{% for option in openvpn_option %} +{# Remove the '--' prefix from variable if it is presented #} +{% if option.startswith('--') %} +{% set option = option.split('--', maxsplit=1)[1] %} +{% endif %} +{# Workaround to pass '--push' options properly. Previously openvpn accepted this option without values in double-quotes #} +{# But now it stopped doing this, so we need to add them for compatibility #} +{# HOWEVER! This is a raw option and we do not promise that this or any other trick will work for all the cases. #} +{# Using 'openvpn-option' you take all responsibility for compatibility for yourself. #} +{% if option.startswith('push') and not (option.startswith('push "') and option.endswith('"')) %} +{% set option = 'push \"%s\"'|format(option.split('push ', maxsplit=1)[1]) %} +{% endif %} + --{{ option }} +{%- endfor %} +{% endif %} + diff --git a/debian/control b/debian/control index 87a0258d2..8cafd8257 100644 --- a/debian/control +++ b/debian/control @@ -141,6 +141,7 @@ Depends: usbutils, vyatta-bash, vyatta-cfg, + vyos-http-api-tools, vyos-utils, wide-dhcpv6-client, wireguard-tools, diff --git a/interface-definitions/dhcp-server.xml.in b/interface-definitions/dhcp-server.xml.in index 47bdc4db1..d1ed579e9 100644 --- a/interface-definitions/dhcp-server.xml.in +++ b/interface-definitions/dhcp-server.xml.in @@ -254,9 +254,9 @@ <properties> <help>DHCP lease range</help> <constraint> - <regex>[-_a-zA-Z0-9.]+</regex> + <regex>^[-_a-zA-Z0-9.]+$</regex> </constraint> - <constraintErrorMessage>Invalid DHCP lease range name. May only contain letters, numbers and .-_</constraintErrorMessage> + <constraintErrorMessage>Invalid range name, may only be alphanumeric, dot and hyphen</constraintErrorMessage> </properties> <children> <leafNode name="start"> @@ -289,9 +289,9 @@ <properties> <help>Name of static mapping</help> <constraint> - <regex>[-_a-zA-Z0-9.]+</regex> + <regex>^[-_a-zA-Z0-9.]+$</regex> </constraint> - <constraintErrorMessage>Invalid static mapping name. May only contain letters, numbers and .-_</constraintErrorMessage> + <constraintErrorMessage>Invalid static mapping name, may only be alphanumeric, dot and hyphen</constraintErrorMessage> </properties> <children> #include <include/generic-disable-node.xml.i> diff --git a/interface-definitions/include/accel-ppp/auth-local-users.xml.i b/interface-definitions/include/accel-ppp/auth-local-users.xml.i index 308d6510d..1b40a9ea7 100644 --- a/interface-definitions/include/accel-ppp/auth-local-users.xml.i +++ b/interface-definitions/include/accel-ppp/auth-local-users.xml.i @@ -18,6 +18,9 @@ <leafNode name="static-ip"> <properties> <help>Static client IP address</help> + <constraint> + <validator name="ipv4-address"/> + </constraint> </properties> <defaultValue>*</defaultValue> </leafNode> diff --git a/interface-definitions/include/accel-ppp/client-ipv6-pool.xml.i b/interface-definitions/include/accel-ppp/client-ipv6-pool.xml.i index bd3dadf8d..a692f2335 100644 --- a/interface-definitions/include/accel-ppp/client-ipv6-pool.xml.i +++ b/interface-definitions/include/accel-ppp/client-ipv6-pool.xml.i @@ -27,6 +27,7 @@ <validator name="numeric" argument="--range 48-128"/> </constraint> </properties> + <defaultValue>64</defaultValue> </leafNode> </children> </tagNode> diff --git a/interface-definitions/include/accel-ppp/ppp-options-ipv4.xml.i b/interface-definitions/include/accel-ppp/ppp-options-ipv4.xml.i new file mode 100644 index 000000000..3e065329d --- /dev/null +++ b/interface-definitions/include/accel-ppp/ppp-options-ipv4.xml.i @@ -0,0 +1,23 @@ +<!-- include start from accel-ppp/ppp-options-ipv4.xml.i --> +<leafNode name="ipv4"> + <properties> + <help>IPv4 negotiation algorithm</help> + <constraint> + <regex>^(deny|allow)$</regex> + </constraint> + <constraintErrorMessage>invalid value</constraintErrorMessage> + <valueHelp> + <format>deny</format> + <description>Do not negotiate IPv4</description> + </valueHelp> + <valueHelp> + <format>allow</format> + <description>Negotiate IPv4 only if client requests</description> + </valueHelp> + <completionHelp> + <list>deny allow</list> + </completionHelp> + </properties> + <defaultValue>allow</defaultValue> +</leafNode> +<!-- include end --> diff --git a/interface-definitions/include/accel-ppp/ppp-options-ipv6.xml.i b/interface-definitions/include/accel-ppp/ppp-options-ipv6.xml.i new file mode 100644 index 000000000..b9fbac5c6 --- /dev/null +++ b/interface-definitions/include/accel-ppp/ppp-options-ipv6.xml.i @@ -0,0 +1,31 @@ +<!-- include start from accel-ppp/ppp-options-ipv6.xml.i --> +<leafNode name="ipv6"> + <properties> + <help>IPv6 (IPCP6) negotiation algorithm</help> + <constraint> + <regex>^(deny|allow|prefer|require)$</regex> + </constraint> + <constraintErrorMessage>invalid value</constraintErrorMessage> + <valueHelp> + <format>deny</format> + <description>Do not negotiate IPv6</description> + </valueHelp> + <valueHelp> + <format>allow</format> + <description>Negotiate IPv6 only if client requests</description> + </valueHelp> + <valueHelp> + <format>prefer</format> + <description>Ask client for IPv6 negotiation, do not fail if it rejects</description> + </valueHelp> + <valueHelp> + <format>require</format> + <description>Require IPv6 negotiation</description> + </valueHelp> + <completionHelp> + <list>deny allow prefer require</list> + </completionHelp> + </properties> + <defaultValue>deny</defaultValue> +</leafNode> +<!-- include end --> diff --git a/interface-definitions/interfaces-bonding.xml.in b/interface-definitions/interfaces-bonding.xml.in index c63453588..5a4f08bef 100644 --- a/interface-definitions/interfaces-bonding.xml.in +++ b/interface-definitions/interfaces-bonding.xml.in @@ -177,6 +177,13 @@ <completionHelp> <script>${vyos_completion_dir}/list_interfaces.py --bondable</script> </completionHelp> + <valueHelp> + <format>txt</format> + <description>Interface name</description> + </valueHelp> + <constraint> + <validator name="interface-name"/> + </constraint> <multi/> </properties> </leafNode> @@ -189,6 +196,13 @@ <completionHelp> <script>${vyos_completion_dir}/list_interfaces.py --bondable</script> </completionHelp> + <valueHelp> + <format>txt</format> + <description>Interface name</description> + </valueHelp> + <constraint> + <validator name="interface-name"/> + </constraint> </properties> </leafNode> #include <include/interface/vif-s.xml.i> diff --git a/interface-definitions/system-console.xml.in b/interface-definitions/system-console.xml.in index 88f7f82a9..2897e5e97 100644 --- a/interface-definitions/system-console.xml.in +++ b/interface-definitions/system-console.xml.in @@ -74,6 +74,7 @@ <regex>^(1200|2400|4800|9600|19200|38400|57600|115200)$</regex> </constraint> </properties> + <defaultValue>115200</defaultValue> </leafNode> </children> </tagNode> diff --git a/interface-definitions/vpn_l2tp.xml.in b/interface-definitions/vpn_l2tp.xml.in index 8bcede159..ff3219866 100644 --- a/interface-definitions/vpn_l2tp.xml.in +++ b/interface-definitions/vpn_l2tp.xml.in @@ -33,6 +33,11 @@ <help>Tunnel password used to authenticate the client (LAC)</help> </properties> </leafNode> + <leafNode name="host-name"> + <properties> + <help>Sent to the client (LAC) in the Host-Name attribute</help> + </properties> + </leafNode> </children> </node> <leafNode name="ccp-disable"> diff --git a/interface-definitions/vpn_sstp.xml.in b/interface-definitions/vpn_sstp.xml.in index 5406ede41..ad905a1f0 100644 --- a/interface-definitions/vpn_sstp.xml.in +++ b/interface-definitions/vpn_sstp.xml.in @@ -43,6 +43,8 @@ </properties> <children> #include <include/accel-ppp/ppp-mppe.xml.i> + #include <include/accel-ppp/ppp-options-ipv4.xml.i> + #include <include/accel-ppp/ppp-options-ipv6.xml.i> #include <include/accel-ppp/lcp-echo-interval-failure.xml.i> #include <include/accel-ppp/lcp-echo-timeout.xml.i> </children> diff --git a/python/vyos/configdict.py b/python/vyos/configdict.py index 73986e9af..8e5781b81 100644 --- a/python/vyos/configdict.py +++ b/python/vyos/configdict.py @@ -155,18 +155,15 @@ def get_removed_vlans(conf, dict): D.set_level(conf.get_level()) # get_child_nodes() will return dict_keys(), mangle this into a list with PEP448 keys = D.get_child_nodes_diff(['vif'], expand_nodes=Diff.DELETE)['delete'].keys() - if keys: - dict.update({'vif_remove': [*keys]}) + if keys: dict['vif_remove'] = [*keys] # get_child_nodes() will return dict_keys(), mangle this into a list with PEP448 keys = D.get_child_nodes_diff(['vif-s'], expand_nodes=Diff.DELETE)['delete'].keys() - if keys: - dict.update({'vif_s_remove': [*keys]}) + if keys: dict['vif_s_remove'] = [*keys] for vif in dict.get('vif_s', {}).keys(): keys = D.get_child_nodes_diff(['vif-s', vif, 'vif-c'], expand_nodes=Diff.DELETE)['delete'].keys() - if keys: - dict.update({'vif_s': { vif : {'vif_c_remove': [*keys]}}}) + if keys: dict['vif_s'][vif]['vif_c_remove'] = [*keys] return dict @@ -522,6 +519,11 @@ def get_accel_dict(config, base, chap_secrets): if dict_search('authentication.local_users.username', default_values): del default_values['authentication']['local_users']['username'] + # T2665: defaults include IPv6 client-pool mask per TAG node which need to be + # added to individual local users instead - so we can simply delete them + if dict_search('client_ipv6_pool.prefix.mask', default_values): + del default_values['client_ipv6_pool']['prefix']['mask'] + dict = dict_merge(default_values, dict) # set CPUs cores to process requests @@ -565,4 +567,13 @@ def get_accel_dict(config, base, chap_secrets): dict['authentication']['local_users']['username'][username] = dict_merge( default_values, dict['authentication']['local_users']['username'][username]) + # Add individual IPv6 client-pool default mask if required + if dict_search('client_ipv6_pool.prefix', dict): + # T2665 + default_values = defaults(base + ['client-ipv6-pool', 'prefix']) + + for prefix in dict_search('client_ipv6_pool.prefix', dict): + dict['client_ipv6_pool']['prefix'][prefix] = dict_merge( + default_values, dict['client_ipv6_pool']['prefix'][prefix]) + return dict diff --git a/python/vyos/defaults.py b/python/vyos/defaults.py index ca5e02834..dacdbdef2 100644 --- a/python/vyos/defaults.py +++ b/python/vyos/defaults.py @@ -23,7 +23,10 @@ directories = { "migrate": "/opt/vyatta/etc/config-migrate/migrate", "log": "/var/log/vyatta", "templates": "/usr/share/vyos/templates/", - "certbot": "/config/auth/letsencrypt" + "certbot": "/config/auth/letsencrypt", + "api_schema": "/usr/libexec/vyos/services/api/graphql/graphql/schema/", + "api_templates": "/usr/libexec/vyos/services/api/graphql/recipes/templates/" + } cfg_group = 'vyattacfg' diff --git a/python/vyos/template.py b/python/vyos/template.py index b58f641e1..f9e754357 100644 --- a/python/vyos/template.py +++ b/python/vyos/template.py @@ -29,13 +29,17 @@ _FILTERS = {} # reuse Environments with identical settings to improve performance @functools.lru_cache(maxsize=2) -def _get_environment(): +def _get_environment(location=None): + if location is None: + loc_loader=FileSystemLoader(directories["templates"]) + else: + loc_loader=FileSystemLoader(location) env = Environment( # Don't check if template files were modified upon re-rendering auto_reload=False, # Cache up to this number of templates for quick re-rendering cache_size=100, - loader=FileSystemLoader(directories["templates"]), + loader=loc_loader, trim_blocks=True, ) env.filters.update(_FILTERS) @@ -63,7 +67,7 @@ def register_filter(name, func=None): return func -def render_to_string(template, content, formater=None): +def render_to_string(template, content, formater=None, location=None): """Render a template from the template directory, raise on any errors. :param template: the path to the template relative to the template folder @@ -78,7 +82,7 @@ def render_to_string(template, content, formater=None): package is build (recovering the load time and overhead caused by having the file out of the code). """ - template = _get_environment().get_template(template) + template = _get_environment(location).get_template(template) rendered = template.render(content) if formater is not None: rendered = formater(rendered) @@ -93,6 +97,7 @@ def render( permission=None, user=None, group=None, + location=None, ): """Render a template from the template directory to a file, raise on any errors. @@ -109,7 +114,7 @@ def render( # As we are opening the file with 'w', we are performing the rendering before # calling open() to not accidentally erase the file if rendering fails - rendered = render_to_string(template, content, formater) + rendered = render_to_string(template, content, formater, location) # Write to file with open(destination, "w") as file: diff --git a/smoketest/configs/bgp-dmvpn-hub b/smoketest/configs/bgp-dmvpn-hub new file mode 100644 index 000000000..fc5aadd8f --- /dev/null +++ b/smoketest/configs/bgp-dmvpn-hub @@ -0,0 +1,174 @@ +interfaces { + ethernet eth0 { + address 100.64.10.1/31 + } + ethernet eth1 { + } + loopback lo { + } + tunnel tun0 { + address 192.168.254.62/26 + encapsulation gre + multicast enable + parameters { + ip { + key 1 + } + } + source-address 100.64.10.1 + } +} +protocols { + bgp 65000 { + address-family { + ipv4-unicast { + network 172.20.0.0/16 { + } + } + } + neighbor 192.168.254.1 { + peer-group DMVPN + remote-as 65001 + } + neighbor 192.168.254.2 { + peer-group DMVPN + remote-as 65002 + } + neighbor 192.168.254.3 { + peer-group DMVPN + remote-as 65003 + } + parameters { + default { + no-ipv4-unicast + } + log-neighbor-changes + } + peer-group DMVPN { + address-family { + ipv4-unicast { + } + } + } + timers { + holdtime 30 + keepalive 10 + } + } + nhrp { + tunnel tun0 { + cisco-authentication secret + holding-time 300 + multicast dynamic + redirect + shortcut + } + } + static { + route 0.0.0.0/0 { + next-hop 100.64.10.0 { + } + } + route 172.20.0.0/16 { + blackhole { + distance 200 + } + } + } +} +system { + config-management { + commit-revisions 100 + } + conntrack { + modules { + ftp + h323 + nfs + pptp + sip + sqlnet + tftp + } + } + console { + device ttyS0 { + speed 115200 + } + } + host-name cpe-4 + login { + user vyos { + authentication { + encrypted-password $6$r/Yw/07NXNY$/ZB.Rjf9jxEV.BYoDyLdH.kH14rU52pOBtrX.4S34qlPt77chflCHvpTCq9a6huLzwaMR50rEICzA5GoIRZlM0 + plaintext-password "" + } + } + } + name-server 1.1.1.1 + name-server 8.8.8.8 + name-server 9.9.9.9 + ntp { + server time1.vyos.net { + } + server time2.vyos.net { + } + server time3.vyos.net { + } + } + syslog { + global { + facility all { + level info + } + facility protocols { + level debug + } + } + } +} +vpn { + ipsec { + esp-group ESP-DMVPN { + compression disable + lifetime 1800 + mode transport + pfs dh-group2 + proposal 1 { + encryption aes256 + hash sha1 + } + } + ike-group IKE-DMVPN { + close-action none + ikev2-reauth no + key-exchange ikev1 + lifetime 3600 + proposal 1 { + dh-group 2 + encryption aes256 + hash sha1 + } + } + ipsec-interfaces { + interface eth0 + } + profile NHRPVPN { + authentication { + mode pre-shared-secret + pre-shared-secret VyOS-topsecret + } + bind { + tunnel tun0 + } + esp-group ESP-DMVPN + ike-group IKE-DMVPN + } + } +} + + +// Warning: Do not remove the following line. +// vyos-config-version: "broadcast-relay@1:cluster@1:config-management@1:conntrack@3:conntrack-sync@2:dhcp-relay@2:dhcp-server@6:dhcpv6-server@1:dns-forwarding@3:firewall@5:https@2:interfaces@22:ipoe-server@1:ipsec@5:isis@1:l2tp@3:lldp@1:mdns@1:nat@5:ntp@1:pppoe-server@5:pptp@2:qos@1:quagga@8:rpki@1:salt@1:snmp@2:ssh@2:sstp@3:system@21:vrrp@2:vyos-accel-ppp@2:wanloadbalance@3:webproxy@2:zone-policy@1" +// Release version: 1.3.0-epa3 + diff --git a/smoketest/configs/bgp-dmvpn-spoke b/smoketest/configs/bgp-dmvpn-spoke new file mode 100644 index 000000000..3d7503a9b --- /dev/null +++ b/smoketest/configs/bgp-dmvpn-spoke @@ -0,0 +1,201 @@ +interfaces { + ethernet eth0 { + vif 7 { + description PPPoE-UPLINK + } + } + ethernet eth1 { + address 172.17.1.1/24 + } + loopback lo { + } + pppoe pppoe1 { + authentication { + password cpe-1 + user cpe-1 + } + no-peer-dns + source-interface eth0.7 + } + tunnel tun0 { + address 192.168.254.1/26 + encapsulation gre + multicast enable + parameters { + ip { + key 1 + } + } + source-address 0.0.0.0 + } +} +nat { + source { + rule 10 { + log enable + outbound-interface pppoe1 + source { + address 172.17.0.0/16 + } + translation { + address masquerade + } + } + } +} +protocols { + bgp 65001 { + address-family { + ipv4-unicast { + network 172.17.0.0/16 { + } + } + } + neighbor 192.168.254.62 { + address-family { + ipv4-unicast { + } + } + remote-as 65000 + } + parameters { + default { + no-ipv4-unicast + } + log-neighbor-changes + } + timers { + holdtime 30 + keepalive 10 + } + } + nhrp { + tunnel tun0 { + cisco-authentication secret + holding-time 300 + map 192.168.254.62/26 { + nbma-address 100.64.10.1 + register + } + multicast nhs + redirect + shortcut + } + } + static { + route 172.17.0.0/16 { + blackhole { + distance 200 + } + } + } +} +service { + dhcp-server { + shared-network-name LAN-3 { + subnet 172.17.1.0/24 { + default-router 172.17.1.1 + name-server 172.17.1.1 + range 0 { + start 172.17.1.100 + stop 172.17.1.200 + } + } + } + } +} +system { + config-management { + commit-revisions 100 + } + conntrack { + modules { + ftp + h323 + nfs + pptp + sip + sqlnet + tftp + } + } + console { + device ttyS0 { + speed 115200 + } + } + host-name cpe-1 + login { + user vyos { + authentication { + encrypted-password $6$r/Yw/07NXNY$/ZB.Rjf9jxEV.BYoDyLdH.kH14rU52pOBtrX.4S34qlPt77chflCHvpTCq9a6huLzwaMR50rEICzA5GoIRZlM0 + plaintext-password "" + } + } + } + name-server 1.1.1.1 + name-server 8.8.8.8 + name-server 9.9.9.9 + ntp { + server time1.vyos.net { + } + server time2.vyos.net { + } + server time3.vyos.net { + } + } + syslog { + global { + facility all { + level info + } + facility protocols { + level debug + } + } + } +} +vpn { + ipsec { + esp-group ESP-DMVPN { + compression disable + lifetime 1800 + mode transport + pfs dh-group2 + proposal 1 { + encryption aes256 + hash sha1 + } + } + ike-group IKE-DMVPN { + close-action none + ikev2-reauth no + key-exchange ikev1 + lifetime 3600 + proposal 1 { + dh-group 2 + encryption aes256 + hash sha1 + } + } + ipsec-interfaces { + interface pppoe1 + } + profile NHRPVPN { + authentication { + mode pre-shared-secret + pre-shared-secret VyOS-topsecret + } + bind { + tunnel tun0 + } + esp-group ESP-DMVPN + ike-group IKE-DMVPN + } + } +} + + +// Warning: Do not remove the following line. +// vyos-config-version: "broadcast-relay@1:cluster@1:config-management@1:conntrack@3:conntrack-sync@2:dhcp-relay@2:dhcp-server@6:dhcpv6-server@1:dns-forwarding@3:firewall@5:https@2:interfaces@22:ipoe-server@1:ipsec@5:isis@1:l2tp@3:lldp@1:mdns@1:nat@5:ntp@1:pppoe-server@5:pptp@2:qos@1:quagga@8:rpki@1:salt@1:snmp@2:ssh@2:sstp@3:system@21:vrrp@2:vyos-accel-ppp@2:wanloadbalance@3:webproxy@2:zone-policy@1" +// Release version: 1.3.0-epa3 diff --git a/smoketest/scripts/cli/base_interfaces_test.py b/smoketest/scripts/cli/base_interfaces_test.py index a508404de..60cad32bb 100644 --- a/smoketest/scripts/cli/base_interfaces_test.py +++ b/smoketest/scripts/cli/base_interfaces_test.py @@ -419,6 +419,16 @@ class BasicInterfaceTest: tmp = read_file(f'/sys/class/net/{vif}/mtu') self.assertEqual(tmp, self._mtu) + # T3972: remove vif-c interfaces from vif-s + for interface in self._interfaces: + base = self._base_path + [interface] + for vif_s in self._qinq_range: + base = self._base_path + [interface, 'vif-s', vif_s, 'vif-c'] + self.cli_delete(base) + + self.cli_commit() + + def test_interface_ip_options(self): if not self._test_ip: self.skipTest('not supported') diff --git a/smoketest/scripts/cli/test_service_snmp.py b/smoketest/scripts/cli/test_service_snmp.py index 008271102..e15d186bc 100755 --- a/smoketest/scripts/cli/test_service_snmp.py +++ b/smoketest/scripts/cli/test_service_snmp.py @@ -22,6 +22,7 @@ from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSession from vyos.configsession import ConfigSessionError from vyos.template import is_ipv4 +from vyos.template import address_from_cidr from vyos.util import read_file from vyos.util import process_named_running @@ -36,16 +37,29 @@ def get_config_value(key): return tmp[0] class TestSNMPService(VyOSUnitTestSHIM.TestCase): - def setUp(self): + @classmethod + def setUpClass(cls): + super(cls, cls).setUpClass() + # ensure we can also run this test on a live system - so lets clean # out the current configuration :) + cls.cli_delete(cls, base_path) + + def tearDown(self): + # delete testing SNMP config self.cli_delete(base_path) + self.cli_commit() def test_snmp_basic(self): + dummy_if = 'dum7312' + dummy_addr = '100.64.0.1/32' + self.cli_set(['interfaces', 'dummy', dummy_if, 'address', dummy_addr]) + # Check if SNMP can be configured and service runs clients = ['192.0.2.1', '2001:db8::1'] networks = ['192.0.2.128/25', '2001:db8:babe::/48'] - listen = ['127.0.0.1', '::1'] + listen = ['127.0.0.1', '::1', address_from_cidr(dummy_addr)] + port = '5000' for auth in ['ro', 'rw']: community = 'VyOS' + auth @@ -56,7 +70,7 @@ class TestSNMPService(VyOSUnitTestSHIM.TestCase): self.cli_set(base_path + ['community', community, 'network', network]) for addr in listen: - self.cli_set(base_path + ['listen-address', addr]) + self.cli_set(base_path + ['listen-address', addr, 'port', port]) self.cli_set(base_path + ['contact', 'maintainers@vyos.io']) self.cli_set(base_path + ['location', 'qemu']) @@ -68,16 +82,18 @@ class TestSNMPService(VyOSUnitTestSHIM.TestCase): # thus we need to transfor this into a proper list config = get_config_value('agentaddress') expected = 'unix:/run/snmpd.socket' + self.assertIn(expected, config) + for addr in listen: if is_ipv4(addr): - expected += ',udp:{}:161'.format(addr) + expected = f'udp:{addr}:{port}' else: - expected += ',udp6:[{}]:161'.format(addr) - - self.assertTrue(expected in config) + expected = f'udp6:[{addr}]:{port}' + self.assertIn(expected, config) # Check for running process self.assertTrue(process_named_running(PROCESS_NAME)) + self.cli_delete(['interfaces', 'dummy', dummy_if]) def test_snmpv3_sha(self): @@ -86,7 +102,7 @@ class TestSNMPService(VyOSUnitTestSHIM.TestCase): self.cli_set(base_path + ['v3', 'engineid', '000000000000000000000002']) self.cli_set(base_path + ['v3', 'group', 'default', 'mode', 'ro']) - # check validate() - a view must be created before this can be comitted + # check validate() - a view must be created before this can be committed with self.assertRaises(ConfigSessionError): self.cli_commit() @@ -152,4 +168,3 @@ class TestSNMPService(VyOSUnitTestSHIM.TestCase): if __name__ == '__main__': unittest.main(verbosity=2) - diff --git a/src/conf_mode/flow_accounting_conf.py b/src/conf_mode/flow_accounting_conf.py index 9cae29481..0a4559ade 100755 --- a/src/conf_mode/flow_accounting_conf.py +++ b/src/conf_mode/flow_accounting_conf.py @@ -306,7 +306,7 @@ def verify(config): source_ip_presented = True break if not source_ip_presented: - raise ConfigError("Your \"netflow source-ip\" does not exist in the system") + print("Warning: your \"netflow source-ip\" does not exist in the system") # check if engine-id compatible with selected protocol version if config['netflow']['engine-id']: diff --git a/src/conf_mode/http-api.py b/src/conf_mode/http-api.py index 472eb77e4..7e4b117c8 100755 --- a/src/conf_mode/http-api.py +++ b/src/conf_mode/http-api.py @@ -19,6 +19,7 @@ import sys import os import json +import time from copy import deepcopy import vyos.defaults @@ -34,11 +35,6 @@ config_file = '/etc/vyos/http-api.conf' vyos_conf_scripts_dir=vyos.defaults.directories['conf_mode'] -# XXX: this model will need to be extended for tag nodes -dependencies = [ - 'https.py', -] - def get_config(config=None): http_api = deepcopy(vyos.defaults.api_data) x = http_api.get('api_keys') @@ -103,8 +99,10 @@ def apply(http_api): else: call('systemctl stop vyos-http-api.service') - for dep in dependencies: - cmd(f'{vyos_conf_scripts_dir}/{dep}', raising=ConfigError) + # Let uvicorn settle before restarting Nginx + time.sleep(2) + + cmd(f'{vyos_conf_scripts_dir}/https.py', raising=ConfigError) if __name__ == '__main__': try: diff --git a/src/conf_mode/interfaces-openvpn.py b/src/conf_mode/interfaces-openvpn.py index 5d537dadf..ae35ed3c4 100755 --- a/src/conf_mode/interfaces-openvpn.py +++ b/src/conf_mode/interfaces-openvpn.py @@ -51,6 +51,7 @@ user = 'openvpn' group = 'openvpn' cfg_file = '/run/openvpn/{ifname}.conf' +service_file = '/run/systemd/system/openvpn@{ifname}.service.d/20-override.conf' def checkCertHeader(header, filename): """ @@ -434,6 +435,11 @@ def generate(openvpn): if os.path.isdir(ccd_dir): rmtree(ccd_dir, ignore_errors=True) + # Remove systemd directories with overrides + service_dir = os.path.dirname(service_file.format(**openvpn)) + if os.path.isdir(service_dir): + rmtree(service_dir, ignore_errors=True) + if 'deleted' in openvpn or 'disable' in openvpn: return None @@ -477,14 +483,20 @@ def generate(openvpn): render(cfg_file.format(**openvpn), 'openvpn/server.conf.tmpl', openvpn, formater=lambda _: _.replace(""", '"'), user=user, group=group) + # Render 20-override.conf for OpenVPN service + render(service_file.format(**openvpn), 'openvpn/service-override.conf.tmpl', openvpn, + formater=lambda _: _.replace(""", '"'), user=user, group=group) + # Reload systemd services config to apply an override + call(f'systemctl daemon-reload') + return None def apply(openvpn): interface = openvpn['ifname'] - call(f'systemctl stop openvpn@{interface}.service') # Do some cleanup when OpenVPN is disabled/deleted if 'deleted' in openvpn or 'disable' in openvpn: + call(f'systemctl stop openvpn@{interface}.service') for cleanup_file in glob(f'/run/openvpn/{interface}.*'): if os.path.isfile(cleanup_file): os.unlink(cleanup_file) @@ -496,7 +508,7 @@ def apply(openvpn): # No matching OpenVPN process running - maybe it got killed or none # existed - nevertheless, spawn new OpenVPN process - call(f'systemctl start openvpn@{interface}.service') + call(f'systemctl reload-or-restart openvpn@{interface}.service') conf = VTunIf.get_config() conf['device_type'] = openvpn['device_type'] diff --git a/src/conf_mode/interfaces-tunnel.py b/src/conf_mode/interfaces-tunnel.py index 4db564e6d..2798d321f 100755 --- a/src/conf_mode/interfaces-tunnel.py +++ b/src/conf_mode/interfaces-tunnel.py @@ -87,6 +87,10 @@ def verify(tunnel): # Check pairs tunnel source-address/encapsulation/key with exists tunnels. # Prevent the same key for 2 tunnels with same source-address/encap. T2920 for tunnel_if in Section.interfaces('tunnel'): + # It makes no sense to run the test for re-used GRE keys on our + # own interface we are currently working on + if tunnel['ifname'] == tunnel_if: + continue tunnel_cfg = get_interface_config(tunnel_if) # no match on encapsulation - bail out if dict_search('linkinfo.info_kind', tunnel_cfg) != tunnel['encapsulation']: diff --git a/src/conf_mode/snmp.py b/src/conf_mode/snmp.py index 3990e5735..0fbe90cce 100755 --- a/src/conf_mode/snmp.py +++ b/src/conf_mode/snmp.py @@ -20,13 +20,17 @@ from sys import exit from vyos.config import Config from vyos.configverify import verify_vrf -from vyos.snmpv3_hashgen import plaintext_to_md5, plaintext_to_sha1, random +from vyos.snmpv3_hashgen import plaintext_to_md5 +from vyos.snmpv3_hashgen import plaintext_to_sha1 +from vyos.snmpv3_hashgen import random from vyos.template import render from vyos.template import is_ipv4 -from vyos.util import call, chmod_755 +from vyos.util import call +from vyos.util import chmod_755 from vyos.validate import is_addr_assigned from vyos.version import get_version_data -from vyos import ConfigError, airbag +from vyos import ConfigError +from vyos import airbag airbag.enable() config_file_client = r'/etc/snmp/snmp.conf' @@ -401,19 +405,20 @@ def verify(snmp): addr = listen[0] port = listen[1] + tmp = None if is_ipv4(addr): # example: udp:127.0.0.1:161 - listen = 'udp:' + addr + ':' + port + tmp = f'udp:{addr}:{port}' elif snmp['ipv6_enabled']: # example: udp6:[::1]:161 - listen = 'udp6:' + '[' + addr + ']' + ':' + port + tmp = f'udp6:[{addr}]:{port}' # We only wan't to configure addresses that exist on the system. # Hint the user if they don't exist if is_addr_assigned(addr): - snmp['listen_on'].append(listen) + if tmp: snmp['listen_on'].append(tmp) else: - print('WARNING: SNMP listen address {0} not configured!'.format(addr)) + print(f'WARNING: SNMP listen address {addr} not configured!') verify_vrf(snmp) diff --git a/src/conf_mode/system-login-banner.py b/src/conf_mode/system-login-banner.py index 569010735..2220d7b66 100755 --- a/src/conf_mode/system-login-banner.py +++ b/src/conf_mode/system-login-banner.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2020 VyOS maintainers and contributors +# Copyright (C) 2020-2021 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -22,12 +22,13 @@ from vyos import airbag airbag.enable() motd=""" -The programs included with the Debian GNU/Linux system are free software; -the exact distribution terms for each program are described in the -individual files in /usr/share/doc/*/copyright. +Check out project news at https://blog.vyos.io +and feel free to report bugs at https://phabricator.vyos.net -Debian GNU/Linux comes with ABSOLUTELY NO WARRANTY, to the extent -permitted by applicable law. +You can change this banner using "set system login banner post-login" command. + +VyOS is a free software distribution that includes multiple components, +you can check individual component licenses under /usr/share/doc/*/copyright """ @@ -36,7 +37,7 @@ PRELOGIN_NET_FILE = r'/etc/issue.net' POSTLOGIN_FILE = r'/etc/motd' default_config_data = { - 'issue': 'Welcome to VyOS - \n \l\n', + 'issue': 'Welcome to VyOS - \\n \\l\n\n', 'issue_net': 'Welcome to VyOS\n', 'motd': motd } diff --git a/src/conf_mode/system_console.py b/src/conf_mode/system_console.py index 33a546bd3..19b252513 100755 --- a/src/conf_mode/system_console.py +++ b/src/conf_mode/system_console.py @@ -18,9 +18,14 @@ import os import re from vyos.config import Config -from vyos.util import call, read_file, write_file +from vyos.configdict import dict_merge +from vyos.util import call +from vyos.util import read_file +from vyos.util import write_file from vyos.template import render -from vyos import ConfigError, airbag +from vyos.xml import defaults +from vyos import ConfigError +from vyos import airbag airbag.enable() by_bus_dir = '/dev/serial/by-bus' @@ -36,21 +41,27 @@ def get_config(config=None): console = conf.get_config_dict(base, get_first_key=True) # bail out early if no serial console is configured - if 'device' not in console.keys(): + if 'device' not in console: return console # convert CLI values to system values - for device in console['device'].keys(): - # no speed setting has been configured - use default value - if not 'speed' in console['device'][device].keys(): - tmp = { 'speed': '' } - if device.startswith('hvc'): - tmp['speed'] = 38400 - else: - tmp['speed'] = 115200 + default_values = defaults(base + ['device']) + for device, device_config in console['device'].items(): + if 'speed' not in device_config and device.startswith('hvc'): + # XEN console has a different default console speed + console['device'][device]['speed'] = 38400 + else: + # Merge in XML defaults - the proper way to do it + console['device'][device] = dict_merge(default_values, + console['device'][device]) + + return console - console['device'][device].update(tmp) +def verify(console): + if not console or 'device' not in console: + return None + for device in console['device']: if device.startswith('usb'): # It is much easiert to work with the native ttyUSBn name when using # getty, but that name may change across reboots - depending on the @@ -58,13 +69,13 @@ def get_config(config=None): # to its dynamic device file - and create a new dict entry for it. by_bus_device = f'{by_bus_dir}/{device}' if os.path.isdir(by_bus_dir) and os.path.exists(by_bus_device): - tmp = os.path.basename(os.readlink(by_bus_device)) - # updating the dict must come as last step in the loop! - console['device'][tmp] = console['device'].pop(device) + device = os.path.basename(os.readlink(by_bus_device)) - return console + # If the device name still starts with usbXXX no matching tty was found + # and it can not be used as a serial interface + if device.startswith('usb'): + raise ConfigError(f'Device {device} does not support beeing used as tty') -def verify(console): return None def generate(console): @@ -76,20 +87,29 @@ def generate(console): call(f'systemctl stop {basename}') os.unlink(os.path.join(root, basename)) - if not console: + if not console or 'device' not in console: return None - for device in console['device'].keys(): + for device, device_config in console['device'].items(): + if device.startswith('usb'): + # It is much easiert to work with the native ttyUSBn name when using + # getty, but that name may change across reboots - depending on the + # amount of connected devices. We will resolve the fixed device name + # to its dynamic device file - and create a new dict entry for it. + by_bus_device = f'{by_bus_dir}/{device}' + if os.path.isdir(by_bus_dir) and os.path.exists(by_bus_device): + device = os.path.basename(os.readlink(by_bus_device)) + config_file = base_dir + f'/serial-getty@{device}.service' getty_wants_symlink = base_dir + f'/getty.target.wants/serial-getty@{device}.service' - render(config_file, 'getty/serial-getty.service.tmpl', console['device'][device]) + render(config_file, 'getty/serial-getty.service.tmpl', device_config) os.symlink(config_file, getty_wants_symlink) # GRUB # For existing serial line change speed (if necessary) # Only applys to ttyS0 - if 'ttyS0' not in console['device'].keys(): + if 'ttyS0' not in console['device']: return None speed = console['device']['ttyS0']['speed'] @@ -98,7 +118,6 @@ def generate(console): return None lines = read_file(grub_config).split('\n') - p = re.compile(r'^(.* console=ttyS0),[0-9]+(.*)$') write = False newlines = [] @@ -122,9 +141,8 @@ def generate(console): return None def apply(console): - # reset screen blanking + # Reset screen blanking call('/usr/bin/setterm -blank 0 -powersave off -powerdown 0 -term linux </dev/tty1 >/dev/tty1 2>&1') - # Reload systemd manager configuration call('systemctl daemon-reload') @@ -136,11 +154,11 @@ def apply(console): call('/usr/bin/setterm -blank 15 -powersave powerdown -powerdown 60 -term linux </dev/tty1 >/dev/tty1 2>&1') # Start getty process on configured serial interfaces - for device in console['device'].keys(): + for device in console['device']: # Only start console if it exists on the running system. If a user # detaches a USB serial console and reboots - it should not fail! if os.path.exists(f'/dev/{device}'): - call(f'systemctl start serial-getty@{device}.service') + call(f'systemctl restart serial-getty@{device}.service') return None diff --git a/src/conf_mode/vpn_l2tp.py b/src/conf_mode/vpn_l2tp.py index e970d2ef5..86aa9af09 100755 --- a/src/conf_mode/vpn_l2tp.py +++ b/src/conf_mode/vpn_l2tp.py @@ -291,6 +291,8 @@ def get_config(config=None): # LNS secret if conf.exists(['lns', 'shared-secret']): l2tp['lns_shared_secret'] = conf.return_value(['lns', 'shared-secret']) + if conf.exists(['lns', 'host-name']): + l2tp['lns_host_name'] = conf.return_value(['lns', 'host-name']) if conf.exists(['ccp-disable']): l2tp['ccp_disable'] = True diff --git a/src/etc/dhcp/dhclient-enter-hooks.d/02-vyos-stopdhclient b/src/etc/dhcp/dhclient-enter-hooks.d/02-vyos-stopdhclient index f737148dc..ae6bf9f16 100644 --- a/src/etc/dhcp/dhclient-enter-hooks.d/02-vyos-stopdhclient +++ b/src/etc/dhcp/dhclient-enter-hooks.d/02-vyos-stopdhclient @@ -23,10 +23,12 @@ if [ -z ${CONTROLLED_STOP} ] ; then if ([ $dhclient -ne $current_dhclient ] && [ $dhclient -ne $master_dhclient ]); then # get path to PID-file of dhclient process local dhclient_pidfile=`ps --no-headers --format args --pid $dhclient | awk 'match(\$0, ".*-pf (/.*pid) .*", PF) { print PF[1] }'` + # get path to lease-file of dhclient process + local dhclient_leasefile=`ps --no-headers --format args --pid $dhclient | awk 'match(\$0, ".*-lf (/\\\S*leases) .*", LF) { print LF[1] }'` # stop dhclient with native command - this will run dhclient-script with correct reason unlike simple kill - logmsg info "Stopping dhclient with PID: ${dhclient}, PID file: $dhclient_pidfile" + logmsg info "Stopping dhclient with PID: ${dhclient}, PID file: ${dhclient_pidfile}, Leases file: ${dhclient_leasefile}" if [[ -e $dhclient_pidfile ]]; then - dhclient -e CONTROLLED_STOP=yes -x -pf $dhclient_pidfile + dhclient -e CONTROLLED_STOP=yes -x -pf $dhclient_pidfile -lf $dhclient_leasefile else logmsg error "PID file $dhclient_pidfile does not exists, killing dhclient with SIGTERM signal" kill -s 15 ${dhclient} diff --git a/src/etc/systemd/system/openvpn@.service.d/override.conf b/src/etc/systemd/system/openvpn@.service.d/10-override.conf index 03fe6b587..775a2d7ba 100644 --- a/src/etc/systemd/system/openvpn@.service.d/override.conf +++ b/src/etc/systemd/system/openvpn@.service.d/10-override.conf @@ -7,6 +7,7 @@ WorkingDirectory= WorkingDirectory=/run/openvpn ExecStart= ExecStart=/usr/sbin/openvpn --daemon openvpn-%i --config %i.conf --status %i.status 30 --writepid %i.pid +ExecReload=/bin/kill -HUP $MAINPID User=openvpn Group=openvpn AmbientCapabilities=CAP_IPC_LOCK CAP_NET_ADMIN CAP_NET_BIND_SERVICE CAP_NET_RAW CAP_SETGID CAP_SETUID CAP_SYS_CHROOT CAP_DAC_OVERRIDE CAP_AUDIT_WRITE diff --git a/src/etc/udev/rules.d/90-vyos-serial.rules b/src/etc/udev/rules.d/90-vyos-serial.rules index 3f10f4924..5cca89e89 100644 --- a/src/etc/udev/rules.d/90-vyos-serial.rules +++ b/src/etc/udev/rules.d/90-vyos-serial.rules @@ -22,7 +22,7 @@ IMPORT{builtin}="path_id", IMPORT{builtin}="usb_id" # (tr -d -) does the replacement # - Replace the first group after ":" to represent the bus relation (sed -e 0,/:/s//b/) indicated by "b" # - Replace the next group after ":" to represent the port relation (sed -e 0,/:/s//p/) indicated by "p" -ENV{ID_PATH}=="?*", ENV{.ID_PORT}=="", PROGRAM="/bin/sh -c 'D=$env{ID_PATH}; echo ${D:17} | tr -d - | sed -e 0,/:/s//b/ | sed -e 0,/:/s//p/'", SYMLINK+="serial/by-bus/$result" -ENV{ID_PATH}=="?*", ENV{.ID_PORT}=="?*", PROGRAM="/bin/sh -c 'D=$env{ID_PATH}; echo ${D:17} | tr -d - | sed -e 0,/:/s//b/ | sed -e 0,/:/s//p/'", SYMLINK+="serial/by-bus/$result" +ENV{ID_PATH}=="?*", ENV{.ID_PORT}=="", PROGRAM="/bin/sh -c 'echo $env{ID_PATH} | cut -d- -f3- | tr -d - | sed -e 0,/:/s//b/ | sed -e 0,/:/s//p/'", SYMLINK+="serial/by-bus/$result" +ENV{ID_PATH}=="?*", ENV{.ID_PORT}=="?*", PROGRAM="/bin/sh -c 'echo $env{ID_PATH} | cut -d- -f3- | tr -d - | sed -e 0,/:/s//b/ | sed -e 0,/:/s//p/'", SYMLINK+="serial/by-bus/$result" LABEL="serial_end" diff --git a/src/services/api/graphql/README.graphql b/src/services/api/graphql/README.graphql new file mode 100644 index 000000000..c91b70782 --- /dev/null +++ b/src/services/api/graphql/README.graphql @@ -0,0 +1,140 @@ + +Example using GraphQL mutations to configure a DHCP server: + +This assumes that the http-api is running: + +'set service https api' + +One can configure an address on an interface, and configure the DHCP server +to run with that address as default router by requesting these 'mutations' +in the GraphQL playground: + +mutation { + createInterfaceEthernet (data: {interface: "eth1", + address: "192.168.0.1/24", + description: "BOB"}) { + success + errors + data { + address + } + } +} + +mutation { + createDhcpServer(data: {sharedNetworkName: "BOB", + subnet: "192.168.0.0/24", + defaultRouter: "192.168.0.1", + nameServer: "192.168.0.1", + domainName: "vyos.net", + lease: 86400, + range: 0, + start: "192.168.0.9", + stop: "192.168.0.254", + dnsForwardingAllowFrom: "192.168.0.0/24", + dnsForwardingCacheSize: 0, + dnsForwardingListenAddress: "192.168.0.1"}) { + success + errors + data { + defaultRouter + } + } +} + +mutation { + saveConfigFile(data: {fileName: "/config/config.boot"}) { + success + errors + data { + fileName + } + } +} + +N.B. fileName can be empty (fileName: "") or data can be empty (data: {}) to save to +/config/config.boot; to save to an alternative path, specify fileName. + +mutation { + loadConfigFile(data: {fileName: "/home/vyos/config.boot"}) { + success + errors + data { + fileName + } + } +} + + +The GraphQL playground will be found at: + +https://{{ host_address }}/graphql + +An equivalent curl command to the first example above would be: + +curl -k 'https://192.168.100.168/graphql' -H 'Content-Type: application/json' --data-binary '{"query": "mutation {createInterfaceEthernet (data: {interface: \"eth1\", address: \"192.168.0.1/24\", description: \"BOB\"}) {success errors data {address}}}"}' + +Note that the 'mutation' term is prefaced by 'query' in the curl command. + +What's here: + +services +├── api +│  └── graphql +│  ├── graphql +│  │  ├── directives.py +│  │  ├── __init__.py +│  │  ├── mutations.py +│  │  └── schema +│  │  ├── dhcp_server.graphql +│  │  ├── interface_ethernet.graphql +│  │  └── schema.graphql +│  ├── recipes +│  │  ├── dhcp_server.py +│  │  ├── __init__.py +│  │  ├── interface_ethernet.py +│  │  ├── recipe.py +│  │  └── templates +│  │  ├── dhcp_server.tmpl +│  │  └── interface_ethernet.tmpl +│  └── state.py +├── vyos-configd +├── vyos-hostsd +└── vyos-http-api-server + +The GraphQL library that we are using, Ariadne, advertises itself as a +'schema-first' implementation: define the schema; define resolvers +(handlers) for declared Query and Mutation types (Subscription types are not +currently used). + +In the current approach to a high-level API, we consider the +Jinja2-templated collection of configuration mode 'set'/'delete' commands as +the Ur-data; the GraphQL schema is produced from those files, located in +'api/graphql/recipes/templates'. + +Resolvers for the schema Mutation fields are dynamically generated using a +'directive' added to the respective schema field. The directive, +'@generate', is handled by the class 'DataDirective' in +'api/graphql/graphql/directives.py', which calls the 'make_resolver' function in +'api/graphql/graphql/mutations.py'; the produced resolver calls the appropriate +wrapper in 'api/graphql/recipes', with base class doing the (overridable) +configuration steps of calling all defined 'set'/'delete' commands. + +Integrating the above with vyos-http-api-server is ~10 lines of code. + +What needs to be done: + +• automate generation of schema and wrappers from templated configuration +commands + +• investigate whether the subclassing provided by the named wrappers in +'api/graphql/recipes' is sufficient for use cases which need to modify data + +• encapsulate the manipulation of 'canonical names' which transforms the +prefixed camel-case schema names to various snake-case file/function names + +• consider mechanism for migration of templates: offline vs. on-the-fly + +• define the naming convention for those schema fields that refer to +configuration mode parameters: e.g. how much of the path is needed as prefix +to uniquely define the term diff --git a/src/services/api/graphql/bindings.py b/src/services/api/graphql/bindings.py new file mode 100644 index 000000000..c123f68d8 --- /dev/null +++ b/src/services/api/graphql/bindings.py @@ -0,0 +1,14 @@ +import vyos.defaults +from . graphql.mutations import mutation +from . graphql.directives import DataDirective, ConfigFileDirective + +from ariadne import make_executable_schema, load_schema_from_path, snake_case_fallback_resolvers + +def generate_schema(): + api_schema_dir = vyos.defaults.directories['api_schema'] + + type_defs = load_schema_from_path(api_schema_dir) + + schema = make_executable_schema(type_defs, mutation, snake_case_fallback_resolvers, directives={"generate": DataDirective, "configfile": ConfigFileDirective}) + + return schema diff --git a/src/services/api/graphql/graphql/__init__.py b/src/services/api/graphql/graphql/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/services/api/graphql/graphql/__init__.py diff --git a/src/services/api/graphql/graphql/directives.py b/src/services/api/graphql/graphql/directives.py new file mode 100644 index 000000000..85d514de4 --- /dev/null +++ b/src/services/api/graphql/graphql/directives.py @@ -0,0 +1,32 @@ +from ariadne import SchemaDirectiveVisitor, ObjectType +from . mutations import make_resolver, make_config_file_resolver + +class DataDirective(SchemaDirectiveVisitor): + """ + Class providing implementation of 'generate' directive in schema. + + """ + def visit_field_definition(self, field, object_type): + name = f'{field.type}' + # field.type contains the return value of the mutation; trim value + # to produce canonical name + name = name.replace('Result', '', 1) + + func = make_resolver(name) + field.resolve = func + return field + +class ConfigFileDirective(SchemaDirectiveVisitor): + """ + Class providing implementation of 'configfile' directive in schema. + + """ + def visit_field_definition(self, field, object_type): + name = f'{field.type}' + # field.type contains the return value of the mutation; trim value + # to produce canonical name + name = name.replace('Result', '', 1) + + func = make_config_file_resolver(name) + field.resolve = func + return field diff --git a/src/services/api/graphql/graphql/mutations.py b/src/services/api/graphql/graphql/mutations.py new file mode 100644 index 000000000..2eb0a0b4a --- /dev/null +++ b/src/services/api/graphql/graphql/mutations.py @@ -0,0 +1,109 @@ + +from importlib import import_module +from typing import Any, Dict +from ariadne import ObjectType, convert_kwargs_to_snake_case, convert_camel_case_to_snake +from graphql import GraphQLResolveInfo +from makefun import with_signature + +from .. import state + +mutation = ObjectType("Mutation") + +def make_resolver(mutation_name): + """Dynamically generate a resolver for the mutation named in the + schema by 'mutation_name'. + + Dynamic generation is provided using the package 'makefun' (via the + decorator 'with_signature'), which provides signature-preserving + function wrappers; it provides several improvements over, say, + functools.wraps. + + :raise Exception: + encapsulating ConfigErrors, or internal errors + """ + class_name = mutation_name.replace('create', '', 1).replace('delete', '', 1) + func_base_name = convert_camel_case_to_snake(class_name) + resolver_name = f'resolve_create_{func_base_name}' + func_sig = '(obj: Any, info: GraphQLResolveInfo, data: Dict)' + + @mutation.field(mutation_name) + @convert_kwargs_to_snake_case + @with_signature(func_sig, func_name=resolver_name) + async def func_impl(*args, **kwargs): + try: + if 'data' not in kwargs: + return { + "success": False, + "errors": ['missing data'] + } + + data = kwargs['data'] + session = state.settings['app'].state.vyos_session + + mod = import_module(f'api.graphql.recipes.{func_base_name}') + klass = getattr(mod, class_name) + k = klass(session, data) + k.configure() + + return { + "success": True, + "data": data + } + except Exception as error: + return { + "success": False, + "errors": [str(error)] + } + + return func_impl + +def make_config_file_resolver(mutation_name): + op = '' + if 'save' in mutation_name: + op = 'save' + elif 'load' in mutation_name: + op = 'load' + + class_name = mutation_name.replace('save', '', 1).replace('load', '', 1) + func_base_name = convert_camel_case_to_snake(class_name) + resolver_name = f'resolve_{func_base_name}' + func_sig = '(obj: Any, info: GraphQLResolveInfo, data: Dict)' + + @mutation.field(mutation_name) + @convert_kwargs_to_snake_case + @with_signature(func_sig, func_name=resolver_name) + async def func_impl(*args, **kwargs): + try: + if 'data' not in kwargs: + return { + "success": False, + "errors": ['missing data'] + } + + data = kwargs['data'] + session = state.settings['app'].state.vyos_session + + mod = import_module(f'api.graphql.recipes.{func_base_name}') + klass = getattr(mod, class_name) + k = klass(session, data) + if op == 'save': + k.save() + elif op == 'load': + k.load() + else: + return { + "success": False, + "errors": ["Input must be saveConfigFile | loadConfigFile"] + } + + return { + "success": True, + "data": data + } + except Exception as error: + return { + "success": False, + "errors": [str(error)] + } + + return func_impl diff --git a/src/services/api/graphql/graphql/schema/config_file.graphql b/src/services/api/graphql/graphql/schema/config_file.graphql new file mode 100644 index 000000000..3096cf743 --- /dev/null +++ b/src/services/api/graphql/graphql/schema/config_file.graphql @@ -0,0 +1,27 @@ +input saveConfigFileInput { + fileName: String +} + +type saveConfigFile { + fileName: String +} + +type saveConfigFileResult { + data: saveConfigFile + success: Boolean! + errors: [String] +} + +input loadConfigFileInput { + fileName: String! +} + +type loadConfigFile { + fileName: String! +} + +type loadConfigFileResult { + data: loadConfigFile + success: Boolean! + errors: [String] +} diff --git a/src/services/api/graphql/graphql/schema/dhcp_server.graphql b/src/services/api/graphql/graphql/schema/dhcp_server.graphql new file mode 100644 index 000000000..9f741a0a5 --- /dev/null +++ b/src/services/api/graphql/graphql/schema/dhcp_server.graphql @@ -0,0 +1,35 @@ +input dhcpServerConfigInput { + sharedNetworkName: String + subnet: String + defaultRouter: String + nameServer: String + domainName: String + lease: Int + range: Int + start: String + stop: String + dnsForwardingAllowFrom: String + dnsForwardingCacheSize: Int + dnsForwardingListenAddress: String +} + +type dhcpServerConfig { + sharedNetworkName: String + subnet: String + defaultRouter: String + nameServer: String + domainName: String + lease: Int + range: Int + start: String + stop: String + dnsForwardingAllowFrom: String + dnsForwardingCacheSize: Int + dnsForwardingListenAddress: String +} + +type createDhcpServerResult { + data: dhcpServerConfig + success: Boolean! + errors: [String] +} diff --git a/src/services/api/graphql/graphql/schema/interface_ethernet.graphql b/src/services/api/graphql/graphql/schema/interface_ethernet.graphql new file mode 100644 index 000000000..fdcf97bad --- /dev/null +++ b/src/services/api/graphql/graphql/schema/interface_ethernet.graphql @@ -0,0 +1,18 @@ +input interfaceEthernetConfigInput { + interface: String + address: String + replace: Boolean = true + description: String +} + +type interfaceEthernetConfig { + interface: String + address: String + description: String +} + +type createInterfaceEthernetResult { + data: interfaceEthernetConfig + success: Boolean! + errors: [String] +} diff --git a/src/services/api/graphql/graphql/schema/schema.graphql b/src/services/api/graphql/graphql/schema/schema.graphql new file mode 100644 index 000000000..70fe0d726 --- /dev/null +++ b/src/services/api/graphql/graphql/schema/schema.graphql @@ -0,0 +1,18 @@ +schema { + query: Query + mutation: Mutation +} + +type Query { + _dummy: String +} + +directive @generate on FIELD_DEFINITION +directive @configfile on FIELD_DEFINITION + +type Mutation { + createDhcpServer(data: dhcpServerConfigInput) : createDhcpServerResult @generate + createInterfaceEthernet(data: interfaceEthernetConfigInput) : createInterfaceEthernetResult @generate + saveConfigFile(data: saveConfigFileInput) : saveConfigFileResult @configfile + loadConfigFile(data: loadConfigFileInput) : loadConfigFileResult @configfile +} diff --git a/src/services/api/graphql/recipes/__init__.py b/src/services/api/graphql/recipes/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/services/api/graphql/recipes/__init__.py diff --git a/src/services/api/graphql/recipes/config_file.py b/src/services/api/graphql/recipes/config_file.py new file mode 100644 index 000000000..850e5326e --- /dev/null +++ b/src/services/api/graphql/recipes/config_file.py @@ -0,0 +1,16 @@ + +from . recipe import Recipe + +class ConfigFile(Recipe): + def __init__(self, session, command_file): + super().__init__(session, command_file) + + # Define any custom processing of parameters here by overriding + # save/load: + # + # def save(self): + # self.data = transform_data(self.data) + # super().save() + # def load(self): + # self.data = transform_data(self.data) + # super().load() diff --git a/src/services/api/graphql/recipes/dhcp_server.py b/src/services/api/graphql/recipes/dhcp_server.py new file mode 100644 index 000000000..3edb3028e --- /dev/null +++ b/src/services/api/graphql/recipes/dhcp_server.py @@ -0,0 +1,13 @@ + +from . recipe import Recipe + +class DhcpServer(Recipe): + def __init__(self, session, command_file): + super().__init__(session, command_file) + + # Define any custom processing of parameters here by overriding + # configure: + # + # def configure(self): + # self.data = transform_data(self.data) + # super().configure() diff --git a/src/services/api/graphql/recipes/interface_ethernet.py b/src/services/api/graphql/recipes/interface_ethernet.py new file mode 100644 index 000000000..f88f5924f --- /dev/null +++ b/src/services/api/graphql/recipes/interface_ethernet.py @@ -0,0 +1,13 @@ + +from . recipe import Recipe + +class InterfaceEthernet(Recipe): + def __init__(self, session, command_file): + super().__init__(session, command_file) + + # Define any custom processing of parameters here by overriding + # configure: + # + # def configure(self): + # self.data = transform_data(self.data) + # super().configure() diff --git a/src/services/api/graphql/recipes/recipe.py b/src/services/api/graphql/recipes/recipe.py new file mode 100644 index 000000000..91d8bd67a --- /dev/null +++ b/src/services/api/graphql/recipes/recipe.py @@ -0,0 +1,68 @@ +from ariadne import convert_camel_case_to_snake +import vyos.defaults +from vyos.template import render + +class Recipe(object): + def __init__(self, session, data): + self._session = session + self.data = data + self._name = convert_camel_case_to_snake(type(self).__name__) + + @property + def data(self): + return self.__data + + @data.setter + def data(self, data): + if isinstance(data, dict): + self.__data = data + else: + raise ValueError("data must be of type dict") + + def configure(self): + session = self._session + data = self.data + func_base_name = self._name + + tmpl_file = f'{func_base_name}.tmpl' + cmd_file = f'/tmp/{func_base_name}.cmds' + tmpl_dir = vyos.defaults.directories['api_templates'] + + try: + render(cmd_file, tmpl_file, data, location=tmpl_dir) + commands = [] + with open(cmd_file) as f: + lines = f.readlines() + for line in lines: + commands.append(line.split()) + for cmd in commands: + if cmd[0] == 'set': + session.set(cmd[1:]) + elif cmd[0] == 'delete': + session.delete(cmd[1:]) + else: + raise ValueError('Operation must be "set" or "delete"') + session.commit() + except Exception as error: + raise error + + def save(self): + session = self._session + data = self.data + if 'file_name' not in data or not data['file_name']: + data['file_name'] = '/config/config.boot' + + try: + session.save_config(data['file_name']) + except Exception as error: + raise error + + def load(self): + session = self._session + data = self.data + + try: + session.load_config(data['file_name']) + session.commit() + except Exception as error: + raise error diff --git a/src/services/api/graphql/recipes/templates/dhcp_server.tmpl b/src/services/api/graphql/recipes/templates/dhcp_server.tmpl new file mode 100644 index 000000000..70de43183 --- /dev/null +++ b/src/services/api/graphql/recipes/templates/dhcp_server.tmpl @@ -0,0 +1,9 @@ +set service dhcp-server shared-network-name {{ shared_network_name }} subnet {{ subnet }} default-router {{ default_router }} +set service dhcp-server shared-network-name {{ shared_network_name }} subnet {{ subnet }} name-server {{ name_server }} +set service dhcp-server shared-network-name {{ shared_network_name }} subnet {{ subnet }} domain-name {{ domain_name }} +set service dhcp-server shared-network-name {{ shared_network_name }} subnet {{ subnet }} lease {{ lease }} +set service dhcp-server shared-network-name {{ shared_network_name }} subnet {{ subnet }} range {{ range }} start {{ start }} +set service dhcp-server shared-network-name {{ shared_network_name }} subnet {{ subnet }} range {{ range }} stop {{ stop }} +set service dns forwarding allow-from {{ dns_forwarding_allow_from }} +set service dns forwarding cache-size {{ dns_forwarding_cache_size }} +set service dns forwarding listen-address {{ dns_forwarding_listen_address }} diff --git a/src/services/api/graphql/recipes/templates/interface_ethernet.tmpl b/src/services/api/graphql/recipes/templates/interface_ethernet.tmpl new file mode 100644 index 000000000..d9d7ed691 --- /dev/null +++ b/src/services/api/graphql/recipes/templates/interface_ethernet.tmpl @@ -0,0 +1,5 @@ +{% if replace %} +delete interfaces ethernet {{ interface }} address +{% endif %} +set interfaces ethernet {{ interface }} address {{ address }} +set interfaces ethernet {{ interface }} description {{ description }} diff --git a/src/services/api/graphql/state.py b/src/services/api/graphql/state.py new file mode 100644 index 000000000..63db9f4ef --- /dev/null +++ b/src/services/api/graphql/state.py @@ -0,0 +1,4 @@ + +def init(): + global settings + settings = {} diff --git a/src/services/vyos-http-api-server b/src/services/vyos-http-api-server index 703628558..aa7ac6708 100755 --- a/src/services/vyos-http-api-server +++ b/src/services/vyos-http-api-server @@ -1,6 +1,6 @@ -#!/usr/bin/env python3 +#!/usr/share/vyos-http-api-tools/bin/python3 # -# Copyright (C) 2019 VyOS maintainers and contributors +# Copyright (C) 2019-2021 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -19,25 +19,43 @@ import os import sys import grp +import copy import json +import logging import traceback import threading -import signal +from typing import List, Union, Callable, Dict -import vyos.config - -from flask import Flask, request -from waitress import serve +import uvicorn +from fastapi import FastAPI, Depends, Request, Response, HTTPException +from fastapi.responses import HTMLResponse +from fastapi.exceptions import RequestValidationError +from fastapi.routing import APIRoute +from pydantic import BaseModel, StrictStr, validator +from starlette.datastructures import FormData +from starlette.formparsers import FormParser, MultiPartParser +from multipart.multipart import parse_options_header -from functools import wraps +from ariadne.asgi import GraphQL +import vyos.config from vyos.configsession import ConfigSession, ConfigSessionError +import api.graphql.state DEFAULT_CONFIG_FILE = '/etc/vyos/http-api.conf' CFG_GROUP = 'vyattacfg' -app = Flask(__name__) +debug = True + +logger = logging.getLogger(__name__) +logs_handler = logging.StreamHandler() +logger.addHandler(logs_handler) + +if debug: + logger.setLevel(logging.DEBUG) +else: + logger.setLevel(logging.INFO) # Giant lock! lock = threading.Lock() @@ -48,63 +66,347 @@ def load_server_config(): return config def check_auth(key_list, key): - id = None + key_id = None for k in key_list: if k['key'] == key: - id = k['id'] - return id + key_id = k['id'] + return key_id def error(code, msg): resp = {"success": False, "error": msg, "data": None} - return json.dumps(resp), code + resp = json.dumps(resp) + return HTMLResponse(resp, status_code=code) def success(data): resp = {"success": True, "data": data, "error": None} - return json.dumps(resp) - -def get_command(f): - @wraps(f) - def decorated_function(*args, **kwargs): - cmd = request.form.get("data") - if not cmd: - return error(400, "Non-empty data field is required") - try: - cmd = json.loads(cmd) - except Exception as e: - return error(400, "Failed to parse JSON: {0}".format(e)) - return f(cmd, *args, **kwargs) - - return decorated_function - -def auth_required(f): - @wraps(f) - def decorated_function(*args, **kwargs): - key = request.form.get("key") - api_keys = app.config['vyos_keys'] - id = check_auth(api_keys, key) - if not id: - return error(401, "Valid API key is required") - return f(*args, **kwargs) - - return decorated_function - -@app.route('/configure', methods=['POST']) -@get_command -@auth_required -def configure_op(commands): - session = app.config['vyos_session'] + resp = json.dumps(resp) + return HTMLResponse(resp) + +# Pydantic models for validation +# Pydantic will cast when possible, so use StrictStr +# validators added as needed for additional constraints +# schema_extra adds anotations to OpenAPI, to add examples + +class ApiModel(BaseModel): + key: StrictStr + +class BaseConfigureModel(BaseModel): + op: StrictStr + path: List[StrictStr] + value: StrictStr = None + + @validator("path", pre=True, always=True) + def check_non_empty(cls, path): + assert len(path) > 0 + return path + +class ConfigureModel(ApiModel): + op: StrictStr + path: List[StrictStr] + value: StrictStr = None + + @validator("path", pre=True, always=True) + def check_non_empty(cls, path): + assert len(path) > 0 + return path + + class Config: + schema_extra = { + "example": { + "key": "id_key", + "op": "set | delete | comment", + "path": ['config', 'mode', 'path'], + } + } + +class ConfigureListModel(ApiModel): + commands: List[BaseConfigureModel] + + class Config: + schema_extra = { + "example": { + "key": "id_key", + "commands": "list of commands", + } + } + +class RetrieveModel(ApiModel): + op: StrictStr + path: List[StrictStr] + configFormat: StrictStr = None + + class Config: + schema_extra = { + "example": { + "key": "id_key", + "op": "returnValue | returnValues | exists | showConfig", + "path": ['config', 'mode', 'path'], + "configFormat": "json (default) | json_ast | raw", + + } + } + +class ConfigFileModel(ApiModel): + op: StrictStr + file: StrictStr = None + + class Config: + schema_extra = { + "example": { + "key": "id_key", + "op": "save | load", + "file": "filename", + } + } + +class ImageModel(ApiModel): + op: StrictStr + url: StrictStr = None + name: StrictStr = None + + class Config: + schema_extra = { + "example": { + "key": "id_key", + "op": "add | delete", + "url": "imagelocation", + "name": "imagename", + } + } + +class GenerateModel(ApiModel): + op: StrictStr + path: List[StrictStr] + + class Config: + schema_extra = { + "example": { + "key": "id_key", + "op": "generate", + "path": ["op", "mode", "path"], + } + } + +class ShowModel(ApiModel): + op: StrictStr + path: List[StrictStr] + + class Config: + schema_extra = { + "example": { + "key": "id_key", + "op": "show", + "path": ["op", "mode", "path"], + } + } + +class Success(BaseModel): + success: bool + data: Union[str, bool, Dict] + error: str + +class Error(BaseModel): + success: bool = False + data: Union[str, bool, Dict] + error: str + +responses = { + 200: {'model': Success}, + 400: {'model': Error}, + 422: {'model': Error, 'description': 'Validation Error'}, + 500: {'model': Error} +} + +def auth_required(data: ApiModel): + key = data.key + api_keys = app.state.vyos_keys + key_id = check_auth(api_keys, key) + if not key_id: + raise HTTPException(status_code=401, detail="Valid API key is required") + app.state.vyos_id = key_id + +# override Request and APIRoute classes in order to convert form request to json; +# do all explicit validation here, for backwards compatability of error messages; +# the explicit validation may be dropped, if desired, in favor of native +# validation by FastAPI/Pydantic, as is used for application/json requests +class MultipartRequest(Request): + ERR_MISSING_KEY = False + ERR_MISSING_DATA = False + ERR_NOT_JSON = False + ERR_NOT_DICT = False + ERR_NO_OP = False + ERR_NO_PATH = False + ERR_EMPTY_PATH = False + ERR_PATH_NOT_LIST = False + ERR_VALUE_NOT_STRING = False + ERR_PATH_NOT_LIST_OF_STR = False + offending_command = {} + exception = None + + @property + def orig_headers(self): + self._orig_headers = super().headers + return self._orig_headers + + @property + def headers(self): + self._headers = super().headers.mutablecopy() + self._headers['content-type'] = 'application/json' + return self._headers + + async def form(self) -> FormData: + if not hasattr(self, "_form"): + assert ( + parse_options_header is not None + ), "The `python-multipart` library must be installed to use form parsing." + content_type_header = self.orig_headers.get("Content-Type") + content_type, options = parse_options_header(content_type_header) + if content_type == b"multipart/form-data": + multipart_parser = MultiPartParser(self.orig_headers, self.stream()) + self._form = await multipart_parser.parse() + elif content_type == b"application/x-www-form-urlencoded": + form_parser = FormParser(self.orig_headers, self.stream()) + self._form = await form_parser.parse() + else: + self._form = FormData() + return self._form + + async def body(self) -> bytes: + if not hasattr(self, "_body"): + forms = {} + merge = {} + body = await super().body() + self._body = body + + form_data = await self.form() + if form_data: + logger.debug("processing form data") + for k, v in form_data.multi_items(): + forms[k] = v + + if 'data' not in forms: + self.ERR_MISSING_DATA = True + else: + try: + tmp = json.loads(forms['data']) + except json.JSONDecodeError as e: + self.ERR_NOT_JSON = True + self.exception = e + tmp = {} + if isinstance(tmp, list): + merge['commands'] = tmp + else: + merge = tmp + + if 'commands' in merge: + cmds = merge['commands'] + else: + cmds = copy.deepcopy(merge) + cmds = [cmds] + + for c in cmds: + if not isinstance(c, dict): + self.ERR_NOT_DICT = True + self.offending_command = c + elif 'op' not in c: + self.ERR_NO_OP = True + self.offending_command = c + elif 'path' not in c: + self.ERR_NO_PATH = True + self.offending_command = c + elif not c['path']: + self.ERR_EMPTY_PATH = True + self.offending_command = c + elif not isinstance(c['path'], list): + self.ERR_PATH_NOT_LIST = True + self.offending_command = c + elif not all(isinstance(el, str) for el in c['path']): + self.ERR_PATH_NOT_LIST_OF_STR = True + self.offending_command = c + elif 'value' in c and not isinstance(c['value'], str): + self.ERR_VALUE_NOT_STRING = True + self.offending_command = c + + if 'key' not in forms and 'key' not in merge: + self.ERR_MISSING_KEY = True + if 'key' in forms and 'key' not in merge: + merge['key'] = forms['key'] + + new_body = json.dumps(merge) + new_body = new_body.encode() + self._body = new_body + + return self._body + +class MultipartRoute(APIRoute): + def get_route_handler(self) -> Callable: + original_route_handler = super().get_route_handler() + + async def custom_route_handler(request: Request) -> Response: + request = MultipartRequest(request.scope, request.receive) + endpoint = request.url.path + try: + response: Response = await original_route_handler(request) + except HTTPException as e: + return error(e.status_code, e.detail) + except Exception as e: + if request.ERR_MISSING_KEY: + return error(422, "Valid API key is required") + if request.ERR_MISSING_DATA: + return error(422, "Non-empty data field is required") + if request.ERR_NOT_JSON: + return error(400, "Failed to parse JSON: {0}".format(request.exception)) + if endpoint == '/configure': + if request.ERR_NOT_DICT: + return error(400, "Malformed command \"{0}\": any command must be a dict".format(json.dumps(request.offending_command))) + if request.ERR_NO_OP: + return error(400, "Malformed command \"{0}\": missing \"op\" field".format(json.dumps(request.offending_command))) + if request.ERR_NO_PATH: + return error(400, "Malformed command \"{0}\": missing \"path\" field".format(json.dumps(request.offending_command))) + if request.ERR_EMPTY_PATH: + return error(400, "Malformed command \"{0}\": empty path".format(json.dumps(request.offending_command))) + if request.ERR_PATH_NOT_LIST: + return error(400, "Malformed command \"{0}\": \"path\" field must be a list".format(json.dumps(request.offending_command))) + if request.ERR_VALUE_NOT_STRING: + return error(400, "Malformed command \"{0}\": \"value\" field must be a string".format(json.dumps(request.offending_command))) + if request.ERR_PATH_NOT_LIST_OF_STR: + return error(400, "Malformed command \"{0}\": \"path\" field must be a list of strings".format(json.dumps(request.offending_command))) + if endpoint in ('/retrieve','/generate','/show'): + if request.ERR_NO_OP or request.ERR_NO_PATH: + return error(400, "Missing required field. \"op\" and \"path\" fields are required") + if endpoint in ('/config-file', '/image'): + if request.ERR_NO_OP: + return error(400, "Missing required field \"op\"") + + raise e + + return response + + return custom_route_handler + +app = FastAPI(debug=True, + title="VyOS API", + version="0.1.0", + responses={**responses}, + dependencies=[Depends(auth_required)]) + +app.router.route_class = MultipartRoute + +@app.exception_handler(RequestValidationError) +async def validation_exception_handler(request, exc): + return error(400, str(exc.errors()[0])) + +@app.post('/configure') +def configure_op(data: Union[ConfigureModel, ConfigureListModel]): + session = app.state.vyos_session env = session.get_session_env() config = vyos.config.Config(session_env=env) - strict_field = request.form.get("strict") - if strict_field == "true": - strict = True - else: - strict = False - # Allow users to pass just one command - if not isinstance(commands, list): - commands = [commands] + if not isinstance(data, ConfigureListModel): + data = [data] + else: + data = data.commands # We don't want multiple people/apps to be able to commit at once, # or modify the shared session while someone else is doing the same, @@ -114,53 +416,25 @@ def configure_op(commands): status = 200 error_msg = None try: - for c in commands: - # What we've got may not even be a dict - if not isinstance(c, dict): - raise ConfigSessionError("Malformed command \"{0}\": any command must be a dict".format(json.dumps(c))) - - # Missing op or path is a show stopper - if not ('op' in c): - raise ConfigSessionError("Malformed command \"{0}\": missing \"op\" field".format(json.dumps(c))) - if not ('path' in c): - raise ConfigSessionError("Malformed command \"{0}\": missing \"path\" field".format(json.dumps(c))) - - # Missing value is fine, substitute for empty string - if 'value' in c: - value = c['value'] - else: - value = "" - - op = c['op'] - path = c['path'] - - if not path: - raise ConfigSessionError("Malformed command \"{0}\": empty path".format(json.dumps(c))) - - # Type checking - if not isinstance(path, list): - raise ConfigSessionError("Malformed command \"{0}\": \"path\" field must be a list".format(json.dumps(c))) - - if not isinstance(value, str): - raise ConfigSessionError("Malformed command \"{0}\": \"value\" field must be a string".format(json.dumps(c))) + for c in data: + op = c.op + path = c.path - # Account for the case when value field is present and set to null - if not value: + if c.value: + value = c.value + else: value = "" - # For vyos.configsessios calls that have no separate value arguments, + # For vyos.configsession calls that have no separate value arguments, # and for type checking too - try: - cfg_path = " ".join(path + [value]).strip() - except TypeError: - raise ConfigSessionError("Malformed command \"{0}\": \"path\" field must be a list of strings".format(json.dumps(c))) + cfg_path = " ".join(path + [value]).strip() if op == 'set': # XXX: it would be nice to do a strict check for "path already exists", # but there's probably no way to do that session.set(path, value=value) elif op == 'delete': - if strict and not config.exists(cfg_path): + if app.state.vyos_strict and not config.exists(cfg_path): raise ConfigSessionError("Cannot delete [{0}]: path/value does not exist".format(cfg_path)) session.delete(path, value=value) elif op == 'comment': @@ -169,16 +443,16 @@ def configure_op(commands): raise ConfigSessionError("\"{0}\" is not a valid operation".format(op)) # end for session.commit() - print("Configuration modified via HTTP API using key \"{0}\"".format(id)) + logger.info(f"Configuration modified via HTTP API using key '{app.state.vyos_id}'") except ConfigSessionError as e: session.discard() status = 400 - if app.config['vyos_debug']: - print(traceback.format_exc(), file=sys.stderr) + if app.state.vyos_debug: + logger.critical(f"ConfigSessionError:\n {traceback.format_exc()}") error_msg = str(e) except Exception as e: session.discard() - print(traceback.format_exc(), file=sys.stderr) + logger.critical(traceback.format_exc()) status = 500 # Don't give the details away to the outer world @@ -188,22 +462,17 @@ def configure_op(commands): if status != 200: return error(status, error_msg) - else: - return success(None) -@app.route('/retrieve', methods=['POST']) -@get_command -@auth_required -def retrieve_op(command): - session = app.config['vyos_session'] + return success(None) + +@app.post("/retrieve") +def retrieve_op(data: RetrieveModel): + session = app.state.vyos_session env = session.get_session_env() config = vyos.config.Config(session_env=env) - try: - op = command['op'] - path = " ".join(command['path']) - except KeyError: - return error(400, "Missing required field. \"op\" and \"path\" fields are required") + op = data.op + path = " ".join(data.path) try: if op == 'returnValue': @@ -214,10 +483,10 @@ def retrieve_op(command): res = config.exists(path) elif op == 'showConfig': config_format = 'json' - if 'configFormat' in command: - config_format = command['configFormat'] + if data.configFormat: + config_format = data.configFormat - res = session.show_config(path=command['path']) + res = session.show_config(path=data.path) if config_format == 'json': config_tree = vyos.configtree.ConfigTree(res) res = json.loads(config_tree.to_json()) @@ -233,33 +502,28 @@ def retrieve_op(command): except ConfigSessionError as e: return error(400, str(e)) except Exception as e: - print(traceback.format_exc(), file=sys.stderr) + logger.critical(traceback.format_exc()) return error(500, "An internal error occured. Check the logs for details.") return success(res) -@app.route('/config-file', methods=['POST']) -@get_command -@auth_required -def config_file_op(command): - session = app.config['vyos_session'] +@app.post('/config-file') +def config_file_op(data: ConfigFileModel): + session = app.state.vyos_session - try: - op = command['op'] - except KeyError: - return error(400, "Missing required field \"op\"") + op = data.op try: if op == 'save': - try: - path = command['file'] - except KeyError: + if data.file: + path = data.file + else: path = '/config/config.boot' res = session.save_config(path) elif op == 'load': - try: - path = command['file'] - except KeyError: + if data.file: + path = data.file + else: return error(400, "Missing required field \"file\"") res = session.migrate_and_load_config(path) res = session.commit() @@ -268,33 +532,28 @@ def config_file_op(command): except ConfigSessionError as e: return error(400, str(e)) except Exception as e: - print(traceback.format_exc(), file=sys.stderr) + logger.critical(traceback.format_exc()) return error(500, "An internal error occured. Check the logs for details.") return success(res) -@app.route('/image', methods=['POST']) -@get_command -@auth_required -def image_op(command): - session = app.config['vyos_session'] +@app.post('/image') +def image_op(data: ImageModel): + session = app.state.vyos_session - try: - op = command['op'] - except KeyError: - return error(400, "Missing required field \"op\"") + op = data.op try: if op == 'add': - try: - url = command['url'] - except KeyError: + if data.url: + url = data.url + else: return error(400, "Missing required field \"url\"") res = session.install_image(url) elif op == 'delete': - try: - name = command['name'] - except KeyError: + if data.name: + name = data.name + else: return error(400, "Missing required field \"name\"") res = session.remove_image(name) else: @@ -302,26 +561,17 @@ def image_op(command): except ConfigSessionError as e: return error(400, str(e)) except Exception as e: - print(traceback.format_exc(), file=sys.stderr) + logger.critical(traceback.format_exc()) return error(500, "An internal error occured. Check the logs for details.") return success(res) +@app.post('/generate') +def generate_op(data: GenerateModel): + session = app.state.vyos_session -@app.route('/generate', methods=['POST']) -@get_command -@auth_required -def generate_op(command): - session = app.config['vyos_session'] - - try: - op = command['op'] - path = command['path'] - except KeyError: - return error(400, "Missing required field. \"op\" and \"path\" fields are required") - - if not isinstance(path, list): - return error(400, "Malformed command: \"path\" field must be a list of strings") + op = data.op + path = data.path try: if op == 'generate': @@ -331,25 +581,17 @@ def generate_op(command): except ConfigSessionError as e: return error(400, str(e)) except Exception as e: - print(traceback.format_exc(), file=sys.stderr) + logger.critical(traceback.format_exc()) return error(500, "An internal error occured. Check the logs for details.") return success(res) -@app.route('/show', methods=['POST']) -@get_command -@auth_required -def show_op(command): - session = app.config['vyos_session'] +@app.post('/show') +def show_op(data: ShowModel): + session = app.state.vyos_session - try: - op = command['op'] - path = command['path'] - except KeyError: - return error(400, "Missing required field. \"op\" and \"path\" fields are required") - - if not isinstance(path, list): - return error(400, "Malformed command: \"path\" field must be a list of strings") + op = data.op + path = data.path try: if op == 'show': @@ -359,13 +601,24 @@ def show_op(command): except ConfigSessionError as e: return error(400, str(e)) except Exception as e: - print(traceback.format_exc(), file=sys.stderr) + logger.critical(traceback.format_exc()) return error(500, "An internal error occured. Check the logs for details.") return success(res) -def shutdown(): - raise KeyboardInterrupt +### +# GraphQL integration +### + +from api.graphql.bindings import generate_schema + +api.graphql.state.init() + +schema = generate_schema() + +app.add_route('/graphql', GraphQL(schema, debug=True)) + +### if __name__ == '__main__': # systemd's user and group options don't work, do it by hand here, @@ -379,22 +632,23 @@ if __name__ == '__main__': try: server_config = load_server_config() - except Exception as e: - print("Failed to load the HTTP API server config: {0}".format(e)) + except Exception as err: + logger.critical(f"Failed to load the HTTP API server config: {err}") - session = ConfigSession(os.getpid()) + config_session = ConfigSession(os.getpid()) - app.config['vyos_session'] = session - app.config['vyos_keys'] = server_config['api_keys'] - app.config['vyos_debug'] = server_config['debug'] + app.state.vyos_session = config_session + app.state.vyos_keys = server_config['api_keys'] - def sig_handler(signum, frame): - shutdown() + app.state.vyos_debug = bool(server_config['debug'] == 'true') + app.state.vyos_strict = bool(server_config['strict'] == 'true') - signal.signal(signal.SIGTERM, sig_handler) + api.graphql.state.settings['app'] = app try: - serve(app, host=server_config["listen_address"], - port=server_config["port"]) - except OSError as e: - print(f"OSError {e}") + uvicorn.run(app, host=server_config["listen_address"], + port=int(server_config["port"]), + proxy_headers=True) + except OSError as err: + logger.critical(f"OSError {err}") + sys.exit(1) diff --git a/src/systemd/vyos-http-api.service b/src/systemd/vyos-http-api.service index 4fa68b4ff..ba5df5984 100644 --- a/src/systemd/vyos-http-api.service +++ b/src/systemd/vyos-http-api.service @@ -5,9 +5,8 @@ Requires=vyos-router.service [Service] ExecStartPre=/usr/libexec/vyos/init/vyos-config -ExecStart=/usr/bin/python3 -u /usr/libexec/vyos/services/vyos-http-api-server +ExecStart=/usr/libexec/vyos/services/vyos-http-api-server Type=idle -KillMode=process SyslogIdentifier=vyos-http-api SyslogFacility=daemon |