Merge remote-tracking branch 'upstream/current' into current

7 files changed, 168 insertions, 262 deletions

diff --git a/src/conf_mode/dns_forwarding.py b/src/conf_mode/dns_forwarding.py
index 51631dc16..53bc37882 100755
--- a/src/conf_mode/dns_forwarding.py
+++ b/src/conf_mode/dns_forwarding.py
@@ -127,6 +127,7 @@ def verify(conf, dns):
                     f'Error: No server configured for domain {domain}'))
 
     no_system_nameservers = False
+    conf.set_level([])
     if dns['system'] and not (
             conf.exists(['system', 'name-server']) or
             conf.exists(['system', 'name-servers-dhcp']) ):
diff --git a/src/conf_mode/dynamic_dns.py b/src/conf_mode/dynamic_dns.py
index 57c910a68..93e995b78 100755
--- a/src/conf_mode/dynamic_dns.py
+++ b/src/conf_mode/dynamic_dns.py
@@ -17,14 +17,13 @@
 import os
 
 from sys import exit
-from copy import deepcopy
-from stat import S_IRUSR, S_IWUSR
 
 from vyos.config import Config
-from vyos import ConfigError
-from vyos.util import call
+from vyos.configdict import dict_merge
 from vyos.template import render
-
+from vyos.util import call
+from vyos.xml import defaults
+from vyos import ConfigError
 from vyos import airbag
 airbag.enable()
 
@@ -45,197 +44,101 @@ default_service_protocol = {
     'zoneedit': 'zoneedit1'
 }
 
-default_config_data = {
-    'interfaces': [],
-    'deleted': False
-}
-
 def get_config(config=None):
-    dyndns = deepcopy(default_config_data)
     if config:
         conf = config
     else:
         conf = Config()
-    base_level = ['service', 'dns', 'dynamic']
 
+    base_level = ['service', 'dns', 'dynamic']
     if not conf.exists(base_level):
-        dyndns['deleted'] = True
-        return dyndns
-
-    for interface in conf.list_nodes(base_level + ['interface']):
-        node = {
-            'interface': interface,
-            'rfc2136': [],
-            'service': [],
-            'web_skip': '',
-            'web_url': ''
-        }
-
-        # set config level to e.g. "service dns dynamic interface eth0"
-        conf.set_level(base_level + ['interface', interface])
-        # Handle RFC2136 - Dynamic Updates in the Domain Name System
-        for rfc2136 in conf.list_nodes(['rfc2136']):
-            rfc = {
-                'name': rfc2136,
-                'keyfile': '',
-                'record': [],
-                'server': '',
-                'ttl': '600',
-                'zone': ''
-            }
-
-            # set config level
-            conf.set_level(base_level + ['interface', interface, 'rfc2136', rfc2136])
-
-            if conf.exists(['key']):
-                rfc['keyfile'] = conf.return_value(['key'])
-
-            if conf.exists(['record']):
-                rfc['record'] = conf.return_values(['record'])
-
-            if conf.exists(['server']):
-                rfc['server'] = conf.return_value(['server'])
-
-            if conf.exists(['ttl']):
-                rfc['ttl'] = conf.return_value(['ttl'])
-
-            if conf.exists(['zone']):
-                rfc['zone'] = conf.return_value(['zone'])
-
-            node['rfc2136'].append(rfc)
-
-        # set config level to e.g. "service dns dynamic interface eth0"
-        conf.set_level(base_level + ['interface', interface])
-        # Handle DynDNS service providers
-        for service in conf.list_nodes(['service']):
-            srv = {
-                'provider': service,
-                'host': [],
-                'login': '',
-                'password': '',
-                'protocol': '',
-                'server': '',
-                'custom' : False,
-                'zone' : ''
-            }
-
-            # set config level
-            conf.set_level(base_level + ['interface', interface, 'service', service])
-
-            # preload protocol from default service mapping
-            if service in default_service_protocol.keys():
-                srv['protocol'] = default_service_protocol[service]
-            else:
-                srv['custom'] = True
-
-            if conf.exists(['login']):
-                srv['login'] = conf.return_value(['login'])
-
-            if conf.exists(['host-name']):
-                srv['host'] = conf.return_values(['host-name'])
-
-            if conf.exists(['protocol']):
-                srv['protocol'] = conf.return_value(['protocol'])
-
-            if conf.exists(['password']):
-                srv['password'] = conf.return_value(['password'])
-
-            if conf.exists(['server']):
-                srv['server'] = conf.return_value(['server'])
-
-            if conf.exists(['zone']):
-                srv['zone'] = conf.return_value(['zone'])
-            elif srv['provider'] == 'cloudflare':
-                # default populate zone entry with bar.tld if
-                # host-name is foo.bar.tld
-                srv['zone'] = srv['host'][0].split('.',1)[1]
-
-            node['service'].append(srv)
-
-        # Set config back to appropriate level for these options
-        conf.set_level(base_level + ['interface', interface])
-
-        # Additional settings in CLI
-        if conf.exists(['use-web', 'skip']):
-            node['web_skip'] = conf.return_value(['use-web', 'skip'])
-
-        if conf.exists(['use-web', 'url']):
-            node['web_url'] = conf.return_value(['use-web', 'url'])
-
-        # set config level back to top level
-        conf.set_level(base_level)
-
-        dyndns['interfaces'].append(node)
+        return None
+
+    dyndns = conf.get_config_dict(base_level, key_mangling=('-', '_'), get_first_key=True)
+
+    # We have gathered the dict representation of the CLI, but there are default
+    # options which we need to update into the dictionary retrived.
+    for interface in dyndns['interface']:
+        if 'service' in dyndns['interface'][interface]:
+            # 'Autodetect' protocol used by DynDNS service
+            for service in dyndns['interface'][interface]['service']:
+                if service in default_service_protocol:
+                    dyndns['interface'][interface]['service'][service].update(
+                        {'protocol' : default_service_protocol.get(service)})
+                else:
+                    dyndns['interface'][interface]['service'][service].update(
+                        {'custom': ''})
+
+        if 'rfc2136' in dyndns['interface'][interface]:
+            default_values = defaults(base_level + ['interface', 'rfc2136'])
+            for rfc2136 in dyndns['interface'][interface]['rfc2136']:
+                dyndns['interface'][interface]['rfc2136'][rfc2136] = dict_merge(
+                    default_values, dyndns['interface'][interface]['rfc2136'][rfc2136])
 
     return dyndns
 
 def verify(dyndns):
     # bail out early - looks like removal from running config
-    if dyndns['deleted']:
+    if not dyndns:
         return None
 
     # A 'node' corresponds to an interface
-    for node in dyndns['interfaces']:
+    if 'interface' not in dyndns:
+        return None
 
+    for interface in dyndns['interface']:
         # RFC2136 - configuration validation
-        for rfc2136 in node['rfc2136']:
-            if not rfc2136['record']:
-                raise ConfigError('Set key for service "{0}" to send DDNS updates for interface "{1}"'.format(rfc2136['name'], node['interface']))
+        if 'rfc2136' in dyndns['interface'][interface]:
+            for rfc2136, config in dyndns['interface'][interface]['rfc2136'].items():
 
-            if not rfc2136['zone']:
-                raise ConfigError('Set zone for service "{0}" to send DDNS updates for interface "{1}"'.format(rfc2136['name'], node['interface']))
+                for tmp in ['record', 'zone', 'server', 'key']:
+                    if tmp not in config:
+                        raise ConfigError(f'"{tmp}" required for rfc2136 based '
+                                          f'DynDNS service on "{interface}"')
 
-            if not rfc2136['keyfile']:
-                raise ConfigError('Set keyfile for service "{0}" to send DDNS updates for interface "{1}"'.format(rfc2136['name'], node['interface']))
-            else:
-                if not os.path.isfile(rfc2136['keyfile']):
-                    raise ConfigError('Keyfile for service "{0}" to send DDNS updates for interface "{1}" does not exist'.format(rfc2136['name'], node['interface']))
-
-            if not rfc2136['server']:
-                raise ConfigError('Set server for service "{0}" to send DDNS updates for interface "{1}"'.format(rfc2136['name'], node['interface']))
+                if not os.path.isfile(config['key']):
+                    raise ConfigError(f'"key"-file not found for rfc2136 based '
+                                      f'DynDNS service on "{interface}"')
 
         # DynDNS service provider - configuration validation
-        for service in node['service']:
-            if not service['host']:
-                raise ConfigError('Set host-name for service "{0}" to send DDNS updates for interface "{1}"'.format(service['provider'], node['interface']))
+        if 'service' in dyndns['interface'][interface]:
+            for service, config in dyndns['interface'][interface]['service'].items():
+                error_msg = f'required for DynDNS service "{service}" on "{interface}"'
+                if 'host_name' not in config:
+                    raise ConfigError(f'"host-name" {error_msg}')
 
-            if not service['login']:
-                raise ConfigError('Set login for service "{0}" to send DDNS updates for interface "{1}"'.format(service['provider'], node['interface']))
+                if 'login' not in config:
+                    raise ConfigError(f'"login" (username) {error_msg}')
 
-            if not service['password']:
-                raise ConfigError('Set password for service "{0}" to send DDNS updates for interface "{1}"'.format(service['provider'], node['interface']))
+                if 'password' not in config:
+                    raise ConfigError(f'"password" {error_msg}')
 
-            if service['custom'] is True:
-                if not service['protocol']:
-                    raise ConfigError('Set protocol for service "{0}" to send DDNS updates for interface "{1}"'.format(service['provider'], node['interface']))
+                if 'zone' in config:
+                    if service != 'cloudflare':
+                        raise ConfigError(f'"zone" option only supported with CloudFlare')
 
-                if not service['server']:
-                    raise ConfigError('Set server for service "{0}" to send DDNS updates for interface "{1}"'.format(service['provider'], node['interface']))
+                if 'custom' in config:
+                    if 'protocol' not in config:
+                        raise ConfigError(f'"protocol" {error_msg}')
 
-            if service['zone']:
-                if service['provider'] != 'cloudflare':
-                    raise ConfigError('Zone option not allowed for "{0}", it can only be used for CloudFlare'.format(service['provider']))
+                    if 'server' not in config:
+                        raise ConfigError(f'"server" {error_msg}')
 
     return None
 
 def generate(dyndns):
     # bail out early - looks like removal from running config
-    if dyndns['deleted']:
+    if not dyndns:
         return None
 
-    render(config_file, 'dynamic-dns/ddclient.conf.tmpl', dyndns)
-
-    # Config file must be accessible only by its owner
-    os.chmod(config_file, S_IRUSR | S_IWUSR)
-
+    render(config_file, 'dynamic-dns/ddclient.conf.tmpl', dyndns, trim_blocks=True, permission=0o600)
     return None
 
 def apply(dyndns):
-    if dyndns['deleted']:
+    if not dyndns:
         call('systemctl stop ddclient.service')
         if os.path.exists(config_file):
             os.unlink(config_file)
-
     else:
         call('systemctl restart ddclient.service')
 
diff --git a/src/conf_mode/intel_qat.py b/src/conf_mode/intel_qat.py
index 1e5101a9f..86dbccaf0 100755
--- a/src/conf_mode/intel_qat.py
+++ b/src/conf_mode/intel_qat.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2019 VyOS maintainers and contributors
+# Copyright (C) 2019-2020 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -13,94 +13,87 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-#
 
-import sys
 import os
 import re
 
+from sys import exit
+
 from vyos.config import Config
-from vyos import ConfigError
 from vyos.util import popen, run
-
+from vyos import ConfigError
 from vyos import airbag
 airbag.enable()
 
-# Define for recovering
-gl_ipsec_conf = None
+qat_init_script = '/etc/init.d/qat_service'
 
 def get_config(config=None):
-  if config:
-      c = config
-  else:
-      c = Config()
-  config_data = {
-    'qat_conf'     : None,
-    'ipsec_conf'   : None,
-    'openvpn_conf' : None,
-  }
-
-  if c.exists('system acceleration qat'):
-    config_data['qat_conf'] = True
-
-  if c.exists('vpn ipsec '):
-    gl_ipsec_conf = True
-    config_data['ipsec_conf'] = True
-
-  if c.exists('interfaces openvpn'):
-    config_data['openvpn_conf'] = True
-
-  return config_data
-
-# Control configured VPN service which can use QAT
-def vpn_control(action):
-  # XXX: Should these commands report failure
-  if action == 'restore' and gl_ipsec_conf:
-    return run('ipsec start')
-  return run(f'ipsec {action}')
-
-def verify(c):
-  # Check if QAT service installed
-  if not os.path.exists('/etc/init.d/qat_service'):
-    raise ConfigError("Warning: QAT init file not found")
-
-  if c['qat_conf'] == None:
-    return
-
-  # Check if QAT device exist
-  output, err = popen('lspci -nn', decode='utf-8')
-  if not err:
-    data = re.findall('(8086:19e2)|(8086:37c8)|(8086:0435)|(8086:6f54)', output)
-    #If QAT devices found
-    if not data:
-      print("\t No QAT acceleration device found")
-      sys.exit(1)
-
-def apply(c):
-  if c['ipsec_conf']:
+    if config:
+        conf = config
+    else:
+        conf = Config()
+
+    data = {}
+
+    if conf.exists(['system', 'acceleration', 'qat']):
+        data.update({'qat_enable' : ''})
+
+    if conf.exists(['vpn', 'ipsec']):
+        data.update({'ipsec' : ''})
+
+    if conf.exists(['interfaces', 'openvpn']):
+        data.update({'openvpn' : ''})
+
+    return data
+
+
+def vpn_control(action, force_ipsec=False):
+    # XXX: Should these commands report failure?
+    if action == 'restore' and force_ipsec:
+        return run('ipsec start')
+
+    return run(f'ipsec {action}')
+
+
+def verify(qat):
+    if 'qat_enable' not in qat:
+        return
+
+    # Check if QAT service installed
+    if not os.path.exists(qat_init_script):
+        raise ConfigError('QAT init script not found')
+
+    # Check if QAT device exist
+    output, err = popen('lspci -nn', decode='utf-8')
+    if not err:
+        data = re.findall(
+            '(8086:19e2)|(8086:37c8)|(8086:0435)|(8086:6f54)', output)
+        # If QAT devices found
+        if not data:
+            raise ConfigError('No QAT acceleration device found')
+
+def apply(qat):
     # Shutdown VPN service which can use QAT
-    vpn_control('stop')
+    if 'ipsec' in qat:
+        vpn_control('stop')
+
+    # Enable/Disable QAT service
+    if 'qat_enable' in qat:
+        run(f'{qat_init_script} start')
+    else:
+        run(f'{qat_init_script} stop')
 
-  # Disable QAT service
-  if c['qat_conf'] == None:
-    run('/etc/init.d/qat_service stop')
-    if c['ipsec_conf']:
-      vpn_control('start')
-    return
+    # Recover VPN service
+    if 'ipsec' in qat:
+        vpn_control('start')
 
-  # Run qat init.d script
-  run('/etc/init.d/qat_service start')
-  if c['ipsec_conf']:
-    # Recovery VPN service
-    vpn_control('start')
 
 if __name__ == '__main__':
-  try:
-    c = get_config()
-    verify(c)
-    apply(c)
-  except ConfigError as e:
-    print(e)
-    vpn_control('restore')
-    sys.exit(1)
+    try:
+        c = get_config()
+        verify(c)
+        apply(c)
+    except ConfigError as e:
+        print(e)
+        vpn_control('restore', force_ipsec=('ipsec' in c))
+        exit(1)
diff --git a/src/conf_mode/interfaces-bonding.py b/src/conf_mode/interfaces-bonding.py
index 16e6e4f6e..a9679b47c 100755
--- a/src/conf_mode/interfaces-bonding.py
+++ b/src/conf_mode/interfaces-bonding.py
@@ -29,6 +29,7 @@ from vyos.configverify import verify_source_interface
 from vyos.configverify import verify_vlan_config
 from vyos.configverify import verify_vrf
 from vyos.ifconfig import BondIf
+from vyos.ifconfig import Section
 from vyos.validate import is_member
 from vyos.validate import has_address_configured
 from vyos import ConfigError
@@ -69,31 +70,33 @@ def get_config(config=None):
     # into a dictionary - we will use this to add additional information
     # later on for wach member
     if 'member' in bond and 'interface' in bond['member']:
-        # first convert it to a list if only one member is given
-        if isinstance(bond['member']['interface'], str):
-            bond['member']['interface'] = [bond['member']['interface']]
-
-        tmp={}
-        for interface in bond['member']['interface']:
-            tmp.update({interface: {}})
-
-        bond['member']['interface'] = tmp
+        # convert list if member interfaces to a dictionary
+        bond['member']['interface'] = dict.fromkeys(
+            bond['member']['interface'], {})
 
     if 'mode' in bond:
         bond['mode'] = get_bond_mode(bond['mode'])
 
     tmp = leaf_node_changed(conf, ['mode'])
-    if tmp:
-        bond.update({'shutdown_required': ''})
+    if tmp: bond.update({'shutdown_required': {}})
 
     # determine which members have been removed
-    tmp = leaf_node_changed(conf, ['member', 'interface'])
-    if tmp:
-        bond.update({'shutdown_required': ''})
-        if 'member' in bond:
-            bond['member'].update({'interface_remove': tmp })
-        else:
-            bond.update({'member': {'interface_remove': tmp }})
+    interfaces_removed = leaf_node_changed(conf, ['member', 'interface'])
+    if interfaces_removed:
+        bond.update({'shutdown_required': {}})
+        if 'member' not in bond:
+            bond.update({'member': {}})
+
+        tmp = {}
+        for interface in interfaces_removed:
+            section = Section.section(interface) # this will be 'ethernet' for 'eth0'
+            if conf.exists(['insterfaces', section, interface, 'disable']):
+                tmp.update({interface : {'disable': ''}})
+            else:
+                tmp.update({interface : {}})
+
+        # also present the interfaces to be removed from the bond as dictionary
+        bond['member'].update({'interface_remove': tmp})
 
     if 'member' in bond and 'interface' in bond['member']:
         for interface, interface_config in bond['member']['interface'].items():
@@ -109,8 +112,7 @@ def get_config(config=None):
 
             # bond members must not have an assigned address
             tmp = has_address_configured(conf, interface)
-            if tmp:
-                interface_config.update({'has_address' : ''})
+            if tmp: interface_config.update({'has_address' : ''})
 
     return bond
 
diff --git a/src/conf_mode/interfaces-l2tpv3.py b/src/conf_mode/interfaces-l2tpv3.py
index 8250a3df8..144cee5fe 100755
--- a/src/conf_mode/interfaces-l2tpv3.py
+++ b/src/conf_mode/interfaces-l2tpv3.py
@@ -56,10 +56,11 @@ def get_config(config=None):
     # To delete an l2tpv3 interface we need the current tunnel and session-id
     if 'deleted' in l2tpv3:
         tmp = leaf_node_changed(conf, ['tunnel-id'])
-        l2tpv3.update({'tunnel_id': tmp})
+        # leaf_node_changed() returns a list
+        l2tpv3.update({'tunnel_id': tmp[0]})
 
         tmp = leaf_node_changed(conf, ['session-id'])
-        l2tpv3.update({'session_id': tmp})
+        l2tpv3.update({'session_id': tmp[0]})
 
     return l2tpv3
 
diff --git a/src/conf_mode/interfaces-wireless.py b/src/conf_mode/interfaces-wireless.py
index 9861f72db..c6c843e7b 100755
--- a/src/conf_mode/interfaces-wireless.py
+++ b/src/conf_mode/interfaces-wireless.py
@@ -33,6 +33,7 @@ from vyos.configverify import verify_vrf
 from vyos.ifconfig import WiFiIf
 from vyos.template import render
 from vyos.util import call
+from vyos.util import vyos_dict_search
 from vyos import ConfigError
 from vyos import airbag
 airbag.enable()
@@ -213,6 +214,11 @@ def generate(wifi):
             mac.dialect = mac_unix_expanded
             wifi['mac'] = str(mac)
 
+    # XXX: Jinja2 can not operate on a dictionary key when it starts of with a number
+    if '40mhz_incapable' in (vyos_dict_search('capabilities.ht', wifi) or []):
+        wifi['capabilities']['ht']['fourtymhz_incapable'] = wifi['capabilities']['ht']['40mhz_incapable']
+        del wifi['capabilities']['ht']['40mhz_incapable']
+
     # render appropriate new config files depending on access-point or station mode
     if wifi['type'] == 'access-point':
         render(hostapd_conf.format(**wifi), 'wifi/hostapd.conf.tmpl', wifi, trim_blocks=True)
diff --git a/src/conf_mode/vpn_anyconnect.py b/src/conf_mode/vpn_openconnect.py
index 158e1a117..af8604972 100755
--- a/src/conf_mode/vpn_anyconnect.py
+++ b/src/conf_mode/vpn_openconnect.py
@@ -42,7 +42,7 @@ def get_hash(password):
 
 def get_config():
     conf = Config()
-    base = ['vpn', 'anyconnect']
+    base = ['vpn', 'openconnect']
     if not conf.exists(base):
         return None
 
@@ -61,24 +61,24 @@ def verify(ocserv):
         if "mode" in ocserv["authentication"]:
             if "local" in ocserv["authentication"]["mode"]:
                 if not ocserv["authentication"]["local_users"] or not ocserv["authentication"]["local_users"]["username"]:
-                    raise ConfigError('Anyconect mode local required at leat one user')
+                    raise ConfigError('openconnect mode local required at leat one user')
                 else:
                     for user in ocserv["authentication"]["local_users"]["username"]:
                         if not "password" in ocserv["authentication"]["local_users"]["username"][user]:
                             raise ConfigError(f'password required for user {user}')
         else:
-            raise ConfigError('anyconnect authentication mode required')
+            raise ConfigError('openconnect authentication mode required')
     else:
-        raise ConfigError('anyconnect authentication credentials required')
+        raise ConfigError('openconnect authentication credentials required')
 
     # Check ssl
     if "ssl" in ocserv:
         req_cert = ['ca_cert_file', 'cert_file', 'key_file']
         for cert in req_cert:
             if not cert in ocserv["ssl"]:
-                raise ConfigError('anyconnect ssl {0} required'.format(cert.replace('_', '-')))
+                raise ConfigError('openconnect ssl {0} required'.format(cert.replace('_', '-')))
     else:
-        raise ConfigError('anyconnect ssl required')
+        raise ConfigError('openconnect ssl required')
 
     # Check network settings
     if "network_settings" in ocserv:
@@ -90,7 +90,7 @@ def verify(ocserv):
         else:
             ocserv["network_settings"]["push_route"] = "default"  
     else:
-        raise ConfigError('anyconnect network settings required')
+        raise ConfigError('openconnect network settings required')
 
 
 def generate(ocserv):