Merge branch 'current' of github.com:vyos/vyos-1x into equuleus

* 'current' of github.com:vyos/vyos-1x:
  T1749: support multiple ranges in the numeric validator.
  dhcp-server: T1745: bugfix corner case on static-assignments
  system-proxy: T1741 - Add system wide proxy setting
  wireguard - remove endpoint check to enable roaming connections
  system-proxy: T1741 - Add system wide proxy setting CLI implementation
  Python/ifconfig: T1712: always start DHCP when configured
  Python/ifconfig: T1557: get_status() must use admin state not operstate
  bgp: T1490: fix migrator file permissions
  snmp: T1737: add missing completion helpers
  Revert "Python/ifconfig: T1712: wait when changing interface state"
  snmpd: T1705 - High CPU usage by bgpd when snmp is active
  Revert "snmpd: T1705 - High CPU usage by bgpd when snmp is active"
  openvpn: T1548: clean out import statements
  ssh.py: check if file exists before deleting it
  [BGP] T1490: Added migration for obsoleted 'bgp scan-time' parameter

11 files changed, 282 insertions, 60 deletions

diff --git a/interface-definitions/snmp.xml b/interface-definitions/snmp.xml
index 8b2fa8a93..1634bfb64 100644
--- a/interface-definitions/snmp.xml
+++ b/interface-definitions/snmp.xml
@@ -21,6 +21,9 @@
               <leafNode name="authorization">
                 <properties>
                   <help>Authorization type (default: 'ro')</help>
+                  <completionHelp>
+                    <list>ro rw</list>
+                  </completionHelp>
                   <valueHelp>
                     <format>ro</format>
                     <description>read only</description>
@@ -193,6 +196,9 @@
                   <leafNode name="mode">
                     <properties>
                       <help>Define group access permission (default: 'ro')</help>
+                      <completionHelp>
+                        <list>ro rw</list>
+                      </completionHelp>
                       <valueHelp>
                         <format>ro</format>
                         <description>read only</description>
@@ -210,6 +216,9 @@
                   <leafNode name="seclevel">
                     <properties>
                       <help>Security levels</help>
+                      <completionHelp>
+                        <list>noauth auth priv2</list>
+                      </completionHelp>
                       <valueHelp>
                         <format>noauth</format>
                         <description>Messages not authenticated and not encrypted (noAuthNoPriv)</description>
@@ -280,6 +289,9 @@
                       <leafNode name="type">
                         <properties>
                           <help>Defines the protocol used for authentication (default: 'md5')</help>
+                          <completionHelp>
+                            <list>md5 sha</list>
+                          </completionHelp>
                           <valueHelp>
                             <format>md5</format>
                             <description>Message Digest 5</description>
@@ -343,6 +355,9 @@
                       <leafNode name="type">
                         <properties>
                           <help>Defines the protocol for privacy (default: 'des')</help>
+                          <completionHelp>
+                            <list>des aes</list>
+                          </completionHelp>
                           <valueHelp>
                             <format>des</format>
                             <description>Data Encryption Standard</description>
@@ -361,6 +376,9 @@
                   <leafNode name="protocol">
                     <properties>
                       <help>Defines protocol for notification between TCP and UDP</help>
+                      <completionHelp>
+                        <list>tcp udp</list>
+                      </completionHelp>
                       <valueHelp>
                         <format>tcp</format>
                         <description>Use Transmission Control Protocol for notifications</description>
@@ -377,6 +395,9 @@
                   <leafNode name="type">
                     <properties>
                       <help>Specifies the type of notification between inform and trap (default: 'inform')</help>
+                      <completionHelp>
+                        <list>inform trap</list>
+                      </completionHelp>
                       <valueHelp>
                         <format>inform</format>
                         <description>Use INFORM</description>
@@ -464,6 +485,9 @@
                       <leafNode name="type">
                         <properties>
                           <help>Defines the protocol used for authentication (default: 'md5')</help>
+                          <completionHelp>
+                            <list>md5 sha</list>
+                          </completionHelp>
                           <valueHelp>
                             <format>md5</format>
                             <description>Message Digest 5</description>
@@ -499,6 +523,9 @@
                   <leafNode name="mode">
                     <properties>
                       <help>Define users access permission (default: 'ro')</help>
+                      <completionHelp>
+                        <list>ro rw</list>
+                      </completionHelp>
                       <valueHelp>
                         <format>ro</format>
                         <description>read only</description>
@@ -539,6 +566,9 @@
                       <leafNode name="type">
                         <properties>
                           <help>Defines the protocol for privacy (default: 'des')</help>
+                          <completionHelp>
+                            <list>des aes</list>
+                          </completionHelp>
                           <valueHelp>
                             <format>des</format>
                             <description>Data Encryption Standard</description>
@@ -610,13 +640,13 @@
                 </properties>
                 <children>
                   <leafNode name="script">
-                  <properties>
-                    <help>Script location and name</help>
-                    <completionHelp>
-                      <script>ls /config/user-data</script>
-                    </completionHelp>
-                  </properties>
-                </leafNode>
+                    <properties>
+                      <help>Script location and name</help>
+                      <completionHelp>
+                        <script>ls /config/user-data</script>
+                      </completionHelp>
+                    </properties>
+                  </leafNode>
                 </children>
               </tagNode>
             </children>
diff --git a/interface-definitions/system-proxy.xml b/interface-definitions/system-proxy.xml
new file mode 100644
index 000000000..f43702fc8
--- /dev/null
+++ b/interface-definitions/system-proxy.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0"?>
+<interfaceDefinition>
+  <node name="system">
+    <children>
+      <node name="proxy" owner="${vyos_conf_scripts_dir}/system-proxy.py">
+        <properties>
+          <help>Sets a proxy for system wide use</help>
+        </properties>
+        <children>
+          <leafNode name="url">
+            <properties>
+              <help>Proxy URL</help>
+              <constraint>
+                <regex>^http://[a-z0-9\.]+$</regex>
+              </constraint>
+            </properties>
+          </leafNode>
+          <leafNode name="port">
+            <properties>
+              <help>Proxy port</help>
+              <constraint>
+                <validator name="numeric" argument="--range 1-65535"/>
+              </constraint>
+            </properties>
+          </leafNode>
+          <leafNode name="username">
+            <properties>
+              <help>Proxy username</help>
+              <constraint>
+                <regex>^[a-z0-9-_\.]{1,100}$</regex>
+              </constraint>
+            </properties>
+          </leafNode>
+          <leafNode name="password">
+            <properties>
+              <help>Proxy password</help>
+            </properties>
+          </leafNode>
+        </children>
+      </node>
+    </children>
+  </node>
+</interfaceDefinition>
diff --git a/python/vyos/ifconfig.py b/python/vyos/ifconfig.py
index 3225971ef..3470b3aa6 100644
--- a/python/vyos/ifconfig.py
+++ b/python/vyos/ifconfig.py
@@ -16,6 +16,7 @@
 import os
 import re
 import jinja2
+import json
 
 from vyos.validate import *
 from ipaddress import IPv4Network, IPv6Address
@@ -71,7 +72,6 @@ class Interface:
         >>> i = Interface('eth0')
         """
         self._ifname = str(ifname)
-        self._statechange_wait = True
 
         if not os.path.exists('/sys/class/net/{}'.format(ifname)) and not type:
             raise Exception('interface "{}" not found'.format(self._ifname))
@@ -117,7 +117,7 @@ class Interface:
             self._debug_msg("returned:\n{}".format(tmp.decode()))
 
         # do we need some error checking code here?
-        return tmp
+        return tmp.decode()
 
     def _read_sysfs(self, filename):
         """
@@ -301,8 +301,10 @@ class Interface:
         >>> Interface('eth0').get_state()
         'up'
         """
-        return self._read_sysfs('/sys/class/net/{}/operstate'
-                                .format(self._ifname))
+        cmd = 'ip -json link show dev {}'.format(self._ifname)
+        tmp = self._cmd(cmd)
+        out = json.loads(tmp)
+        return out[0]['operstate'].lower()
 
     def set_state(self, state):
         """
@@ -320,22 +322,7 @@ class Interface:
         # Assemble command executed on system. Unfortunately there is no way
         # to up/down an interface via sysfs
         cmd = 'ip link set dev {} {}'.format(self._ifname, state)
-        tmp = self._cmd(cmd)
-
-        if self._statechange_wait:
-            # better safe then sorry - wait until the interface is really up
-            # but only for a given period of time to avoid potential deadlocks!
-            cnt = 0
-            while self.get_state() != state:
-                cnt += 1
-                if cnt == 50:
-                    print('Interface {} could not be brought up in time ...'.format(self._ifname))
-                    break
-
-                # sleep 250ms
-                sleep(0.250)
-
-        return tmp
+        return self._cmd(cmd)
 
     def set_proxy_arp(self, enable):
         """
@@ -534,14 +521,13 @@ class Interface:
         with open(self._dhcp_cfg_file, 'w') as f:
             f.write(dhcp_text)
 
-        if self.get_state() == 'up':
-            cmd  = 'start-stop-daemon --start --quiet --pidfile ' + \
-                self._dhcp_pid_file
-            cmd += ' --exec /sbin/dhclient --'
-            # now pass arguments to dhclient binary
-            cmd += ' -4 -nw -cf {} -pf {} -lf {} {}'.format(
-                self._dhcp_cfg_file, self._dhcp_pid_file, self._dhcp_lease_file, self._ifname)
-            return self._cmd(cmd)
+        cmd  = 'start-stop-daemon --start --quiet --pidfile ' + \
+            self._dhcp_pid_file
+        cmd += ' --exec /sbin/dhclient --'
+        # now pass arguments to dhclient binary
+        cmd += ' -4 -nw -cf {} -pf {} -lf {} {}'.format(
+            self._dhcp_cfg_file, self._dhcp_pid_file, self._dhcp_lease_file, self._ifname)
+        return self._cmd(cmd)
 
 
     def _del_dhcp(self):
@@ -1408,7 +1394,6 @@ class WireGuardIf(Interface):
 
     def __init__(self, ifname):
         super().__init__(ifname, type='wireguard')
-        self._statechange_wait = False
 
         self.config = {
             'port': 0,
diff --git a/src/conf_mode/dhcp_server.py b/src/conf_mode/dhcp_server.py
index f19bcb250..af803a696 100755
--- a/src/conf_mode/dhcp_server.py
+++ b/src/conf_mode/dhcp_server.py
@@ -303,7 +303,8 @@ def dhcp_slice_range(exclude_list, range_list):
                       'start': str(ip_address(e) + 1),
                       'stop': str(range_stop)
                     }
-                    output.append(r)
+                    if not (ip_address(r['start']) > ip_address(r['stop'])):
+                        output.append(r)
             else:
               # if we have no exclude in the whole range - we just take the range
               # as it is
diff --git a/src/conf_mode/interfaces-openvpn.py b/src/conf_mode/interfaces-openvpn.py
index 5345bf7a2..cdd133904 100755
--- a/src/conf_mode/interfaces-openvpn.py
+++ b/src/conf_mode/interfaces-openvpn.py
@@ -16,11 +16,11 @@
 
 import os
 import re
-import sys
-import stat
-import jinja2
 
+from jinja2 import Template
 from copy import deepcopy
+from sys import exit
+from stat import S_IRUSR,S_IRWXU,S_IRGRP,S_IXGRP,S_IROTH,S_IXOTH
 from grp import getgrnam
 from ipaddress import ip_address,ip_network,IPv4Interface
 from netifaces import interfaces
@@ -331,12 +331,12 @@ def openvpn_mkdir(directory):
         os.mkdir(directory)
 
     # fix permissions - corresponds to mode 755
-    os.chmod(directory, stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP|stat.S_IROTH|stat.S_IXOTH)
+    os.chmod(directory, S_IRWXU|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH)
     uid = getpwnam(user).pw_uid
     gid = getgrnam(group).gr_gid
     os.chown(directory, uid, gid)
 
-def fixup_permission(filename, permission=stat.S_IRUSR):
+def fixup_permission(filename, permission=S_IRUSR):
     """
     Check if the given file exists and change ownershit to root/vyattacfg
     and appripriate file access permissions - default is user and group readable
@@ -737,7 +737,7 @@ def verify(openvpn):
     if openvpn['shared_secret_file']:
         if openvpn['encryption'] in ['aes128gcm', 'aes192gcm', 'aes256gcm']:
             raise ConfigError('GCM encryption with shared-secret-key-file is not supported')
-        
+
         if not checkCertHeader('-----BEGIN OpenVPN Static key V1-----', openvpn['shared_secret_file']):
             raise ConfigError('Specified shared-secret-key-file "{}" is not valid'.format(openvpn['shared_secret_file']))
 
@@ -851,13 +851,13 @@ def generate(openvpn):
     # Generate client specific configuration
     for client in openvpn['client']:
         client_file = directory + '/ccd/' + interface + '/' + client['name']
-        tmpl = jinja2.Template(client_tmpl)
+        tmpl = Template(client_tmpl)
         client_text = tmpl.render(client)
         with open(client_file, 'w') as f:
             f.write(client_text)
         os.chown(client_file, uid, gid)
 
-    tmpl = jinja2.Template(config_tmpl)
+    tmpl = Template(config_tmpl)
     config_text = tmpl.render(openvpn)
 
     # we need to support quoting of raw parameters from OpenVPN CLI
@@ -957,4 +957,4 @@ if __name__ == '__main__':
         apply(c)
     except ConfigError as e:
         print(e)
-        sys.exit(1)
+        exit(1)
diff --git a/src/conf_mode/interfaces-wireguard.py b/src/conf_mode/interfaces-wireguard.py
index 7a684bafa..013a07f32 100755
--- a/src/conf_mode/interfaces-wireguard.py
+++ b/src/conf_mode/interfaces-wireguard.py
@@ -190,9 +190,6 @@ def verify(c):
                 raise ConfigError("ERROR: allowed-ips required for peer " + p)
             if not c['peer'][p]['pubkey']:
                 raise ConfigError("peer pubkey required for peer " + p)
-            if not c['peer'][p]['endpoint']:
-                raise ConfigError("peer endpoint required for peer " + p)
-
 
 def apply(c):
     # no wg configs left, remove all interface from system
diff --git a/src/conf_mode/snmp.py b/src/conf_mode/snmp.py
index 60e4c343d..cba1fe319 100755
--- a/src/conf_mode/snmp.py
+++ b/src/conf_mode/snmp.py
@@ -224,7 +224,7 @@ init_config_tmpl = """
 SNMPDRUN=yes
 
 # snmpd options (use syslog, close stdin/out/err).
-SNMPDOPTS='-LSed -u snmp -g snmp -I -ipCidrRouteTable, inetCidrRouteTable -p /run/snmpd.pid'
+SNMPDOPTS='-LSed -u snmp -g snmp -I -ipCidrRouteTable,inetCidrRouteTable -p /run/snmpd.pid'
 """
 
 default_config_data = {
diff --git a/src/conf_mode/ssh.py b/src/conf_mode/ssh.py
index e761d75ff..9fe22bfee 100755
--- a/src/conf_mode/ssh.py
+++ b/src/conf_mode/ssh.py
@@ -262,7 +262,8 @@ def apply(ssh):
     else:
         # SSH access is removed in the commit
         os.system("sudo systemctl stop ssh.service")
-        os.unlink(config_file)
+        if os.path.isfile(config_file):
+            os.unlink(config_file)
 
     return None
 
diff --git a/src/conf_mode/system-proxy.py b/src/conf_mode/system-proxy.py
new file mode 100755
index 000000000..cf72a1f96
--- /dev/null
+++ b/src/conf_mode/system-proxy.py
@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2018 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import sys
+import os
+import re
+
+from vyos import ConfigError
+from vyos.config import Config
+
+proxy_def = r'/etc/profile.d/vyos-system-proxy.sh'
+
+
+def get_config():
+    c = Config()
+    if not c.exists('system proxy'):
+        return None
+
+    c.set_level('system proxy')
+
+    cnf = {
+        'url': None,
+      'port': None,
+      'usr': None,
+      'passwd': None
+    }
+
+    if c.exists('url'):
+        cnf['url'] = c.return_value('url')
+    if c.exists('port'):
+        cnf['port'] = c.return_value('port')
+    if c.exists('username'):
+        cnf['usr'] = c.return_value('username')
+    if c.exists('password'):
+        cnf['passwd'] = c.return_value('password')
+
+    return cnf
+
+
+def verify(c):
+    if not c:
+        return None
+    if not c['url'] or not c['port']:
+        raise ConfigError("proxy url and port requires a value")
+    elif c['usr'] and not c['passwd']:
+        raise ConfigError("proxy password requires a value")
+    elif not c['usr'] and c['passwd']:
+        raise ConfigError("proxy username requires a value")
+
+
+def generate(c):
+    if not c:
+        return None
+    if not c['usr']:
+        return str("export http_proxy={url}:{port}\nexport https_proxy=$http_proxy\nexport ftp_proxy=$http_proxy"
+                   .format(url=c['url'], port=c['port']))
+    else:
+        return str("export http_proxy=http://{usr}:{passwd}@{url}:{port}\nexport https_proxy=$http_proxy\nexport ftp_proxy=$http_proxy"
+                   .format(url=re.sub('http://', '', c['url']), port=c['port'], usr=c['usr'], passwd=c['passwd']))
+
+
+def apply(ln):
+    if not ln and os.path.exists(proxy_def):
+        os.remove(proxy_def)
+    else:
+        open(proxy_def, 'w').write(
+            "# generated by system-proxy.py\n{}\n".format(ln))
+
+if __name__ == '__main__':
+    try:
+        c = get_config()
+        verify(c)
+        ln = generate(c)
+        apply(ln)
+    except ConfigError as e:
+        print(e)
+        sys.exit(1)
diff --git a/src/migration-scripts/quagga/3-to-4 b/src/migration-scripts/quagga/3-to-4
new file mode 100755
index 000000000..f8c87ce8c
--- /dev/null
+++ b/src/migration-scripts/quagga/3-to-4
@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2019 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import sys
+
+from vyos.configtree import ConfigTree
+
+
+if (len(sys.argv) < 1):
+    print("Must specify file name!")
+    sys.exit(1)
+
+file_name = sys.argv[1]
+
+with open(file_name, 'r') as f:
+    config_file = f.read()
+
+config = ConfigTree(config_file)
+
+if not config.exists(['protocols', 'bgp']):
+    # Nothing to do
+    sys.exit(0)
+else:
+    # Check if BGP is actually configured and obtain the ASN
+    asn_list = config.list_nodes(['protocols', 'bgp'])
+    if asn_list:
+        # There's always just one BGP node, if any
+        asn = asn_list[0]
+    else:
+        # There's actually no BGP, just its empty shell
+        sys.exit(0)
+
+    # Check if BGP scan-time parameter exist
+    scan_time_param = ['protocols', 'bgp', asn, 'parameters', 'scan-time']
+    if config.exists(scan_time_param):
+        # Delete BGP scan-time parameter
+        config.delete(scan_time_param)
+    else:
+        # Do nothing
+        sys.exit(0)
+
+    # Save a new configuration file
+    try:
+        with open(file_name, 'w') as f:
+            f.write(config.to_string())
+    except OSError as e:
+        print("Failed to save the modified config: {}".format(e))
+        sys.exit(1)
diff --git a/src/validators/numeric b/src/validators/numeric
index ffe84a234..0a2d83d14 100755
--- a/src/validators/numeric
+++ b/src/validators/numeric
@@ -24,7 +24,7 @@ import re
 parser = argparse.ArgumentParser()
 parser.add_argument("-f", "--float", action="store_true", help="Accept floating point values")
 group = parser.add_mutually_exclusive_group()
-group.add_argument("-r", "--range", type=str, help="Check if the number is within range (inclusive), example: 1024-65535")
+group.add_argument("-r", "--range", type=str, help="Check if the number is within range (inclusive), example: 1024-65535", action='append')
 group.add_argument("-n", "--non-negative", action="store_true", help="Check if the number is non-negative (>= 0)")
 group.add_argument("-p", "--positive", action="store_true", help="Check if the number is positive (> 0)")
 parser.add_argument("number", type=str, help="Number to validate")
@@ -47,15 +47,25 @@ else:
         sys.exit(1)
 
 if args.range:
-    try:
-        lower, upper = re.match(r'(\d+)\s*\-\s*(\d+)', args.range).groups()
-        lower, upper = int(lower), int(upper)
-    except:
-        print("{0} is not a valid number range",format(args.range), file=sys.stderr)
-        sys.exit(1)
+    valid = False
+    for r in args.range:
+        try:
+            lower, upper = re.match(r'(\d+)\s*\-\s*(\d+)', r).groups()
+            lower, upper = int(lower), int(upper)
+        except:
+            print("{0} is not a valid number range",format(args.range), file=sys.stderr)
+            sys.exit(1)
+
+        if (number >= lower) and (number <= upper):
+            valid = True
+    # end for
 
-    if (number < lower) or (number > upper):
-        print("Number {0} is not in the {1} range".format(number, args.range), file=sys.stderr)
+    if not valid:
+        if len(args.range) > 1:
+            err_msg = "Number {0} is not in any of the ranges {1}".format(number, args.range)
+        else:
+            err_msg = "Number {0} is not in the range {1}".format(number, args.range[0])
+        print(err_msg, file=sys.stderr)
         sys.exit(1)
 elif args.non_negative:
     if number < 0: