Merge branch 'current' into equuleus

8 files changed, 266 insertions, 101 deletions

diff --git a/src/conf_mode/dhcp_server.py b/src/conf_mode/dhcp_server.py
index 3e1381cd0..f19bcb250 100755
--- a/src/conf_mode/dhcp_server.py
+++ b/src/conf_mode/dhcp_server.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2018 VyOS maintainers and contributors
+# Copyright (C) 2018-2019 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -13,18 +13,16 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-#
 
 import sys
 import os
-import ipaddress
 import jinja2
 import socket
 import struct
 
 import vyos.validate
 
+from ipaddress import ip_address, ip_network
 from vyos.config import Config
 from vyos import ConfigError
 
@@ -253,6 +251,68 @@ default_config_data = {
     'shared_network': [],
 }
 
+def dhcp_slice_range(exclude_list, range_list):
+    """
+    This function is intended to slice a DHCP range. What does it mean?
+
+    Lets assume we have a DHCP range from '192.0.2.1' to '192.0.2.100'
+    but want to exclude address '192.0.2.74' and '192.0.2.75'. We will
+    pass an input 'range_list' in the format:
+      [{'start' : '192.0.2.1', 'stop' : '192.0.2.100' }]
+    and we will receive an output list of:
+      [{'start' : '192.0.2.1' , 'stop' : '192.0.2.73'  },
+       {'start' : '192.0.2.76', 'stop' : '192.0.2.100' }]
+    The resulting list can then be used in turn to build the proper dhcpd
+    configuration file.
+    """
+    output = []
+    # exclude list must be sorted for this to work
+    exclude_list = sorted(exclude_list)
+    for ra in range_list:
+        range_start = ra['start']
+        range_stop = ra['stop']
+        range_last_exclude = ''
+
+        for e in exclude_list:
+            if (ip_address(e) >= ip_address(range_start)) and \
+               (ip_address(e) <= ip_address(range_stop)):
+                range_last_exclude = e
+
+        for e in exclude_list:
+            if (ip_address(e) >= ip_address(range_start)) and \
+               (ip_address(e) <= ip_address(range_stop)):
+
+                # Build new IP address range ending one IP address before exclude address
+                r = {
+                    'start' : range_start,
+                    'stop' : str(ip_address(e) -1)
+                }
+                # On the next run our IP address range will start one address after the exclude address
+                range_start = str(ip_address(e) + 1)
+
+                # on subsequent exclude addresses we can not
+                # append them to our output
+                if not (ip_address(r['start']) > ip_address(r['stop'])):
+                    # Everything is fine, add range to result
+                    output.append(r)
+
+                # Take care of last IP address range spanning from the last exclude
+                # address (+1) to the end of the initial configured range
+                if ip_address(e) == ip_address(range_last_exclude):
+                    r = {
+                      'start': str(ip_address(e) + 1),
+                      'stop': str(range_stop)
+                    }
+                    output.append(r)
+            else:
+              # if we have no exclude in the whole range - we just take the range
+              # as it is
+              if not range_last_exclude:
+                  if ra not in output:
+                      output.append(ra)
+
+    return output
+
 def get_config():
     dhcp = default_config_data
     conf = Config()
@@ -327,8 +387,8 @@ def get_config():
                     conf.set_level('service dhcp-server shared-network-name {0} subnet {1}'.format(network, net))
                     subnet = {
                         'network': net,
-                        'address': str(ipaddress.ip_network(net).network_address),
-                        'netmask': str(ipaddress.ip_network(net).netmask),
+                        'address': str(ip_network(net).network_address),
+                        'netmask': str(ip_network(net).netmask),
                         'bootfile_name': '',
                         'bootfile_server': '',
                         'client_prefix_length': '',
@@ -460,54 +520,12 @@ def get_config():
 
                     # IP address that needs to be excluded from DHCP lease range
                     if conf.exists('exclude'):
-                        # We have no need to store the exclude addresses. Exclude addresses
-                        # are recalculated into several ranges
-                        exclude = []
                         subnet['exclude'] = conf.return_values('exclude')
-                        for addr in subnet['exclude']:
-                            exclude.append(ipaddress.ip_address(addr))
-
-                        # sort excluded IP addresses ascending
-                        exclude = sorted(exclude)
-
-                        # calculate multipe ranges based on the excluded IP addresses
-                        output = []
-                        for range in subnet['range']:
-                            range_start = range['start']
-                            range_stop = range['stop']
-
-                            for i in exclude:
-                                # Excluded IP address must be in out specified range
-                                if (i >= ipaddress.ip_address(range_start)) and (i <= ipaddress.ip_address(range_stop)):
-                                    # Build up new IP address range ending one IP address before
-                                    # our exclude address
-                                    range = {
-                                        'start': str(range_start),
-                                        'stop': str(i - 1)
-                                    }
-                                    # Our next IP address range will start one address after
-                                    # our exclude address
-                                    range_start = i + 1
-                                    output.append(range)
-
-                                    # Take care of last IP address range spanning from the last exclude
-                                    # address (+1) to the end of the initial configured range
-                                    if i is exclude[-1]:
-                                        last = {
-                                            'start': str(i + 1),
-                                            'stop': str(range_stop)
-                                        }
-                                        output.append(last)
-                                else:
-                                    # IP address not inside search range, take range is it is
-                                    output.append(range)
-
-                        # We successfully build up a new list containing several IP address
-                        # ranges, replace IP address range in our dictionary
-                        subnet['range'] = output
+                        subnet['range'] = dhcp_slice_range(subnet['exclude'], subnet['range'])
 
                     # Static DHCP leases
                     if conf.exists('static-mapping'):
+                        addresses_for_exclude = []
                         for mapping in conf.list_nodes('static-mapping'):
                             conf.set_level('service dhcp-server shared-network-name {0} subnet {1} static-mapping {2}'.format(network, net, mapping))
                             mapping = {
@@ -525,6 +543,7 @@ def get_config():
                             # IP address used for this DHCP client
                             if conf.exists('ip-address'):
                                 mapping['ip_address'] = conf.return_value('ip-address')
+                                addresses_for_exclude.append(mapping['ip_address'])
 
                             # MAC address of requesting DHCP client
                             if conf.exists('mac-address'):
@@ -543,6 +562,13 @@ def get_config():
                             # append static-mapping configuration to subnet list
                             subnet['static_mapping'].append(mapping)
 
+                        # Now we have all static DHCP leases - we also need to slice them
+                        # out of our DHCP ranges to avoid ISC DHCPd warnings as:
+                        #   dhcpd: Dynamic and static leases present for 192.0.2.51.
+                        #   dhcpd: Remove host declaration DMZ_PC1 or remove 192.0.2.51
+                        #   dhcpd: from the dynamic address pool for DMZ
+                        subnet['range'] = dhcp_slice_range(addresses_for_exclude, subnet['range'])
+
                     # Reset config level to matching hirachy
                     conf.set_level('service dhcp-server shared-network-name {0} subnet {1}'.format(network, net))
 
@@ -562,7 +588,7 @@ def get_config():
                         # Option format is:
                         # <netmask>, <network-byte1>, <network-byte2>, <network-byte3>, <router-byte1>, <router-byte2>, <router-byte3>
                         # where bytes with the value 0 are omitted.
-                        net = ipaddress.ip_network(subnet['static_subnet'])
+                        net = ip_network(subnet['static_subnet'])
                         # add netmask
                         string = str(net.prefixlen) + ','
                         # add network bytes
@@ -682,17 +708,17 @@ def verify(dhcp):
                     raise ConfigError('DHCP range stop address for start {0} is not defined!'.format(start))
 
                 # Start address must be inside network
-                if not ipaddress.ip_address(start) in ipaddress.ip_network(subnet['network']):
+                if not ip_address(start) in ip_network(subnet['network']):
                     raise ConfigError('DHCP range start address {0} is not in subnet {1}\n' \
                                       'specified for shared network {2}!'.format(start, subnet['network'], network['name']))
 
                 # Stop address must be inside network
-                if not ipaddress.ip_address(stop) in ipaddress.ip_network(subnet['network']):
+                if not ip_address(stop) in ip_network(subnet['network']):
                     raise ConfigError('DHCP range stop address {0} is not in subnet {1}\n' \
                                       'specified for shared network {2}!'.format(stop, subnet['network'], network['name']))
 
                 # Stop address must be greater or equal to start address
-                if not ipaddress.ip_address(stop) >= ipaddress.ip_address(start):
+                if not ip_address(stop) >= ip_address(start):
                     raise ConfigError('DHCP range stop address {0} must be greater or equal\n' \
                                       'to the range start address {1}!'.format(stop, start))
 
@@ -712,7 +738,7 @@ def verify(dhcp):
 
             # Exclude addresses must be in bound
             for exclude in subnet['exclude']:
-                if not ipaddress.ip_address(exclude) in ipaddress.ip_network(subnet['network']):
+                if not ip_address(exclude) in ip_network(subnet['network']):
                     raise ConfigError('Exclude IP address {0} is outside of the DHCP lease network {1}\n' \
                                       'under shared network {2}!'.format(exclude, subnet['network'], network['name']))
 
@@ -735,7 +761,7 @@ def verify(dhcp):
 
                 if mapping['ip_address']:
                     # Static IP address must be in bound
-                    if not ipaddress.ip_address(mapping['ip_address']) in ipaddress.ip_network(subnet['network']):
+                    if not ip_address(mapping['ip_address']) in ip_network(subnet['network']):
                         raise ConfigError('DHCP static lease IP address {0} for static mapping {1}\n' \
                                           'in shared network {2} is outside DHCP lease subnet {3}!' \
                                           .format(mapping['ip_address'], mapping['name'], network['name'], subnet['network']))
@@ -758,9 +784,9 @@ def verify(dhcp):
                 subnets.append(subnet['network'])
 
             # Check for overlapping subnets
-            net = ipaddress.ip_network(subnet['network'])
+            net = ip_network(subnet['network'])
             for n in subnets:
-                net2 = ipaddress.ip_network(n)
+                net2 = ip_network(n)
                 if (net != net2):
                     if net.overlaps(net2):
                         raise ConfigError('DHCP conflicting subnet ranges: {0} overlaps {1}'.format(net, net2))
diff --git a/src/conf_mode/interface-ethernet.py b/src/conf_mode/interface-ethernet.py
index 99450b19e..317da5772 100755
--- a/src/conf_mode/interface-ethernet.py
+++ b/src/conf_mode/interface-ethernet.py
@@ -254,7 +254,7 @@ def verify(eth):
     for bond in conf.list_nodes('interfaces bonding'):
         if conf.exists('interfaces bonding ' + bond + ' member interface'):
             bond_member = conf.return_values('interfaces bonding ' + bond + ' member interface')
-            if eth['name'] in bond_member:
+            if eth['intf'] in bond_member:
                 if eth['address']:
                     raise ConfigError('Can not assign address to interface {} which is a member of {}').format(eth['intf'], bond)
 
diff --git a/src/conf_mode/interface-openvpn.py b/src/conf_mode/interface-openvpn.py
index a988e1ab1..5345bf7a2 100755
--- a/src/conf_mode/interface-openvpn.py
+++ b/src/conf_mode/interface-openvpn.py
@@ -207,10 +207,16 @@ keysize 128
 {%- elif 'bf256' in encryption %}
 cipher bf-cbc
 keysize 25
+{%- elif 'aes128gcm' in encryption %}
+cipher aes-128-gcm
 {%- elif 'aes128' in encryption %}
 cipher aes-128-cbc
+{%- elif 'aes192gcm' in encryption %}
+cipher aes-192-gcm
 {%- elif 'aes192' in encryption %}
 cipher aes-192-cbc
+{%- elif 'aes256gcm' in encryption %}
+cipher aes-256-gcm
 {%- elif 'aes256' in encryption %}
 cipher aes-256-cbc
 {% endif %}
@@ -729,6 +735,9 @@ def verify(openvpn):
     # TLS/encryption
     #
     if openvpn['shared_secret_file']:
+        if openvpn['encryption'] in ['aes128gcm', 'aes192gcm', 'aes256gcm']:
+            raise ConfigError('GCM encryption with shared-secret-key-file is not supported')
+        
         if not checkCertHeader('-----BEGIN OpenVPN Static key V1-----', openvpn['shared_secret_file']):
             raise ConfigError('Specified shared-secret-key-file "{}" is not valid'.format(openvpn['shared_secret_file']))
 
diff --git a/src/conf_mode/interface-wireguard.py b/src/conf_mode/interface-wireguard.py
index 0be8b7b0f..0dcce6b1c 100755
--- a/src/conf_mode/interface-wireguard.py
+++ b/src/conf_mode/interface-wireguard.py
@@ -224,10 +224,10 @@ def apply(c):
         intfc.add_addr(ip)
 
     # interface mtu
-    intfc.mtu = int(c['mtu'])
+    intfc.set_mtu(int(c['mtu']))
 
     # ifalias for snmp from description
-    intfc.ifalias = str(c['descr'])
+    intfc.set_alias(str(c['descr']))
 
     # remove peers
     if c['peer_remove']:
@@ -267,7 +267,7 @@ def apply(c):
         intfc.update()
 
     # interface state
-    intfc.state = c['state']
+    intfc.set_state(c['state'])
 
     return None
 
diff --git a/src/helpers/vyos-load-config.py b/src/helpers/vyos-load-config.py
new file mode 100755
index 000000000..4e6d67efa
--- /dev/null
+++ b/src/helpers/vyos-load-config.py
@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2019 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+"""Load config file from within config session.
+Config file specified by URI or path (without scheme prefix).
+Example: load https://somewhere.net/some.config
+        or
+         load /tmp/some.config
+"""
+
+import sys
+import tempfile
+import vyos.defaults
+import vyos.remote
+from vyos.config import Config, VyOSError
+from vyos.migrator import Migrator, MigratorError
+
+system_config_file = 'config.boot'
+
+class LoadConfig(Config):
+    """A subclass for calling 'loadFile'.
+    This does not belong in config.py, and only has a single caller.
+    """
+    def load_config(self, file_path):
+        cmd = [self._cli_shell_api, 'loadFile', file_path]
+        self._run(cmd)
+
+if len(sys.argv) > 1:
+    file_name = sys.argv[1]
+else:
+    file_name = system_config_file
+
+configdir = vyos.defaults.directories['config']
+
+protocols = ['scp', 'sftp', 'http', 'https', 'ftp', 'tftp']
+
+if any(x in file_name for x in protocols):
+    config_file = vyos.remote.get_remote_config(file_name)
+    if not config_file:
+        sys.exit("No config file by that name.")
+else:
+    canonical_path = '{0}/{1}'.format(configdir, file_name)
+    try:
+        with open(canonical_path, 'r') as f:
+            config_file = f.read()
+    except OSError as err1:
+        try:
+            with open(file_name, 'r') as f:
+                config_file = f.read()
+        except OSError as err2:
+            sys.exit('{0}\n{1}'.format(err1, err2))
+
+config = LoadConfig()
+
+print("Loading configuration from '{}'".format(file_name))
+
+with tempfile.NamedTemporaryFile() as fp:
+    with open(fp.name, 'w') as fd:
+        fd.write(config_file)
+
+    migration = Migrator(fp.name)
+    try:
+        migration.run()
+    except MigratorError as err:
+        sys.exit('{}'.format(err))
+
+    try:
+        config.load_config(fp.name)
+    except VyOSError as err:
+        sys.exit('{}'.format(err))
+
+if config.session_changed():
+    print("Load complete. Use 'commit' to make changes effective.")
+else:
+    print("No configuration changes to commit.")
diff --git a/src/op_mode/reset_openvpn.py b/src/op_mode/reset_openvpn.py
new file mode 100755
index 000000000..7043ac261
--- /dev/null
+++ b/src/op_mode/reset_openvpn.py
@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2018 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import sys
+import os
+
+from psutil import pid_exists
+from subprocess import Popen, PIPE
+from time import sleep
+from netifaces import interfaces
+
+def get_config_name(intf):
+    cfg_file = r'/opt/vyatta/etc/openvpn/openvpn-{}.conf'.format(intf)
+    return cfg_file
+
+def get_pid_file(intf):
+    pid_file = r'/var/run/openvpn/{}.pid'.format(intf)
+    return pid_file
+
+def subprocess_cmd(command):
+    p = Popen(command, stdout=PIPE, shell=True)
+    p.communicate()
+
+if __name__ == '__main__':
+    if (len(sys.argv) < 1):
+        print("Must specify OpenVPN interface name!")
+        sys.exit(1)
+
+    interface = sys.argv[1]
+    if os.path.isfile(get_config_name(interface)):
+        pidfile = '/var/run/openvpn/{}.pid'.format(interface)
+        if os.path.isfile(pidfile):
+            pid = 0
+            with open(pidfile, 'r') as f:
+                pid = int(f.read())
+
+            if pid_exists(pid):
+                cmd  = 'start-stop-daemon --stop --quiet'
+                cmd += ' --pidfile ' + pidfile
+                subprocess_cmd(cmd)
+
+        # When stopping OpenVPN we need to wait for the 'old' interface to
+        # vanish from the Kernel, if it is not gone, OpenVPN will report:
+        # ERROR: Cannot ioctl TUNSETIFF vtun10: Device or resource busy (errno=16)
+        while interface in interfaces():
+            sleep(0.250) # 250ms
+
+        # re-start OpenVPN process
+        cmd  = 'start-stop-daemon --start --quiet'
+        cmd += ' --pidfile ' + get_pid_file(interface)
+        cmd += ' --exec /usr/sbin/openvpn'
+        # now pass arguments to openvpn binary
+        cmd += ' --'
+        cmd += ' --config ' + get_config_name(interface)
+
+        subprocess_cmd(cmd)
+    else:
+        print("OpenVPN interface {} does not exist!".format(interface))
+        sys.exit(1)
diff --git a/src/services/vyos-hostsd b/src/services/vyos-hostsd
index e7ecd8573..5c2ea71c8 100755
--- a/src/services/vyos-hostsd
+++ b/src/services/vyos-hostsd
@@ -166,9 +166,9 @@ def delete_name_servers(data, tag):
 def set_host_name(state, data):
     if data['host_name']:
         state['host_name'] = data['host_name']
-    if data['domain_name']:
+    if 'domain_name' in data:
         state['domain_name'] = data['domain_name']
-    if data['search_domains']:
+    if 'search_domains' in data:
         state['search_domains'] = data['search_domains']
 
 def get_name_servers(state, tag):
diff --git a/src/system/on-dhcp-event.sh b/src/system/on-dhcp-event.sh
index 02bbd4c3c..70a563d4c 100755
--- a/src/system/on-dhcp-event.sh
+++ b/src/system/on-dhcp-event.sh
@@ -37,50 +37,18 @@ fi
 
 case "$action" in
   commit) # add mapping for new lease
-    echo "- new lease event, setting static mapping for host "\
-         "$client_fqdn_name (MAC=$client_mac, IP=$client_ip)"
-    #
-    # grep fails miserably with \t in the search expression.
-    # In the following line one <Ctrl-V> <TAB> is used after $client_search_expr
-    # followed by a single space
-    grep -q " $client_search_expr	 #on-dhcp-event " $file
-    if [ $? == 0 ]; then
-       echo pattern found, removing
-       wc1=`cat $file | wc -l`
-       sudo sed -i "/ $client_search_expr\t #on-dhcp-event /d" $file
-       wc2=`cat $file | wc -l`
-       if [ "$wc1" -eq "$wc2" ]; then
-         echo No change
-       fi
-    else
-       echo pattern NOT found
-    fi
-
-    # check if hostname already exists (e.g. a static host mapping)
-    # if so don't overwrite
-    grep -q " $client_search_expr	 " $file
+    grep -q "   $client_search_expr     " $file
     if [ $? == 0 ]; then
        echo host $client_fqdn_name already exists, exiting
        exit 1
     fi
-
-    line="$client_ip\t $client_fqdn_name\t #on-dhcp-event $client_mac"
-    sudo sh -c "echo -e '$line' >> $file"
-    ((changes++))
-    echo Entry was added
+    # add host
+    /usr/bin/vyos-hostsd-client --add-hosts --tag "DHCP-$client_ip" --host "$client_fqdn_name,$client_ip"
     ;;
 
   release) # delete mapping for released address
-    echo "- lease release event, deleting static mapping for host $client_fqdn_name"
-    wc1=`cat $file | wc -l`
-    sudo sed -i "/ $client_search_expr\t #on-dhcp-event /d" $file
-    wc2=`cat $file | wc -l`
-    if [ "$wc1" -eq "$wc2" ]; then
-      echo No change
-    else
-      echo Entry was removed
-      ((changes++))
-    fi
+    # delete host
+    /usr/bin/vyos-hostsd-client --delete-hosts --tag "DHCP-$client_ip"
     ;;
 
   *)