Merge branch 'vyos:current' into current

11 files changed, 291 insertions, 35 deletions

diff --git a/python/vyos/base.py b/python/vyos/base.py
index 4e23714e5..c78045548 100644
--- a/python/vyos/base.py
+++ b/python/vyos/base.py
@@ -1,4 +1,4 @@
-# Copyright 2018 VyOS maintainers and contributors <maintainers@vyos.io>
+# Copyright 2018-2021 VyOS maintainers and contributors <maintainers@vyos.io>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -13,6 +13,11 @@
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library.  If not, see <http://www.gnu.org/licenses/>.
 
+from textwrap import fill
 
 class ConfigError(Exception):
-    pass
+    def __init__(self, message):
+        # Reformat the message and trim it to 72 characters in length
+        message = fill(message, width=72)
+        # Call the base class constructor with the parameters it needs
+        super().__init__(message)
diff --git a/python/vyos/configdict.py b/python/vyos/configdict.py
index 425a2e416..d974a7565 100644
--- a/python/vyos/configdict.py
+++ b/python/vyos/configdict.py
@@ -459,7 +459,10 @@ def get_interface_dict(config, base, ifname=''):
         # Only add defaults if interface is not about to be deleted - this is
         # to keep a cleaner config dict.
         if 'deleted' not in dict:
-            dict['vif'][vif] = dict_merge(default_vif_values, vif_config)
+            address = leaf_node_changed(config, ['vif', vif, 'address'])
+            if address: dict['vif'][vif].update({'address_old' : address})
+
+            dict['vif'][vif] = dict_merge(default_vif_values, dict['vif'][vif])
             # XXX: T2665: blend in proper DHCPv6-PD default values
             dict['vif'][vif] = T2665_set_dhcpv6pd_defaults(dict['vif'][vif])
 
@@ -480,7 +483,11 @@ def get_interface_dict(config, base, ifname=''):
         # Only add defaults if interface is not about to be deleted - this is
         # to keep a cleaner config dict.
         if 'deleted' not in dict:
-            dict['vif_s'][vif_s] = dict_merge(default_vif_s_values, vif_s_config)
+            address = leaf_node_changed(config, ['vif-s', vif_s, 'address'])
+            if address: dict['vif_s'][vif_s].update({'address_old' : address})
+
+            dict['vif_s'][vif_s] = dict_merge(default_vif_s_values,
+                    dict['vif_s'][vif_s])
             # XXX: T2665: blend in proper DHCPv6-PD default values
             dict['vif_s'][vif_s] = T2665_set_dhcpv6pd_defaults(dict['vif_s'][vif_s])
 
@@ -499,8 +506,12 @@ def get_interface_dict(config, base, ifname=''):
             # Only add defaults if interface is not about to be deleted - this is
             # to keep a cleaner config dict.
             if 'deleted' not in dict:
+                address = leaf_node_changed(config, ['vif-s', vif_s, 'vif-c', vif_c, 'address'])
+                if address: dict['vif_s'][vif_s]['vif_c'][vif_c].update(
+                        {'address_old' : address})
+
                 dict['vif_s'][vif_s]['vif_c'][vif_c] = dict_merge(
-                    default_vif_c_values, vif_c_config)
+                    default_vif_c_values, dict['vif_s'][vif_s]['vif_c'][vif_c])
                 # XXX: T2665: blend in proper DHCPv6-PD default values
                 dict['vif_s'][vif_s]['vif_c'][vif_c] = T2665_set_dhcpv6pd_defaults(
                     dict['vif_s'][vif_s]['vif_c'][vif_c])
diff --git a/python/vyos/configquery.py b/python/vyos/configquery.py
index 1cdcbcf39..5b097b312 100644
--- a/python/vyos/configquery.py
+++ b/python/vyos/configquery.py
@@ -18,16 +18,15 @@ A small library that allows querying existence or value(s) of config
 settings from op mode, and execution of arbitrary op mode commands.
 '''
 
-import re
-import json
-from copy import deepcopy
+import os
 from subprocess import STDOUT
 
-import vyos.util
-import vyos.xml
+from  vyos.util import popen, boot_configuration_complete
 from vyos.config import Config
-from vyos.configtree import ConfigTree
-from vyos.configsource import ConfigSourceSession
+from vyos.configsource import ConfigSourceSession, ConfigSourceString
+from vyos.defaults import directories
+
+config_file = os.path.join(directories['config'], 'config.boot')
 
 class ConfigQueryError(Exception):
     pass
@@ -58,21 +57,21 @@ class CliShellApiConfigQuery(GenericConfigQuery):
 
     def exists(self, path: list):
         cmd = ' '.join(path)
-        (_, err) = vyos.util.popen(f'cli-shell-api existsActive {cmd}')
+        (_, err) = popen(f'cli-shell-api existsActive {cmd}')
         if err:
             return False
         return True
 
     def value(self, path: list):
         cmd = ' '.join(path)
-        (out, err) = vyos.util.popen(f'cli-shell-api returnActiveValue {cmd}')
+        (out, err) = popen(f'cli-shell-api returnActiveValue {cmd}')
         if err:
             raise ConfigQueryError('No value for given path')
         return out
 
     def values(self, path: list):
         cmd = ' '.join(path)
-        (out, err) = vyos.util.popen(f'cli-shell-api returnActiveValues {cmd}')
+        (out, err) = popen(f'cli-shell-api returnActiveValues {cmd}')
         if err:
             raise ConfigQueryError('No values for given path')
         return out
@@ -81,25 +80,36 @@ class ConfigTreeQuery(GenericConfigQuery):
     def __init__(self):
         super().__init__()
 
-        config_source = ConfigSourceSession()
-        self.configtree = Config(config_source=config_source)
+        if boot_configuration_complete():
+            config_source = ConfigSourceSession()
+            self.config = Config(config_source=config_source)
+        else:
+            try:
+                with open(config_file) as f:
+                    config_string = f.read()
+            except OSError as err:
+                raise ConfigQueryError('No config file available') from err
+
+            config_source = ConfigSourceString(running_config_text=config_string,
+                                               session_config_text=config_string)
+            self.config = Config(config_source=config_source)
 
     def exists(self, path: list):
-        return self.configtree.exists(path)
+        return self.config.exists(path)
 
     def value(self, path: list):
-        return self.configtree.return_value(path)
+        return self.config.return_value(path)
 
     def values(self, path: list):
-        return self.configtree.return_values(path)
+        return self.config.return_values(path)
 
     def list_nodes(self, path: list):
-        return self.configtree.list_nodes(path)
+        return self.config.list_nodes(path)
 
     def get_config_dict(self, path=[], effective=False, key_mangling=None,
                         get_first_key=False, no_multi_convert=False,
                         no_tag_node_value_mangle=False):
-        return self.configtree.get_config_dict(path, effective=effective,
+        return self.config.get_config_dict(path, effective=effective,
                 key_mangling=key_mangling, get_first_key=get_first_key,
                 no_multi_convert=no_multi_convert,
                 no_tag_node_value_mangle=no_tag_node_value_mangle)
@@ -110,7 +120,7 @@ class VbashOpRun(GenericOpRun):
 
     def run(self, path: list, **kwargs):
         cmd = ' '.join(path)
-        (out, err) = vyos.util.popen(f'.  /opt/vyatta/share/vyatta-op/functions/interpreter/vyatta-op-run; _vyatta_op_run {cmd}', stderr=STDOUT, **kwargs)
+        (out, err) = popen(f'/opt/vyatta/bin/vyatta-op-cmd-wrapper {cmd}', stderr=STDOUT, **kwargs)
         if err:
             raise ConfigQueryError(out)
         return out
diff --git a/python/vyos/configsource.py b/python/vyos/configsource.py
index b0981d25e..a0f6a46b5 100644
--- a/python/vyos/configsource.py
+++ b/python/vyos/configsource.py
@@ -19,6 +19,7 @@ import re
 import subprocess
 
 from vyos.configtree import ConfigTree
+from vyos.util import boot_configuration_complete
 
 class VyOSError(Exception):
     """
@@ -117,7 +118,7 @@ class ConfigSourceSession(ConfigSource):
         # Running config can be obtained either from op or conf mode, it always succeeds
         # once the config system is initialized during boot;
         # before initialization, set to empty string
-        if os.path.isfile('/tmp/vyos-config-status'):
+        if boot_configuration_complete():
             try:
                 running_config_text = self._run([self._cli_shell_api, '--show-active-only', '--show-show-defaults', '--show-ignore-edit', 'showConfig'])
             except VyOSError:
diff --git a/python/vyos/defaults.py b/python/vyos/defaults.py
index 00b14a985..c77b695bd 100644
--- a/python/vyos/defaults.py
+++ b/python/vyos/defaults.py
@@ -29,6 +29,8 @@ directories = {
   "vyos_udev_dir": "/run/udev/vyos"
 }
 
+config_status = '/tmp/vyos-config-status'
+
 cfg_group = 'vyattacfg'
 
 cfg_vintage = 'vyos'
@@ -44,8 +46,9 @@ https_data = {
 api_data = {
     'listen_address' : '127.0.0.1',
     'port' : '8080',
-    'strict' : 'false',
-    'debug' : 'false',
+    'socket' : False,
+    'strict' : False,
+    'debug' : False,
     'api_keys' : [ {"id": "testapp", "key": "qwerty"} ]
 }
 
diff --git a/python/vyos/hostsd_client.py b/python/vyos/hostsd_client.py
index 303b6ea47..f31ef51cf 100644
--- a/python/vyos/hostsd_client.py
+++ b/python/vyos/hostsd_client.py
@@ -79,6 +79,18 @@ class Client(object):
         msg = {'type': 'forward_zones', 'op': 'get'}
         return self._communicate(msg)
 
+    def add_authoritative_zones(self, data):
+        msg = {'type': 'authoritative_zones', 'op': 'add', 'data': data}
+        self._communicate(msg)
+
+    def delete_authoritative_zones(self, data):
+        msg = {'type': 'authoritative_zones', 'op': 'delete', 'data': data}
+        self._communicate(msg)
+
+    def get_authoritative_zones(self):
+        msg = {'type': 'authoritative_zones', 'op': 'get'}
+        return self._communicate(msg)
+
     def add_search_domains(self, data):
         msg = {'type': 'search_domains', 'op': 'add', 'data': data}
         self._communicate(msg)
diff --git a/python/vyos/ifconfig/interface.py b/python/vyos/ifconfig/interface.py
index 58d130ef6..bcb692697 100755
--- a/python/vyos/ifconfig/interface.py
+++ b/python/vyos/ifconfig/interface.py
@@ -37,6 +37,7 @@ from vyos.util import mac2eui64
 from vyos.util import dict_search
 from vyos.util import read_file
 from vyos.util import get_interface_config
+from vyos.util import get_interface_namespace
 from vyos.util import is_systemd_service_active
 from vyos.template import is_ipv4
 from vyos.template import is_ipv6
@@ -135,6 +136,9 @@ class Interface(Control):
             'validate': assert_mtu,
             'shellcmd': 'ip link set dev {ifname} mtu {value}',
         },
+        'netns': {
+            'shellcmd': 'ip link set dev {ifname} netns {value}',
+        },
         'vrf': {
             'convert': lambda v: f'master {v}' if v else 'nomaster',
             'shellcmd': 'ip link set dev {ifname} {value}',
@@ -512,6 +516,35 @@ class Interface(Control):
         if prev_state == 'up':
             self.set_admin_state('up')
 
+    def del_netns(self, netns):
+        """
+        Remove interface from given NETNS.
+        """
+
+        # If NETNS does not exist then there is nothing to delete
+        if not os.path.exists(f'/run/netns/{netns}'):
+            return None
+
+        # As a PoC we only allow 'dummy' interfaces
+        if 'dum' not in self.ifname:
+            return None
+
+        # Check if interface realy exists in namespace
+        if get_interface_namespace(self.ifname) != None:
+            self._cmd(f'ip netns exec {get_interface_namespace(self.ifname)} ip link del dev {self.ifname}')
+            return
+
+    def set_netns(self, netns):
+        """
+        Add interface from given NETNS.
+
+        Example:
+        >>> from vyos.ifconfig import Interface
+        >>> Interface('dum0').set_netns('foo')
+        """
+
+        self.set_interface('netns', netns)
+
     def set_vrf(self, vrf):
         """
         Add/Remove interface from given VRF instance.
@@ -1353,6 +1386,16 @@ class Interface(Control):
             if mac:
                 self.set_mac(mac)
 
+        # If interface is connected to NETNS we don't have to check all other
+        # settings like MTU/IPv6/sysctl values, etc.
+        # Since the interface is pushed onto a separate logical stack
+        # Configure NETNS
+        if dict_search('netns', config) != None:
+            self.set_netns(config.get('netns', ''))
+            return
+        else:
+            self.del_netns(config.get('netns', ''))
+
         # Update interface description
         self.set_alias(config.get('description', ''))
 
diff --git a/python/vyos/ifconfig/section.py b/python/vyos/ifconfig/section.py
index 0e4447b9e..91f667b65 100644
--- a/python/vyos/ifconfig/section.py
+++ b/python/vyos/ifconfig/section.py
@@ -52,12 +52,12 @@ class Section:
         name: name of the interface
         vlan: if vlan is True, do not stop at the vlan number
         """
-        name = name.rstrip('0123456789')
-        name = name.rstrip('.')
-        if vlan:
-            name = name.rstrip('0123456789.')
         if vrrp:
-            name = name.rstrip('0123456789v')
+            name = re.sub(r'\d(\d|v|\.)*$', '', name)
+        elif vlan:
+            name = re.sub(r'\d(\d|\.)*$', '', name)
+        else:
+            name = re.sub(r'\d+$', '', name)
         return name
 
     @classmethod
diff --git a/python/vyos/ifconfig/vxlan.py b/python/vyos/ifconfig/vxlan.py
index d73fb47b8..9615f396d 100644
--- a/python/vyos/ifconfig/vxlan.py
+++ b/python/vyos/ifconfig/vxlan.py
@@ -54,18 +54,20 @@ class VXLANIf(Interface):
         # arguments used by iproute2. For more information please refer to:
         # - https://man7.org/linux/man-pages/man8/ip-link.8.html
         mapping = {
-            'source_address'             : 'local',
-            'source_interface'           : 'dev',
-            'remote'                     : 'remote',
             'group'                      : 'group',
+            'external'                   : 'external',
             'parameters.ip.dont_fragment': 'df set',
             'parameters.ip.tos'          : 'tos',
             'parameters.ip.ttl'          : 'ttl',
             'parameters.ipv6.flowlabel'  : 'flowlabel',
             'parameters.nolearning'      : 'nolearning',
+            'remote'                     : 'remote',
+            'source_address'             : 'local',
+            'source_interface'           : 'dev',
+            'vni'                        : 'id',
         }
 
-        cmd = 'ip link add {ifname} type {type} id {vni} dstport {port}'
+        cmd = 'ip link add {ifname} type {type} dstport {port}'
         for vyos_key, iproute2_key in mapping.items():
             # dict_search will return an empty dict "{}" for valueless nodes like
             # "parameters.nolearning" - thus we need to test the nodes existence
diff --git a/python/vyos/range_regex.py b/python/vyos/range_regex.py
new file mode 100644
index 000000000..a8190d140
--- /dev/null
+++ b/python/vyos/range_regex.py
@@ -0,0 +1,142 @@
+'''Copyright (c) 2013, Dmitry Voronin
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+'''
+import math
+
+# coding=utf8
+
+#  Split range to ranges that has its unique pattern.
+#  Example for 12-345:
+#
+#  12- 19: 1[2-9]
+#  20- 99: [2-9]\d
+# 100-299: [1-2]\d{2}
+# 300-339: 3[0-3]\d
+# 340-345: 34[0-5]
+
+def range_to_regex(inpt_range):
+    if isinstance(inpt_range, str):
+        range_list = inpt_range.split('-')
+        # Check input arguments
+        if len(range_list) == 2:
+            # The first element in range must be higher then the second
+            if int(range_list[0]) < int(range_list[1]):
+                return regex_for_range(int(range_list[0]), int(range_list[1]))
+
+    return None
+
+def bounded_regex_for_range(min_, max_):
+    return r'\b({})\b'.format(regex_for_range(min_, max_))
+
+def regex_for_range(min_, max_):
+    """
+    > regex_for_range(12, 345)
+    '1[2-9]|[2-9]\d|[1-2]\d{2}|3[0-3]\d|34[0-5]'
+    """
+    positive_subpatterns = []
+    negative_subpatterns = []
+
+    if min_ < 0:
+        min__ = 1
+        if max_ < 0:
+            min__ = abs(max_)
+        max__ = abs(min_)
+
+        negative_subpatterns = split_to_patterns(min__, max__)
+        min_ = 0
+
+    if max_ >= 0:
+        positive_subpatterns = split_to_patterns(min_, max_)    
+
+    negative_only_subpatterns = ['-' + val for val in negative_subpatterns if val not in positive_subpatterns]
+    positive_only_subpatterns = [val for val in positive_subpatterns if val not in negative_subpatterns]
+    intersected_subpatterns = ['-?' + val for val in negative_subpatterns if val in positive_subpatterns]
+
+    subpatterns = negative_only_subpatterns + intersected_subpatterns + positive_only_subpatterns
+    return '|'.join(subpatterns)
+
+
+def split_to_patterns(min_, max_):
+    subpatterns = []
+
+    start = min_
+    for stop in split_to_ranges(min_, max_):
+        subpatterns.append(range_to_pattern(start, stop))
+        start = stop + 1
+
+    return subpatterns
+
+
+def split_to_ranges(min_, max_):
+    stops = {max_}
+
+    nines_count = 1
+    stop = fill_by_nines(min_, nines_count)
+    while min_ <= stop < max_:
+        stops.add(stop)
+
+        nines_count += 1
+        stop = fill_by_nines(min_, nines_count)
+
+    zeros_count = 1
+    stop = fill_by_zeros(max_ + 1, zeros_count) - 1
+    while min_ < stop <= max_:
+        stops.add(stop)
+
+        zeros_count += 1
+        stop = fill_by_zeros(max_ + 1, zeros_count) - 1
+
+    stops = list(stops)
+    stops.sort()
+
+    return stops
+
+
+def fill_by_nines(integer, nines_count):
+    return int(str(integer)[:-nines_count] + '9' * nines_count)
+
+
+def fill_by_zeros(integer, zeros_count):
+    return integer - integer % 10 ** zeros_count
+
+
+def range_to_pattern(start, stop):
+    pattern = ''
+    any_digit_count = 0
+
+    for start_digit, stop_digit in zip(str(start), str(stop)):
+        if start_digit == stop_digit:
+            pattern += start_digit
+        elif start_digit != '0' or stop_digit != '9':
+            pattern += '[{}-{}]'.format(start_digit, stop_digit)
+        else:
+            any_digit_count += 1
+
+    if any_digit_count:
+        pattern += r'\d'
+
+    if any_digit_count > 1:
+        pattern += '{{{}}}'.format(any_digit_count)
+
+    return pattern
\ No newline at end of file
diff --git a/python/vyos/util.py b/python/vyos/util.py
index d8e83ab8d..954c6670d 100644
--- a/python/vyos/util.py
+++ b/python/vyos/util.py
@@ -794,6 +794,24 @@ def get_interface_address(interface):
     tmp = loads(cmd(f'ip -d -j addr show {interface}'))[0]
     return tmp
 
+def get_interface_namespace(iface):
+    """
+       Returns wich netns the interface belongs to
+    """
+    from json import loads
+    # Check if netns exist
+    tmp = loads(cmd(f'ip --json netns ls'))
+    if len(tmp) == 0:
+        return None
+
+    for ns in tmp:
+        namespace = f'{ns["name"]}'
+        # Search interface in each netns
+        data = loads(cmd(f'ip netns exec {namespace} ip -j link show'))
+        for compare in data:
+            if iface == compare["ifname"]:
+                return namespace
+
 def get_all_vrfs():
     """ Return a dictionary of all system wide known VRF instances """
     from json import loads
@@ -961,3 +979,12 @@ def is_wwan_connected(interface):
 
     # return True/False if interface is in connected state
     return dict_search('modem.generic.state', tmp) == 'connected'
+
+def boot_configuration_complete() -> bool:
+    """ Check if the boot config loader has completed
+    """
+    from vyos.defaults import config_status
+
+    if os.path.isfile(config_status):
+        return True
+    return False