Merge branch 't31-vrf' of github.com:c-po/vyos-1x into current

* 't31-vrf' of github.com:c-po/vyos-1x:
  vrf: T31: enable vrf support for dummy interface
  templates: T2099: make op-mode path completion helper working
  vrf: T31: reorder routing table lookups
  vrf: T31: adding unreachable routes to the routing tables
  vrf: T31: prior to the v4.8 kernel iif and oif rules are needed
  vrf: T31: create iproute2 table to name mapping reference
  vrf: T31: rename 'vrf disable-bind-to-all ipv4' to 'vrf bind-to-all'
  vrf: T31: support add/remove of interfaces from vrf
  vrf: T31: remove superfluous vyos.vrf library functions
  vrf: T31: reduce script complexity
  vrf: T31: no need to use sudo calls in vrf.py
  vrf: T31: make 'show vrf' command behave like other 'show interface commands'
  xml: include: description: adjust help message
  vrf: T31: improve help for routing table
  vrf: T31: reuse interface-description.xml.i for instance description
  vrf: T31: use embedded regex on 'vrf name' instead of python script
  vrf: T31: initial support for a VRF backend in XML/Python
  ifconfig: T2057: generic interface option setting

10 files changed, 414 insertions, 5 deletions

diff --git a/interface-definitions/include/interface-description.xml.i b/interface-definitions/include/interface-description.xml.i
index 7a7a37871..961533e26 100644
--- a/interface-definitions/include/interface-description.xml.i
+++ b/interface-definitions/include/interface-description.xml.i
@@ -1,9 +1,9 @@
 <leafNode name="description">
   <properties>
-    <help>Interface description</help>
+    <help>Interface specific description</help>
     <constraint>
       <regex>.{1,256}$</regex>
     </constraint>
-    <constraintErrorMessage>Interface description too long (limit 256 characters)</constraintErrorMessage>
+    <constraintErrorMessage>Description too long (limit 256 characters)</constraintErrorMessage>
   </properties>
 </leafNode>
diff --git a/interface-definitions/include/interface-vrf.xml.i b/interface-definitions/include/interface-vrf.xml.i
new file mode 100644
index 000000000..355e7f0f3
--- /dev/null
+++ b/interface-definitions/include/interface-vrf.xml.i
@@ -0,0 +1,12 @@
+<leafNode name="vrf">
+  <properties>
+    <help>VRF instance name</help>
+    <valueHelp>
+      <format>text</format>
+      <description>VRF instance name</description>
+    </valueHelp>
+    <completionHelp>
+      <path>vrf name</path>
+    </completionHelp>
+  </properties>
+</leafNode>
diff --git a/interface-definitions/interfaces-dummy.xml.in b/interface-definitions/interfaces-dummy.xml.in
index 39809a610..5229e602a 100644
--- a/interface-definitions/interfaces-dummy.xml.in
+++ b/interface-definitions/interfaces-dummy.xml.in
@@ -19,6 +19,7 @@
           #include <include/address-ipv4-ipv6.xml.i>
           #include <include/interface-description.xml.i>
           #include <include/interface-disable.xml.i>
+          #include <include/interface-vrf.xml.i>
         </children>
       </tagNode>
     </children>
diff --git a/interface-definitions/vrf.xml.in b/interface-definitions/vrf.xml.in
new file mode 100644
index 000000000..f1895598e
--- /dev/null
+++ b/interface-definitions/vrf.xml.in
@@ -0,0 +1,47 @@
+<?xml version="1.0"?>
+<interfaceDefinition>
+  <node name="vrf" owner="${vyos_conf_scripts_dir}/vrf.py">
+    <properties>
+      <help>Virtual Routing and Forwarding</help>
+      <!-- must be before any interface creation -->
+      <priority>210</priority>
+    </properties>
+    <children>
+      <leafNode name="bind-to-all">
+        <properties>
+          <help>Enable binding services to all VRFs</help>
+          <valueless/>
+        </properties>
+      </leafNode>
+      <tagNode name="name">
+        <properties>
+          <help>VRF instance name</help>
+          <constraint>
+            <regex>[^/\s]{1,16}$</regex>
+          </constraint>
+          <constraintErrorMessage>VRF instance name must be 16 characters or less</constraintErrorMessage>
+          <valueHelp>
+            <format>name</format>
+            <description>Instance name</description>
+          </valueHelp>
+        </properties>
+        <children>
+          <leafNode name="table">
+            <properties>
+              <help>Routing table associated with this instance</help>
+              <constraint>
+                <validator name="numeric" argument="--range 1-2147483647"/>
+              </constraint>
+              <constraintErrorMessage>Invalid kernel table number</constraintErrorMessage>
+              <valueHelp>
+                <format>1-2147483647</format>
+                <description>Routing table ID</description>
+              </valueHelp>
+            </properties>
+          </leafNode>
+          #include <include/interface-description.xml.i>
+        </children>
+      </tagNode>
+    </children>
+  </node>
+</interfaceDefinition>
\ No newline at end of file
diff --git a/op-mode-definitions/show-vrf.xml b/op-mode-definitions/show-vrf.xml
new file mode 100644
index 000000000..360153d8e
--- /dev/null
+++ b/op-mode-definitions/show-vrf.xml
@@ -0,0 +1,22 @@
+<?xml version="1.0"?>
+<interfaceDefinition>
+  <node name="show">
+    <children>
+      <node name="vrf">
+        <properties>
+          <help>Show VRF information</help>
+        </properties>
+        <command>${vyos_op_scripts_dir}/show_vrf.py -e</command>
+      </node>
+      <tagNode name="vrf">
+        <properties>
+          <help>Show information on specific VRF instance</help>
+            <completionHelp>
+              <path>vrf name</path>
+            </completionHelp>
+        </properties>
+        <command>${vyos_op_scripts_dir}/show_vrf.py -e "$3"</command>
+      </tagNode>
+    </children>
+  </node>
+</interfaceDefinition>
diff --git a/python/vyos/ifconfig.py b/python/vyos/ifconfig.py
index afcf4e96f..7f03befeb 100644
--- a/python/vyos/ifconfig.py
+++ b/python/vyos/ifconfig.py
@@ -202,6 +202,12 @@ class Interface(Control):
             'validate': assert_mac,
             'shellcmd': 'ip link set dev {ifname} address {value}',
         },
+        'add_vrf': {
+            'shellcmd': 'ip link set dev {ifname} master {value}',
+        },
+        'del_vrf': {
+            'shellcmd': 'ip link set dev {ifname} nomaster {value}',
+        },
     }
 
     _sysfs_get = {
@@ -344,7 +350,7 @@ class Interface(Control):
             self.del_addr(addr)
 
         # ---------------------------------------------------------------------
-        # A code refactoring is required as this type check is present as 
+        # A code refactoring is required as this type check is present as
         # Interface implement behaviour for one of it's sub-class.
 
         # It is required as the current pattern for vlan is:
@@ -404,6 +410,26 @@ class Interface(Control):
         """
         self.set_interface('mac', mac)
 
+    def add_vrf(self, vrf):
+        """
+        Add interface to given VRF instance.
+
+        Example:
+        >>> from vyos.ifconfig import Interface
+        >>> Interface('eth0').add_vrf('foo')
+        """
+        self.set_interface('add_vrf', vrf)
+
+    def del_vrf(self, vrf):
+        """
+        Remove interface from given VRF instance.
+
+        Example:
+        >>> from vyos.ifconfig import Interface
+        >>> Interface('eth0').del_vrf('foo')
+        """
+        self.set_interface('del_vrf', vrf)
+
     def set_arp_cache_tmo(self, tmo):
         """
         Set ARP cache timeout value in seconds. Internal Kernel representation
diff --git a/scripts/build-command-op-templates b/scripts/build-command-op-templates
index 7630ba23a..689d19ece 100755
--- a/scripts/build-command-op-templates
+++ b/scripts/build-command-op-templates
@@ -111,7 +111,7 @@ def get_properties(p):
             for i in lists:
                 comp_exprs.append("echo \"{0}\"".format(i.text))
             for i in paths:
-                comp_exprs.append("/bin/cli-shell-api listActiveNodes {0}".format(i.text))
+                comp_exprs.append("/bin/cli-shell-api listActiveNodes {0} | sed -e \"s/'//g\"".format(i.text))
             for i in scripts:
                 comp_exprs.append("{0}".format(i.text))
             comp_help = " && ".join(comp_exprs)
diff --git a/src/conf_mode/interfaces-dummy.py b/src/conf_mode/interfaces-dummy.py
index e79e6222d..10cae5d7d 100755
--- a/src/conf_mode/interfaces-dummy.py
+++ b/src/conf_mode/interfaces-dummy.py
@@ -18,6 +18,7 @@ import os
 
 from copy import deepcopy
 from sys import exit
+from netifaces import interfaces
 
 from vyos.ifconfig import DummyIf
 from vyos.configdict import list_diff
@@ -30,7 +31,8 @@ default_config_data = {
     'deleted': False,
     'description': '',
     'disable': False,
-    'intf': ''
+    'intf': '',
+    'vrf': ''
 }
 
 def get_config():
@@ -69,9 +71,17 @@ def get_config():
     act_addr = conf.return_values('address')
     dummy['address_remove'] = list_diff(eff_addr, act_addr)
 
+    # retrieve VRF instance
+    if conf.exists('vrf'):
+        dummy['vrf'] = conf.return_value('vrf')
+
     return dummy
 
 def verify(dummy):
+    vrf_name = dummy['vrf']
+    if vrf_name and vrf_name not in interfaces():
+        raise ConfigError(f'VRF "{vrf_name}" does not exist')
+
     return None
 
 def generate(dummy):
@@ -95,6 +105,12 @@ def apply(dummy):
         for addr in dummy['address']:
             d.add_addr(addr)
 
+        # assign to VRF
+        if dummy['vrf']:
+            d.add_vrf(dummy['vrf'])
+        else:
+            d.del_vrf(dummy['vrf'])
+
         # disable interface on demand
         if dummy['disable']:
             d.set_state('down')
diff --git a/src/conf_mode/vrf.py b/src/conf_mode/vrf.py
new file mode 100755
index 000000000..a39366126
--- /dev/null
+++ b/src/conf_mode/vrf.py
@@ -0,0 +1,237 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2020 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import jinja2
+
+from sys import exit
+from copy import deepcopy
+from subprocess import check_call, CalledProcessError
+from vyos.config import Config
+from vyos.configdict import list_diff
+from vyos import ConfigError
+
+config_file = r'/etc/iproute2/rt_tables.d/vyos-vrf.conf'
+
+# Please be careful if you edit the template.
+config_tmpl = """
+### Autogenerated by vrf.py ###
+#
+# Routing table ID to name mapping reference
+
+# id       vrf name         comment
+{% for vrf in vrf_add -%}
+{{ "%-10s" | format(vrf.table) }} {{ "%-16s" | format(vrf.name) }} # {{ vrf.description }}
+{% endfor -%}
+
+"""
+
+default_config_data = {
+    'bind_to_all': 0,
+    'deleted': False,
+    'vrf_add': [],
+    'vrf_existing': [],
+    'vrf_remove': []
+}
+
+def _cmd(command):
+    """
+    Run any arbitrary command on the system
+    """
+    try:
+        check_call(command.split())
+    except CalledProcessError as e:
+        pass
+        raise ConfigError(f'Error operationg on VRF: {e}')
+
+def interfaces_with_vrf(match):
+    matched = []
+    config = Config()
+    section = config.get_config_dict('interfaces')
+    for type in section:
+        interfaces = section[type]
+        for name in interfaces:
+            interface = interfaces[name]
+            if 'vrf' in interface:
+                v = interface.get('vrf', '')
+                if v == match:
+                    matched.append(name)
+    return matched
+
+def get_config():
+    conf = Config()
+    vrf_config = deepcopy(default_config_data)
+
+    cfg_base = ['vrf']
+    if not conf.exists(cfg_base):
+        # get all currently effetive VRFs and mark them for deletion
+        vrf_config['vrf_remove'] = conf.list_effective_nodes(cfg_base + ['name'])
+    else:
+
+        # Should services be allowed to bind to all VRFs?
+        if conf.exists(['bind-to-all']):
+            vrf_config['bind_to_all'] = 1
+
+        # Determine vrf interfaces (currently effective) - to determine which
+        # vrf interface is no longer present and needs to be removed
+        eff_vrf = conf.list_effective_nodes(cfg_base + ['name'])
+        act_vrf = conf.list_nodes(cfg_base + ['name'])
+        vrf_config['vrf_remove'] = list_diff(eff_vrf, act_vrf)
+
+        # read in individual VRF definition and build up
+        # configuration
+        for name in conf.list_nodes(cfg_base + ['name']):
+            vrf_inst = {
+                'description' : '\0',
+                'members': [],
+                'name' : name,
+                'table' : '',
+                'table_mod': False
+            }
+            conf.set_level(cfg_base + ['name', name])
+
+            if conf.exists(['table']):
+                # VRF table can't be changed on demand, thus we need to read in the
+                # current and the effective routing table number
+                act_table = conf.return_value(['table'])
+                eff_table = conf.return_effective_value(['table'])
+                vrf_inst['table'] = act_table
+                if eff_table and eff_table != act_table:
+                    vrf_inst['table_mod'] = True
+
+            if conf.exists(['description']):
+                vrf_inst['description'] = conf.return_value(['description'])
+
+            # find member interfaces of this particulat VRF
+            vrf_inst['members'] = interfaces_with_vrf(name)
+
+            # append individual VRF configuration to global configuration list
+            vrf_config['vrf_add'].append(vrf_inst)
+
+    # check VRFs which need to be removed as they are not allowed to have
+    # interfaces attached
+    tmp = []
+    for name in vrf_config['vrf_remove']:
+        vrf_inst = {
+            'members': [],
+            'name' : name,
+        }
+
+        # find member interfaces of this particulat VRF
+        vrf_inst['members'] = interfaces_with_vrf(name)
+
+        # append individual VRF configuration to temporary configuration list
+        tmp.append(vrf_inst)
+
+    # replace values in vrf_remove with list of dictionaries
+    # as we need it in verify() - we can't delete a VRF with members attached
+    vrf_config['vrf_remove'] = tmp
+    return vrf_config
+
+def verify(vrf_config):
+    # ensure VRF is not assigned to any interface
+    for vrf in vrf_config['vrf_remove']:
+        if len(vrf['members']) > 0:
+            raise ConfigError('VRF "{}" can not be deleted as it has active members'.format(vrf['name']))
+
+    # routing table id can't be changed - OS restriction
+    for vrf in vrf_config['vrf_add']:
+        if vrf['table_mod']:
+            raise ConfigError('VRF routing table id modification is not possible')
+
+    # add test to see if routing table already exists or not?
+
+    return None
+
+def generate(vrf_config):
+    tmpl = jinja2.Template(config_tmpl)
+    config_text = tmpl.render(vrf_config)
+    with open(config_file, 'w') as f:
+        f.write(config_text)
+
+    return None
+
+def apply(vrf_config):
+    # https://github.com/torvalds/linux/blob/master/Documentation/networking/vrf.txt
+
+    # set the default VRF global behaviour
+    bind_all = vrf_config['bind_to_all']
+    _cmd(f'sysctl -wq net.ipv4.tcp_l3mdev_accept={bind_all}')
+    _cmd(f'sysctl -wq net.ipv4.udp_l3mdev_accept={bind_all}')
+
+    for vrf_name in vrf_config['vrf_remove']:
+        if os.path.isdir(f'/sys/class/net/{vrf_name}'):
+            _cmd(f'ip link delete dev {vrf_name}')
+
+    for vrf in vrf_config['vrf_add']:
+        name = vrf['name']
+        table = vrf['table']
+
+        if not os.path.isdir(f'/sys/class/net/{name}'):
+            # For each VRF apart from your default context create a VRF
+            # interface with a separate routing table
+            _cmd(f'ip link add {name} type vrf table {table}')
+            # Start VRf
+            _cmd(f'ip link set dev {name} up')
+            # The kernel Documentation/networking/vrf.txt also recommends
+            # adding unreachable routes to the VRF routing tables so that routes
+            # afterwards are taken.
+            _cmd(f'ip -4 route add vrf {name} unreachable default metric 4278198272')
+            _cmd(f'ip -6 route add vrf {name} unreachable default metric 4278198272')
+
+        # set VRF description for e.g. SNMP monitoring
+        with open(f'/sys/class/net/{name}/ifalias', 'w') as f:
+            f.write(vrf['description'])
+
+    # Linux routing uses rules to find tables - routing targets are then
+    # looked up in those tables. If the lookup got a matching route, the
+    # process ends.
+    #
+    # TL;DR; first table with a matching entry wins!
+    #
+    # You can see your routing table lookup rules using "ip rule", sadly the
+    # local lookup is hit before any VRF lookup. Pinging an addresses from the
+    # VRF will usually find a hit in the local table, and never reach the VRF
+    # routing table - this is usually not what you want. Thus we will
+    # re-arrange the tables and move the local lookup furhter down once VRFs
+    # are enabled.
+
+    # set "normal" non VRF table lookups
+    add_pref = '0'
+    del_pref = '32765'
+
+    # Lookup table is adjusted if we are in VRF mode
+    if vrf_config['vrf_add']:
+        add_pref = '32765'
+        del_pref = '0'
+
+    # Configure table lookups
+    _cmd(f'ip -4 rule add pref {add_pref} table local')
+    _cmd(f'ip -4 rule del pref {del_pref}')
+    _cmd(f'ip -6 rule add pref {add_pref} table local')
+    _cmd(f'ip -6 rule del pref {del_pref}')
+
+    return None
+
+if __name__ == '__main__':
+    try:
+        c = get_config()
+        verify(c)
+        generate(c)
+        apply(c)
+    except ConfigError as e:
+        print(e)
+        exit(1)
diff --git a/src/op_mode/show_vrf.py b/src/op_mode/show_vrf.py
new file mode 100755
index 000000000..ec894d572
--- /dev/null
+++ b/src/op_mode/show_vrf.py
@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2020 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import argparse
+
+from subprocess import check_output
+from json import loads
+
+def list_vrfs():
+    command = 'ip -j -br link show type vrf'
+    answer = loads(check_output(command.split()).decode())
+    return [_ for _ in answer if _]
+
+parser = argparse.ArgumentParser()
+group = parser.add_mutually_exclusive_group()
+group.add_argument("-e", "--extensive", action="store_true",
+                   help="provide detailed vrf informatio")
+parser.add_argument('interface', metavar='I', type=str, nargs='?',
+                    help='interface to display')
+
+args = parser.parse_args()
+
+if args.extensive:
+    print('{:16}  {:7}  {:17}  {}'.format('interface', 'state', 'mac', 'flags'))
+    print('{:16}  {:7}  {:17}  {}'.format('---------', '-----', '---', '-----'))
+    for vrf in list_vrfs():
+        name = vrf['ifname']
+        if args.interface and name != args.interface:
+            continue
+        state = vrf['operstate'].lower()
+        mac = vrf['address'].lower()
+        info = ','.join([_.lower() for _ in vrf['flags']])
+        print(f'{name:16}  {state:7}  {mac:17}  {info}')
+else:
+    print(" ".join([vrf['ifname'] for vrf in list_vrfs()]))