Merge branch 'current' of github.com:vyos/vyos-1x into equuleus

* 'current' of github.com:vyos/vyos-1x:
  [conf completion]: T1779: Add tunnels to completion
  [XML templates] T1772: Changed old hacks to proper regex, according to the fix
  Add a function for retrieving config dicts.
  snmp: make script extension code more readable
  snmp: use proper stat literals on chmod()
  snmp: fix verify() indent on script extensions
  snmp: fix verify() bail out early order
  snmp: T1738: cleanup import statements
  T1759: Fixing dependency bug from previous commit
  T1773, T1774: add a show config operation with JSON and raw options.
  T1759: Merging interface.py into ifconfig.py
  Allow list arguments in the vyos.config show_config() function.
  Replace the try and wait for segfault approach with explicit inSession check.
  T1773: add a script for converting the config to JSON. It also exposes those functions in vyos.configtree
  [XML templates] T1772: Add escaping of `\` symbol in `<regex>`

4 files changed, 129 insertions, 158 deletions

diff --git a/python/vyos/config.py b/python/vyos/config.py
index 7f65a9397..5bd8fb072 100644
--- a/python/vyos/config.py
+++ b/python/vyos/config.py
@@ -65,6 +65,7 @@ In operational mode, all functions return values from the running config.
 
 import os
 import re
+import json
 import subprocess
 
 import vyos.configtree
@@ -104,12 +105,10 @@ class Config(object):
                 running_config_text = f.read()
 
         # Session config ("active") only exists in conf mode.
-        # Trying to obtain it from op mode will cause a fatal cli-shell-api error.
-        # If that happens, we assume that a script is running from op mode and use the running config
-        # for the "session config" variable as well.
-        try:
+        # In op mode, we'll just use the same running config for both active and session configs.
+        if self.in_session():
             session_config_text = self._run([self._cli_shell_api, '--show-working-only', '--show-show-defaults', 'showConfig'])
-        except VyOSError:
+        else:
             session_config_text = running_config_text
 
         self._session_config = vyos.configtree.ConfigTree(session_config_text)
@@ -224,21 +223,33 @@ class Config(object):
         except VyOSError:
             return False
 
-    def show_config(self, path='', default=None):
+    def show_config(self, path=[], default=None):
         """
         Args:
-            path (str): Configuration tree path, or empty
+            path (str list): Configuration tree path, or empty
             default (str): Default value to return
 
         Returns:
             str: working configuration
         """
+        if isinstance(path, list):
+            path = " ".join(path)
         try:
             out = self._run(self._make_command('showConfig', path))
             return out
         except VyOSError:
             return(default)
 
+    def get_config_dict(self, path=[], effective=False):
+        """
+        Args: path (str list): Configuration tree path, can be empty
+        Returns: a dict representation of the config
+        """
+        res = self.show_config(self._make_path(path))
+        config_tree = vyos.configtree.ConfigTree(res)
+        config_dict = json.loads(config_tree.to_json())
+        return config_dict
+
     def is_multi(self, path):
         """
         Args:
diff --git a/python/vyos/configtree.py b/python/vyos/configtree.py
index 8832a5a63..77cffe90b 100644
--- a/python/vyos/configtree.py
+++ b/python/vyos/configtree.py
@@ -105,6 +105,14 @@ class ConfigTree(object):
         self.__to_commands.argtypes = [c_void_p]
         self.__to_commands.restype = c_char_p
 
+        self.__to_json = self.__lib.to_json
+        self.__to_json.argtypes = [c_void_p]
+        self.__to_json.restype = c_char_p
+
+        self.__to_json_ast = self.__lib.to_json_ast
+        self.__to_json_ast.argtypes = [c_void_p]
+        self.__to_json_ast.restype = c_char_p
+
         self.__set_add_value = self.__lib.set_add_value
         self.__set_add_value.argtypes = [c_void_p, c_char_p, c_char_p]
         self.__set_add_value.restype = c_int
@@ -184,6 +192,12 @@ class ConfigTree(object):
     def to_commands(self):
         return self.__to_commands(self.__config).decode()
 
+    def to_json(self):
+        return self.__to_json(self.__config).decode()
+
+    def to_json_ast(self):
+        return self.__to_json_ast(self.__config).decode()
+
     def set(self, path, value=None, replace=True):
         """Set new entry in VyOS configuration.
         path: configuration path e.g. 'system dns forwarding listen-address'
diff --git a/python/vyos/ifconfig.py b/python/vyos/ifconfig.py
index 3470b3aa6..df86e3c93 100644
--- a/python/vyos/ifconfig.py
+++ b/python/vyos/ifconfig.py
@@ -17,12 +17,20 @@ import os
 import re
 import jinja2
 import json
+import glob
+import time
 
+import vyos.interfaces
 from vyos.validate import *
+from vyos.config import Config
 from ipaddress import IPv4Network, IPv6Address
 from netifaces import ifaddresses, AF_INET, AF_INET6
 from subprocess import Popen, PIPE, STDOUT
 from time import sleep
+from os.path import isfile
+from tabulate import tabulate
+from hurry.filesize import size,alternative
+from datetime import timedelta
 
 dhclient_base = r'/var/lib/dhcp/dhclient_'
 dhcp_cfg = """
@@ -671,6 +679,31 @@ class Interface:
         if os.path.isfile(self._dhcpv6_lease_file):
             os.remove(self._dhcpv6_lease_file)
 
+    def op_show_interface_stats(self):
+        stats = self.get_interface_stats()
+        rx = [['bytes','packets','errors','dropped','overrun','mcast'],[stats['rx_bytes'],stats['rx_packets'],stats['rx_errors'],stats['rx_dropped'],stats['rx_over_errors'],stats['multicast']]]
+        tx = [['bytes','packets','errors','dropped','carrier','collisions'],[stats['tx_bytes'],stats['tx_packets'],stats['tx_errors'],stats['tx_dropped'],stats['tx_carrier_errors'],stats['collisions']]]
+        output = "RX: \n"
+        output += tabulate(rx,headers="firstrow",numalign="right",tablefmt="plain")
+        output += "\n\nTX: \n"
+        output += tabulate(tx,headers="firstrow",numalign="right",tablefmt="plain")
+        print('  '.join(('\n'+output.lstrip()).splitlines(True)))
+
+    def get_interface_stats(self):
+        interface_stats = dict()
+        devices = [f for f in glob.glob("/sys/class/net/**/statistics")]
+        for dev_path in devices:
+            metrics = [f for f in glob.glob(dev_path +"/**")]
+            dev = re.findall(r"/sys/class/net/(.*)/statistics",dev_path)[0]
+            dev_dict = dict()
+            for metric_path in metrics:
+                metric = metric_path.replace(dev_path+"/","")
+                if isfile(metric_path):
+                    data = open(metric_path, 'r').read()[:-1]
+                    dev_dict[metric] = int(data)
+            interface_stats[dev] = dev_dict
+
+        return interface_stats[self._ifname]
 
 class LoopbackIf(Interface):
 
@@ -1371,7 +1404,6 @@ class BondIf(VLANIf):
         return self._write_sysfs('/sys/class/net/{}/bonding/mode'
                                  .format(self._ifname), mode)
 
-
 class WireGuardIf(Interface):
     """
     Wireguard interface class, contains a comnfig dictionary since
@@ -1447,6 +1479,70 @@ class WireGuardIf(Interface):
         cmd = "wg set {0} peer {1} remove".format(
             self._ifname, str(peerkey))
         return self._cmd(cmd)
+    
+    def op_show_interface(self):
+        wgdump = vyos.interfaces.wireguard_dump().get(self._ifname,None)
+
+        c = Config()
+        c.set_level(["interfaces","wireguard",self._ifname])
+        description = c.return_effective_value(["description"])
+        ips = c.return_effective_values(["address"])
+
+        print ("interface: {}".format(self._ifname))
+        if (description):
+            print ("  description: {}".format(description))
+
+        if (ips):
+            print ("  address: {}".format(", ".join(ips)))
+        print ("  public key: {}".format(wgdump['public_key']))
+        print ("  private key: (hidden)")
+        print ("  listening port: {}".format(wgdump['listen_port']))
+        print ()
+
+        for peer in c.list_effective_nodes(["peer"]):
+            if wgdump['peers']:
+                pubkey = c.return_effective_value(["peer",peer,"pubkey"])
+                if pubkey in wgdump['peers']:
+                    wgpeer = wgdump['peers'][pubkey] 
+
+                    print ("  peer: {}".format(peer))
+                    print ("    public key: {}".format(pubkey))
+
+                    """ figure out if the tunnel is recently active or not """
+                    status = "inactive"
+                    if (wgpeer['latest_handshake'] is None):
+                        """ no handshake ever """
+                        status = "inactive"
+                    else:
+                        if int(wgpeer['latest_handshake']) > 0:
+                            delta = timedelta(seconds=int(time.time() - wgpeer['latest_handshake']))
+                            print ("    latest handshake: {}".format(delta))
+                            if (time.time() - int(wgpeer['latest_handshake']) < (60*5)):
+                                """ Five minutes and the tunnel is still active """
+                                status = "active"
+                            else:
+                                """ it's been longer than 5 minutes """
+                                status = "inactive"
+                        elif int(wgpeer['latest_handshake']) == 0:
+                            """ no handshake ever """
+                            status = "inactive"
+                        print ("    status: {}".format(status))    
+
+                    if wgpeer['endpoint'] is not None:
+                        print ("    endpoint: {}".format(wgpeer['endpoint']))
+
+                    if wgpeer['allowed_ips'] is not None:
+                        print ("    allowed ips: {}".format(",".join(wgpeer['allowed_ips']).replace(",",", ")))
+                    
+                    if wgpeer['transfer_rx'] > 0 or wgpeer['transfer_tx'] > 0: 
+                        rx_size =size(wgpeer['transfer_rx'],system=alternative)
+                        tx_size =size(wgpeer['transfer_tx'],system=alternative)
+                        print ("    transfer: {} received, {} sent".format(rx_size,tx_size))
+
+                    if wgpeer['persistent_keepalive'] is not None:
+                        print ("    persistent keepalive: every {} seconds".format(wgpeer['persistent_keepalive']))
+                print()
+        super().op_show_interface_stats()
 
 
 class VXLANIf(Interface, ):
diff --git a/python/vyos/interface.py b/python/vyos/interface.py
deleted file mode 100644
index cc726e5cc..000000000
--- a/python/vyos/interface.py
+++ /dev/null
@@ -1,150 +0,0 @@
-# Copyright 2019 VyOS maintainers and contributors <maintainers@vyos.io>
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library.  If not, see <http://www.gnu.org/licenses/>.
-
-import vyos
-from vyos.config import Config
-import vyos.interfaces
-import vyos.ioctl
-from vyos.iflag import IFlag
-
-import re
-import json
-
-import subprocess
-import time
-from datetime import timedelta
-import glob
-from os.path import isfile
-from tabulate import tabulate
-from hurry.filesize import size,alternative
-
-class Interface():
-
-    intf = None
-    intf_type = None
-    valid = False
-    flags = None
-
-    def __init__(self,intf):
-        self.intf = intf
-        self.intf_type = vyos.interfaces.get_type_of_interface(self.intf)
-        self.valid = (self.intf in vyos.interfaces.list_interfaces())
-        if (self.valid):
-            self.flags = vyos.ioctl.get_interface_flags(intf)
-    
-    def up(self):
-        """ return whether interface is up or not """
-        return self.flags & IFlag.IFF_UP
-
-    def print_interface(self):
-
-        if not self.valid:
-            print("Invalid interface [{}]".format(self.intf))
-            return
-
-        if (self.intf_type == 'wireguard'):
-            self.print_wireguard_interface()
-
-        self.print_interface_stats()
-
-    def print_interface_stats(self):
-        stats = self.get_interface_stats()
-        rx = [['bytes','packets','errors','dropped','overrun','mcast'],[stats['rx_bytes'],stats['rx_packets'],stats['rx_errors'],stats['rx_dropped'],stats['rx_over_errors'],stats['multicast']]]
-        tx = [['bytes','packets','errors','dropped','carrier','collisions'],[stats['tx_bytes'],stats['tx_packets'],stats['tx_errors'],stats['tx_dropped'],stats['tx_carrier_errors'],stats['collisions']]]
-        output = "RX: \n"
-        output += tabulate(rx,headers="firstrow",numalign="right",tablefmt="plain")
-        output += "\n\nTX: \n"
-        output += tabulate(tx,headers="firstrow",numalign="right",tablefmt="plain")
-        print('  '.join(('\n'+output.lstrip()).splitlines(True)))
-
-    def get_interface_stats(self):
-        interface_stats = dict()
-        devices = [f for f in glob.glob("/sys/class/net/**/statistics")]
-        for dev_path in devices:
-            metrics = [f for f in glob.glob(dev_path +"/**")]
-            dev = re.findall(r"/sys/class/net/(.*)/statistics",dev_path)[0]
-            dev_dict = dict()
-            for metric_path in metrics:
-                metric = metric_path.replace(dev_path+"/","")
-                if isfile(metric_path):
-                    data = open(metric_path, 'r').read()[:-1]
-                    dev_dict[metric] = int(data)
-            interface_stats[dev] = dev_dict
-
-        return interface_stats[self.intf]
-
-    def print_wireguard_interface(self):
-        
-        wgdump = vyos.interfaces.wireguard_dump().get(self.intf,None)
-
-        c = Config()
-        c.set_level("interfaces wireguard {}".format(self.intf))
-        description = c.return_effective_value("description")
-        ips = c.return_effective_values("address")
-
-        print ("interface: {}".format(self.intf))
-        if (description):
-            print ("  description: {}".format(description))
-
-        if (ips):
-            print ("  address: {}".format(", ".join(ips)))
-        print ("  public key: {}".format(wgdump['public_key']))
-        print ("  private key: (hidden)")
-        print ("  listening port: {}".format(wgdump['listen_port']))
-        print ()
-
-        for peer in c.list_effective_nodes(' peer'):
-            if wgdump['peers']:
-                pubkey = c.return_effective_value("peer {} pubkey".format(peer))
-                if pubkey in wgdump['peers']:
-                    wgpeer = wgdump['peers'][pubkey] 
-
-                    print ("  peer: {}".format(peer))
-                    print ("    public key: {}".format(pubkey))
-
-                    """ figure out if the tunnel is recently active or not """
-                    status = "inactive"
-                    if (wgpeer['latest_handshake'] is None):
-                        """ no handshake ever """
-                        status = "inactive"
-                    else:
-                        if int(wgpeer['latest_handshake']) > 0:
-                            delta = timedelta(seconds=int(time.time() - wgpeer['latest_handshake']))
-                            print ("    latest handshake: {}".format(delta))
-                            if (time.time() - int(wgpeer['latest_handshake']) < (60*5)):
-                                """ Five minutes and the tunnel is still active """
-                                status = "active"
-                            else:
-                                """ it's been longer than 5 minutes """
-                                status = "inactive"
-                        elif int(wgpeer['latest_handshake']) == 0:
-                            """ no handshake ever """
-                            status = "inactive"
-                        print ("    status: {}".format(status))    
-
-                    if wgpeer['endpoint'] is not None:
-                        print ("    endpoint: {}".format(wgpeer['endpoint']))
-
-                    if wgpeer['allowed_ips'] is not None:
-                        print ("    allowed ips: {}".format(",".join(wgpeer['allowed_ips']).replace(",",", ")))
-                    
-                    if wgpeer['transfer_rx'] > 0 or wgpeer['transfer_tx'] > 0: 
-                        rx_size =size(wgpeer['transfer_rx'],system=alternative)
-                        tx_size =size(wgpeer['transfer_tx'],system=alternative)
-                        print ("    transfer: {} received, {} sent".format(rx_size,tx_size))
-
-                    if wgpeer['persistent_keepalive'] is not None:
-                        print ("    persistent keepalive: every {} seconds".format(wgpeer['persistent_keepalive']))
-                print()