Merge branch 'current' of github.com:vyos/vyos-1x into equuleus

* 'current' of github.com:vyos/vyos-1x:
  [conf completion]: T1779: Add tunnels to completion
  [XML templates] T1772: Changed old hacks to proper regex, according to the fix
  Add a function for retrieving config dicts.
  snmp: make script extension code more readable
  snmp: use proper stat literals on chmod()
  snmp: fix verify() indent on script extensions
  snmp: fix verify() bail out early order
  snmp: T1738: cleanup import statements
  T1759: Fixing dependency bug from previous commit
  T1773, T1774: add a show config operation with JSON and raw options.
  T1759: Merging interface.py into ifconfig.py
  Allow list arguments in the vyos.config show_config() function.
  Replace the try and wait for segfault approach with explicit inSession check.
  T1773: add a script for converting the config to JSON. It also exposes those functions in vyos.configtree
  [XML templates] T1772: Add escaping of `\` symbol in `<regex>`

5 files changed, 83 insertions, 29 deletions

diff --git a/src/completion/list_interfaces.py b/src/completion/list_interfaces.py
index 5e444ef78..84d17f89f 100755
--- a/src/completion/list_interfaces.py
+++ b/src/completion/list_interfaces.py
@@ -34,7 +34,8 @@ elif args.bridgeable:
     openvpn = vyos.interfaces.list_interfaces_of_type("openvpn")
     vxlan = vyos.interfaces.list_interfaces_of_type("vxlan")
     wireless = vyos.interfaces.list_interfaces_of_type("wireless")
-    interfaces = eth + bond + l2tpv3 + openvpn + vxlan + wireless
+    tunnel = vyos.interfaces.list_interfaces_of_type("tunnel")
+    interfaces = eth + bond + l2tpv3 + openvpn + vxlan + wireless + tunnel
 
 elif args.bondable:
     eth = vyos.interfaces.list_interfaces_of_type("ethernet")
diff --git a/src/conf_mode/snmp.py b/src/conf_mode/snmp.py
index 74c17fff6..b64cccbfa 100755
--- a/src/conf_mode/snmp.py
+++ b/src/conf_mode/snmp.py
@@ -16,19 +16,18 @@
 
 import sys
 import os
-import shutil
 import stat
 import pwd
-import time
-
 import jinja2
-import random
-import binascii
 import re
 
 import vyos.version
 import vyos.validate
 
+from binascii import hexlify
+from shutil import move
+from time import sleep
+from stat import S_IRWXU,S_IXGRP,S_IXOTH
 from vyos.config import Config
 from vyos import ConfigError
 
@@ -203,7 +202,7 @@ group {{ u.group }} usm {{ u.name }}
 {% if script_ext %}
 # extension scripts
 {%- for ext in script_ext|sort %}
-extend\t{{ext}}\t{{script_ext[ext]}}
+extend {{ ext.name }} {{ ext.script }}
 {%- endfor %}
 {% endif %}
 """
@@ -239,7 +238,7 @@ default_config_data = {
     'v3_traps': [],
     'v3_users': [],
     'v3_views': [],
-    'script_ext': {}
+    'script_ext': []
 }
 
 def rmfile(file):
@@ -257,10 +256,10 @@ def get_config():
     version_data = vyos.version.get_version_data()
     snmp['version'] = version_data['version']
 
-    # create an internal snmpv3 user of the form 'vyattaxxxxxxxxxxxxxxxx'
+    # create an internal snmpv3 user of the form 'vyosxxxxxxxxxxxxxxxx'
     # os.urandom(8) returns 8 bytes of random data
-    snmp['vyos_user'] = 'vyatta' + binascii.hexlify(os.urandom(8)).decode('utf-8')
-    snmp['vyos_user_pass'] = binascii.hexlify(os.urandom(16)).decode('utf-8')
+    snmp['vyos_user'] = 'vyos' + hexlify(os.urandom(8)).decode('utf-8')
+    snmp['vyos_user_pass'] = hexlify(os.urandom(16)).decode('utf-8')
 
     if conf.exists('community'):
         for name in conf.list_nodes('community'):
@@ -348,9 +347,13 @@ def get_config():
     # 'set service snmp script-extensions'
     #
     if conf.exists('script-extensions'):
-      for extname in conf.list_nodes('script-extensions extension-name'):
-        snmp['script_ext'][extname] = '/config/user-data/' + conf.return_value('script-extensions extension-name ' + extname + ' script')
+        for extname in conf.list_nodes('script-extensions extension-name'):
+            extension = {
+                'name': extname,
+                'script' : conf.return_value('script-extensions extension-name {} script'.format(extname))
+            }
 
+            snmp['script_ext'].append(extension)
 
     #########################################################################
     #                ____  _   _ __  __ ____          _____                 #
@@ -545,15 +548,11 @@ def verify(snmp):
 
     ### check if the configured script actually exist under /config/user-data
     if snmp['script_ext']:
-      for ext in snmp['script_ext']:
-        if not os.path.isfile(snmp['script_ext'][ext]):
-          print ("WARNING: script: " + snmp['script_ext'][ext] + " doesn\'t exist")
-        else:
-          os.chmod(snmp['script_ext'][ext], 0o555)
-
-    # bail out early if SNMP v3 is not configured
-    if not snmp['v3_enabled']:
-        return None
+        for ext in snmp['script_ext']:
+            if not os.path.isfile(ext['script']):
+                print ("WARNING: script: {} doesn't exist".format(ext['script']))
+            else:
+                os.chmod(ext['script'], S_IRWXU|S_IXGRP|S_IXOTH)
 
     for listen in snmp['listen_address']:
         addr = listen[0]
@@ -573,6 +572,10 @@ def verify(snmp):
         else:
             print('WARNING: SNMP listen address {0} not configured!'.format(addr))
 
+    # bail out early if SNMP v3 is not configured
+    if not snmp['v3_enabled']:
+        return None
+
     if 'v3_groups' in snmp.keys():
         for group in snmp['v3_groups']:
             #
@@ -723,7 +726,7 @@ def apply(snmp):
             if os.path.exists(volatiledir) and os.path.isdir(volatiledir):
                 files = os.listdir(volatiledir)
                 for f in files:
-                    shutil.move(volatiledir + '/' + f, nonvolatiledir)
+                    move(volatiledir + '/' + f, nonvolatiledir)
                     os.chmod(nonvolatiledir + '/' + f, stat.S_IWUSR | stat.S_IRUSR)
 
                 os.rmdir(volatiledir)
@@ -744,7 +747,7 @@ def apply(snmp):
         snmpReady = False
         while not snmpReady:
             while not os.path.exists(config_file_user):
-                time.sleep(1)
+                sleep(1)
 
             with open(config_file_user, 'r') as f:
                 for line in f:
diff --git a/src/op_mode/wireguard.py b/src/op_mode/wireguard.py
index 6860aa3ea..38c061cf4 100755
--- a/src/op_mode/wireguard.py
+++ b/src/op_mode/wireguard.py
@@ -24,7 +24,7 @@ import subprocess
 import syslog as sl
 import re
 
-from vyos.interface import Interface
+from vyos.ifconfig import WireGuardIf
 
 from vyos import ConfigError
 from vyos.config import Config
@@ -150,8 +150,8 @@ if __name__ == '__main__':
         if args.listkdir:
             list_key_dirs()
         if args.showinterface:
-            intf = Interface(args.showinterface)
-            intf.print_interface()
+            intf = WireGuardIf(args.showinterface)
+            intf.op_show_interface()
         if args.delkdir:
             if args.location:
                 del_key_dir(args.location)
diff --git a/src/services/vyos-http-api-server b/src/services/vyos-http-api-server
index 04c44c2be..571ec1258 100755
--- a/src/services/vyos-http-api-server
+++ b/src/services/vyos-http-api-server
@@ -207,11 +207,21 @@ def get_value():
         elif op == 'exists':
             res = config.exists(path)
         elif op == 'showConfig':
-            config_format = 'raw'
+            config_format = 'json'
             if 'configFormat' in command:
                 config_format = command['configFormat']
 
-            res = session.show_config(command['path'], format=config_format)
+            res = session.show_config(path=command['path'])
+            if config_format == 'json':
+                config_tree = vyos.configtree.ConfigTree(res)
+                res = json.loads(config_tree.to_json())
+            elif config_format == 'json_ast':
+                config_tree = vyos.configtree.ConfigTree(res)
+                res = json.loads(config_tree.to_json_ast())
+            elif config_format == 'raw':
+                pass
+            else:
+                return error(400, "\"{0}\" is not a valid config format")
         else:
             return error(400, "\"{0}\" is not a valid operation".format(op))
     except VyOSError as e:
diff --git a/src/utils/vyos-config-to-json b/src/utils/vyos-config-to-json
new file mode 100755
index 000000000..e03fd6a59
--- /dev/null
+++ b/src/utils/vyos-config-to-json
@@ -0,0 +1,40 @@
+#!/usr/bin/python3
+
+import sys
+import json
+
+from signal import signal, SIGPIPE, SIG_DFL
+from vyos.configtree import ConfigTree
+
+signal(SIGPIPE,SIG_DFL)
+
+config_string = None
+if (len(sys.argv) == 1):
+    # If no argument given, act as a pipe
+    config_string = sys.stdin.read()
+else:
+    file_name = sys.argv[1]
+    try:
+        with open(file_name, 'r') as f:
+            config_string = f.read()
+    except OSError as e:
+        print("Could not read config file {0}: {1}".format(file_name, e), file=sys.stderr)
+
+# This script is usually called with the output of "cli-shell-api showCfg", which does not
+# escape backslashes. "ConfigTree()" expects escaped backslashes when parsing a config
+# string (and also prints them itself). Therefore this script would fail.
+# Manually escape backslashes here to handle backslashes in any configuration strings
+# properly. The alternative would be to modify the output of "cli-shell-api showCfg",
+# but that may be break other things who rely on that specific output.
+config_string = config_string.replace("\\", "\\\\")
+
+try:
+    config = ConfigTree(config_string)
+    json_str = config.to_json()
+    # Pretty print
+    json_str = json.dumps(json.loads(json_str), indent=4, sort_keys=True)
+except ValueError as e:
+    print("Could not parse the config file: {0}".format(e), file=sys.stderr)
+    sys.exit(1)
+
+print(json_str)