Merge branch 'current' of https://github.com/vyos/vyos-1x into current

7 files changed, 321 insertions, 72 deletions

diff --git a/python/vyos/configsession.py b/python/vyos/configsession.py
index 09fae78a1..ed6288939 100644
--- a/python/vyos/configsession.py
+++ b/python/vyos/configsession.py
@@ -25,6 +25,9 @@ COMMIT = '/opt/vyatta/sbin/my_commit'
 DISCARD = '/opt/vyatta/sbin/my_discard'
 SHOW_CONFIG = ['/bin/cli-shell-api', 'showConfig']
 LOAD_CONFIG = ['/bin/cli-shell-api', 'loadFile']
+SAVE_CONFIG = ['/opt/vyatta/sbin/vyatta-save-config.pl']
+INSTALL_IMAGE = ['/opt/vyatta/sbin/install-image']
+REMOVE_IMAGE = ['/opt/vyatta/bin/vyatta-boot-image.pl', '--del']
 
 # Default "commit via" string
 APP = "vyos-http-api"
@@ -36,6 +39,8 @@ APP = "vyos-http-api"
 def inject_vyos_env(env):
     env['VYATTA_CFG_GROUP_NAME'] = 'vyattacfg'
     env['VYATTA_USER_LEVEL_DIR'] = '/opt/vyatta/etc/shell/level/admin'
+    env['VYATTA_PROCESS_CLIENT'] = 'gui2_rest'
+    env['VYOS_HEADLESS_CLIENT'] = 'vyos_http_api'
     env['vyatta_bindir']= '/opt/vyatta/bin'
     env['vyatta_cfg_templates'] = '/opt/vyatta/share/vyatta-cfg/templates'
     env['vyatta_configdir'] = '/opt/vyatta/config'
@@ -160,3 +165,16 @@ class ConfigSession(object):
     def load_config(self, file_path):
         out = self.__run_command(LOAD_CONFIG + [file_path])
         return out
+
+    def save_config(self, file_path):
+        out = self.__run_command(SAVE_CONFIG + [file_path])
+        return out
+
+    def install_image(self, url):
+        out = self.__run_command(INSTALL_IMAGE + [url])
+        return out
+
+    def remove_image(self, name):
+        out = self.__run_command(REMOVE_IMAGE + [name])
+        return out
+
diff --git a/python/vyos/interface.py b/python/vyos/interface.py
new file mode 100644
index 000000000..6d57d972b
--- /dev/null
+++ b/python/vyos/interface.py
@@ -0,0 +1,131 @@
+# Copyright 2019 VyOS maintainers and contributors <maintainers@vyos.io>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library.  If not, see <http://www.gnu.org/licenses/>.
+
+import vyos
+from vyos.config import Config
+import vyos.interfaces
+
+import re
+import json
+
+import subprocess
+import time
+from datetime import timedelta
+import glob
+from os.path import isfile
+from tabulate import tabulate
+from hurry.filesize import size,alternative
+
+class Interface():
+
+    intf = None
+    intf_type = None
+
+    def __init__(self,intf):
+        self.intf = intf
+        self.intf_type = vyos.interfaces.get_type_of_interface(self.intf)
+
+    def print_interface(self):
+        if (self.intf_type == 'wireguard'):
+            self.print_wireguard_interface()
+
+        self.print_interface_stats()
+
+    def print_interface_stats(self):
+        stats = self.get_interface_stats()
+        rx = [['bytes','packets','errors','dropped','overrun','mcast'],[stats['rx_bytes'],stats['rx_packets'],stats['rx_errors'],stats['rx_dropped'],stats['rx_over_errors'],stats['multicast']]]
+        tx = [['bytes','packets','errors','dropped','carrier','collisions'],[stats['tx_bytes'],stats['tx_packets'],stats['tx_errors'],stats['tx_dropped'],stats['tx_carrier_errors'],stats['collisions']]]
+        output = "RX: \n"
+        output += tabulate(rx,headers="firstrow",numalign="right",tablefmt="plain")
+        output += "\n\nTX: \n"
+        output += tabulate(tx,headers="firstrow",numalign="right",tablefmt="plain")
+        print('  '.join(('\n'+output.lstrip()).splitlines(True)))
+
+    def get_interface_stats(self):
+        interface_stats = dict()
+        devices = [f for f in glob.glob("/sys/class/net/**/statistics")]
+        for dev_path in devices:
+            metrics = [f for f in glob.glob(dev_path +"/**")]
+            dev = re.findall(r"/sys/class/net/(.*)/statistics",dev_path)[0]
+            dev_dict = dict()
+            for metric_path in metrics:
+                metric = metric_path.replace(dev_path+"/","")
+                if isfile(metric_path):
+                    data = open(metric_path, 'r').read()[:-1]
+                    dev_dict[metric] = int(data)
+            interface_stats[dev] = dev_dict
+        return interface_stats[self.intf]
+
+    def print_wireguard_interface(self):
+        output = subprocess.check_output(["wg", "show", self.intf], universal_newlines=True)
+        wgdump = vyos.interfaces.wireguard_dump()[self.intf]
+        c = Config()
+        c.set_level("interfaces wireguard {}".format(self.intf))
+        description = c.return_effective_value("description")
+
+        print ("interface: {}".format(self.intf))
+        """ if the interface has a description, modify the output to include it """
+        if (description):
+            print ("  description: {}".format(description))
+            output = re.sub(r"interface: {}".format(re.escape(self.intf)),"interface: {}\n  Description: {}".format(self.intf,description),output)
+
+        print ("  public key: {}".format(wgdump['public_key']))
+        print ("  private key: (hidden)")
+        print ("  listening port: {}".format(wgdump['listen_port']))
+        print ()
+
+        for peer in c.list_effective_nodes(' peer'):
+            if wgdump['peers']:
+                pubkey = c.return_effective_value("peer {} pubkey".format(peer))
+                if pubkey in wgdump['peers']:
+                    wgpeer = wgdump['peers'][pubkey] 
+
+                    print ("  peer: {}".format(peer))
+                    print ("    public key: {}".format(pubkey))
+
+                    """ figure out if the tunnel is recently active or not """
+                    status = "inactive"
+                    if (wgpeer['latest_handshake'] is None):
+                        """ no handshake ever """
+                        status = "inactive"
+                    else:
+                        if int(wgpeer['latest_handshake']) > 0:
+                            delta = timedelta(seconds=int(time.time() - wgpeer['latest_handshake']))
+                            print ("    latest handshake: {}".format(delta))
+                            if (time.time() - int(wgpeer['latest_handshake']) < (60*5)):
+                                """ Five minutes and the tunnel is still active """
+                                status = "active"
+                            else:
+                                """ it's been longer than 5 minutes """
+                                status = "inactive"
+                        elif int(wgpeer['latest_handshake']) == 0:
+                            """ no handshake ever """
+                            status = "inactive"
+                        print ("    status: {}".format(status))    
+
+                    if wgpeer['endpoint'] is not None:
+                        print ("    endpoint: {}".format(wgpeer['endpoint']))
+
+                    if wgpeer['allowed_ips'] is not None:
+                        print ("    allowed ips: {}".format(",".join(wgpeer['allowed_ips']).replace(",",", ")))
+                    
+                    if wgpeer['transfer_rx'] > 0 or wgpeer['transfer_tx'] > 0: 
+                        rx_size =size(wgpeer['transfer_rx'],system=alternative)
+                        tx_size =size(wgpeer['transfer_tx'],system=alternative)
+                        print ("    transfer: {} received, {} sent".format(rx_size,tx_size))
+
+                    if wgpeer['persistent_keepalive'] is not None:
+                        print ("    persistent keepalive: every {} seconds".format(wgpeer['persistent_keepalive']))
+                print()
diff --git a/python/vyos/interfaces.py b/python/vyos/interfaces.py
index d69ce9d04..ecf061d17 100644
--- a/python/vyos/interfaces.py
+++ b/python/vyos/interfaces.py
@@ -16,9 +16,9 @@
 import re
 import json
 
+import subprocess
 import netifaces
 
-
 intf_type_data_file = '/usr/share/vyos/interface-types.json'
 
 def list_interfaces():
@@ -54,3 +54,46 @@ def get_type_of_interface(intf):
             return key
 
     raise ValueError("No type found for interface name: {0}".format(intf))
+
+def wireguard_dump():
+    """Dump wireguard data in a python friendly way."""
+    last_device=None
+    output = {}
+    
+    # Dump wireguard connection data
+    _f = subprocess.check_output(["wg", "show", "all", "dump"]).decode()
+    for line in _f.split('\n'):
+        if not line:
+          # Skip empty lines and last line
+          continue
+        items = line.split('\t')
+
+        if last_device != items[0]:
+            # We are currently entering a new node
+            device, private_key, public_key, listen_port, fw_mark = items
+            last_device = device
+            
+            output[device] = {
+                'private_key': None if private_key == '(none)' else private_key,
+                'public_key': None if public_key == '(none)' else public_key,
+                'listen_port': int(listen_port),
+                'fw_mark': None if fw_mark == 'off' else int(fw_mark),
+                'peers': {},
+                } 
+        else:
+            # We are entering a peer
+            device, public_key, preshared_key, endpoint, allowed_ips, latest_handshake, transfer_rx, transfer_tx, persistent_keepalive = items
+            if allowed_ips == '(none)':
+                allowed_ips = []
+            else:
+                allowed_ips = allowed_ips.split('\t')
+            output[device]['peers'][public_key] = {
+                'preshared_key': None if preshared_key == '(none)' else preshared_key,
+                'endpoint': None if endpoint == '(none)' else endpoint,
+                'allowed_ips': allowed_ips,
+                'latest_handshake': None if latest_handshake == '0' else int(latest_handshake),
+                'transfer_rx': int(transfer_rx),
+                'transfer_tx': int(transfer_tx),
+                'persistent_keepalive': None if persistent_keepalive == 'off' else int(persistent_keepalive),
+           } 
+    return output
diff --git a/src/conf_mode/dynamic_dns.py b/src/conf_mode/dynamic_dns.py
index ff3c1f825..c4a95f5f1 100755
--- a/src/conf_mode/dynamic_dns.py
+++ b/src/conf_mode/dynamic_dns.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2018 VyOS maintainers and contributors
+# Copyright (C) 2018-2019 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -13,8 +13,6 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-#
 
 import os
 import sys
@@ -23,16 +21,17 @@ import jinja2
 from vyos.config import Config
 from vyos import ConfigError
 
-config_file = r'/etc/ddclient.conf'
+config_file = r'/etc/ddclient/ddclient.conf'
 cache_file = r'/var/cache/ddclient/ddclient.cache'
+pid_file = r'/var/run/ddclient/ddclient.pid'
 
 config_tmpl = """
 ### Autogenerated by dynamic_dns.py ###
 daemon=1m
 syslog=yes
 ssl=yes
-pid=/var/run/ddclient/ddclient.pid
-cache=/var/cache/ddclient/ddclient.cache
+pid={{ pid_file }}
+cache={{ cache_file }}
 
 {% for interface in interfaces -%}
 
@@ -48,11 +47,11 @@ use=if, if={{ interface.interface }}
 {% for rfc in interface.rfc2136 -%}
 {% for record in rfc.record %}
 # RFC2136 dynamic DNS configuration for {{ record }}.{{ rfc.zone }}
-server={{ rfc.server }}
-protocol=nsupdate
-password={{ rfc.keyfile }}
-ttl={{ rfc.ttl }}
-zone={{ rfc.zone }}
+server={{ rfc.server }},
+protocol=nsupdate,
+password={{ rfc.keyfile }},
+ttl={{ rfc.ttl }},
+zone={{ rfc.zone }},
 {{ record }}
 {% endfor -%}
 {% endfor -%}
@@ -60,12 +59,12 @@ zone={{ rfc.zone }}
 {% for srv in interface.service %}
 {% for host in srv.host %}
 # DynDNS provider configuration for {{ host }}
-protocol={{ srv.protocol }}
-max-interval=28d
-login={{ srv.login }}
-password='{{ srv.password }}'
+protocol={{ srv.protocol }},
+max-interval=28d,
+login={{ srv.login }},
+password='{{ srv.password }}',
 {% if srv.server -%}
-server={{ srv.server }}
+server={{ srv.server }},
 {% endif -%}
 {{ host }}
 {% endfor %}
@@ -91,6 +90,8 @@ default_service_protocol = {
 
 default_config_data = {
     'interfaces': [],
+    'cache_file': cache_file,
+    'pid_file': pid_file
 }
 
 def get_config():
@@ -237,8 +238,15 @@ def generate(dyndns):
     if dyndns is None:
         return None
 
-    tmpl = jinja2.Template(config_tmpl)
+    dirname = os.path.dirname(dyndns['pid_file'])
+    if not os.path.exists(dirname):
+        os.mkdir(dirname)
 
+    dirname = os.path.dirname(config_file)
+    if not os.path.exists(dirname):
+        os.mkdir(dirname)
+
+    tmpl = jinja2.Template(config_tmpl)
     config_text = tmpl.render(dyndns)
     with open(config_file, 'w') as f:
         f.write(config_text)
@@ -246,11 +254,16 @@ def generate(dyndns):
     return None
 
 def apply(dyndns):
-    if os.path.exists(cache_file):
-        os.unlink(cache_file)
+    if os.path.exists(dyndns['cache_file']):
+        os.unlink(dyndns['cache_file'])
+
+    if os.path.exists('/etc/ddclient.conf'):
+        os.unlink('/etc/ddclient.conf')
 
     if dyndns is None:
         os.system('/etc/init.d/ddclient stop')
+        if os.path.exists(dyndns['pid_file']):
+            os.unlink(dyndns['pid_file'])
     else:
         os.system('/etc/init.d/ddclient restart')
 
diff --git a/src/conf_mode/https.py b/src/conf_mode/https.py
index f948063e9..233c815bc 100755
--- a/src/conf_mode/https.py
+++ b/src/conf_mode/https.py
@@ -67,7 +67,7 @@ server {
 {% endif %}
 
         # proxy settings for HTTP API, if enabled; 503, if not
-        location ~ /(retrieve|configure) {
+        location ~ /(retrieve|configure|config-file|image) {
 {% if api %}
                 proxy_pass http://localhost:{{ api.port }};
                 proxy_buffering off;
diff --git a/src/op_mode/wireguard.py b/src/op_mode/wireguard.py
index f6978554d..6860aa3ea 100755
--- a/src/op_mode/wireguard.py
+++ b/src/op_mode/wireguard.py
@@ -23,8 +23,8 @@ import shutil
 import subprocess
 import syslog as sl
 import re
-import time
 
+from vyos.interface import Interface
 
 from vyos import ConfigError
 from vyos.config import Config
@@ -40,41 +40,6 @@ def check_kmod():
             sl.syslog(sl.LOG_ERR, "modprobe wireguard failed")
             raise ConfigError("modprobe wireguard failed")
 
-
-def showint(interface):
-    output = subprocess.check_output(["wg", "show", interface], universal_newlines=True)
-    c = Config()
-    c.set_level("interfaces wireguard {}".format(interface))
-    description = c.return_effective_value("description".format(interface))
-    """ if the interface has a description, modify the output to include it """
-    if (description):
-        output = re.sub(r"interface: {}".format(re.escape(interface)),"interface: {}\n  Description: {}".format(interface,description),output)
-    
-    """ pull the last handshake times.  Assume if the handshake was greater than 5 minutes, the tunnel is down """
-    peer_timeouts = {}
-    last_hs_output = subprocess.check_output(["wg", "show", interface, "latest-handshakes"], universal_newlines=True)
-    for match in re.findall(r'(\S+)\s+(\d+)',last_hs_output): 
-        peer_timeouts[match[0]] = match[1]
-
-    """ modify all the peers, reformat to provide VyOS config provided peername, whether the tunnel is up/down """
-    for peer in c.list_effective_nodes(' peer'):
-        pubkey = c.return_effective_value("peer {} pubkey".format(peer))
-        status = ""
-        if int(peer_timeouts[pubkey]) > 0:
-            #Five minutes and the tunnel is still up
-            if (time.time() - int(peer_timeouts[pubkey]) < (60*5)):
-                status = "UP"
-            else:
-                status = "DOWN"
-        elif (peer_timeouts[pubkey] is None):
-            status = "DOWN"
-        elif (int(peer_timeouts[pubkey]) == 0):
-            status = "DOWN"
-
-        output = re.sub(r"peer: {}".format(re.escape(pubkey)),"peer: {}\n  Status: {}\n  public key: {}".format(peer,status,pubkey),output)
-
-    print(output)
-    
 def generate_keypair(pk, pub):
     """ generates a keypair which is stored in /config/auth/wireguard """
     old_umask = os.umask(0o027)
@@ -185,7 +150,8 @@ if __name__ == '__main__':
         if args.listkdir:
             list_key_dirs()
         if args.showinterface:
-            showint(args.showinterface)
+            intf = Interface(args.showinterface)
+            intf.print_interface()
         if args.delkdir:
             if args.location:
                 del_key_dir(args.location)
diff --git a/src/services/vyos-http-api-server b/src/services/vyos-http-api-server
index afab9be70..04c44c2be 100755
--- a/src/services/vyos-http-api-server
+++ b/src/services/vyos-http-api-server
@@ -27,12 +27,13 @@ import vyos.config
 
 import bottle
 
+from functools import wraps
+
 from vyos.configsession import ConfigSession, ConfigSessionError
 from vyos.config import VyOSError
 
 
 DEFAULT_CONFIG_FILE = '/etc/vyos/http-api.conf'
-
 CFG_GROUP = 'vyattacfg'
 
 app = bottle.default_app()
@@ -61,16 +62,23 @@ def success(data):
     resp = {"success": True, "data": data, "error": None}
     return json.dumps(resp)
 
+def auth_required(f):
+    @wraps(f)
+    def decorated_function(*args, **kwargs):
+        key = bottle.request.forms.get("key")
+        api_keys = app.config['vyos_keys']
+        id = check_auth(api_keys, key)
+        if not id:
+            return error(401, "Valid API key is required")
+        return f(*args, **kwargs)
+
+    return decorated_function
+
 @app.route('/configure', method='POST')
+@auth_required
 def configure():
     session = app.config['vyos_session']
     config = app.config['vyos_config']
-    api_keys = app.config['vyos_keys']
-
-    key = bottle.request.forms.get("key")
-    id = check_auth(api_keys, key)
-    if not id:
-        return error(401, "Valid API key is required")
 
     strict_field = bottle.request.forms.get("strict")
     if strict_field == "true":
@@ -177,17 +185,11 @@ def configure():
         return success(None)
 
 @app.route('/retrieve', method='POST')
+@auth_required
 def get_value():
     config = app.config['vyos_config']
     session = app.config['vyos_session']
 
-    api_keys = app.config['vyos_keys']
-
-    key = bottle.request.forms.get("key")
-    id = check_auth(api_keys, key)
-    if not id:
-        return error(401, "Valid API key is required")
-
     command = bottle.request.forms.get("data")
     command = json.loads(command)
 
@@ -220,6 +222,82 @@ def get_value():
 
     return success(res)
 
+@app.route('/config-file', method='POST')
+@auth_required
+def config_file_op():
+    config = app.config['vyos_config']
+    session = app.config['vyos_session']
+
+    command = bottle.request.forms.get("data")
+    command = json.loads(command)
+
+    try:
+        op = command['op']
+    except KeyError:
+        return error(400, "Missing required field \"op\"")
+
+    try:
+        if op == 'save':
+            try:
+                path = command['file']
+            except KeyError:
+                path = '/config/config.boot'
+            res = session.save_config(path)
+        elif op == 'load':
+            try:
+                path = command['file']
+            except KeyError:
+                return error(400, "Missing required field \"file\"")
+            res = session.load_config(path)
+            res = session.commit()
+        else:
+            return error(400, "\"{0}\" is not a valid operation".format(op))
+    except VyOSError as e:
+        return error(400, str(e))
+    except Exception as e:
+        print(traceback.format_exc(), file=sys.stderr)
+        return error(500, "An internal error occured. Check the logs for details.")
+
+    return success(res)
+
+@app.route('/image', method='POST')
+@auth_required
+def config_file_op():
+    config = app.config['vyos_config']
+    session = app.config['vyos_session']
+
+    command = bottle.request.forms.get("data")
+    command = json.loads(command)
+
+    try:
+        op = command['op']
+    except KeyError:
+        return error(400, "Missing required field \"op\"")
+
+    try:
+        if op == 'add':
+            try:
+                url = command['url']
+            except KeyError:
+                return error(400, "Missing required field \"url\"")
+            res = session.install_image(url)
+        elif op == 'delete':
+            try:
+                name = command['name']
+            except KeyError:
+                return error(400, "Missing required field \"name\"")
+            res = session.remove_image(name)
+        else:
+            return error(400, "\"{0}\" is not a valid operation".format(op))
+    except VyOSError as e:
+        return error(400, str(e))
+    except Exception as e:
+        print(traceback.format_exc(), file=sys.stderr)
+        return error(500, "An internal error occured. Check the logs for details.")
+
+    return success(res)
+
+
 if __name__ == '__main__':
     # systemd's user and group options don't work, do it by hand here,
     # else no one else will be able to commit