Merge branch 'current' of github.com:vyos/vyos-1x into equuleus

* 'current' of github.com:vyos/vyos-1x:
  T1762: adjust the set_level() calls to use the new list representation.
  [vyos.config] T1764: support both string and list arguments in config functions.
  T1759: bug fixes, missing interface IP
  [vyos.config] T1758: use vyos.configtree for reading values, instead of calling cli-shell-api.
  [HTTP API] Add endpoints for config file and image management.
  ddclient: T1030: add cloudflare zone config entry
  [service https] T1443: organize internal data by server block
  [vyos.config] T1758: check that config setup has completed before calling showConfig, else, default to config.boot
  [HTTP API] Use a decorator for functions that require authentication.
  ddclient: T1030: adjust to latest syntax
  ddclient: T1030: auto create runtime directories
  ddclient: T1030: use new default configuration file path
  T1759: Migrating interfaces
  T1755: fixes issue with 'show vpn ipsec sa' command where lack of keysize (encr-keysize) will result in KeyError - such as for CHACHA20_POLY1305
  T1755: fixes issue with 'show vpn ipsec sa' command where lack of hash (integ-alg) will result in KeyError - such as with GCM based options

6 files changed, 195 insertions, 138 deletions

diff --git a/src/conf_mode/dynamic_dns.py b/src/conf_mode/dynamic_dns.py
index ff3c1f825..027a7f7e3 100755
--- a/src/conf_mode/dynamic_dns.py
+++ b/src/conf_mode/dynamic_dns.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2018 VyOS maintainers and contributors
+# Copyright (C) 2018-2019 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -13,8 +13,6 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-#
 
 import os
 import sys
@@ -23,16 +21,17 @@ import jinja2
 from vyos.config import Config
 from vyos import ConfigError
 
-config_file = r'/etc/ddclient.conf'
+config_file = r'/etc/ddclient/ddclient.conf'
 cache_file = r'/var/cache/ddclient/ddclient.cache'
+pid_file = r'/var/run/ddclient/ddclient.pid'
 
 config_tmpl = """
 ### Autogenerated by dynamic_dns.py ###
 daemon=1m
 syslog=yes
 ssl=yes
-pid=/var/run/ddclient/ddclient.pid
-cache=/var/cache/ddclient/ddclient.cache
+pid={{ pid_file }}
+cache={{ cache_file }}
 
 {% for interface in interfaces -%}
 
@@ -48,11 +47,11 @@ use=if, if={{ interface.interface }}
 {% for rfc in interface.rfc2136 -%}
 {% for record in rfc.record %}
 # RFC2136 dynamic DNS configuration for {{ record }}.{{ rfc.zone }}
-server={{ rfc.server }}
-protocol=nsupdate
-password={{ rfc.keyfile }}
-ttl={{ rfc.ttl }}
-zone={{ rfc.zone }}
+server={{ rfc.server }},
+protocol=nsupdate,
+password={{ rfc.keyfile }},
+ttl={{ rfc.ttl }},
+zone={{ rfc.zone }},
 {{ record }}
 {% endfor -%}
 {% endfor -%}
@@ -60,12 +59,16 @@ zone={{ rfc.zone }}
 {% for srv in interface.service %}
 {% for host in srv.host %}
 # DynDNS provider configuration for {{ host }}
-protocol={{ srv.protocol }}
-max-interval=28d
-login={{ srv.login }}
-password='{{ srv.password }}'
+protocol={{ srv.protocol }},
+max-interval=28d,
+login={{ srv.login }},
+password='{{ srv.password }}',
 {% if srv.server -%}
-server={{ srv.server }}
+server={{ srv.server }},
+{% endif -%}
+{% if 'cloudflare' in srv.protocol -%}
+{% set zone = host.split('.',1) -%}
+zone={{ zone[1] }},
 {% endif -%}
 {{ host }}
 {% endfor %}
@@ -91,6 +94,8 @@ default_service_protocol = {
 
 default_config_data = {
     'interfaces': [],
+    'cache_file': cache_file,
+    'pid_file': pid_file
 }
 
 def get_config():
@@ -237,8 +242,15 @@ def generate(dyndns):
     if dyndns is None:
         return None
 
-    tmpl = jinja2.Template(config_tmpl)
+    dirname = os.path.dirname(dyndns['pid_file'])
+    if not os.path.exists(dirname):
+        os.mkdir(dirname)
 
+    dirname = os.path.dirname(config_file)
+    if not os.path.exists(dirname):
+        os.mkdir(dirname)
+
+    tmpl = jinja2.Template(config_tmpl)
     config_text = tmpl.render(dyndns)
     with open(config_file, 'w') as f:
         f.write(config_text)
@@ -246,11 +258,16 @@ def generate(dyndns):
     return None
 
 def apply(dyndns):
-    if os.path.exists(cache_file):
-        os.unlink(cache_file)
+    if os.path.exists(dyndns['cache_file']):
+        os.unlink(dyndns['cache_file'])
+
+    if os.path.exists('/etc/ddclient.conf'):
+        os.unlink('/etc/ddclient.conf')
 
     if dyndns is None:
         os.system('/etc/init.d/ddclient stop')
+        if os.path.exists(dyndns['pid_file']):
+            os.unlink(dyndns['pid_file'])
     else:
         os.system('/etc/init.d/ddclient restart')
 
diff --git a/src/conf_mode/https.py b/src/conf_mode/https.py
index f948063e9..d7fcb74de 100755
--- a/src/conf_mode/https.py
+++ b/src/conf_mode/https.py
@@ -30,34 +30,34 @@ config_file = '/etc/nginx/sites-available/default'
 # Please be careful if you edit the template.
 config_tmpl = """
 
-### Autogenerated by http-api.py ###
+### Autogenerated by https.py ###
 # Default server configuration
 #
 server {
         listen 80 default_server;
         listen [::]:80 default_server;
         server_name _;
-        return 302 https://$server_name$request_uri;
+        return 301 https://$server_name$request_uri;
 }
 
-{% for addr, names in listen_addresses.items() %}
+{% for server in server_block_list %}
 server {
 
         # SSL configuration
         #
-{% if addr == '*' %}
-        listen 443 ssl default_server;
-        listen [::]:443 ssl default_server;
+{% if server.address == '*' %}
+        listen 443 ssl;
+        listen [::]:443 ssl;
 {% else %}
-        listen {{ addr }}:443 ssl;
+        listen {{ server.address }}:443 ssl;
 {% endif %}
 
-{% for name in names %}
+{% for name in server.name %}
         server_name {{ name }};
 {% endfor %}
 
-{% if vyos_cert %}
-        include {{ vyos_cert.conf }};
+{% if server.vyos_cert %}
+        include {{ server.vyos_cert.conf }};
 {% else %}
         #
         # Self signed certs generated by the ssl-cert package
@@ -67,46 +67,9 @@ server {
 {% endif %}
 
         # proxy settings for HTTP API, if enabled; 503, if not
-        location ~ /(retrieve|configure) {
-{% if api %}
-                proxy_pass http://localhost:{{ api.port }};
-                proxy_buffering off;
-{% else %}
-                return 503;
-{% endif %}
-        }
-
-        error_page 501 502 503 =200 @50*_json;
-
-        location @50*_json {
-                default_type application/json;
-                return 200 '{"error": "Start service in configuration mode: set service https api"}';
-        }
-
-}
-{% else %}
-server {
-        # SSL configuration
-        #
-        listen 443 ssl default_server;
-        listen [::]:443 ssl default_server;
-
-        server_name _;
-
-{% if vyos_cert %}
-        include {{ vyos_cert.conf }};
-{% else %}
-        #
-        # Self signed certs generated by the ssl-cert package
-        # Don't use them in a production server!
-        #
-        include snippets/snakeoil.conf;
-{% endif %}
-
-        # proxy settings for HTTP API, if enabled; 503, if not
-        location ~ /(retrieve|configure) {
-{% if api %}
-                proxy_pass http://localhost:{{ api.port }};
+        location ~ /(retrieve|configure|config-file|image) {
+{% if server.api %}
+                proxy_pass http://localhost:{{ server.api.port }};
                 proxy_buffering off;
 {% else %}
                 return 503;
@@ -125,8 +88,16 @@ server {
 {% endfor %}
 """
 
+default_server_block = {
+    'address'   : '*',
+    'name'      : ['_'],
+    # api       :
+    # vyos_cert :
+    # le_cert   :
+}
+
 def get_config():
-    https = vyos.defaults.https_data
+    server_block_list = []
     conf = Config()
     if not conf.exists('service https'):
         return None
@@ -134,25 +105,36 @@ def get_config():
         conf.set_level('service https')
 
     if conf.exists('listen-address'):
-        addrs = {}
         for addr in conf.list_nodes('listen-address'):
-            addrs[addr] = ['_']
+            server_block = {'address' : addr}
+            server_block['name'] = ['_']
             if conf.exists('listen-address {0} server-name'.format(addr)):
                 names = conf.return_values('listen-address {0} server-name'.format(addr))
-                addrs[addr] = names[:]
-        https['listen_addresses'] = addrs
+                server_block['name'] = names[:]
+            server_block_list.append(server_block)
 
+    if not server_block_list:
+        server_block_list.append(default_server_block)
+
+    vyos_cert_data = {}
     if conf.exists('certificates'):
         if conf.exists('certificates system-generated-certificate'):
-            https['vyos_cert'] = vyos.defaults.vyos_cert_data
+            vyos_cert_data = vyos.defaults.vyos_cert_data
+    if vyos_cert_data:
+        for block in server_block_list:
+            block['vyos_cert'] = vyos_cert_data
 
+    api_data = {}
     if conf.exists('api'):
-        https['api'] = vyos.defaults.api_data
-
-    if conf.exists('api port'):
-        port = conf.return_value('api port')
-        https['api']['port'] = port
-
+        api_data = vyos.defaults.api_data
+        if conf.exists('api port'):
+            port = conf.return_value('api port')
+            api_data['port'] = port
+    if api_data:
+        for block in server_block_list:
+            block['api'] = api_data
+
+    https = {'server_block_list' : server_block_list}
     return https
 
 def verify(https):
@@ -162,6 +144,9 @@ def generate(https):
     if https is None:
         return None
 
+    if 'server_block_list' not in https or not https['server_block_list']:
+        https['server_block_list'] = [default_server_block]
+
     tmpl = jinja2.Template(config_tmpl, trim_blocks=True)
     config_text = tmpl.render(https)
     with open(config_file, 'w') as f:
diff --git a/src/conf_mode/interfaces-ethernet.py b/src/conf_mode/interfaces-ethernet.py
index cd40aff3e..a9ed6bfb6 100755
--- a/src/conf_mode/interfaces-ethernet.py
+++ b/src/conf_mode/interfaces-ethernet.py
@@ -130,7 +130,7 @@ def get_config():
         print("Interface not specified")
 
     # check if ethernet interface has been removed
-    cfg_base = 'interfaces ethernet ' + eth['intf']
+    cfg_base = ['interfaces', 'ethernet', eth['intf']]
     if not conf.exists(cfg_base):
         eth['deleted'] = True
         # we can not bail out early as ethernet interface can not be removed
@@ -249,7 +249,7 @@ def get_config():
     if conf.exists('vif-s'):
         for vif_s in conf.list_nodes('vif-s'):
             # set config level to vif-s interface
-            conf.set_level(cfg_base + ' vif-s ' + vif_s)
+            conf.set_level(cfg_base + ['vif-s', vif_s])
             eth['vif_s'].append(vlan_to_dict(conf))
 
     # re-set configuration level to parse new nodes
@@ -263,7 +263,7 @@ def get_config():
     if conf.exists('vif'):
         for vif in conf.list_nodes('vif'):
             # set config level to vif interface
-            conf.set_level(cfg_base + ' vif ' + vif)
+            conf.set_level(cfg_base + ['vif', vif])
             eth['vif'].append(vlan_to_dict(conf))
 
     return eth
diff --git a/src/op_mode/show_ipsec_sa.py b/src/op_mode/show_ipsec_sa.py
index 0828743e8..e319cc38d 100755
--- a/src/op_mode/show_ipsec_sa.py
+++ b/src/op_mode/show_ipsec_sa.py
@@ -82,13 +82,24 @@ for sa in sas:
                 pkts_str = re.sub(r'B', r'', pkts_str)
 
                 enc = isa["encr-alg"].decode()
-                key_size = isa["encr-keysize"].decode()
-                hash = isa["integ-alg"].decode()
+                if "encr-keysize" in isa:
+                    key_size = isa["encr-keysize"].decode()
+                else:
+                    key_size = ""
+                if "integ-alg" in isa:
+                    hash = isa["integ-alg"].decode()
+                else:
+                    hash = ""
                 if "dh-group" in isa:
                     dh_group = isa["dh-group"].decode()
                 else:
                     dh_group = ""
-                proposal = "{0}_{1}/{2}".format(enc, key_size, hash)
+
+                proposal = enc
+                if key_size:
+                    proposal = "{0}_{1}".format(proposal, key_size)
+                if hash:
+                    proposal = "{0}/{1}".format(proposal, hash)
                 if dh_group:
                     proposal = "{0}/{1}".format(proposal, dh_group)
 
diff --git a/src/op_mode/wireguard.py b/src/op_mode/wireguard.py
index f6978554d..6860aa3ea 100755
--- a/src/op_mode/wireguard.py
+++ b/src/op_mode/wireguard.py
@@ -23,8 +23,8 @@ import shutil
 import subprocess
 import syslog as sl
 import re
-import time
 
+from vyos.interface import Interface
 
 from vyos import ConfigError
 from vyos.config import Config
@@ -40,41 +40,6 @@ def check_kmod():
             sl.syslog(sl.LOG_ERR, "modprobe wireguard failed")
             raise ConfigError("modprobe wireguard failed")
 
-
-def showint(interface):
-    output = subprocess.check_output(["wg", "show", interface], universal_newlines=True)
-    c = Config()
-    c.set_level("interfaces wireguard {}".format(interface))
-    description = c.return_effective_value("description".format(interface))
-    """ if the interface has a description, modify the output to include it """
-    if (description):
-        output = re.sub(r"interface: {}".format(re.escape(interface)),"interface: {}\n  Description: {}".format(interface,description),output)
-    
-    """ pull the last handshake times.  Assume if the handshake was greater than 5 minutes, the tunnel is down """
-    peer_timeouts = {}
-    last_hs_output = subprocess.check_output(["wg", "show", interface, "latest-handshakes"], universal_newlines=True)
-    for match in re.findall(r'(\S+)\s+(\d+)',last_hs_output): 
-        peer_timeouts[match[0]] = match[1]
-
-    """ modify all the peers, reformat to provide VyOS config provided peername, whether the tunnel is up/down """
-    for peer in c.list_effective_nodes(' peer'):
-        pubkey = c.return_effective_value("peer {} pubkey".format(peer))
-        status = ""
-        if int(peer_timeouts[pubkey]) > 0:
-            #Five minutes and the tunnel is still up
-            if (time.time() - int(peer_timeouts[pubkey]) < (60*5)):
-                status = "UP"
-            else:
-                status = "DOWN"
-        elif (peer_timeouts[pubkey] is None):
-            status = "DOWN"
-        elif (int(peer_timeouts[pubkey]) == 0):
-            status = "DOWN"
-
-        output = re.sub(r"peer: {}".format(re.escape(pubkey)),"peer: {}\n  Status: {}\n  public key: {}".format(peer,status,pubkey),output)
-
-    print(output)
-    
 def generate_keypair(pk, pub):
     """ generates a keypair which is stored in /config/auth/wireguard """
     old_umask = os.umask(0o027)
@@ -185,7 +150,8 @@ if __name__ == '__main__':
         if args.listkdir:
             list_key_dirs()
         if args.showinterface:
-            showint(args.showinterface)
+            intf = Interface(args.showinterface)
+            intf.print_interface()
         if args.delkdir:
             if args.location:
                 del_key_dir(args.location)
diff --git a/src/services/vyos-http-api-server b/src/services/vyos-http-api-server
index afab9be70..04c44c2be 100755
--- a/src/services/vyos-http-api-server
+++ b/src/services/vyos-http-api-server
@@ -27,12 +27,13 @@ import vyos.config
 
 import bottle
 
+from functools import wraps
+
 from vyos.configsession import ConfigSession, ConfigSessionError
 from vyos.config import VyOSError
 
 
 DEFAULT_CONFIG_FILE = '/etc/vyos/http-api.conf'
-
 CFG_GROUP = 'vyattacfg'
 
 app = bottle.default_app()
@@ -61,16 +62,23 @@ def success(data):
     resp = {"success": True, "data": data, "error": None}
     return json.dumps(resp)
 
+def auth_required(f):
+    @wraps(f)
+    def decorated_function(*args, **kwargs):
+        key = bottle.request.forms.get("key")
+        api_keys = app.config['vyos_keys']
+        id = check_auth(api_keys, key)
+        if not id:
+            return error(401, "Valid API key is required")
+        return f(*args, **kwargs)
+
+    return decorated_function
+
 @app.route('/configure', method='POST')
+@auth_required
 def configure():
     session = app.config['vyos_session']
     config = app.config['vyos_config']
-    api_keys = app.config['vyos_keys']
-
-    key = bottle.request.forms.get("key")
-    id = check_auth(api_keys, key)
-    if not id:
-        return error(401, "Valid API key is required")
 
     strict_field = bottle.request.forms.get("strict")
     if strict_field == "true":
@@ -177,17 +185,11 @@ def configure():
         return success(None)
 
 @app.route('/retrieve', method='POST')
+@auth_required
 def get_value():
     config = app.config['vyos_config']
     session = app.config['vyos_session']
 
-    api_keys = app.config['vyos_keys']
-
-    key = bottle.request.forms.get("key")
-    id = check_auth(api_keys, key)
-    if not id:
-        return error(401, "Valid API key is required")
-
     command = bottle.request.forms.get("data")
     command = json.loads(command)
 
@@ -220,6 +222,82 @@ def get_value():
 
     return success(res)
 
+@app.route('/config-file', method='POST')
+@auth_required
+def config_file_op():
+    config = app.config['vyos_config']
+    session = app.config['vyos_session']
+
+    command = bottle.request.forms.get("data")
+    command = json.loads(command)
+
+    try:
+        op = command['op']
+    except KeyError:
+        return error(400, "Missing required field \"op\"")
+
+    try:
+        if op == 'save':
+            try:
+                path = command['file']
+            except KeyError:
+                path = '/config/config.boot'
+            res = session.save_config(path)
+        elif op == 'load':
+            try:
+                path = command['file']
+            except KeyError:
+                return error(400, "Missing required field \"file\"")
+            res = session.load_config(path)
+            res = session.commit()
+        else:
+            return error(400, "\"{0}\" is not a valid operation".format(op))
+    except VyOSError as e:
+        return error(400, str(e))
+    except Exception as e:
+        print(traceback.format_exc(), file=sys.stderr)
+        return error(500, "An internal error occured. Check the logs for details.")
+
+    return success(res)
+
+@app.route('/image', method='POST')
+@auth_required
+def config_file_op():
+    config = app.config['vyos_config']
+    session = app.config['vyos_session']
+
+    command = bottle.request.forms.get("data")
+    command = json.loads(command)
+
+    try:
+        op = command['op']
+    except KeyError:
+        return error(400, "Missing required field \"op\"")
+
+    try:
+        if op == 'add':
+            try:
+                url = command['url']
+            except KeyError:
+                return error(400, "Missing required field \"url\"")
+            res = session.install_image(url)
+        elif op == 'delete':
+            try:
+                name = command['name']
+            except KeyError:
+                return error(400, "Missing required field \"name\"")
+            res = session.remove_image(name)
+        else:
+            return error(400, "\"{0}\" is not a valid operation".format(op))
+    except VyOSError as e:
+        return error(400, str(e))
+    except Exception as e:
+        print(traceback.format_exc(), file=sys.stderr)
+        return error(500, "An internal error occured. Check the logs for details.")
+
+    return success(res)
+
+
 if __name__ == '__main__':
     # systemd's user and group options don't work, do it by hand here,
     # else no one else will be able to commit