Merge branch 'current' of github.com:vyos/vyos-1x into equuleus

* 'current' of github.com:vyos/vyos-1x:
  T1762: adjust the set_level() calls to use the new list representation.
  [vyos.config] T1764: support both string and list arguments in config functions.
  T1759: bug fixes, missing interface IP
  [vyos.config] T1758: use vyos.configtree for reading values, instead of calling cli-shell-api.
  [HTTP API] Add endpoints for config file and image management.
  ddclient: T1030: add cloudflare zone config entry
  [service https] T1443: organize internal data by server block
  [vyos.config] T1758: check that config setup has completed before calling showConfig, else, default to config.boot
  [HTTP API] Use a decorator for functions that require authentication.
  ddclient: T1030: adjust to latest syntax
  ddclient: T1030: auto create runtime directories
  ddclient: T1030: use new default configuration file path
  T1759: Migrating interfaces
  T1755: fixes issue with 'show vpn ipsec sa' command where lack of keysize (encr-keysize) will result in KeyError - such as for CHACHA20_POLY1305
  T1755: fixes issue with 'show vpn ipsec sa' command where lack of hash (integ-alg) will result in KeyError - such as with GCM based options

11 files changed, 526 insertions, 256 deletions

diff --git a/python/vyos/config.py b/python/vyos/config.py
index c9c73b971..3a340b2da 100644
--- a/python/vyos/config.py
+++ b/python/vyos/config.py
@@ -1,4 +1,4 @@
-# Copyright 2017 VyOS maintainers and contributors <maintainers@vyos.io>
+# Copyright 2017, 2019 VyOS maintainers and contributors <maintainers@vyos.io>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -17,7 +17,7 @@
 A library for reading VyOS running config data.
 
 This library is used internally by all config scripts of VyOS,
-but its API should be considered stable and it is safe to use
+but its API should be considered stable and safe to use
 in user scripts.
 
 Note that this module will not work outside VyOS.
@@ -43,8 +43,8 @@ For example, under "system", the names of all valid child nodes are predefined
 
 To the contrary, children of the "system task-scheduler task" node can have arbitrary names.
 Such nodes are called *tag nodes*. This terminology is confusing but we keep using it for lack
-of a better word. The knowledge of whether in "task Foo" the "tag" is "task" or "Foo" is lost
-in time, luckily, the distinction is irrelevant in practice.
+of a better word. No one remembers if the "tag" in "task Foo" is "task" or "Foo",
+but the distinction is irrelevant in practice.
 
 Configuration modes
 ###################
@@ -53,21 +53,21 @@ VyOS has two distinct modes: operational mode and configuration mode. When a use
 the CLI is in the operational mode. In this mode, only the running (effective) config is accessible for reading.
 
 When a user enters the "configure" command, a configuration session is setup. Every config session
-has its *proposed* config built on top of the current running config. When changes are commited, if commit succeeds,
+has its *proposed* (or *session*) config built on top of the current running config. When changes are commited, if commit succeeds,
 the proposed config is merged into the running config.
 
-For this reason, this library has two sets of functions. The base versions, such as ``exists`` or ``return_value``
-are only usable in configuration mode. They take all nodes into account, in both proposed and running configs.
-Configuration scripts require access to uncommited changes for obvious reasons. Configuration mode completion helpers
-should also use these functions because not having nodes you've just created in completion is annoying.
+In configuration mode, "base" functions like `exists`, `return_value` return values from the session config,
+while functions prefixed "effective" return values from the running config.
+
+In operational mode, all functions return values from the running config.
 
-However, in operational mode, only the running config is available. Currently, you need to use special functions
-for reading it from operational mode scripts, they can be distinguished by the word "effective" in their names.
-In the future base versions may be made to detect if they are called from a config session or not.
 """
 
-import subprocess
+import os
 import re
+import subprocess
+
+import vyos.configtree
 
 
 class VyOSError(Exception):
@@ -89,17 +89,51 @@ class Config(object):
     """
     def __init__(self, session_env=None):
         self._cli_shell_api = "/bin/cli-shell-api"
-        self._level = ""
+        self._level = []
         if session_env:
             self.__session_env = session_env
         else:
             self.__session_env = None
 
+        # Running config can be obtained either from op or conf mode, it always succeeds
+        # (if config system is initialized at all).
+        if os.path.isfile('/tmp/vyos-config-status'):
+            running_config_text = self._run([self._cli_shell_api, '--show-active-only', '--show-show-defaults', 'showConfig'])
+        else:
+            with open('/opt/vyatta/etc/config/config.boot') as f:
+                running_config_text = f.read()
+
+        # Session config ("active") only exists in conf mode.
+        # Trying to obtain it from op mode will cause a fatal cli-shell-api error.
+        # If that happens, we assume that a script is running from op mode and use the running config
+        # for the "session config" variable as well.
+        try:
+            session_config_text = self._run([self._cli_shell_api, '--show-working-only', '--show-show-defaults', 'showConfig'])
+        except VyOSError:
+            session_config_text = running_config_text
+
+        self._session_config = vyos.configtree.ConfigTree(session_config_text)
+        self._running_config = vyos.configtree.ConfigTree(running_config_text)
+
     def _make_command(self, op, path):
         args = path.split()
         cmd = [self._cli_shell_api, op] + args
         return cmd
 
+    def _make_path(self, path):
+        # Backwards-compatibility stuff: original implementation used string paths
+        # libvyosconfig paths are lists, but since node names cannot contain whitespace,
+        # splitting at whitespace is reasonably safe.
+        # It may cause problems with exists() when it's used for checking values,
+        # since values may contain whitespace.
+        if isinstance(path, str):
+            path = re.split(r'\s*', path)
+        elif isinstance(path, list):
+            pass
+        else:
+            raise TypeError("Path must be a whitespace-separated string or a list")
+        return (self._level + path)
+
     def _run(self, cmd):
         if self.__session_env:
             p = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=self.__session_env)
@@ -126,7 +160,12 @@ class Config(object):
         # Make sure there's always a space between default path (level)
         # and path supplied as method argument
         # XXX: for small strings in-place concatenation is not a problem
-        self._level = path + " "
+        if isinstance(path, str):
+            self._level = re.split(r'\s*', path)
+        elif isinstance(path, list):
+            self._level = path
+        else:
+            raise TypeError("Level path must be either a whitespace-separated string or a list")
 
     def get_level(self):
         """
@@ -135,7 +174,7 @@ class Config(object):
         Returns:
             str: current edit level
         """
-        return(self._level.strip())
+        return(self._level)
 
     def exists(self, path):
         """
@@ -148,11 +187,20 @@ class Config(object):
             This function cannot be used outside a configuration sessions.
             In operational mode scripts, use ``exists_effective``.
         """
-        try:
-            self._run(self._make_command('exists', self._level + path))
+        if self._session_config.exists(self._make_path(path)):
             return True
-        except VyOSError:
-            return False
+        else:
+            # libvyosconfig exists() works only for _nodes_, not _values_
+            # libvyattacfg one also worked for values, so we emulate that case here
+            path = re.split(r'\s*', path)
+            path_without_value = path[:-1]
+            path_str = " ".join(path_without_value)
+            try:
+                value = self._session_config.return_value(self._make_path(path_str))
+                return (value == path[-1])
+            except vyos.configtree.ConfigTreeError:
+                # node doesn't exist at all
+                return False
 
     def session_changed(self):
         """
@@ -203,7 +251,8 @@ class Config(object):
             It also returns False if node doesn't exist.
         """
         try:
-            self._run(self._make_command('isMulti', self._level + path))
+            path = " ".join(self._level) + " " + path
+            self._run(self._make_command('isMulti', path))
             return True
         except VyOSError:
             return False
@@ -220,7 +269,8 @@ class Config(object):
             It also returns False if node doesn't exist.
         """
         try:
-            self._run(self._make_command('isTag', self._level + path))
+            path = " ".join(self._level) + " " + path
+            self._run(self._make_command('isTag', path))
             return True
         except VyOSError:
             return False
@@ -237,7 +287,8 @@ class Config(object):
             It also returns False if node doesn't exist.
         """
         try:
-            self._run(self._make_command('isLeaf', self._level + path))
+            path = " ".join(self._level) + " " + path
+            self._run(self._make_command('isLeaf', path))
             return True
         except VyOSError:
             return False
@@ -254,9 +305,6 @@ class Config(object):
             str: Node value, if it has any
             None: if node is valueless *or* if it doesn't exist
 
-        Raises:
-            VyOSError: if node is not a single-value leaf node
-
         Note:
             Due to the issue with treatment of valueless nodes by this function,
             valueless nodes should be checked with ``exists`` instead.
@@ -264,17 +312,15 @@ class Config(object):
             This function cannot be used outside a configuration session.
             In operational mode scripts, use ``return_effective_value``.
         """
-        full_path = self._level + path
-        if self.is_multi(path):
-            raise VyOSError("Cannot use return_value on multi node: {0}".format(full_path))
-        elif not self.is_leaf(path):
-            raise VyOSError("Cannot use return_value on non-leaf node: {0}".format(full_path))
+        try:
+            value = self._session_config.return_value(self._make_path(path))
+        except vyos.configtree.ConfigTreeError:
+            value = None
+
+        if not value:
+            return(default)
         else:
-            try:
-                out = self._run(self._make_command('returnValue', full_path))
-                return out
-            except VyOSError:
-                return(default)
+            return(value)
 
     def return_values(self, path, default=[]):
         """
@@ -285,27 +331,21 @@ class Config(object):
 
         Returns:
             str list: Node values, if it has any
-            None: if node does not exist
-
-        Raises:
-            VyOSError: if node is not a multi-value leaf node
+            []: if node does not exist
 
         Note:
             This function cannot be used outside a configuration session.
             In operational mode scripts, use ``return_effective_values``.
         """
-        full_path = self._level + path
-        if not self.is_multi(path):
-            raise VyOSError("Cannot use return_values on non-multi node: {0}".format(full_path))
-        elif not self.is_leaf(path):
-            raise VyOSError("Cannot use return_values on non-leaf node: {0}".format(full_path))
+        try:
+            values = self._session_config.return_values(self._make_path(path))
+        except vyos.configtree.ConfigTreeError:
+            values = []
+
+        if not values:
+            return(default)
         else:
-            try:
-                out = self._run(self._make_command('returnValues', full_path))
-                values = re.findall(r"\'(.*?)\'", out)
-                return values
-            except VyOSError:
-                return(default)
+            return(values)
 
     def list_nodes(self, path, default=[]):
         """
@@ -317,26 +357,16 @@ class Config(object):
         Returns:
             string list: child node names
 
-        Raises:
-            VyOSError: if the node is not a tag node
-
-        Note:
-            There is no way to list all children of a non-tag node in
-            the current config backend.
-
-            This function cannot be used outside a configuration session.
-            In operational mode scripts, use ``list_effective_nodes``.
         """
-        full_path = self._level + path
-        if self.is_tag(path):
-            try:
-                out = self._run(self._make_command('listNodes', full_path))
-                values = re.findall(r"\'(.*?)\'", out)
-                return values
-            except VyOSError:
-                return(default)
+        try:
+            nodes = self._session_config.list_nodes(self._make_path(path))
+        except vyos.configtree.ConfigTreeError:
+            nodes = []
+
+        if not nodes:
+            return(default)
         else:
-            raise VyOSError("Cannot use list_nodes on a non-tag node: {0}".format(full_path))
+            return(nodes)
 
     def exists_effective(self, path):
         """
@@ -352,11 +382,7 @@ class Config(object):
             This function is safe to use in operational mode. In configuration mode,
             it ignores uncommited changes.
         """
-        try:
-            self._run(self._make_command('existsEffective', self._level + path))
-            return True
-        except VyOSError:
-            return False
+        return(self._running_config.exists(self._make_path(path)))
 
     def return_effective_value(self, path, default=None):
         """
@@ -368,21 +394,17 @@ class Config(object):
 
         Returns:
             str: Node value
-
-        Raises:
-            VyOSError: if node is not a multi-value leaf node
         """
-        full_path = self._level + path
-        if self.is_multi(path):
-            raise VyOSError("Cannot use return_effective_value on multi node: {0}".format(full_path))
-        elif not self.is_leaf(path):
-            raise VyOSError("Cannot use return_effective_value on non-leaf node: {0}".format(full_path))
+        try:
+            value = self._running_config.return_value(self._make_path(path))
+        except vyos.configtree.ConfigTreeError:
+            value = None
+
+        if not value:
+            return(default)
         else:
-            try:
-                out = self._run(self._make_command('returnEffectiveValue', full_path))
-                return out
-            except VyOSError:
-                return(default)
+            return(value)
+
 
     def return_effective_values(self, path, default=[]):
         """
@@ -393,22 +415,16 @@ class Config(object):
 
         Returns:
             str list: A list of values
-
-        Raises:
-            VyOSError: if node is not a multi-value leaf node
         """
-        full_path = self._level + path
-        if not self.is_multi(path):
-            raise VyOSError("Cannot use return_effective_values on non-multi node: {0}".format(full_path))
-        elif not self.is_leaf(path):
-            raise VyOSError("Cannot use return_effective_values on non-leaf node: {0}".format(full_path))
+        try:
+            values = self._running_config.return_values(self._make_path(path))
+        except vyos.configtree.ConfigTreeError:
+            values = []
+
+        if not values:
+            return(default)
         else:
-            try:
-                out = self._run(self._make_command('returnEffectiveValues', full_path))
-                values = re.findall(r"\'(.*?)\'", out)
-                return values
-            except VyOSError:
-                return(default)
+            return(values)
 
     def list_effective_nodes(self, path, default=[]):
         """
@@ -422,18 +438,13 @@ class Config(object):
 
         Raises:
             VyOSError: if the node is not a tag node
-
-        Note:
-            There is no way to list all children of a non-tag node in
-            the current config backend.
         """
-        full_path = self._level + path
-        if self.is_tag(path):
-            try:
-                out = self._run(self._make_command('listEffectiveNodes', full_path))
-                values = out.split()
-                return list(map(lambda x: re.sub(r'^\'(.*)\'$', r'\1',x), values))
-            except VyOSError:
-                return(default)
+        try:
+            nodes = self._running_config.list_nodes(self._make_path(path))
+        except vyos.configtree.ConfigTreeError:
+            nodes = []
+
+        if not nodes:
+            return(default)
         else:
-            raise VyOSError("Cannot use list_effective_nodes on a non-tag node: {0}".format(full_path))
+            return(nodes)
diff --git a/python/vyos/configdict.py b/python/vyos/configdict.py
index 983906923..e8e52b33d 100644
--- a/python/vyos/configdict.py
+++ b/python/vyos/configdict.py
@@ -106,7 +106,7 @@ def vlan_to_dict(conf):
     Function call's itself recursively if a vif-s/vif-c pair is detected.
     """
     vlan = {
-        'id': conf.get_level().split()[-1], # get the '100' in 'interfaces bonding bond0 vif-s 100'
+        'id': conf.get_level()[-1], # get the '100' in 'interfaces bonding bond0 vif-s 100'
         'address': [],
         'address_remove': [],
         'description': '',
@@ -192,7 +192,7 @@ def vlan_to_dict(conf):
 
     # ethertype is mandatory on vif-s nodes and only exists here!
     # check if this is a vif-s node at all:
-    if conf.get_level().split()[-2] == 'vif-s':
+    if conf.get_level()[-2] == 'vif-s':
         vlan['vif_c'] = []
         vlan['vif_c_remove'] = []
 
@@ -215,7 +215,7 @@ def vlan_to_dict(conf):
             # add new key (vif-c) to dictionary
             for vif in conf.list_nodes('vif-c'):
                 # set config level to vif interface
-                conf.set_level(cfg_level + ' vif-c ' + vif)
+                conf.set_level(cfg_level + ['vif-c', vif])
                 vlan['vif_c'].append(vlan_to_dict(conf))
 
     return vlan
diff --git a/python/vyos/configsession.py b/python/vyos/configsession.py
index 09fae78a1..ed6288939 100644
--- a/python/vyos/configsession.py
+++ b/python/vyos/configsession.py
@@ -25,6 +25,9 @@ COMMIT = '/opt/vyatta/sbin/my_commit'
 DISCARD = '/opt/vyatta/sbin/my_discard'
 SHOW_CONFIG = ['/bin/cli-shell-api', 'showConfig']
 LOAD_CONFIG = ['/bin/cli-shell-api', 'loadFile']
+SAVE_CONFIG = ['/opt/vyatta/sbin/vyatta-save-config.pl']
+INSTALL_IMAGE = ['/opt/vyatta/sbin/install-image']
+REMOVE_IMAGE = ['/opt/vyatta/bin/vyatta-boot-image.pl', '--del']
 
 # Default "commit via" string
 APP = "vyos-http-api"
@@ -36,6 +39,8 @@ APP = "vyos-http-api"
 def inject_vyos_env(env):
     env['VYATTA_CFG_GROUP_NAME'] = 'vyattacfg'
     env['VYATTA_USER_LEVEL_DIR'] = '/opt/vyatta/etc/shell/level/admin'
+    env['VYATTA_PROCESS_CLIENT'] = 'gui2_rest'
+    env['VYOS_HEADLESS_CLIENT'] = 'vyos_http_api'
     env['vyatta_bindir']= '/opt/vyatta/bin'
     env['vyatta_cfg_templates'] = '/opt/vyatta/share/vyatta-cfg/templates'
     env['vyatta_configdir'] = '/opt/vyatta/config'
@@ -160,3 +165,16 @@ class ConfigSession(object):
     def load_config(self, file_path):
         out = self.__run_command(LOAD_CONFIG + [file_path])
         return out
+
+    def save_config(self, file_path):
+        out = self.__run_command(SAVE_CONFIG + [file_path])
+        return out
+
+    def install_image(self, url):
+        out = self.__run_command(INSTALL_IMAGE + [url])
+        return out
+
+    def remove_image(self, name):
+        out = self.__run_command(REMOVE_IMAGE + [name])
+        return out
+
diff --git a/python/vyos/interface.py b/python/vyos/interface.py
new file mode 100644
index 000000000..1aae6db60
--- /dev/null
+++ b/python/vyos/interface.py
@@ -0,0 +1,141 @@
+# Copyright 2019 VyOS maintainers and contributors <maintainers@vyos.io>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library.  If not, see <http://www.gnu.org/licenses/>.
+
+import vyos
+from vyos.config import Config
+import vyos.interfaces
+
+import re
+import json
+
+import subprocess
+import time
+from datetime import timedelta
+import glob
+from os.path import isfile
+from tabulate import tabulate
+from hurry.filesize import size,alternative
+
+class Interface():
+
+    intf = None
+    intf_type = None
+    valid = False
+
+    def __init__(self,intf):
+        self.intf = intf
+        self.intf_type = vyos.interfaces.get_type_of_interface(self.intf)
+        self.valid = (self.intf in vyos.interfaces.list_interfaces())
+
+    def print_interface(self):
+
+        if not self.valid:
+            print("Invalid interface [{}]".format(self.intf))
+            return
+
+        if (self.intf_type == 'wireguard'):
+            self.print_wireguard_interface()
+
+        self.print_interface_stats()
+
+    def print_interface_stats(self):
+        stats = self.get_interface_stats()
+        rx = [['bytes','packets','errors','dropped','overrun','mcast'],[stats['rx_bytes'],stats['rx_packets'],stats['rx_errors'],stats['rx_dropped'],stats['rx_over_errors'],stats['multicast']]]
+        tx = [['bytes','packets','errors','dropped','carrier','collisions'],[stats['tx_bytes'],stats['tx_packets'],stats['tx_errors'],stats['tx_dropped'],stats['tx_carrier_errors'],stats['collisions']]]
+        output = "RX: \n"
+        output += tabulate(rx,headers="firstrow",numalign="right",tablefmt="plain")
+        output += "\n\nTX: \n"
+        output += tabulate(tx,headers="firstrow",numalign="right",tablefmt="plain")
+        print('  '.join(('\n'+output.lstrip()).splitlines(True)))
+
+    def get_interface_stats(self):
+        interface_stats = dict()
+        devices = [f for f in glob.glob("/sys/class/net/**/statistics")]
+        for dev_path in devices:
+            metrics = [f for f in glob.glob(dev_path +"/**")]
+            dev = re.findall(r"/sys/class/net/(.*)/statistics",dev_path)[0]
+            dev_dict = dict()
+            for metric_path in metrics:
+                metric = metric_path.replace(dev_path+"/","")
+                if isfile(metric_path):
+                    data = open(metric_path, 'r').read()[:-1]
+                    dev_dict[metric] = int(data)
+            interface_stats[dev] = dev_dict
+
+        return interface_stats[self.intf]
+
+    def print_wireguard_interface(self):
+        
+        wgdump = vyos.interfaces.wireguard_dump().get(self.intf,None)
+
+        c = Config()
+        c.set_level("interfaces wireguard {}".format(self.intf))
+        description = c.return_effective_value("description")
+        ips = c.return_effective_values("address")
+
+        print ("interface: {}".format(self.intf))
+        if (description):
+            print ("  description: {}".format(description))
+
+        if (ips):
+            print ("  address: {}".format(", ".join(ips)))
+        print ("  public key: {}".format(wgdump['public_key']))
+        print ("  private key: (hidden)")
+        print ("  listening port: {}".format(wgdump['listen_port']))
+        print ()
+
+        for peer in c.list_effective_nodes(' peer'):
+            if wgdump['peers']:
+                pubkey = c.return_effective_value("peer {} pubkey".format(peer))
+                if pubkey in wgdump['peers']:
+                    wgpeer = wgdump['peers'][pubkey] 
+
+                    print ("  peer: {}".format(peer))
+                    print ("    public key: {}".format(pubkey))
+
+                    """ figure out if the tunnel is recently active or not """
+                    status = "inactive"
+                    if (wgpeer['latest_handshake'] is None):
+                        """ no handshake ever """
+                        status = "inactive"
+                    else:
+                        if int(wgpeer['latest_handshake']) > 0:
+                            delta = timedelta(seconds=int(time.time() - wgpeer['latest_handshake']))
+                            print ("    latest handshake: {}".format(delta))
+                            if (time.time() - int(wgpeer['latest_handshake']) < (60*5)):
+                                """ Five minutes and the tunnel is still active """
+                                status = "active"
+                            else:
+                                """ it's been longer than 5 minutes """
+                                status = "inactive"
+                        elif int(wgpeer['latest_handshake']) == 0:
+                            """ no handshake ever """
+                            status = "inactive"
+                        print ("    status: {}".format(status))    
+
+                    if wgpeer['endpoint'] is not None:
+                        print ("    endpoint: {}".format(wgpeer['endpoint']))
+
+                    if wgpeer['allowed_ips'] is not None:
+                        print ("    allowed ips: {}".format(",".join(wgpeer['allowed_ips']).replace(",",", ")))
+                    
+                    if wgpeer['transfer_rx'] > 0 or wgpeer['transfer_tx'] > 0: 
+                        rx_size =size(wgpeer['transfer_rx'],system=alternative)
+                        tx_size =size(wgpeer['transfer_tx'],system=alternative)
+                        print ("    transfer: {} received, {} sent".format(rx_size,tx_size))
+
+                    if wgpeer['persistent_keepalive'] is not None:
+                        print ("    persistent keepalive: every {} seconds".format(wgpeer['persistent_keepalive']))
+                print()
diff --git a/python/vyos/interfaces.py b/python/vyos/interfaces.py
index d69ce9d04..ecf061d17 100644
--- a/python/vyos/interfaces.py
+++ b/python/vyos/interfaces.py
@@ -16,9 +16,9 @@
 import re
 import json
 
+import subprocess
 import netifaces
 
-
 intf_type_data_file = '/usr/share/vyos/interface-types.json'
 
 def list_interfaces():
@@ -54,3 +54,46 @@ def get_type_of_interface(intf):
             return key
 
     raise ValueError("No type found for interface name: {0}".format(intf))
+
+def wireguard_dump():
+    """Dump wireguard data in a python friendly way."""
+    last_device=None
+    output = {}
+    
+    # Dump wireguard connection data
+    _f = subprocess.check_output(["wg", "show", "all", "dump"]).decode()
+    for line in _f.split('\n'):
+        if not line:
+          # Skip empty lines and last line
+          continue
+        items = line.split('\t')
+
+        if last_device != items[0]:
+            # We are currently entering a new node
+            device, private_key, public_key, listen_port, fw_mark = items
+            last_device = device
+            
+            output[device] = {
+                'private_key': None if private_key == '(none)' else private_key,
+                'public_key': None if public_key == '(none)' else public_key,
+                'listen_port': int(listen_port),
+                'fw_mark': None if fw_mark == 'off' else int(fw_mark),
+                'peers': {},
+                } 
+        else:
+            # We are entering a peer
+            device, public_key, preshared_key, endpoint, allowed_ips, latest_handshake, transfer_rx, transfer_tx, persistent_keepalive = items
+            if allowed_ips == '(none)':
+                allowed_ips = []
+            else:
+                allowed_ips = allowed_ips.split('\t')
+            output[device]['peers'][public_key] = {
+                'preshared_key': None if preshared_key == '(none)' else preshared_key,
+                'endpoint': None if endpoint == '(none)' else endpoint,
+                'allowed_ips': allowed_ips,
+                'latest_handshake': None if latest_handshake == '0' else int(latest_handshake),
+                'transfer_rx': int(transfer_rx),
+                'transfer_tx': int(transfer_tx),
+                'persistent_keepalive': None if persistent_keepalive == 'off' else int(persistent_keepalive),
+           } 
+    return output
diff --git a/src/conf_mode/dynamic_dns.py b/src/conf_mode/dynamic_dns.py
index ff3c1f825..027a7f7e3 100755
--- a/src/conf_mode/dynamic_dns.py
+++ b/src/conf_mode/dynamic_dns.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2018 VyOS maintainers and contributors
+# Copyright (C) 2018-2019 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -13,8 +13,6 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-#
 
 import os
 import sys
@@ -23,16 +21,17 @@ import jinja2
 from vyos.config import Config
 from vyos import ConfigError
 
-config_file = r'/etc/ddclient.conf'
+config_file = r'/etc/ddclient/ddclient.conf'
 cache_file = r'/var/cache/ddclient/ddclient.cache'
+pid_file = r'/var/run/ddclient/ddclient.pid'
 
 config_tmpl = """
 ### Autogenerated by dynamic_dns.py ###
 daemon=1m
 syslog=yes
 ssl=yes
-pid=/var/run/ddclient/ddclient.pid
-cache=/var/cache/ddclient/ddclient.cache
+pid={{ pid_file }}
+cache={{ cache_file }}
 
 {% for interface in interfaces -%}
 
@@ -48,11 +47,11 @@ use=if, if={{ interface.interface }}
 {% for rfc in interface.rfc2136 -%}
 {% for record in rfc.record %}
 # RFC2136 dynamic DNS configuration for {{ record }}.{{ rfc.zone }}
-server={{ rfc.server }}
-protocol=nsupdate
-password={{ rfc.keyfile }}
-ttl={{ rfc.ttl }}
-zone={{ rfc.zone }}
+server={{ rfc.server }},
+protocol=nsupdate,
+password={{ rfc.keyfile }},
+ttl={{ rfc.ttl }},
+zone={{ rfc.zone }},
 {{ record }}
 {% endfor -%}
 {% endfor -%}
@@ -60,12 +59,16 @@ zone={{ rfc.zone }}
 {% for srv in interface.service %}
 {% for host in srv.host %}
 # DynDNS provider configuration for {{ host }}
-protocol={{ srv.protocol }}
-max-interval=28d
-login={{ srv.login }}
-password='{{ srv.password }}'
+protocol={{ srv.protocol }},
+max-interval=28d,
+login={{ srv.login }},
+password='{{ srv.password }}',
 {% if srv.server -%}
-server={{ srv.server }}
+server={{ srv.server }},
+{% endif -%}
+{% if 'cloudflare' in srv.protocol -%}
+{% set zone = host.split('.',1) -%}
+zone={{ zone[1] }},
 {% endif -%}
 {{ host }}
 {% endfor %}
@@ -91,6 +94,8 @@ default_service_protocol = {
 
 default_config_data = {
     'interfaces': [],
+    'cache_file': cache_file,
+    'pid_file': pid_file
 }
 
 def get_config():
@@ -237,8 +242,15 @@ def generate(dyndns):
     if dyndns is None:
         return None
 
-    tmpl = jinja2.Template(config_tmpl)
+    dirname = os.path.dirname(dyndns['pid_file'])
+    if not os.path.exists(dirname):
+        os.mkdir(dirname)
 
+    dirname = os.path.dirname(config_file)
+    if not os.path.exists(dirname):
+        os.mkdir(dirname)
+
+    tmpl = jinja2.Template(config_tmpl)
     config_text = tmpl.render(dyndns)
     with open(config_file, 'w') as f:
         f.write(config_text)
@@ -246,11 +258,16 @@ def generate(dyndns):
     return None
 
 def apply(dyndns):
-    if os.path.exists(cache_file):
-        os.unlink(cache_file)
+    if os.path.exists(dyndns['cache_file']):
+        os.unlink(dyndns['cache_file'])
+
+    if os.path.exists('/etc/ddclient.conf'):
+        os.unlink('/etc/ddclient.conf')
 
     if dyndns is None:
         os.system('/etc/init.d/ddclient stop')
+        if os.path.exists(dyndns['pid_file']):
+            os.unlink(dyndns['pid_file'])
     else:
         os.system('/etc/init.d/ddclient restart')
 
diff --git a/src/conf_mode/https.py b/src/conf_mode/https.py
index f948063e9..d7fcb74de 100755
--- a/src/conf_mode/https.py
+++ b/src/conf_mode/https.py
@@ -30,34 +30,34 @@ config_file = '/etc/nginx/sites-available/default'
 # Please be careful if you edit the template.
 config_tmpl = """
 
-### Autogenerated by http-api.py ###
+### Autogenerated by https.py ###
 # Default server configuration
 #
 server {
         listen 80 default_server;
         listen [::]:80 default_server;
         server_name _;
-        return 302 https://$server_name$request_uri;
+        return 301 https://$server_name$request_uri;
 }
 
-{% for addr, names in listen_addresses.items() %}
+{% for server in server_block_list %}
 server {
 
         # SSL configuration
         #
-{% if addr == '*' %}
-        listen 443 ssl default_server;
-        listen [::]:443 ssl default_server;
+{% if server.address == '*' %}
+        listen 443 ssl;
+        listen [::]:443 ssl;
 {% else %}
-        listen {{ addr }}:443 ssl;
+        listen {{ server.address }}:443 ssl;
 {% endif %}
 
-{% for name in names %}
+{% for name in server.name %}
         server_name {{ name }};
 {% endfor %}
 
-{% if vyos_cert %}
-        include {{ vyos_cert.conf }};
+{% if server.vyos_cert %}
+        include {{ server.vyos_cert.conf }};
 {% else %}
         #
         # Self signed certs generated by the ssl-cert package
@@ -67,46 +67,9 @@ server {
 {% endif %}
 
         # proxy settings for HTTP API, if enabled; 503, if not
-        location ~ /(retrieve|configure) {
-{% if api %}
-                proxy_pass http://localhost:{{ api.port }};
-                proxy_buffering off;
-{% else %}
-                return 503;
-{% endif %}
-        }
-
-        error_page 501 502 503 =200 @50*_json;
-
-        location @50*_json {
-                default_type application/json;
-                return 200 '{"error": "Start service in configuration mode: set service https api"}';
-        }
-
-}
-{% else %}
-server {
-        # SSL configuration
-        #
-        listen 443 ssl default_server;
-        listen [::]:443 ssl default_server;
-
-        server_name _;
-
-{% if vyos_cert %}
-        include {{ vyos_cert.conf }};
-{% else %}
-        #
-        # Self signed certs generated by the ssl-cert package
-        # Don't use them in a production server!
-        #
-        include snippets/snakeoil.conf;
-{% endif %}
-
-        # proxy settings for HTTP API, if enabled; 503, if not
-        location ~ /(retrieve|configure) {
-{% if api %}
-                proxy_pass http://localhost:{{ api.port }};
+        location ~ /(retrieve|configure|config-file|image) {
+{% if server.api %}
+                proxy_pass http://localhost:{{ server.api.port }};
                 proxy_buffering off;
 {% else %}
                 return 503;
@@ -125,8 +88,16 @@ server {
 {% endfor %}
 """
 
+default_server_block = {
+    'address'   : '*',
+    'name'      : ['_'],
+    # api       :
+    # vyos_cert :
+    # le_cert   :
+}
+
 def get_config():
-    https = vyos.defaults.https_data
+    server_block_list = []
     conf = Config()
     if not conf.exists('service https'):
         return None
@@ -134,25 +105,36 @@ def get_config():
         conf.set_level('service https')
 
     if conf.exists('listen-address'):
-        addrs = {}
         for addr in conf.list_nodes('listen-address'):
-            addrs[addr] = ['_']
+            server_block = {'address' : addr}
+            server_block['name'] = ['_']
             if conf.exists('listen-address {0} server-name'.format(addr)):
                 names = conf.return_values('listen-address {0} server-name'.format(addr))
-                addrs[addr] = names[:]
-        https['listen_addresses'] = addrs
+                server_block['name'] = names[:]
+            server_block_list.append(server_block)
 
+    if not server_block_list:
+        server_block_list.append(default_server_block)
+
+    vyos_cert_data = {}
     if conf.exists('certificates'):
         if conf.exists('certificates system-generated-certificate'):
-            https['vyos_cert'] = vyos.defaults.vyos_cert_data
+            vyos_cert_data = vyos.defaults.vyos_cert_data
+    if vyos_cert_data:
+        for block in server_block_list:
+            block['vyos_cert'] = vyos_cert_data
 
+    api_data = {}
     if conf.exists('api'):
-        https['api'] = vyos.defaults.api_data
-
-    if conf.exists('api port'):
-        port = conf.return_value('api port')
-        https['api']['port'] = port
-
+        api_data = vyos.defaults.api_data
+        if conf.exists('api port'):
+            port = conf.return_value('api port')
+            api_data['port'] = port
+    if api_data:
+        for block in server_block_list:
+            block['api'] = api_data
+
+    https = {'server_block_list' : server_block_list}
     return https
 
 def verify(https):
@@ -162,6 +144,9 @@ def generate(https):
     if https is None:
         return None
 
+    if 'server_block_list' not in https or not https['server_block_list']:
+        https['server_block_list'] = [default_server_block]
+
     tmpl = jinja2.Template(config_tmpl, trim_blocks=True)
     config_text = tmpl.render(https)
     with open(config_file, 'w') as f:
diff --git a/src/conf_mode/interfaces-ethernet.py b/src/conf_mode/interfaces-ethernet.py
index cd40aff3e..a9ed6bfb6 100755
--- a/src/conf_mode/interfaces-ethernet.py
+++ b/src/conf_mode/interfaces-ethernet.py
@@ -130,7 +130,7 @@ def get_config():
         print("Interface not specified")
 
     # check if ethernet interface has been removed
-    cfg_base = 'interfaces ethernet ' + eth['intf']
+    cfg_base = ['interfaces', 'ethernet', eth['intf']]
     if not conf.exists(cfg_base):
         eth['deleted'] = True
         # we can not bail out early as ethernet interface can not be removed
@@ -249,7 +249,7 @@ def get_config():
     if conf.exists('vif-s'):
         for vif_s in conf.list_nodes('vif-s'):
             # set config level to vif-s interface
-            conf.set_level(cfg_base + ' vif-s ' + vif_s)
+            conf.set_level(cfg_base + ['vif-s', vif_s])
             eth['vif_s'].append(vlan_to_dict(conf))
 
     # re-set configuration level to parse new nodes
@@ -263,7 +263,7 @@ def get_config():
     if conf.exists('vif'):
         for vif in conf.list_nodes('vif'):
             # set config level to vif interface
-            conf.set_level(cfg_base + ' vif ' + vif)
+            conf.set_level(cfg_base + ['vif', vif])
             eth['vif'].append(vlan_to_dict(conf))
 
     return eth
diff --git a/src/op_mode/show_ipsec_sa.py b/src/op_mode/show_ipsec_sa.py
index 0828743e8..e319cc38d 100755
--- a/src/op_mode/show_ipsec_sa.py
+++ b/src/op_mode/show_ipsec_sa.py
@@ -82,13 +82,24 @@ for sa in sas:
                 pkts_str = re.sub(r'B', r'', pkts_str)
 
                 enc = isa["encr-alg"].decode()
-                key_size = isa["encr-keysize"].decode()
-                hash = isa["integ-alg"].decode()
+                if "encr-keysize" in isa:
+                    key_size = isa["encr-keysize"].decode()
+                else:
+                    key_size = ""
+                if "integ-alg" in isa:
+                    hash = isa["integ-alg"].decode()
+                else:
+                    hash = ""
                 if "dh-group" in isa:
                     dh_group = isa["dh-group"].decode()
                 else:
                     dh_group = ""
-                proposal = "{0}_{1}/{2}".format(enc, key_size, hash)
+
+                proposal = enc
+                if key_size:
+                    proposal = "{0}_{1}".format(proposal, key_size)
+                if hash:
+                    proposal = "{0}/{1}".format(proposal, hash)
                 if dh_group:
                     proposal = "{0}/{1}".format(proposal, dh_group)
 
diff --git a/src/op_mode/wireguard.py b/src/op_mode/wireguard.py
index f6978554d..6860aa3ea 100755
--- a/src/op_mode/wireguard.py
+++ b/src/op_mode/wireguard.py
@@ -23,8 +23,8 @@ import shutil
 import subprocess
 import syslog as sl
 import re
-import time
 
+from vyos.interface import Interface
 
 from vyos import ConfigError
 from vyos.config import Config
@@ -40,41 +40,6 @@ def check_kmod():
             sl.syslog(sl.LOG_ERR, "modprobe wireguard failed")
             raise ConfigError("modprobe wireguard failed")
 
-
-def showint(interface):
-    output = subprocess.check_output(["wg", "show", interface], universal_newlines=True)
-    c = Config()
-    c.set_level("interfaces wireguard {}".format(interface))
-    description = c.return_effective_value("description".format(interface))
-    """ if the interface has a description, modify the output to include it """
-    if (description):
-        output = re.sub(r"interface: {}".format(re.escape(interface)),"interface: {}\n  Description: {}".format(interface,description),output)
-    
-    """ pull the last handshake times.  Assume if the handshake was greater than 5 minutes, the tunnel is down """
-    peer_timeouts = {}
-    last_hs_output = subprocess.check_output(["wg", "show", interface, "latest-handshakes"], universal_newlines=True)
-    for match in re.findall(r'(\S+)\s+(\d+)',last_hs_output): 
-        peer_timeouts[match[0]] = match[1]
-
-    """ modify all the peers, reformat to provide VyOS config provided peername, whether the tunnel is up/down """
-    for peer in c.list_effective_nodes(' peer'):
-        pubkey = c.return_effective_value("peer {} pubkey".format(peer))
-        status = ""
-        if int(peer_timeouts[pubkey]) > 0:
-            #Five minutes and the tunnel is still up
-            if (time.time() - int(peer_timeouts[pubkey]) < (60*5)):
-                status = "UP"
-            else:
-                status = "DOWN"
-        elif (peer_timeouts[pubkey] is None):
-            status = "DOWN"
-        elif (int(peer_timeouts[pubkey]) == 0):
-            status = "DOWN"
-
-        output = re.sub(r"peer: {}".format(re.escape(pubkey)),"peer: {}\n  Status: {}\n  public key: {}".format(peer,status,pubkey),output)
-
-    print(output)
-    
 def generate_keypair(pk, pub):
     """ generates a keypair which is stored in /config/auth/wireguard """
     old_umask = os.umask(0o027)
@@ -185,7 +150,8 @@ if __name__ == '__main__':
         if args.listkdir:
             list_key_dirs()
         if args.showinterface:
-            showint(args.showinterface)
+            intf = Interface(args.showinterface)
+            intf.print_interface()
         if args.delkdir:
             if args.location:
                 del_key_dir(args.location)
diff --git a/src/services/vyos-http-api-server b/src/services/vyos-http-api-server
index afab9be70..04c44c2be 100755
--- a/src/services/vyos-http-api-server
+++ b/src/services/vyos-http-api-server
@@ -27,12 +27,13 @@ import vyos.config
 
 import bottle
 
+from functools import wraps
+
 from vyos.configsession import ConfigSession, ConfigSessionError
 from vyos.config import VyOSError
 
 
 DEFAULT_CONFIG_FILE = '/etc/vyos/http-api.conf'
-
 CFG_GROUP = 'vyattacfg'
 
 app = bottle.default_app()
@@ -61,16 +62,23 @@ def success(data):
     resp = {"success": True, "data": data, "error": None}
     return json.dumps(resp)
 
+def auth_required(f):
+    @wraps(f)
+    def decorated_function(*args, **kwargs):
+        key = bottle.request.forms.get("key")
+        api_keys = app.config['vyos_keys']
+        id = check_auth(api_keys, key)
+        if not id:
+            return error(401, "Valid API key is required")
+        return f(*args, **kwargs)
+
+    return decorated_function
+
 @app.route('/configure', method='POST')
+@auth_required
 def configure():
     session = app.config['vyos_session']
     config = app.config['vyos_config']
-    api_keys = app.config['vyos_keys']
-
-    key = bottle.request.forms.get("key")
-    id = check_auth(api_keys, key)
-    if not id:
-        return error(401, "Valid API key is required")
 
     strict_field = bottle.request.forms.get("strict")
     if strict_field == "true":
@@ -177,17 +185,11 @@ def configure():
         return success(None)
 
 @app.route('/retrieve', method='POST')
+@auth_required
 def get_value():
     config = app.config['vyos_config']
     session = app.config['vyos_session']
 
-    api_keys = app.config['vyos_keys']
-
-    key = bottle.request.forms.get("key")
-    id = check_auth(api_keys, key)
-    if not id:
-        return error(401, "Valid API key is required")
-
     command = bottle.request.forms.get("data")
     command = json.loads(command)
 
@@ -220,6 +222,82 @@ def get_value():
 
     return success(res)
 
+@app.route('/config-file', method='POST')
+@auth_required
+def config_file_op():
+    config = app.config['vyos_config']
+    session = app.config['vyos_session']
+
+    command = bottle.request.forms.get("data")
+    command = json.loads(command)
+
+    try:
+        op = command['op']
+    except KeyError:
+        return error(400, "Missing required field \"op\"")
+
+    try:
+        if op == 'save':
+            try:
+                path = command['file']
+            except KeyError:
+                path = '/config/config.boot'
+            res = session.save_config(path)
+        elif op == 'load':
+            try:
+                path = command['file']
+            except KeyError:
+                return error(400, "Missing required field \"file\"")
+            res = session.load_config(path)
+            res = session.commit()
+        else:
+            return error(400, "\"{0}\" is not a valid operation".format(op))
+    except VyOSError as e:
+        return error(400, str(e))
+    except Exception as e:
+        print(traceback.format_exc(), file=sys.stderr)
+        return error(500, "An internal error occured. Check the logs for details.")
+
+    return success(res)
+
+@app.route('/image', method='POST')
+@auth_required
+def config_file_op():
+    config = app.config['vyos_config']
+    session = app.config['vyos_session']
+
+    command = bottle.request.forms.get("data")
+    command = json.loads(command)
+
+    try:
+        op = command['op']
+    except KeyError:
+        return error(400, "Missing required field \"op\"")
+
+    try:
+        if op == 'add':
+            try:
+                url = command['url']
+            except KeyError:
+                return error(400, "Missing required field \"url\"")
+            res = session.install_image(url)
+        elif op == 'delete':
+            try:
+                name = command['name']
+            except KeyError:
+                return error(400, "Missing required field \"name\"")
+            res = session.remove_image(name)
+        else:
+            return error(400, "\"{0}\" is not a valid operation".format(op))
+    except VyOSError as e:
+        return error(400, str(e))
+    except Exception as e:
+        print(traceback.format_exc(), file=sys.stderr)
+        return error(500, "An internal error occured. Check the logs for details.")
+
+    return success(res)
+
+
 if __name__ == '__main__':
     # systemd's user and group options don't work, do it by hand here,
     # else no one else will be able to commit