Merge branch 'current' of github.com:vyos/vyos-1x into equuleus

* 'current' of github.com:vyos/vyos-1x:
  T1762: adjust the set_level() calls to use the new list representation.
  [vyos.config] T1764: support both string and list arguments in config functions.
  T1759: bug fixes, missing interface IP
  [vyos.config] T1758: use vyos.configtree for reading values, instead of calling cli-shell-api.
  [HTTP API] Add endpoints for config file and image management.
  ddclient: T1030: add cloudflare zone config entry
  [service https] T1443: organize internal data by server block
  [vyos.config] T1758: check that config setup has completed before calling showConfig, else, default to config.boot
  [HTTP API] Use a decorator for functions that require authentication.
  ddclient: T1030: adjust to latest syntax
  ddclient: T1030: auto create runtime directories
  ddclient: T1030: use new default configuration file path
  T1759: Migrating interfaces
  T1755: fixes issue with 'show vpn ipsec sa' command where lack of keysize (encr-keysize) will result in KeyError - such as for CHACHA20_POLY1305
  T1755: fixes issue with 'show vpn ipsec sa' command where lack of hash (integ-alg) will result in KeyError - such as with GCM based options

3 files changed, 86 insertions, 84 deletions

diff --git a/src/conf_mode/dynamic_dns.py b/src/conf_mode/dynamic_dns.py
index ff3c1f825..027a7f7e3 100755
--- a/src/conf_mode/dynamic_dns.py
+++ b/src/conf_mode/dynamic_dns.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2018 VyOS maintainers and contributors
+# Copyright (C) 2018-2019 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -13,8 +13,6 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-#
 
 import os
 import sys
@@ -23,16 +21,17 @@ import jinja2
 from vyos.config import Config
 from vyos import ConfigError
 
-config_file = r'/etc/ddclient.conf'
+config_file = r'/etc/ddclient/ddclient.conf'
 cache_file = r'/var/cache/ddclient/ddclient.cache'
+pid_file = r'/var/run/ddclient/ddclient.pid'
 
 config_tmpl = """
 ### Autogenerated by dynamic_dns.py ###
 daemon=1m
 syslog=yes
 ssl=yes
-pid=/var/run/ddclient/ddclient.pid
-cache=/var/cache/ddclient/ddclient.cache
+pid={{ pid_file }}
+cache={{ cache_file }}
 
 {% for interface in interfaces -%}
 
@@ -48,11 +47,11 @@ use=if, if={{ interface.interface }}
 {% for rfc in interface.rfc2136 -%}
 {% for record in rfc.record %}
 # RFC2136 dynamic DNS configuration for {{ record }}.{{ rfc.zone }}
-server={{ rfc.server }}
-protocol=nsupdate
-password={{ rfc.keyfile }}
-ttl={{ rfc.ttl }}
-zone={{ rfc.zone }}
+server={{ rfc.server }},
+protocol=nsupdate,
+password={{ rfc.keyfile }},
+ttl={{ rfc.ttl }},
+zone={{ rfc.zone }},
 {{ record }}
 {% endfor -%}
 {% endfor -%}
@@ -60,12 +59,16 @@ zone={{ rfc.zone }}
 {% for srv in interface.service %}
 {% for host in srv.host %}
 # DynDNS provider configuration for {{ host }}
-protocol={{ srv.protocol }}
-max-interval=28d
-login={{ srv.login }}
-password='{{ srv.password }}'
+protocol={{ srv.protocol }},
+max-interval=28d,
+login={{ srv.login }},
+password='{{ srv.password }}',
 {% if srv.server -%}
-server={{ srv.server }}
+server={{ srv.server }},
+{% endif -%}
+{% if 'cloudflare' in srv.protocol -%}
+{% set zone = host.split('.',1) -%}
+zone={{ zone[1] }},
 {% endif -%}
 {{ host }}
 {% endfor %}
@@ -91,6 +94,8 @@ default_service_protocol = {
 
 default_config_data = {
     'interfaces': [],
+    'cache_file': cache_file,
+    'pid_file': pid_file
 }
 
 def get_config():
@@ -237,8 +242,15 @@ def generate(dyndns):
     if dyndns is None:
         return None
 
-    tmpl = jinja2.Template(config_tmpl)
+    dirname = os.path.dirname(dyndns['pid_file'])
+    if not os.path.exists(dirname):
+        os.mkdir(dirname)
 
+    dirname = os.path.dirname(config_file)
+    if not os.path.exists(dirname):
+        os.mkdir(dirname)
+
+    tmpl = jinja2.Template(config_tmpl)
     config_text = tmpl.render(dyndns)
     with open(config_file, 'w') as f:
         f.write(config_text)
@@ -246,11 +258,16 @@ def generate(dyndns):
     return None
 
 def apply(dyndns):
-    if os.path.exists(cache_file):
-        os.unlink(cache_file)
+    if os.path.exists(dyndns['cache_file']):
+        os.unlink(dyndns['cache_file'])
+
+    if os.path.exists('/etc/ddclient.conf'):
+        os.unlink('/etc/ddclient.conf')
 
     if dyndns is None:
         os.system('/etc/init.d/ddclient stop')
+        if os.path.exists(dyndns['pid_file']):
+            os.unlink(dyndns['pid_file'])
     else:
         os.system('/etc/init.d/ddclient restart')
 
diff --git a/src/conf_mode/https.py b/src/conf_mode/https.py
index f948063e9..d7fcb74de 100755
--- a/src/conf_mode/https.py
+++ b/src/conf_mode/https.py
@@ -30,34 +30,34 @@ config_file = '/etc/nginx/sites-available/default'
 # Please be careful if you edit the template.
 config_tmpl = """
 
-### Autogenerated by http-api.py ###
+### Autogenerated by https.py ###
 # Default server configuration
 #
 server {
         listen 80 default_server;
         listen [::]:80 default_server;
         server_name _;
-        return 302 https://$server_name$request_uri;
+        return 301 https://$server_name$request_uri;
 }
 
-{% for addr, names in listen_addresses.items() %}
+{% for server in server_block_list %}
 server {
 
         # SSL configuration
         #
-{% if addr == '*' %}
-        listen 443 ssl default_server;
-        listen [::]:443 ssl default_server;
+{% if server.address == '*' %}
+        listen 443 ssl;
+        listen [::]:443 ssl;
 {% else %}
-        listen {{ addr }}:443 ssl;
+        listen {{ server.address }}:443 ssl;
 {% endif %}
 
-{% for name in names %}
+{% for name in server.name %}
         server_name {{ name }};
 {% endfor %}
 
-{% if vyos_cert %}
-        include {{ vyos_cert.conf }};
+{% if server.vyos_cert %}
+        include {{ server.vyos_cert.conf }};
 {% else %}
         #
         # Self signed certs generated by the ssl-cert package
@@ -67,46 +67,9 @@ server {
 {% endif %}
 
         # proxy settings for HTTP API, if enabled; 503, if not
-        location ~ /(retrieve|configure) {
-{% if api %}
-                proxy_pass http://localhost:{{ api.port }};
-                proxy_buffering off;
-{% else %}
-                return 503;
-{% endif %}
-        }
-
-        error_page 501 502 503 =200 @50*_json;
-
-        location @50*_json {
-                default_type application/json;
-                return 200 '{"error": "Start service in configuration mode: set service https api"}';
-        }
-
-}
-{% else %}
-server {
-        # SSL configuration
-        #
-        listen 443 ssl default_server;
-        listen [::]:443 ssl default_server;
-
-        server_name _;
-
-{% if vyos_cert %}
-        include {{ vyos_cert.conf }};
-{% else %}
-        #
-        # Self signed certs generated by the ssl-cert package
-        # Don't use them in a production server!
-        #
-        include snippets/snakeoil.conf;
-{% endif %}
-
-        # proxy settings for HTTP API, if enabled; 503, if not
-        location ~ /(retrieve|configure) {
-{% if api %}
-                proxy_pass http://localhost:{{ api.port }};
+        location ~ /(retrieve|configure|config-file|image) {
+{% if server.api %}
+                proxy_pass http://localhost:{{ server.api.port }};
                 proxy_buffering off;
 {% else %}
                 return 503;
@@ -125,8 +88,16 @@ server {
 {% endfor %}
 """
 
+default_server_block = {
+    'address'   : '*',
+    'name'      : ['_'],
+    # api       :
+    # vyos_cert :
+    # le_cert   :
+}
+
 def get_config():
-    https = vyos.defaults.https_data
+    server_block_list = []
     conf = Config()
     if not conf.exists('service https'):
         return None
@@ -134,25 +105,36 @@ def get_config():
         conf.set_level('service https')
 
     if conf.exists('listen-address'):
-        addrs = {}
         for addr in conf.list_nodes('listen-address'):
-            addrs[addr] = ['_']
+            server_block = {'address' : addr}
+            server_block['name'] = ['_']
             if conf.exists('listen-address {0} server-name'.format(addr)):
                 names = conf.return_values('listen-address {0} server-name'.format(addr))
-                addrs[addr] = names[:]
-        https['listen_addresses'] = addrs
+                server_block['name'] = names[:]
+            server_block_list.append(server_block)
 
+    if not server_block_list:
+        server_block_list.append(default_server_block)
+
+    vyos_cert_data = {}
     if conf.exists('certificates'):
         if conf.exists('certificates system-generated-certificate'):
-            https['vyos_cert'] = vyos.defaults.vyos_cert_data
+            vyos_cert_data = vyos.defaults.vyos_cert_data
+    if vyos_cert_data:
+        for block in server_block_list:
+            block['vyos_cert'] = vyos_cert_data
 
+    api_data = {}
     if conf.exists('api'):
-        https['api'] = vyos.defaults.api_data
-
-    if conf.exists('api port'):
-        port = conf.return_value('api port')
-        https['api']['port'] = port
-
+        api_data = vyos.defaults.api_data
+        if conf.exists('api port'):
+            port = conf.return_value('api port')
+            api_data['port'] = port
+    if api_data:
+        for block in server_block_list:
+            block['api'] = api_data
+
+    https = {'server_block_list' : server_block_list}
     return https
 
 def verify(https):
@@ -162,6 +144,9 @@ def generate(https):
     if https is None:
         return None
 
+    if 'server_block_list' not in https or not https['server_block_list']:
+        https['server_block_list'] = [default_server_block]
+
     tmpl = jinja2.Template(config_tmpl, trim_blocks=True)
     config_text = tmpl.render(https)
     with open(config_file, 'w') as f:
diff --git a/src/conf_mode/interfaces-ethernet.py b/src/conf_mode/interfaces-ethernet.py
index cd40aff3e..a9ed6bfb6 100755
--- a/src/conf_mode/interfaces-ethernet.py
+++ b/src/conf_mode/interfaces-ethernet.py
@@ -130,7 +130,7 @@ def get_config():
         print("Interface not specified")
 
     # check if ethernet interface has been removed
-    cfg_base = 'interfaces ethernet ' + eth['intf']
+    cfg_base = ['interfaces', 'ethernet', eth['intf']]
     if not conf.exists(cfg_base):
         eth['deleted'] = True
         # we can not bail out early as ethernet interface can not be removed
@@ -249,7 +249,7 @@ def get_config():
     if conf.exists('vif-s'):
         for vif_s in conf.list_nodes('vif-s'):
             # set config level to vif-s interface
-            conf.set_level(cfg_base + ' vif-s ' + vif_s)
+            conf.set_level(cfg_base + ['vif-s', vif_s])
             eth['vif_s'].append(vlan_to_dict(conf))
 
     # re-set configuration level to parse new nodes
@@ -263,7 +263,7 @@ def get_config():
     if conf.exists('vif'):
         for vif in conf.list_nodes('vif'):
             # set config level to vif interface
-            conf.set_level(cfg_base + ' vif ' + vif)
+            conf.set_level(cfg_base + ['vif', vif])
             eth['vif'].append(vlan_to_dict(conf))
 
     return eth