Merge branch 'current' into equuleus

15 files changed, 1219 insertions, 253 deletions

diff --git a/src/conf_mode/accel_pppoe.py b/src/conf_mode/accel_pppoe.py
index 3b3bf8cac..9c879502a 100755
--- a/src/conf_mode/accel_pppoe.py
+++ b/src/conf_mode/accel_pppoe.py
@@ -47,6 +47,8 @@ pppoe
 ippool
 {% if client_ipv6_pool %}
 ipv6pool
+ipv6_nd
+ipv6_dhcp
 {% endif %}
 chap-secrets
 auth_pap
@@ -81,6 +83,7 @@ master=1
 [client-ip-range]
 disable
 
+{% if ppp_gw %}
 [ip-pool]
 gw-ip-address={{ppp_gw}}
 {% if client_ip_pool %}
@@ -92,6 +95,7 @@ gw-ip-address={{ppp_gw}}
 {{sn}}
 {% endfor %}
 {% endif %}
+{% endif -%}
 
 {% if client_ipv6_pool %}
 [ipv6-pool]
@@ -101,7 +105,7 @@ gw-ip-address={{ppp_gw}}
 {% for prfx in client_ipv6_pool['delegate-prefix']: %}
 delegate={{prfx}}
 {% endfor %}
-{% endif -%}
+{% endif %}
 
 {% if dns %}
 [dns]
@@ -111,14 +115,14 @@ dns1={{dns[0]}}
 {% if dns[1] %}
 dns2={{dns[1]}}
 {% endif -%}
-{% endif -%}
+{% endif %}
 
 {% if dnsv6 %}
-[dnsv6]
+[ipv6-dns]
 {% for srv in dnsv6: %}
-dns={{srv}}
+{{srv}}
 {% endfor %}
-{% endif -%}
+{% endif %}
 
 {% if wins %}
 [wins]
@@ -169,6 +173,9 @@ verbose=1
 [shaper]
 verbose=1
 attr={{authentication['radiusopt']['shaper']['attr']}}
+{% if authentication['radiusopt']['shaper']['vendor'] %}
+vendor={{authentication['radiusopt']['shaper']['vendor']}}
+{% endif -%}
 {% endif -%}
 {% endif %}
 
@@ -208,6 +215,10 @@ lcp-echo-failure=3
 {% if ppp_options['ipv4'] %}
 ipv4={{ppp_options['ipv4']}}
 {% endif %}
+{% if client_ipv6_pool %}
+ipv6=allow
+{% endif %}
+
 {% if ppp_options['ipv6'] %}
 ipv6={{ppp_options['ipv6']}}
 {% if ppp_options['ipv6-intf-id'] %}
@@ -220,6 +231,7 @@ ipv6-peer-intf-id={{ppp_options['ipv6-peer-intf-id']}}
 ipv6-accept-peer-intf-id={{ppp_options['ipv6-accept-peer-intf-id']}}
 {% endif %}
 {% endif %}
+
 mtu={{mtu}}
 
 [pppoe]
@@ -230,13 +242,16 @@ ac-name={{concentrator}}
 {% if interface %}
 {% for int in interface %}
 interface={{int}}
-{% endfor %}
+{% if interface[int]['vlans'] %}
+vlan_mon={{interface[int]['vlans']|join(',')}}
+interface=re:{{int}}\.(409[0-6]|40[0-8][0-9]|[1-3][0-9]{3}|[1-9][0-9]{0,2})
 {% endif %}
+{% endfor -%}
+{% endif -%}
 {% if svc_name %}
 service-name={{svc_name}}
-{% endif %}
+{% endif -%}
 pado-delay=0
-# maybe: called-sid, tr101, padi-limit etc.
 
 {% if limits %}
 [connlimit]
@@ -326,7 +341,7 @@ def get_config():
     'client_ip_pool'    : '',
     'client_ip_subnets' : [],
     'client_ipv6_pool'  : {},
-    'interface'         : [],
+    'interface'         : {},
     'ppp_gw'            : '',
     'svc_name'          : '',
     'dns'               : [],
@@ -345,7 +360,12 @@ def get_config():
   if c.exists('service-name'):
     config_data['svc_name'] = c.return_value('service-name')
   if c.exists('interface'):
-    config_data['interface'] = c.return_values('interface')
+    for intfc in c.list_nodes('interface'):
+      config_data['interface'][intfc] = {'vlans' : []}
+      if c.exists('interface ' + intfc + ' vlan-id'):
+        config_data['interface'][intfc]['vlans'] += c.return_values('interface ' + intfc + ' vlan-id')
+      if c.exists('interface ' + intfc + ' vlan-range'):
+        config_data['interface'][intfc]['vlans'] +=c.return_values('interface ' + intfc + ' vlan-range')
   if c.exists('local-ip'):
     config_data['ppp_gw'] = c.return_value('local-ip')
   if c.exists('dns-servers'):
@@ -476,6 +496,8 @@ def get_config():
           config_data['authentication']['radiusopt']['shaper'] = {
           'attr'  : c.return_value('authentication radius-settings rate-limit attribute')
           }
+        if c.exists('authentication radius-settings rate-limit vendor'):
+          config_data['authentication']['radiusopt']['shaper']['vendor'] = c.return_value('authentication radius-settings rate-limit vendor')
 
   if c.exists('mtu'):
     config_data['mtu'] = c.return_value('mtu')
@@ -543,18 +565,14 @@ def verify(c):
       if c['authentication']['radiussrv'][rsrv]['secret'] == None:
         raise ConfigError('radius server ' + rsrv + ' needs a secret configured')
 
-  ### local ippool and gateway settings
-
-  if not c['ppp_gw']:
-    raise ConfigError('pppoe-server local-ip required')
-
-  if not c['client_ip_subnets'] and not c['client_ip_pool']:
-    print ("Warning: No pppoe client IP pool defined") 
+  ### local ippool and gateway settings config checks
 
-  ### activate as soon as it is clear what to do migrate or depricate. 
-  #if c['client_ip_pool']:
-  #  print ("Warning: client-ip-pool (start|stop) is depricated, please use client-ip-pool subnet")
-  #  sl.syslog(sl.LOG_NOTICE, "client-ip-pool start stop is depricated, please use client-ip-pool subnet")
+  if c['client_ip_subnets'] or c['client_ip_pool']:
+    if not c['ppp_gw']:
+      raise ConfigError('pppoe-server local-ip required')
+      
+  if c['ppp_gw'] and not c['client_ip_subnets'] and not c['client_ip_pool']:
+    print ("Warning: No pppoe client IPv4 pool defined") 
 
 def generate(c):
   if c == None:
diff --git a/src/conf_mode/accel_pptp.py b/src/conf_mode/accel_pptp.py
index 6c53e8dd4..1dd7efb3e 100755
--- a/src/conf_mode/accel_pptp.py
+++ b/src/conf_mode/accel_pptp.py
@@ -84,7 +84,9 @@ wins2={{wins[1]}}
 {% endif %}
 
 [pptp]
+{% if outside_addr %}
 bind={{outside_addr}}
+{% endif %}
 verbose=5
 ppp-max-mtu={{mtu}}
 mppe={{authentication['mppe']}}
@@ -294,7 +296,7 @@ def verify(c):
 
   if c['authentication']['mode'] == 'local':
     if not c['authentication']['local-users']:
-      raise ConfigError('pppoe-server authentication local-users required')
+      raise ConfigError('pptp-server authentication local-users required')
     for usr in c['authentication']['local-users']:
       if not c['authentication']['local-users'][usr]['passwd']:
         raise ConfigError('user ' + usr + ' requires a password')
diff --git a/src/conf_mode/dhcp_server.py b/src/conf_mode/dhcp_server.py
index c8ae2fa91..3e1381cd0 100755
--- a/src/conf_mode/dhcp_server.py
+++ b/src/conf_mode/dhcp_server.py
@@ -186,8 +186,10 @@ shared-network {{ network.name }} {
         {%- endif -%}
         {%- for host in subnet.static_mapping %}
         {% if not host.disabled -%}
-        host {{ network.name }}_{{ host.name }} {
+        host {% if host_decl_name -%} {{ host.name }} {%- else -%} {{ network.name }}_{{ host.name }} {%- endif %} {
+            {%- if host.ip_address %}
             fixed-address {{ host.ip_address }};
+            {%- endif %}
             hardware ethernet {{ host.mac_address }};
             {%- if host.static_parameters %}
             # The following {{ host.static_parameters | length }} line(s) were added as static-mapping-parameters in the CLI and have not been validated
@@ -728,22 +730,19 @@ def verify(dhcp):
                 raise ConfigError('No DHCP address range or active static-mapping set\n' \
                                   'for subnet {0}!'.format(subnet['network']))
 
-            # Static IP address mappings require both an IP address and MAC address
+            # Static mappings require just a MAC address (will use an IP from the dynamic pool if IP is not set)
             for mapping in subnet['static_mapping']:
-                # Static IP address must be configured
-                if not mapping['ip_address']:
-                    raise ConfigError('DHCP static lease IP address not specified for static mapping\n' \
-                                      '{0} under shared network name {1}!'.format(mapping['name'], network['name']))
 
-                # Static IP address must be in bound
-                if not ipaddress.ip_address(mapping['ip_address']) in ipaddress.ip_network(subnet['network']):
-                    raise ConfigError('DHCP static lease IP address {0} for static mapping {1}\n' \
-                                      'in shared network {2} is outside DHCP lease subnet {3}!' \
-                                      .format(mapping['ip_address'], mapping['name'], network['name'], subnet['network']))
+                if mapping['ip_address']:
+                    # Static IP address must be in bound
+                    if not ipaddress.ip_address(mapping['ip_address']) in ipaddress.ip_network(subnet['network']):
+                        raise ConfigError('DHCP static lease IP address {0} for static mapping {1}\n' \
+                                          'in shared network {2} is outside DHCP lease subnet {3}!' \
+                                          .format(mapping['ip_address'], mapping['name'], network['name'], subnet['network']))
 
                 # Static mapping requires MAC address
                 if not mapping['mac_address']:
-                     raise ConfigError('DHCP static lease MAC address not specified for static mapping\n' \
+                    raise ConfigError('DHCP static lease MAC address not specified for static mapping\n' \
                                        '{0} under shared network name {1}!'.format(mapping['name'], network['name']))
 
             # There must be one subnet connected to a listen interface.
@@ -754,7 +753,7 @@ def verify(dhcp):
 
             # Subnets must be non overlapping
             if subnet['network'] in subnets:
-                raise ConfigError('DHCP subnets must be unique! Subnet {0} defined multiple times!'.format(subnet))
+                raise ConfigError('DHCP subnets must be unique! Subnet {0} defined multiple times!'.format(subnet['network']))
             else:
                 subnets.append(subnet['network'])
 
diff --git a/src/conf_mode/dhcpv6_server.py b/src/conf_mode/dhcpv6_server.py
index bb3e6e90d..039321430 100755
--- a/src/conf_mode/dhcpv6_server.py
+++ b/src/conf_mode/dhcpv6_server.py
@@ -27,8 +27,8 @@ import vyos.validate
 from vyos.config import Config
 from vyos import ConfigError
 
-config_file = r'/etc/dhcp/dhcpd6.conf'
-lease_file = r'/config/dhcpd6.leases'
+config_file = r'/etc/dhcp/dhcpdv6.conf'
+lease_file = r'/config/dhcpdv6.leases'
 daemon_config_file = r'/etc/default/isc-dhcpv6-server'
 
 # Please be careful if you edit the template.
@@ -94,13 +94,20 @@ shared-network {{ network.name }} {
         {%- for host in subnet.static_mapping %}
         {% if not host.disabled -%}
         host {{ network.name }}_{{ host.name }} {
-            host-identifier option dhcp6.client-id "{{ host.client_identifier }}";
+            {%- if host.client_identifier %}
+            host-identifier option dhcp6.client-id {{ host.client_identifier }};
+            {%- endif %}
+            {%- if host.ipv6_address %}
             fixed-address6 {{ host.ipv6_address }};
+            {%- endif %}
         }
         {%- endif %}
         {%- endfor %}
     }
     {%- endfor %}
+    on commit {
+        set shared-networkname = "{{ network.name }}";
+    }
 }
 {%- endif %}
 {% endfor %}
@@ -112,8 +119,8 @@ daemon_tmpl = """
 
 # sourced by /etc/init.d/isc-dhcpv6-server
 
-DHCPD_CONF=/etc/dhcp/dhcpd6.conf
-DHCPD_PID=/var/run/dhcpd6.pid
+DHCPD_CONF=/etc/dhcp/dhcpdv6.conf
+DHCPD_PID=/var/run/dhcpdv6.pid
 OPTIONS="-6 -lf {{ lease_file }}"
 INTERFACES=""
 """
@@ -381,7 +388,25 @@ def verify(dhcpv6):
                         raise ConfigError('DHCPv6 prefix {0} is not in subnet {1}\n' \
                                           'specified for shared network {2}!'.format(prefix['prefix'], subnet['network'], network['name']))
 
+            # Static mappings don't require anything (but check if IP is in subnet if it's set)
+            for mapping in subnet['static_mapping']:
+                if mapping['ipv6_address']:
+                    # Static address must be in subnet
+                    if not ipaddress.ip_address(mapping['ipv6_address']) in ipaddress.ip_network(subnet['network']):
+                        raise ConfigError('DHCPv6 static mapping IPv6 address {0} for static mapping {1}\n' \
+                                          'in shared network {2} is outside subnet {3}!' \
+                                          .format(mapping['ipv6_address'], mapping['name'], network['name'], subnet['network']))
+
+            # Subnets must be unique
+            if subnet['network'] in subnets:
+                raise ConfigError('DHCPv6 subnets must be unique! Subnet {0} defined multiple times!'.format(subnet['network']))
+            else:
+                subnets.append(subnet['network'])
+
         # DHCPv6 requires at least one configured address range or one static mapping
+        # (FIXME: is not actually checked right now?)
+
+        # There must be one subnet connected to a listen interface if network is not disabled.
         if not network['disabled']:
             if vyos.validate.is_subnet_connected(subnet['network']):
                 listen_ok = True
diff --git a/src/conf_mode/dns_forwarding.py b/src/conf_mode/dns_forwarding.py
index 135f6fec0..3ca77adee 100755
--- a/src/conf_mode/dns_forwarding.py
+++ b/src/conf_mode/dns_forwarding.py
@@ -19,12 +19,20 @@
 import sys
 import os
 
-import netifaces
+import argparse
 import jinja2
+import netifaces
+
+import vyos.util
 
 from vyos.config import Config
 from vyos import ConfigError
 
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--dhclient", action="store_true",
+                    help="Started from dhclient-script")
+
 config_file = r'/etc/powerdns/recursor.conf'
 
 # XXX: pdns recursor doesn't like whitespace near entry separators,
@@ -54,24 +62,22 @@ export-etc-hosts={{ export_hosts_file }}
 # listen-on
 local-address={{ listen_on | join(',') }}
 
-# domain ... server ...
-{% if domains -%}
-
-forward-zones={% for d in domains %}
-{{ d.name }}={{ d.servers | join(";") }}
-{{- "," if not loop.last -}}
-{% endfor %}
-
-{% endif %}
-
 # dnssec
 dnssec={{ dnssec }}
 
-{% if name_servers -%}
-# name-server
-forward-zones-recurse=.={{ name_servers | join(';') }}
-{% else %}
-# no name-servers specified - start full recursor
+# forward-zones / recursion
+#
+# statement is only inserted if either one forwarding domain or nameserver is configured
+# if nothing is given at all, powerdns will act as a real recursor and resolve all requests by its own
+#
+{% if name_servers or domains %}forward-zones-recurse=
+{%- for d in domains %}
+{{ d.name }}={{ d.servers | join(";") }}
+{{- ", " if not loop.last -}}
+{%- endfor -%}
+{%- if name_servers -%}
+{%- if domains -%}, {% endif -%}.={{ name_servers | join(';') }}
+{% endif %}
 {% endif %}
 
 """
@@ -84,31 +90,36 @@ default_config_data = {
     'name_servers': [],
     'negative_ttl': 3600,
     'domains': [],
-    'dnssec' : 'process-no-validate'
+    'dnssec': 'process-no-validate'
 }
 
 
 # borrowed from: https://github.com/donjajo/py-world/blob/master/resolvconfReader.py, THX!
 def get_resolvers(file):
-    resolvers = []
     try:
         with open(file, 'r') as resolvconf:
-            for line in resolvconf.readlines():
-                line = line.split('#',1)[0];
-                line = line.rstrip();
-                if 'nameserver' in line:
-                    resolvers.append(line.split()[1])
+            lines = [line.split('#', 1)[0].rstrip()
+                     for line in resolvconf.readlines()]
+            resolvers = [line.split()[1]
+                         for line in lines if 'nameserver' in line]
         return resolvers
     except IOError:
         return []
 
-def get_config():
+
+def get_config(arguments):
     dns = default_config_data
     conf = Config()
+
+    if arguments.dhclient:
+        conf.exists = conf.exists_effective
+        conf.return_value = conf.return_effective_value
+        conf.return_values = conf.return_effective_values
+
     if not conf.exists('service dns forwarding'):
         return None
-    else:
-        conf.set_level('service dns forwarding')
+
+    conf.set_level('service dns forwarding')
 
     if conf.exists('cache-size'):
         cache_size = conf.return_value('cache-size')
@@ -139,7 +150,8 @@ def get_config():
         system_name_servers = []
         system_name_servers = conf.return_values('name-server')
         if not system_name_servers:
-            print("DNS forwarding warning: No name-servers set under 'system name-server'\n")
+            print(
+                "DNS forwarding warning: No name-servers set under 'system name-server'\n")
         else:
             dns['name_servers'] = dns['name_servers'] + system_name_servers
         conf.set_level('service dns forwarding')
@@ -171,9 +183,10 @@ def get_config():
             try:
                 addrs = netifaces.ifaddresses(interface)
             except ValueError:
-                print("WARNING: interface {0} does not exist".format(interface))
+                print(
+                    "WARNING: interface {0} does not exist".format(interface))
                 continue
-                
+
             if netifaces.AF_INET in addrs.keys():
                 for ip4 in addrs[netifaces.AF_INET]:
                     listen4.append(ip4['addr'])
@@ -183,7 +196,8 @@ def get_config():
                     listen6.append(ip6['addr'])
 
             if (not listen4) and (not (listen6)):
-                print("WARNING: interface {0} has no configured addresses".format(interface))
+                print(
+                    "WARNING: interface {0} has no configured addresses".format(interface))
 
         dns['listen_on'] = dns['listen_on'] + listen4 + listen6
 
@@ -195,56 +209,69 @@ def get_config():
         interfaces = []
         interfaces = conf.return_values('dhcp')
         for interface in interfaces:
-            dhcp_resolvers = get_resolvers("/etc/resolv.conf.dhclient-new-{0}".format(interface))
+            dhcp_resolvers = get_resolvers(
+                "/etc/resolv.conf.dhclient-new-{0}".format(interface))
             if dhcp_resolvers:
                 dns['name_servers'] = dns['name_servers'] + dhcp_resolvers
 
     return dns
 
+
 def bracketize_ipv6_addrs(addrs):
     """Wraps each IPv6 addr in addrs in [], leaving IPv4 addrs untouched."""
     return ['[{0}]'.format(a) if a.count(':') > 1 else a for a in addrs]
 
+
 def verify(dns):
     # bail out early - looks like removal from running config
     if dns is None:
         return None
 
     if not dns['listen_on']:
-        raise ConfigError("Error: DNS forwarding requires either a listen-address (preferred) or a listen-on option")
+        raise ConfigError(
+            "Error: DNS forwarding requires either a listen-address (preferred) or a listen-on option")
 
     if dns['domains']:
         for domain in dns['domains']:
             if not domain['servers']:
-                raise ConfigError('Error: No server configured for domain {0}'.format(domain['name']))
+                raise ConfigError(
+                    'Error: No server configured for domain {0}'.format(domain['name']))
 
     return None
 
+
 def generate(dns):
     # bail out early - looks like removal from running config
     if dns is None:
         return None
 
     tmpl = jinja2.Template(config_tmpl, trim_blocks=True)
-
     config_text = tmpl.render(dns)
     with open(config_file, 'w') as f:
         f.write(config_text)
     return None
 
+
 def apply(dns):
     if dns is not None:
         os.system("systemctl restart pdns-recursor")
     else:
         # DNS forwarding is removed in the commit
         os.system("systemctl stop pdns-recursor")
-        os.unlink(config_file)
+        if os.path.isfile(config_file):
+            os.unlink(config_file)
 
-    return None
 
 if __name__ == '__main__':
+    args = parser.parse_args()
+
+    if args.dhclient:
+        # There's a big chance it was triggered by a commit still in progress
+        # so we need to wait until the new values are in the running config
+        vyos.util.wait_for_commit_lock()
+
     try:
-        c = get_config()
+        c = get_config(args)
         verify(c)
         generate(c)
         apply(c)
diff --git a/src/conf_mode/firewall_options.py b/src/conf_mode/firewall_options.py
index e2c306904..2be80cdbf 100755
--- a/src/conf_mode/firewall_options.py
+++ b/src/conf_mode/firewall_options.py
@@ -32,7 +32,8 @@ def get_config():
     opts = copy.deepcopy(default_config_data)
     conf = Config()
     if not conf.exists('firewall options'):
-        return None
+        # bail out early
+        return opts
     else:
         conf.set_level('firewall options')
 
diff --git a/src/conf_mode/host_name.py b/src/conf_mode/host_name.py
index 81a52e87f..16467c8df 100755
--- a/src/conf_mode/host_name.py
+++ b/src/conf_mode/host_name.py
@@ -23,20 +23,29 @@ conf-mode script for 'system host-name' and 'system domain-name'.
 import os
 import re
 import sys
-import subprocess
 import copy
-import jinja2
 import glob
+import subprocess
+import argparse
+import jinja2
+
+import vyos.util
 
 from vyos.config import Config
 from vyos import ConfigError
 
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--dhclient", action="store_true",
+                    help="Started from dhclient-script")
+
 config_file_hosts = '/etc/hosts'
 config_file_resolv = '/etc/resolv.conf'
 
 config_tmpl_hosts = """
 ### Autogenerated by host_name.py ###
-127.0.0.1       localhost {{ hostname }}{% if domain_name %}.{{ domain_name }}{% endif %}
+127.0.0.1       localhost
+127.0.1.1       {{ hostname }}{% if domain_name %}.{{ domain_name }}{% endif %}
 
 # The following lines are desirable for IPv6 capable hosts
 ::1             localhost ip6-localhost ip6-loopback
@@ -72,32 +81,6 @@ search {{ domain_search | join(" ") }}
 
 """
 
-# borrowed from: https://github.com/donjajo/py-world/blob/master/resolvconfReader.py, THX!
-def get_resolvers(file):
-  resolvers = []
-  try:
-    with open(file, 'r') as resolvconf:
-      for line in resolvconf.readlines():
-        line = line.split('#',1)[0];
-        line = line.rstrip();
-        if 'nameserver' in line:
-          resolvers.append(line.split()[1])
-      return resolvers
-  except IOError:
-    return []
-
-def get_dhcp_search_doms(file):
-  search_doms = []
-  try:
-    with open(file, 'r') as resolvconf:
-      for line in resolvconf.readlines():
-        line = line.split('#',1)[0];
-        line = line.rstrip();
-        if 'search' in line:
-          return re.sub('^search','',line).lstrip().split()
-  except IOError:
-    return []
-
 default_config_data = {
     'hostname': 'vyos',
     'domain_name': '',
@@ -106,158 +89,218 @@ default_config_data = {
     'no_dhcp_ns': False
 }
 
-def get_config():
-  conf = Config()
-  hosts = copy.deepcopy(default_config_data)
-
-  if conf.exists("system host-name"):
-    hosts['hostname'] = conf.return_value("system host-name")
-
-  if conf.exists("system domain-name"):
-    hosts['domain_name'] = conf.return_value("system domain-name")
-    hosts['domain_search'].append(hosts['domain_name'])
-
-  for search in conf.return_values("system domain-search domain"):
-    hosts['domain_search'].append(search)
+# borrowed from: https://github.com/donjajo/py-world/blob/master/resolvconfReader.py, THX!
+def get_resolvers(file):
+    resolv = {}
+    try:
+        with open(file, 'r') as resolvconf:
+            lines = [line.split('#', 1)[0].rstrip()
+                     for line in resolvconf.readlines()]
+            resolvers = [line.split()[1]
+                         for line in lines if 'nameserver' in line]
+            domains = [line.split()[1] for line in lines if 'search' in line]
+            resolv['resolvers'] = resolvers
+            resolv['domains'] = domains
+        return resolv
+    except IOError:
+        return []
+
+
+def get_config(arguments):
+    conf = Config()
+    hosts = copy.deepcopy(default_config_data)
+
+    if arguments.dhclient:
+        conf.exists = conf.exists_effective
+        conf.return_value = conf.return_effective_value
+        conf.return_values = conf.return_effective_values
+
+    if conf.exists("system host-name"):
+        hosts['hostname'] = conf.return_value("system host-name")
+        # This may happen if the config is not loaded yet,
+        # e.g. if run by cloud-init
+        if not hosts['hostname']:
+            hosts['hostname'] = default_config_data['hostname']
+
+    if conf.exists("system domain-name"):
+        hosts['domain_name'] = conf.return_value("system domain-name")
+        hosts['domain_search'].append(hosts['domain_name'])
+
+    for search in conf.return_values("system domain-search domain"):
+        hosts['domain_search'].append(search)
+
+    if conf.exists("system name-server"):
+        hosts['nameserver'] = conf.return_values("system name-server")
+
+    if conf.exists("system disable-dhcp-nameservers"):
+        hosts['no_dhcp_ns'] = conf.exists('system disable-dhcp-nameservers')
+
+    # system static-host-mapping
+    hosts['static_host_mapping'] = {'hostnames': {}}
+
+    if conf.exists('system static-host-mapping host-name'):
+        for hn in conf.list_nodes('system static-host-mapping host-name'):
+            hosts['static_host_mapping']['hostnames'][hn] = {
+                'ipaddr': conf.return_value('system static-host-mapping host-name ' + hn + ' inet'),
+                'alias': ''
+            }
+
+            if conf.exists('system static-host-mapping host-name ' + hn + ' alias'):
+                a = conf.return_values(
+                    'system static-host-mapping host-name ' + hn + ' alias')
+                hosts['static_host_mapping']['hostnames'][hn]['alias'] = " ".join(a)
+
+    return hosts
 
-  if conf.exists("system name-server"):
-    hosts['nameserver'] = conf.return_values("system name-server")
 
-  if conf.exists("system disable-dhcp-nameservers"):
-    hosts['no_dhcp_ns'] = conf.exists('system disable-dhcp-nameservers')
+def verify(config):
+    if config is None:
+        return None
+
+    # pattern $VAR(@) "^[[:alnum:]][-.[:alnum:]]*[[:alnum:]]$" ; "invalid host name $VAR(@)"
+    hostname_regex = re.compile("^[A-Za-z0-9][-.A-Za-z0-9]*[A-Za-z0-9]$")
+    if not hostname_regex.match(config['hostname']):
+        raise ConfigError('Invalid host name ' + config["hostname"])
+
+    # pattern $VAR(@) "^.{1,63}$" ; "invalid host-name length"
+    length = len(config['hostname'])
+    if length < 1 or length > 63:
+        raise ConfigError(
+            'Invalid host-name length, must be less than 63 characters')
+
+    # The search list is currently limited to six domains with a total of 256 characters.
+    # https://linux.die.net/man/5/resolv.conf
+    if len(config['domain_search']) > 6:
+        raise ConfigError(
+            'The search list is currently limited to six domains')
+
+    tmp = ' '.join(config['domain_search'])
+    if len(tmp) > 256:
+        raise ConfigError(
+            'The search list is currently limited to 256 characters')
+
+    # static mappings alias hostname
+    if config['static_host_mapping']['hostnames']:
+        for hn in config['static_host_mapping']['hostnames']:
+            if not config['static_host_mapping']['hostnames'][hn]['ipaddr']:
+                raise ConfigError('IP address required for ' + hn)
+            for hn_alias in config['static_host_mapping']['hostnames'][hn]['alias'].split(' '):
+                if not hostname_regex.match(hn_alias) and len(hn_alias) != 0:
+                    raise ConfigError('Invalid hostname alias ' + hn_alias)
 
-  ## system static-host-mapping
-  hosts['static_host_mapping'] = { 'hostnames'  : {}}
+    return None
 
-  if conf.exists('system static-host-mapping host-name'):
-    for hn in conf.list_nodes('system static-host-mapping host-name'):
-      hosts['static_host_mapping']['hostnames'][hn] = {
-        'ipaddr'  : conf.return_value('system static-host-mapping host-name ' + hn + ' inet'),
-        'alias'   : '' 
-      }
-      
-      if conf.exists('system static-host-mapping host-name ' + hn + ' alias'):
-        a = conf.return_values('system static-host-mapping host-name ' + hn + ' alias')
-        hosts['static_host_mapping']['hostnames'][hn]['alias'] = " ".join( conf.return_values('system static-host-mapping host-name ' + hn + ' alias') )
 
-  return hosts
+def generate(config):
+    if config is None:
+        return None
+
+    # If "system disable-dhcp-nameservers" is __configured__ all DNS resolvers
+    # received via dhclient should not be added into the final 'resolv.conf'.
+    #
+    # We iterate over every resolver file and retrieve the received nameservers
+    # for later adjustment of the system nameservers
+    dhcp_ns = []
+    dhcp_sd = []
+    for file in glob.glob('/etc/resolv.conf.dhclient-new*'):
+        for key, value in get_resolvers(file).items():
+            ns = [r for r in value if key == 'resolvers']
+            dhcp_ns.extend(ns)
+            sd = [d for d in value if key == 'domains']
+            dhcp_sd.extend(sd)
+
+    if not config['no_dhcp_ns']:
+        config['nameserver'] += dhcp_ns
+        config['domain_search'] += dhcp_sd
+
+    # Prune duplicate values
+    # Not order preserving, but then when multiple DHCP clients are used,
+    # there can't be guarantees about the order anyway
+    dhcp_ns = list(set(dhcp_ns))
+    dhcp_sd = list(set(dhcp_sd))
+
+    # We have third party scripts altering /etc/hosts, too.
+    # One example are the DHCP hostname update scripts thus we need to cache in
+    # every modification first - so changing domain-name, domain-search or hostname
+    # during runtime works
+    old_hosts = ""
+    with open(config_file_hosts, 'r') as f:
+        # Skips text before the beginning of our marker.
+        # NOTE: Marker __MUST__ match the one specified in config_tmpl_hosts
+        for line in f:
+            if line.strip() == '### modifications from other scripts should be added below':
+                break
+
+        for line in f:
+            # This additional line.strip() filters empty lines
+            if line.strip():
+                old_hosts += line
+
+    # Add an additional newline
+    old_hosts += '\n'
+
+    tmpl = jinja2.Template(config_tmpl_hosts)
+    config_text = tmpl.render(config)
+
+    with open(config_file_hosts, 'w') as f:
+        f.write(config_text)
+        f.write(old_hosts)
+
+    tmpl = jinja2.Template(config_tmpl_resolv)
+    config_text = tmpl.render(config)
+    with open(config_file_resolv, 'w') as f:
+        f.write(config_text)
 
-def verify(config):
-  if config is None:
     return None
 
-  # pattern $VAR(@) "^[[:alnum:]][-.[:alnum:]]*[[:alnum:]]$" ; "invalid host name $VAR(@)"
-  hostname_regex = re.compile("^[A-Za-z0-9][-.A-Za-z0-9]*[A-Za-z0-9]$")
-  if not hostname_regex.match(config['hostname']):
-    raise ConfigError('Invalid host name ' + config["hostname"])
 
-  # pattern $VAR(@) "^.{1,63}$" ; "invalid host-name length"
-  length = len(config['hostname'])
-  if length < 1 or length > 63:
-    raise ConfigError('Invalid host-name length, must be less than 63 characters')
+def apply(config):
+    if config is None:
+        return None
 
-  # The search list is currently limited to six domains with a total of 256 characters.
-  # https://linux.die.net/man/5/resolv.conf
-  if len(config['domain_search']) > 6:
-    raise ConfigError('The search list is currently limited to six domains')
+    # No domain name -- the Debian way.
+    hostname_new = config['hostname']
 
-  tmp = ' '.join(config['domain_search'])
-  if len(tmp) > 256:
-    raise ConfigError('The search list is currently limited to 256 characters')
+    # rsyslog runs into a race condition at boot time with systemd
+    # restart rsyslog only if the hostname changed.
+    hostname_old = subprocess.check_output(['hostnamectl', '--static']).decode().strip()
 
-  # static mappings alias hostname
-  if config['static_host_mapping']['hostnames']:
-    for hn in config['static_host_mapping']['hostnames']:
-      if not config['static_host_mapping']['hostnames'][hn]['ipaddr']:
-        raise ConfigError('IP address required for ' + hn)
-      for hn_alias in config['static_host_mapping']['hostnames'][hn]['alias'].split(' '):
-        if not hostname_regex.match(hn_alias) and len (hn_alias) !=0:
-          raise ConfigError('Invalid hostname alias ' + hn_alias)
+    os.system("hostnamectl set-hostname --static {0}".format(hostname_new))
 
-  return None
+    # Restart services that use the hostname
+    if hostname_new != hostname_old:
+        os.system("systemctl restart rsyslog.service")
 
-def generate(config):
-  if config is None:
-    return None
+    # If SNMP is running, restart it too
+    if os.system("pgrep snmpd > /dev/null") == 0:
+        os.system("systemctl restart snmpd.service")
 
-  # If "system disable-dhcp-nameservers" is __configured__ all DNS resolvers
-  # received via dhclient should not be added into the final 'resolv.conf'.
-  #
-  # We iterate over every resolver file and retrieve the received nameservers
-  # for later adjustment of the system nameservers
-  dhcp_ns = []
-  for file in glob.glob('/etc/resolv.conf.dhclient-new*'):
-    for r in get_resolvers(file):
-      dhcp_ns.append(r)
-
-  if not config['no_dhcp_ns']:
-    config['nameserver'] += dhcp_ns
-    for file in glob.glob('/etc/resolv.conf.dhclient-new*'):
-      config['domain_search'] = get_dhcp_search_doms(file)
-
-  # We have third party scripts altering /etc/hosts, too.
-  # One example are the DHCP hostname update scripts thus we need to cache in
-  # every modification first - so changing domain-name, domain-search or hostname
-  # during runtime works
-  old_hosts = ""
-  with open(config_file_hosts, 'r') as f:
-    # Skips text before the beginning of our marker.
-    # NOTE: Marker __MUST__ match the one specified in config_tmpl_hosts
-    for line in f:
-      if line.strip() == '### modifications from other scripts should be added below':
-        break
-
-    for line in f:
-      # This additional line.strip() filters empty lines
-      if line.strip():
-        old_hosts += line
-
-  # Add an additional newline
-  old_hosts += '\n'
-
-  tmpl = jinja2.Template(config_tmpl_hosts)
-  config_text = tmpl.render(config)
-
-  with open(config_file_hosts, 'w') as f:
-    f.write(config_text)
-    f.write(old_hosts)
-
-  tmpl = jinja2.Template(config_tmpl_resolv)
-  config_text = tmpl.render(config)
-  with open(config_file_resolv, 'w') as f:
-    f.write(config_text)
-
-  return None
+    # restart pdns if it is used
+    if os.system("/usr/bin/rec_control ping >/dev/null 2>&1") == 0:
+        os.system("/etc/init.d/pdns-recursor restart >/dev/null")
 
-def apply(config):
-  if config is None:
     return None
 
-  fqdn = config['hostname']
-  if config['domain_name']:
-    fqdn += '.' + config['domain_name']
-
-  os.system("hostnamectl set-hostname --static {0}".format(fqdn.rstrip('.')))
-
-  # Restart services that use the hostname
-  os.system("systemctl restart rsyslog.service")
-
-  # If SNMP is running, restart it too
-  if os.system("pgrep snmpd > /dev/null") == 0:
-    os.system("systemctl restart snmpd.service")
-
-  # restart pdns if it is used
-  if os.system("/usr/bin/rec_control ping >/dev/null 2>&1") == 0:
-    os.system("/etc/init.d/pdns-recursor restart >/dev/null")
-
-  return None
 
 if __name__ == '__main__':
-  try:
-    c = get_config()
-    verify(c)
-    generate(c)
-    apply(c)
-  except ConfigError as e:
-    print(e)
-    sys.exit(1)
+    args = parser.parse_args()
+
+    if args.dhclient:
+        # There's a big chance it was triggered by a commit still in progress
+        # so we need to wait until the new values are in the running config
+        vyos.util.wait_for_commit_lock()
+
+
+    try:
+        c = get_config(args)
+        # If it's called from dhclient, then either:
+        # a) verification was already done at commit time
+        # b) it's run on an unconfigured system, e.g. by cloud-init
+        # Therefore, verification is either redundant or useless
+        if not args.dhclient:
+            verify(c)
+        generate(c)
+        apply(c)
+    except ConfigError as e:
+        print(e)
+        sys.exit(1)
diff --git a/src/conf_mode/http-api.py b/src/conf_mode/http-api.py
new file mode 100755
index 000000000..7d618dded
--- /dev/null
+++ b/src/conf_mode/http-api.py
@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2019 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import sys
+import os
+import subprocess
+import json
+
+import vyos.defaults
+from vyos.config import Config
+from vyos import ConfigError
+
+config_file = '/etc/vyos/http-api.conf'
+
+default_config_data = {
+    'listen_address' : '127.0.0.1',
+    'port' : '8080',
+    'strict' : 'false',
+    'debug' : 'false',
+    'api_keys' : [ {"id": "testapp", "key": "qwerty"} ]
+}
+
+vyos_conf_scripts_dir=vyos.defaults.directories['conf_mode']
+
+# XXX: this model will need to be extended for tag nodes
+dependencies = [
+    'https.py',
+]
+
+def get_config():
+    http_api = default_config_data
+    conf = Config()
+    if not conf.exists('service https api'):
+        return None
+    else:
+        conf.set_level('service https api')
+
+    if conf.exists('strict'):
+        http_api['strict'] = 'true'
+
+    if conf.exists('debug'):
+        http_api['debug'] = 'true'
+
+    if conf.exists('port'):
+        port = conf.return_value('port')
+        http_api['port'] = port
+
+    if conf.exists('keys'):
+        for name in conf.list_nodes('keys id'):
+            if conf.exists('keys id {0} key'.format(name)):
+                key = conf.return_value('keys id {0} key'.format(name))
+                new_key = { 'id': name, 'key': key }
+                http_api['api_keys'].append(new_key)
+
+    return http_api
+
+def verify(http_api):
+    return None
+
+def generate(http_api):
+    if http_api is None:
+        return None
+
+    with open(config_file, 'w') as f:
+        json.dump(http_api, f, indent=2)
+
+    return None
+
+def apply(http_api):
+    if http_api is not None:
+        os.system('sudo systemctl restart vyos-http-api.service')
+        for dep in dependencies:
+            cmd = '{0}/{1}'.format(vyos_conf_scripts_dir, dep)
+            try:
+                subprocess.check_call(cmd, shell=True)
+            except subprocess.CalledProcessError as err:
+                raise ConfigError("{}.".format(err))
+    else:
+        os.system('sudo systemctl stop vyos-http-api.service')
+
+if __name__ == '__main__':
+    try:
+        c = get_config()
+        verify(c)
+        generate(c)
+        apply(c)
+    except ConfigError as e:
+        print(e)
+        sys.exit(1)
diff --git a/src/conf_mode/https.py b/src/conf_mode/https.py
new file mode 100755
index 000000000..dae51dd7d
--- /dev/null
+++ b/src/conf_mode/https.py
@@ -0,0 +1,132 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2019 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import sys
+import os
+
+import jinja2
+
+from vyos.config import Config
+from vyos import ConfigError
+
+config_file = '/etc/nginx/sites-available/default'
+
+# Please be careful if you edit the template.
+config_tmpl = """
+
+### Autogenerated by http-api.py ###
+# Default server configuration
+#
+server {
+        listen 80 default_server;
+        listen [::]:80 default_server;
+        server_name _;
+        return 302 https://$server_name$request_uri;
+}
+
+server {
+
+        # SSL configuration
+        #
+        listen 443 ssl default_server;
+        listen [::]:443 ssl default_server;
+        #
+        # Self signed certs generated by the ssl-cert package
+        # Don't use them in a production server!
+        #
+        include snippets/snakeoil.conf;
+
+{% for l_addr in listen_address %}
+        server_name {{ l_addr }};
+{% endfor %}
+
+        location / {
+{% if api %}
+                proxy_pass http://localhost:{{ api.port }};
+                proxy_buffering off;
+{% endif %}
+        }
+
+        error_page 501 502 503 =200 @50*_json;
+
+        location @50*_json {
+                default_type application/json;
+                return 200 '{"error": "Start service in configuration mode: set service https api"}';
+        }
+
+}
+"""
+
+default_config_data = {
+    'listen_address' : [ '127.0.0.1' ]
+}
+
+default_api_config_data = {
+    'port' : '8080',
+}
+
+def get_config():
+    https = default_config_data
+    conf = Config()
+    if not conf.exists('service https'):
+        return None
+    else:
+        conf.set_level('service https')
+
+    if conf.exists('listen-address'):
+        addrs = conf.return_values('listen-address')
+        https['listen_address'] = addrs[:]
+
+    if conf.exists('api'):
+        https['api'] = default_api_config_data
+
+    if conf.exists('api port'):
+        port = conf.return_value('api port')
+        https['api']['port'] = port
+
+    return https
+
+def verify(https):
+    return None
+
+def generate(https):
+    if https is None:
+        return None
+
+    tmpl = jinja2.Template(config_tmpl, trim_blocks=True)
+    config_text = tmpl.render(https)
+    with open(config_file, 'w') as f:
+        f.write(config_text)
+
+    return None
+
+def apply(https):
+    if https is not None:
+        os.system('sudo systemctl restart nginx.service')
+    else:
+        os.system('sudo systemctl stop nginx.service')
+
+if __name__ == '__main__':
+    try:
+        c = get_config()
+        verify(c)
+        generate(c)
+        apply(c)
+    except ConfigError as e:
+        print(e)
+        sys.exit(1)
diff --git a/src/conf_mode/ipoe_server.py b/src/conf_mode/ipoe_server.py
new file mode 100755
index 000000000..ca6b423e5
--- /dev/null
+++ b/src/conf_mode/ipoe_server.py
@@ -0,0 +1,417 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2018 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import sys
+import os
+import re
+import time
+import socket
+import subprocess
+import jinja2
+import syslog as sl
+
+from vyos.config import Config
+from vyos import ConfigError
+
+ipoe_cnf_dir = r'/etc/accel-ppp/ipoe'
+ipoe_cnf = ipoe_cnf_dir + r'/ipoe.config'
+
+pidfile = r'/var/run/accel_ipoe.pid'
+cmd_port = r'2002'
+
+chap_secrets = ipoe_cnf_dir + '/chap-secrets'
+## accel-pppd -d -c /etc/accel-ppp/pppoe/pppoe.config -p /var/run/accel_pppoe.pid
+
+ipoe_config = '''
+### generated by ipoe.py ### 
+[modules]
+log_syslog
+ippool
+ipoe
+shaper
+ipv6pool
+ipv6_nd
+ipv6_dhcp
+{% if auth['mech'] == 'radius' %}
+radius
+{% endif -%}
+{% if auth['mech'] == 'local' %}
+chap-secrets
+{% endif %}
+
+[core]
+thread-count={{thread_cnt}}
+
+[log]
+syslog=accel-ipoe,daemon
+copy=1
+level=5
+
+[ipoe]
+verbose=1
+{% for intfc in interfaces %}
+interface={{intfc}},\
+shared={{interfaces[intfc]['shared']}},\
+mode={{interfaces[intfc]['mode']}},\
+ifcfg={{interfaces[intfc]['ifcfg']}},\
+range={{interfaces[intfc]['range']}},\
+start={{interfaces[intfc]['sess_start']}},\
+ipv6=1
+{% endfor %}
+{% if auth['mech'] == 'noauth' %}
+noauth=1
+{% endif %}
+{% if auth['mech'] == 'local' %}
+username=ifname
+password=csid
+{% endif %}
+
+{%- for intfc in interfaces %}
+{% if (interfaces[intfc]['shared'] == '0') and (interfaces[intfc]['vlan_mon']) %}
+vlan_mon={{interfaces[intfc]['vlan_mon']|join(',')}}
+interface=re:{{intfc}}\.(409[0-6]|40[0-8][0-9]|[1-3][0-9]{3}|[1-9][0-9]{0,2})
+{% endif %}
+{% endfor %}
+
+{% if (dns['server1']) or (dns['server2']) %}
+[dns]
+{% if dns['server1'] %}
+dns1={{dns['server1']}}
+{% endif -%}
+{% if dns['server2'] %}
+dns2={{dns['server2']}}
+{% endif -%}
+{% endif -%}
+
+{% if (dnsv6['server1']) or (dnsv6['server2']) or (dnsv6['server3']) %}
+[dnsv6]
+dns={{dnsv6['server1']}}
+dns={{dnsv6['server2']}}
+dns={{dnsv6['server3']}}
+{% endif %}
+
+[ipv6-nd]
+verbose=1
+
+[ipv6-dhcp]
+verbose=1
+
+{% if ipv6['prfx'] %}
+[ipv6-pool]
+{% for prfx in ipv6['prfx'] %}
+{{prfx}}
+{% endfor %}
+{% for pd in ipv6['pd'] %}
+delegate={{pd}}
+{% endfor %}
+{% endif %}
+
+{% if auth['mech'] == 'local' %}
+[chap-secrets]
+chap-secrets=/etc/accel-ppp/ipoe/chap-secrets 
+{% endif %}
+
+{% if auth['mech'] == 'radius' %}
+[radius]
+verbose=1
+{% for srv in auth['radius'] %}
+server={{srv}},{{auth['radius'][srv]['secret']}},\
+req-limit={{auth['radius'][srv]['req-limit']}},\
+fail-time={{auth['radius'][srv]['fail-time']}}
+{% endfor %}
+{% if auth['radsettings']['dae-server']['ip-address'] %}
+dae-server={{auth['radsettings']['dae-server']['ip-address']}}:\
+{{auth['radsettings']['dae-server']['port']}},\
+{{auth['radsettings']['dae-server']['secret']}}
+{% endif -%}
+{% if auth['radsettings']['acct-timeout'] %}
+acct-timeout={{auth['radsettings']['acct-timeout']}}
+{% endif -%}
+{% if auth['radsettings']['max-try'] %}
+max-try={{auth['radsettings']['max-try']}}
+{% endif -%}
+{% if auth['radsettings']['timeout'] %}
+timeout={{auth['radsettings']['timeout']}}
+{% endif -%}
+{% if auth['radsettings']['nas-ip-address'] %}
+nas-ip-address={{auth['radsettings']['nas-ip-address']}}
+{% endif -%}
+{% if auth['radsettings']['nas-identifier'] %}
+nas-identifier={{auth['radsettings']['nas-identifier']}}
+{% endif -%}
+{% endif %}
+
+[cli]
+tcp=127.0.0.1:2002
+'''
+
+### pppoe chap secrets
+chap_secrets_conf = '''
+# username  server  password  acceptable local IP addresses   shaper
+{% for aifc in auth['auth_if'] %}
+{% for mac in auth['auth_if'][aifc] %}
+{% if (auth['auth_if'][aifc][mac]['up']) and (auth['auth_if'][aifc][mac]['down']) %}
+{{aifc}}\t*\t{{mac.lower()}}\t*\t{{auth['auth_if'][aifc][mac]['down']}}/{{auth['auth_if'][aifc][mac]['up']}}
+{% else %}
+{{aifc}}\t*\t{{mac.lower()}}\t*
+{% endif %}
+{% endfor %}
+{% endfor %}
+'''
+
+##### Inline functions start ####
+### config path creation
+if not os.path.exists(ipoe_cnf_dir):
+  os.makedirs(ipoe_cnf_dir)
+  sl.syslog(sl.LOG_NOTICE, ipoe_cnf_dir + " created")
+
+def get_cpu():
+  cpu_cnt = 1
+  if os.cpu_count() == 1:
+    cpu_cnt = 1
+  else:
+    cpu_cnt = int(os.cpu_count()/2)
+  return cpu_cnt
+
+def chk_con():
+  cnt = 0
+  s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+  while True:
+    try:
+      s.connect(("127.0.0.1", int(cmd_port)))
+      break
+    except ConnectionRefusedError:
+      time.sleep(0.5)
+      cnt +=1
+      if cnt == 100:
+        raise("failed to start pppoe server")
+        break
+
+def accel_cmd(cmd=''):
+  if not cmd:
+    return None
+  try:
+    ret = subprocess.check_output(['/usr/bin/accel-cmd', '-p', cmd_port, cmd]).decode().strip()
+    return ret
+  except:
+    return 1
+
+### chap_secrets file if auth mode local
+def gen_chap_secrets(c):
+  tmpl = jinja2.Template(chap_secrets_conf, trim_blocks=True)
+  chap_secrets_txt = tmpl.render(c)
+  old_umask = os.umask(0o077)
+  open(chap_secrets,'w').write(chap_secrets_txt)
+  os.umask(old_umask)
+  sl.syslog(sl.LOG_NOTICE, chap_secrets + ' written')
+
+##### Inline functions end ####
+
+def get_config():
+  c = Config()
+  if not c.exists('service ipoe-server'):
+    return None
+
+  config_data = {}
+
+  c.set_level('service ipoe-server')
+  config_data['interfaces'] = {}
+  for intfc in c.list_nodes('interface'):
+    config_data['interfaces'][intfc] = {
+      'mode'        : 'L2',
+      'shared'      : '1',
+      'sess_start'  : 'dhcpv4',  ### may need a conifg option, can be dhcpv4 or up for unclassified pkts
+      'range'       : None,
+      'ifcfg'       : '1',
+      'vlan_mon'    : []
+    }
+    config_data['dns'] = {
+      'server1'     : None,
+      'server2'     : None
+    }
+    config_data['dnsv6'] = {
+      'server1'     : None,
+      'server2'     : None,
+      'server3'     : None
+    }
+    config_data['ipv6'] = {
+      'prfx'        : [],
+      'pd'     : [],
+    }
+    config_data['auth'] = {
+      'auth_if'       : {},
+      'mech'          : 'noauth',
+      'radius'        : {},
+      'radsettings'   : {
+        'dae-server'  : {}
+      }
+    }
+
+    if c.exists('interface ' + intfc + ' network-mode'):
+      config_data['interfaces'][intfc]['mode'] = c.return_value('interface ' + intfc + ' network-mode')
+    if c.return_value('interface ' + intfc + ' network') == 'vlan':
+      config_data['interfaces'][intfc]['shared'] = '0'
+      if c.exists('interface ' + intfc + ' vlan-id'):
+        config_data['interfaces'][intfc]['vlan_mon'] += c.return_values('interface ' + intfc + ' vlan-id')
+      if c.exists('interface ' + intfc + ' vlan-range'):
+        config_data['interfaces'][intfc]['vlan_mon'] += c.return_values('interface ' + intfc + ' vlan-range')
+    if c.exists('interface ' + intfc + ' client-subnet'):
+      config_data['interfaces'][intfc]['range'] = c.return_value('interface ' + intfc + ' client-subnet')
+    if c.exists('dns-server server-1'):
+      config_data['dns']['server1'] = c.return_value('dns-server server-1')
+    if c.exists('dns-server server-2'):
+      config_data['dns']['server2'] = c.return_value('dns-server server-2')
+    if c.exists('dnsv6-server server-1'):
+      config_data['dnsv6']['server1'] = c.return_value('dnsv6-server server-1')
+    if c.exists('dnsv6-server server-2'):
+      config_data['dnsv6']['server2'] = c.return_value('dnsv6-server server-2')
+    if c.exists('dnsv6-server server-3'):
+      config_data['dnsv6']['server3'] = c.return_value('dnsv6-server server-3')
+    if not c.exists('authentication mode noauth'):
+      config_data['auth']['mech'] = c.return_value('authentication mode')
+    if c.exists('authentication mode local'):
+      for auth_int in c.list_nodes('authentication interface'):
+        for mac in c.list_nodes('authentication interface ' + auth_int + ' mac-address'):
+          config_data['auth']['auth_if'][auth_int] = {}
+          if c.exists('authentication interface ' + auth_int + ' mac-address ' + mac + ' rate-limit'):
+            config_data['auth']['auth_if'][auth_int][mac] = {}
+            config_data['auth']['auth_if'][auth_int][mac]['up'] = c.return_value('authentication interface ' + auth_int + ' mac-address ' + mac + ' rate-limit upload') 
+            config_data['auth']['auth_if'][auth_int][mac]['down'] = c.return_value('authentication interface ' + auth_int + ' mac-address ' + mac + ' rate-limit download')
+          else:
+            config_data['auth']['auth_if'][auth_int][mac] = {}
+            config_data['auth']['auth_if'][auth_int][mac]['up'] = None
+            config_data['auth']['auth_if'][auth_int][mac]['down'] = None
+    if c.exists('authentication mode radius'):
+      for rsrv in c.list_nodes('authentication radius-server'):
+        config_data['auth']['radius'][rsrv] = {}
+        if c.exists('authentication radius-server ' + rsrv + ' secret'):
+          config_data['auth']['radius'][rsrv]['secret'] = c.return_value('authentication radius-server ' + rsrv + ' secret')
+        else:
+          config_data['auth']['radius'][rsrv]['secret'] = None
+        if c.exists('authentication radius-server ' + rsrv + ' fail-time'):
+          config_data['auth']['radius'][rsrv]['fail-time'] = c.return_value('authentication radius-server ' + rsrv + ' fail-time')
+        else:
+          config_data['auth']['radius'][rsrv]['fail-time'] = '0'
+        if c.exists('authentication radius-server ' + rsrv + ' req-limit'):
+          config_data['auth']['radius'][rsrv]['req-limit'] = c.return_value('authentication radius-server ' + rsrv + ' req-limit')
+        else:
+          config_data['auth']['radius'][rsrv]['req-limit'] = '0'
+      if c.exists('authentication radius-settings'):
+        if c.exists('authentication radius-settings timeout'):
+          config_data['auth']['radsettings']['timeout'] = c.return_value('authentication radius-settings timeout')
+        if c.exists('authentication radius-settings nas-ip-address'):
+           config_data['auth']['radsettings']['nas-ip-address'] = c.return_value('authentication radius-settings nas-ip-address')
+        if c.exists('authentication radius-settings nas-identifier'):
+          config_data['auth']['radsettings']['nas-identifier'] = c.return_value('authentication radius-settings nas-identifier')
+        if c.exists('authentication radius-settings max-try'):
+          config_data['auth']['radsettings']['max-try'] = c.return_value('authentication radius-settings max-try')
+        if c.exists('authentication radius-settings acct-timeout'):
+          config_data['auth']['radsettings']['acct-timeout'] = c.return_value('authentication radius-settings acct-timeout')
+        if c.exists('authentication radius-settings dae-server ip-address'):
+          config_data['auth']['radsettings']['dae-server']['ip-address'] = c.return_value('authentication radius-settings dae-server ip-address')
+        if c.exists('authentication radius-settings dae-server port'):
+          config_data['auth']['radsettings']['dae-server']['port'] = c.return_value('authentication radius-settings dae-server port')
+        if c.exists('authentication radius-settings dae-server secret'):
+           config_data['auth']['radsettings']['dae-server']['secret'] = c.return_value('authentication radius-settings dae-server secret')
+
+    if c.exists('client-ipv6-pool prefix'):
+      config_data['ipv6']['prfx'] = c.return_values('client-ipv6-pool prefix')
+    if c.exists('client-ipv6-pool delegate-prefix'):
+      config_data['ipv6']['pd'] = c.return_values('client-ipv6-pool delegate-prefix')
+
+  return config_data
+
+def generate(c):
+  if c == None or not c:
+    return None
+  
+  c['thread_cnt'] = get_cpu()
+
+  if c['auth']['mech'] == 'local':
+    gen_chap_secrets(c)
+
+  tmpl = jinja2.Template(ipoe_config, trim_blocks=True)
+  config_text = tmpl.render(c)
+  open(ipoe_cnf,'w').write(config_text)
+  return c
+
+def verify(c):
+  if c == None or not c:
+    return None
+
+  for intfc in c['interfaces']:
+    if not c['interfaces'][intfc]['range']:
+      raise ConfigError("service ipoe-server interface " + intfc + " client-subnet needs a value") 
+
+  if c['auth']['mech'] == 'radius':
+    if not c['auth']['radius']:
+      raise ConfigError("service ipoe-server authentication radius-server requires a value for authentication mode radius")
+    else:
+      for radsrv in c['auth']['radius']:
+        if not c['auth']['radius'][radsrv]['secret']:
+          raise ConfigError("service ipoe-server authentication radius-server " + radsrv + " secret requires a value")
+
+  if c['auth']['radsettings']['dae-server']:
+    try:
+      if c['auth']['radsettings']['dae-server']['ip-address']:
+        pass
+    except:
+      raise ConfigError("service ipoe-server authentication radius-settings dae-server ip-address value required") 
+    try:
+      if c['auth']['radsettings']['dae-server']['secret']:
+        pass
+    except:
+      raise ConfigError("service ipoe-server authentication radius-settings dae-server secret value required")
+    try:
+      if c['auth']['radsettings']['dae-server']['port']:
+        pass
+    except:
+      raise ConfigError("service ipoe-server authentication radius-settings dae-server port value required")
+
+  if len(c['ipv6']['pd']) != 0 and len(c['ipv6']['prfx']) == 0:
+    raise ConfigError("service ipoe-server client-ipv6-pool prefix needs a value")
+
+  return c
+
+def apply(c):
+  if c == None:
+    if os.path.exists(pidfile):
+      accel_cmd('shutdown hard')
+      if os.path.exists(pidfile):
+        os.remove(pidfile)
+    return None
+
+  if not os.path.exists(pidfile):
+    ret = subprocess.call(['/usr/sbin/accel-pppd', '-c', ipoe_cnf, '-p', pidfile, '-d'])
+    chk_con()
+    if ret !=0 and os.path.exists(pidfile):
+      os.remove(pidfile)
+      raise ConfigError('accel-pppd failed to start')
+  else:
+    accel_cmd('restart')
+    sl.syslog(sl.LOG_NOTICE, "reloading config via daemon restart")
+
+if __name__ == '__main__':
+  try:
+    c = get_config()
+    verify(c)
+    generate(c)
+    apply(c)
+  except ConfigError as e:
+    print(e)
+    sys.exit(1)
diff --git a/src/conf_mode/protocols_bfd.py b/src/conf_mode/protocols_bfd.py
new file mode 100755
index 000000000..04549f4b4
--- /dev/null
+++ b/src/conf_mode/protocols_bfd.py
@@ -0,0 +1,166 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2019 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import sys
+import jinja2
+import copy
+import os
+import vyos.validate
+
+from vyos import ConfigError
+from vyos.config import Config
+
+config_file = r'/tmp/bfd.frr'
+
+# Please be careful if you edit the template.
+config_tmpl = """
+!
+bfd
+{% for peer in old_peers -%}
+ no peer {{ peer }}
+{% endfor -%}
+!
+{% for peer in new_peers -%}
+ peer {{ peer.remote }}{% if peer.multihop %} multihop{% endif %}{% if peer.src_addr %} local-address {{ peer.src_addr }}{% endif %}{% if peer.src_if %} interface {{ peer.src_if }}{% endif %}
+ detect-multiplier {{ peer.multiplier }}
+ receive-interval {{ peer.rx_interval }}
+ transmit-interval {{ peer.tx_interval }}
+ {% if not peer.shutdown %}no {% endif %}shutdown
+{% endfor -%}
+!
+"""
+
+default_config_data = {
+    'new_peers': [],
+    'old_peers' : []
+}
+
+def get_config():
+    bfd = copy.deepcopy(default_config_data)
+    conf = Config()
+    if not (conf.exists('protocols bfd') or conf.exists_effective('protocols bfd')):
+        return None
+    else:
+        conf.set_level('protocols bfd')
+
+    # as we have to use vtysh to talk to FRR we also need to know
+    # which peers are gone due to a config removal - thus we read in
+    # all peers (active or to delete)
+    bfd['old_peers'] = conf.list_effective_nodes('peer')
+
+    for peer in conf.list_nodes('peer'):
+        conf.set_level('protocols bfd peer {0}'.format(peer))
+        bfd_peer = {
+            'remote': peer,
+            'shutdown': False,
+            'src_if': '',
+            'src_addr': '',
+            'multiplier': '3',
+            'rx_interval': '300',
+            'tx_interval': '300',
+            'multihop': False
+        }
+
+        # Check if individual peer is disabled
+        if conf.exists('shutdown'):
+            bfd_peer['shutdown'] = True
+
+        # Check if peer has a local source interface configured
+        if conf.exists('source interface'):
+            bfd_peer['src_if'] = conf.return_value('source interface')
+
+        # Check if peer has a local source address configured - this is mandatory for IPv6
+        if conf.exists('source address'):
+            bfd_peer['src_addr'] = conf.return_value('source address')
+
+        # Tell BFD daemon that we should expect packets with TTL less than 254
+        # (because it will take more than one hop) and to listen on the multihop
+        # port (4784)
+        if conf.exists('multihop'):
+            bfd_peer['multihop'] = True
+
+        # Configures the minimum interval that this system is capable of receiving
+        # control packets. The default value is 300 milliseconds.
+        if conf.exists('interval receive'):
+            bfd_peer['rx_interval'] = conf.return_value('interval receive')
+
+        # The minimum transmission interval (less jitter) that this system wants
+        # to use to send BFD control packets.
+        if conf.exists('interval transmit'):
+            bfd_peer['tx_interval'] = conf.return_value('interval transmit')
+
+        # Configures the detection multiplier to determine packet loss. The remote
+        # transmission interval will be multiplied by this value to determine the
+        # connection loss detection timer. The default value is 3.
+        if conf.exists('interval multiplier'):
+            bfd_peer['multiplier'] = conf.return_value('interval multiplier')
+
+        bfd['new_peers'].append(bfd_peer)
+
+    return bfd
+
+def verify(bfd):
+    if bfd is None:
+        return None
+
+    for peer in bfd['new_peers']:
+        # Bail out early if peer is shutdown
+        if peer['shutdown']:
+            continue
+
+        # IPv6 peers require an explicit local address/interface combination
+        if vyos.validate.is_ipv6(peer['remote']):
+            if not (peer['src_if'] and peer['src_addr']):
+                raise ConfigError('BFD IPv6 peers require explicit local address/interface setting')
+
+        # multihop doesn't accept interface names
+        if peer['multihop'] and peer['src_if']:
+            raise ConfigError('multihop does not accept interface names')
+
+
+    return None
+
+def generate(bfd):
+    if bfd is None:
+        return None
+
+    return None
+
+def apply(bfd):
+    if bfd is None:
+        return None
+
+    tmpl = jinja2.Template(config_tmpl)
+    config_text = tmpl.render(bfd)
+    with open(config_file, 'w') as f:
+        f.write(config_text)
+
+    os.system("sudo vtysh -d bfdd -f " + config_file)
+    if os.path.exists(config_file):
+        os.remove(config_file)
+
+    return None
+
+if __name__ == '__main__':
+    try:
+        c = get_config()
+        verify(c)
+        generate(c)
+        apply(c)
+    except ConfigError as e:
+        print(e)
+        sys.exit(1)
diff --git a/src/conf_mode/snmp.py b/src/conf_mode/snmp.py
index 06d2e253a..0ddab2129 100755
--- a/src/conf_mode/snmp.py
+++ b/src/conf_mode/snmp.py
@@ -206,6 +206,13 @@ group {{ u.group }} usm {{ u.name }}
 group {{ u.group }} tsm {{ u.name }}
 {% endfor %}
 {%- endif %}
+
+{% if script_ext %}
+# extension scripts
+{%- for ext in script_ext|sort %}
+extend\t{{ext}}\t{{script_ext[ext]}}
+{%- endfor %}
+{% endif %}
 """
 
 # SNMP template (/etc/default/snmpd) - be careful if you edit the template.
@@ -240,7 +247,8 @@ default_config_data = {
     'v3_tsm_key': '',
     'v3_tsm_port': '10161',
     'v3_users': [],
-    'v3_views': []
+    'v3_views': [],
+    'script_ext': {}
 }
 
 def rmfile(file):
@@ -345,6 +353,14 @@ def get_config():
 
             snmp['trap_targets'].append(trap_tgt)
 
+    #
+    # 'set service snmp script-extensions'
+    #
+    if conf.exists('script-extensions'):
+      for extname in conf.list_nodes('script-extensions extension-name'):
+        snmp['script_ext'][extname] = '/config/user-data/' + conf.return_value('script-extensions extension-name ' + extname + ' script')
+
+
     #########################################################################
     #                ____  _   _ __  __ ____          _____                 #
     #               / ___|| \ | |  \/  |  _ \  __   _|___ /                 #
@@ -581,6 +597,14 @@ def verify(snmp):
     if snmp is None:
         return None
 
+    ### check if the configured script actually exist under /config/user-data
+    if snmp['script_ext']:
+      for ext in snmp['script_ext']:
+        if not os.path.isfile(snmp['script_ext'][ext]):
+          print ("WARNING: script: " + snmp['script_ext'][ext] + " doesn\'t exist")  
+        else:
+          os.chmod(snmp['script_ext'][ext], 0o555)
+
     # bail out early if SNMP v3 is not configured
     if not snmp['v3_enabled']:
         return None
@@ -633,7 +657,6 @@ def verify(snmp):
             if not 'seclevel' in group.keys():
                 raise ConfigError('"seclevel" must be specified')
 
-
     if 'v3_traps' in snmp.keys():
         for trap in snmp['v3_traps']:
             if trap['authPassword'] and trap['authMasterKey']:
diff --git a/src/conf_mode/syslog.py b/src/conf_mode/syslog.py
index c8541a4a0..7b79c701b 100755
--- a/src/conf_mode/syslog.py
+++ b/src/conf_mode/syslog.py
@@ -72,8 +72,8 @@ logrotate_configs = '''
   missingok
   notifempty
   create
-  rotate  {{files[file]['max-files']}}
-  size={{ files[file]['max-size']//1024}}k
+  rotate {{files[file]['max-files']}}
+  size={{files[file]['max-size']//1024}}k
   postrotate
     invoke-rc.d rsyslog rotate > /dev/null
   endscript
@@ -120,8 +120,8 @@ def get_config():
     config_data['files']['global']['selectors'] = generate_selectors(c, 'global facility')
   if c.exists('global archive size'):
     config_data['files']['global']['max-size'] = int(c.return_value('global archive size'))* 1024
-  if c.exists('global archive files'):
-    config_data['files']['global']['max-files'] = c.return_value('global archive files')
+  if c.exists('global archive file'):
+    config_data['files']['global']['max-files'] = c.return_value('global archive file')
   if c.exists('global preserve-fqdn'):
     config_data['files']['global']['preserver_fqdn'] = True
 
@@ -196,7 +196,7 @@ def get_config():
           }
         }
       )
-
+  
   return config_data
 
 def generate_selectors(c, config_node):
@@ -279,7 +279,7 @@ def verify(c):
             raise ConfigError('Invalid logging level ' + s + ' set in '+ conf + ' ' + item)
 
 def apply(c):
-  if not os.path.exits('/var/run/rsyslogd.pid'):
+  if not os.path.exists('/var/run/rsyslogd.pid'):
     os.system("sudo systemctl start rsyslog >/dev/null")
   else:
     os.system("sudo systemctl restart rsyslog >/dev/null")
diff --git a/src/conf_mode/vrrp.py b/src/conf_mode/vrrp.py
index bc833b63f..a08493309 100755
--- a/src/conf_mode/vrrp.py
+++ b/src/conf_mode/vrrp.py
@@ -39,7 +39,7 @@ config_tmpl = """
 
 {% if group.health_check_script -%}
 vrrp_script healthcheck_{{ group.name }} {
-	script {{ group.health_check_script }}
+	script "{{ group.health_check_script }}"
 	interval {{ group.health_check_interval }}
 	fall {{ group.health_check_count }}
 	rise 1
@@ -85,7 +85,7 @@ vrrp_instance {{ group.name }} {
 
     {% if group.auth_password -%}
       authentication {
-        auth_pass {{ group.auth_password }}
+        auth_pass "{{ group.auth_password }}"
         auth_type {{ group.auth_type }}
       }
     {% endif -%}
diff --git a/src/conf_mode/wireguard.py b/src/conf_mode/wireguard.py
index e893dba47..8234fad0b 100755
--- a/src/conf_mode/wireguard.py
+++ b/src/conf_mode/wireguard.py
@@ -63,6 +63,7 @@ def get_config():
                 'lport'       : '',
                 'status'      : 'exists',
                 'state'       : 'enabled',
+                'fwmark'         : 0x00,
                 'mtu'         : '1420',
                 'peer'        : {}
             }
@@ -95,6 +96,9 @@ def get_config():
       ### listen port
       if c.exists(cnf + ' port'):
         config_data['interfaces'][intfc]['lport'] = c.return_value(cnf + ' port')
+      ### fwmark
+      if c.exists(cnf + ' fwmark'):
+        config_data['interfaces'][intfc]['fwmark'] = c.return_value(cnf + ' fwmark')
       ### description
       if c.exists(cnf + ' description'):
         config_data['interfaces'][intfc]['descr'] = c.return_value(cnf + ' description')
@@ -221,7 +225,7 @@ def apply(c):
     ### config updates
     if c['interfaces'][intf]['status'] == 'exists':
       ### IP address change
-      addr_eff = re.sub("\'", "", c_eff.return_effective_values(intf + ' address')).split()
+      addr_eff = c_eff.return_effective_values(intf + ' address')
       addr_rem = list(set(addr_eff) - set(c['interfaces'][intf]['addr']))
       addr_add = list(set(c['interfaces'][intf]['addr']) - set(addr_eff))
 
@@ -296,6 +300,10 @@ def configure_interface(c, intf):
     if c['interfaces'][intf]['lport']:
       wg_config['port'] = c['interfaces'][intf]['lport']
 
+    ## fwmark
+    if c['interfaces'][intf]['fwmark']:
+      wg_config['fwmark'] = c['interfaces'][intf]['fwmark']
+      
     ## endpoint
     if c['interfaces'][intf]['peer'][p]['endpoint']:
       wg_config['endpoint'] = c['interfaces'][intf]['peer'][p]['endpoint']
@@ -314,6 +322,7 @@ def configure_interface(c, intf):
     ### assemble wg command
     cmd = "sudo wg set " + intf
     cmd += " listen-port " + str(wg_config['port'])
+    cmd += " fwmark " + str(wg_config['fwmark'])
     cmd += " private-key " + wg_config['private-key']
     cmd += " peer " + wg_config['pubkey']
     cmd += " preshared-key " + wg_config['psk']