Merge branch 'current' of github.com:vyos/vyos-1x into equuleus

* 'current' of github.com:vyos/vyos-1x:
  openvpn: bridge: T1556: remove obsolete bridge-group definition
  ifconfig: T1849: fix DHCPv6 startup
  Python/VyOS validate: T1849: handle is_ipv6()/is_ipv6() exceptions
  ifconfig: T1793: remove dhcpv6 client debug output
  ddclient: T1853: bugfix TypeError exception
  syslog: T1845: syslog host no longer accepts a port
  syslog: code formatting
  syslog: T1845: syslog host no longer accepts a port
  syslog: renaming files and conf script to fit new scheme
  T1855, T1826: clean up the reboot/shutdown script.
  wireguard: T1853: disable peer doesn't work
  Revert "syslog: T1845: syslog host no longer accepts a port"
  dmvpn: T1784: Add swanctl load call
  syslog: T1845: syslog host no longer accepts a port
  [vyos.config] T1847: correctly set_level for path given as empty string

11 files changed, 207 insertions, 137 deletions

diff --git a/interface-definitions/interfaces-openvpn.xml b/interface-definitions/interfaces-openvpn.xml
index 2c77bcf37..2c2556f45 100644
--- a/interface-definitions/interfaces-openvpn.xml
+++ b/interface-definitions/interfaces-openvpn.xml
@@ -33,45 +33,6 @@
               </leafNode>
             </children>
           </node>
-          <node name="bridge-group">
-            <properties>
-              <help>Interface to be added to a bridge group</help>
-            </properties>
-            <children>
-              <leafNode name="bridge">
-                <properties>
-                  <help>Interface to a bridge-group</help>
-                  <completionHelp>
-                    <script>${vyos_completion_dir}/list_interfaces.py --type bridge</script>
-                  </completionHelp>
-                </properties>
-              </leafNode>
-              <leafNode name="cost">
-                <properties>
-                  <help>Path cost for this port</help>
-                  <valueHelp>
-                    <format>0-2147483647</format>
-                    <description>Path cost for this port</description>
-                  </valueHelp>
-                  <constraint>
-                    <validator name="numeric" argument="--range 0-2147483647"/>
-                  </constraint>
-                </properties>
-              </leafNode>
-              <leafNode name="cost">
-                <properties>
-                  <help>Path priority for this port</help>
-                  <valueHelp>
-                    <format>0-255</format>
-                    <description>Path priority for this port</description>
-                  </valueHelp>
-                  <constraint>
-                    <validator name="numeric" argument="--range 0-255"/>
-                  </constraint>
-                </properties>
-              </leafNode>
-            </children>
-          </node>
           <leafNode name="description">
             <properties>
               <help>Description</help>
diff --git a/interface-definitions/syslog.xml b/interface-definitions/system-syslog.xml
index d5ea4511e..8f4b105c8 100644
--- a/interface-definitions/syslog.xml
+++ b/interface-definitions/system-syslog.xml
@@ -2,7 +2,7 @@
 <interfaceDefinition>
   <node name="system">
     <children>
-      <node name="syslog" owner="${vyos_conf_scripts_dir}/syslog.py">
+      <node name="syslog" owner="${vyos_conf_scripts_dir}/system-syslog.py">
         <properties>
           <help>System logging</help>
           <priority>400</priority>
@@ -191,6 +191,19 @@
               </valueHelp>
             </properties>
             <children>
+              <leafNode name="port">
+                <properties>
+                  <help>Destination port</help>
+                  <valueHelp>
+                    <format>1-65535</format>
+                    <description>Destination port</description>
+                  </valueHelp>
+                  <constraint>
+                    <validator name="numeric" argument="--range 1-65535"/>
+                  </constraint>
+                  <constraintErrorMessage>Invalid destination port value</constraintErrorMessage>
+                </properties>
+              </leafNode>
               <tagNode name="facility">
                 <properties>
                   <help>Facility for logging</help>
diff --git a/python/vyos/config.py b/python/vyos/config.py
index 5bd8fb072..c7bd96e2f 100644
--- a/python/vyos/config.py
+++ b/python/vyos/config.py
@@ -160,7 +160,10 @@ class Config(object):
         # and path supplied as method argument
         # XXX: for small strings in-place concatenation is not a problem
         if isinstance(path, str):
-            self._level = re.split(r'\s+', path)
+            if path:
+                self._level = re.split(r'\s*', path)
+            else:
+                self._level = []
         elif isinstance(path, list):
             self._level = path
         else:
diff --git a/python/vyos/ifconfig.py b/python/vyos/ifconfig.py
index f487e6a5b..72f11c04d 100644
--- a/python/vyos/ifconfig.py
+++ b/python/vyos/ifconfig.py
@@ -615,8 +615,6 @@ class Interface:
         >>> j.set_dhcpv6()
         """
         dhcpv6 = self.get_dhcpv6_options()
-        import pprint
-        pprint.pprint(dhcpv6)
 
         # better save then sorry .. should be checked in interface script
         # but if you missed it we are safe!
@@ -629,33 +627,32 @@ class Interface:
         with open(self._dhcpv6_cfg_file, 'w') as f:
             f.write(dhcpv6_text)
 
-        if self.get_state() == 'up':
-            # https://bugs.launchpad.net/ubuntu/+source/ifupdown/+bug/1447715
-            #
-            # wee need to wait for IPv6 DAD to finish once and interface is added
-            # this suxx :-(
-            sleep(5)
-
-            # no longer accept router announcements on this interface
-            self._write_sysfs('/proc/sys/net/ipv6/conf/{}/accept_ra'
-                  .format(self._ifname), 0)
-
-            # assemble command-line to start DHCPv6 client (dhclient)
-            cmd  = 'start-stop-daemon --start --quiet --pidfile ' + \
-                self._dhcpv6_pid_file
-            cmd += ' --exec /sbin/dhclient --'
-            # now pass arguments to dhclient binary
-            cmd += ' -6 -nw -cf {} -pf {} -lf {}'.format(
-                self._dhcpv6_cfg_file, self._dhcpv6_pid_file, self._dhcpv6_lease_file)
-
-            # add optional arguments
-            if dhcpv6['dhcpv6_prm_only']:
-                cmd += ' -S'
-            if dhcpv6['dhcpv6_temporary']:
-                cmd += ' -T'
-
-            cmd += ' {}'.format(self._ifname)
-            return self._cmd(cmd)
+        # https://bugs.launchpad.net/ubuntu/+source/ifupdown/+bug/1447715
+        #
+        # wee need to wait for IPv6 DAD to finish once and interface is added
+        # this suxx :-(
+        sleep(5)
+
+        # no longer accept router announcements on this interface
+        self._write_sysfs('/proc/sys/net/ipv6/conf/{}/accept_ra'
+              .format(self._ifname), 0)
+
+        # assemble command-line to start DHCPv6 client (dhclient)
+        cmd  = 'start-stop-daemon --start --quiet --pidfile ' + \
+            self._dhcpv6_pid_file
+        cmd += ' --exec /sbin/dhclient --'
+        # now pass arguments to dhclient binary
+        cmd += ' -6 -nw -cf {} -pf {} -lf {}'.format(
+            self._dhcpv6_cfg_file, self._dhcpv6_pid_file, self._dhcpv6_lease_file)
+
+        # add optional arguments
+        if dhcpv6['dhcpv6_prm_only']:
+            cmd += ' -S'
+        if dhcpv6['dhcpv6_temporary']:
+            cmd += ' -T'
+
+        cmd += ' {}'.format(self._ifname)
+        return self._cmd(cmd)
 
 
     def _del_dhcpv6(self):
diff --git a/python/vyos/validate.py b/python/vyos/validate.py
index 258f7f76a..3f3166022 100644
--- a/python/vyos/validate.py
+++ b/python/vyos/validate.py
@@ -29,10 +29,13 @@ def is_ipv4(addr):
 
     # With the below statement we can check for IPv4 networks and host
     # addresses at the same time
-    if ipaddress.ip_address(addr.split(r'/')[0]).version == 4:
-        return True
-    else:
-        return False
+    try:
+        if ipaddress.ip_address(addr.split(r'/')[0]).version == 4:
+            return True
+    except:
+        pass
+
+    return False
 
 def is_ipv6(addr):
     """
@@ -41,10 +44,13 @@ def is_ipv6(addr):
 
     # With the below statement we can check for IPv4 networks and host
     # addresses at the same time
-    if ipaddress.ip_network(addr.split(r'/')[0]).version == 6:
-        return True
-    else:
-        return False
+    try:
+        if ipaddress.ip_network(addr.split(r'/')[0]).version == 6:
+            return True
+    except:
+        pass
+
+    return False
 
 def is_intf_addr_assigned(intf, addr):
     """
diff --git a/src/conf_mode/dynamic_dns.py b/src/conf_mode/dynamic_dns.py
index 7c3b9ff6a..9ba8659a1 100755
--- a/src/conf_mode/dynamic_dns.py
+++ b/src/conf_mode/dynamic_dns.py
@@ -95,6 +95,7 @@ default_service_protocol = {
 default_config_data = {
     'interfaces': [],
     'cache_file': cache_file,
+    'deleted': False,
     'pid_file': pid_file
 }
 
@@ -102,7 +103,8 @@ def get_config():
     dyndns = default_config_data
     conf = Config()
     if not conf.exists('service dns dynamic'):
-        return None
+        dyndns['deleted'] = True
+        return dyndns
     else:
         conf.set_level('service dns dynamic')
 
@@ -194,7 +196,7 @@ def get_config():
 
 def verify(dyndns):
     # bail out early - looks like removal from running config
-    if dyndns is None:
+    if dyndns['deleted']:
         return None
 
     # A 'node' corresponds to an interface
@@ -239,7 +241,10 @@ def verify(dyndns):
 
 def generate(dyndns):
     # bail out early - looks like removal from running config
-    if dyndns is None:
+    if dyndns['deleted']:
+        if os.path.exists(config_file):
+            os.unlink(config_file)
+
         return None
 
     dirname = os.path.dirname(dyndns['pid_file'])
@@ -264,7 +269,7 @@ def apply(dyndns):
     if os.path.exists('/etc/ddclient.conf'):
         os.unlink('/etc/ddclient.conf')
 
-    if dyndns is None:
+    if dyndns['deleted']:
         os.system('/etc/init.d/ddclient stop')
         if os.path.exists(dyndns['pid_file']):
             os.unlink(dyndns['pid_file'])
diff --git a/src/conf_mode/interfaces-wireguard.py b/src/conf_mode/interfaces-wireguard.py
index 013a07f32..cac911c8c 100755
--- a/src/conf_mode/interfaces-wireguard.py
+++ b/src/conf_mode/interfaces-wireguard.py
@@ -169,6 +169,10 @@ def get_config():
                     if key_eff != key_cfg and key_eff != None:
                         wg['peer_remove'].append(key_cfg)
 
+                # if a peer is disabled, we have to exec a remove for it's pubkey
+                else:
+                  peer_key = c.return_value('peer {peer} pubkey'.format(peer=p))
+                  wg['peer_remove'].append(peer_key)
     return wg
 
 
@@ -191,6 +195,7 @@ def verify(c):
             if not c['peer'][p]['pubkey']:
                 raise ConfigError("peer pubkey required for peer " + p)
 
+
 def apply(c):
     # no wg configs left, remove all interface from system
     # maybe move it into ifconfig.py
diff --git a/src/conf_mode/ipsec-settings.py b/src/conf_mode/ipsec-settings.py
index 156bb2edd..331a62316 100755
--- a/src/conf_mode/ipsec-settings.py
+++ b/src/conf_mode/ipsec-settings.py
@@ -21,6 +21,7 @@ import re
 import os
 import jinja2
 import syslog as sl
+import time
 
 import vyos.config
 import vyos.defaults
@@ -38,6 +39,7 @@ server_cert_path = '/etc/ipsec.d/certs'
 server_key_path = '/etc/ipsec.d/private'
 delim_ipsec_l2tp_begin = "### VyOS L2TP VPN Begin ###"
 delim_ipsec_l2tp_end = "### VyOS L2TP VPN End ###"
+charon_pidfile = '/var/run/charon.pid'
 
 l2pt_ipsec_conf = '''
 {{delim_ipsec_l2tp_begin}}
@@ -243,11 +245,22 @@ def generate(data):
         remove_confs(delim_ipsec_l2tp_begin, delim_ipsec_l2tp_end, ipsec_secrets_flie)
         remove_confs(delim_ipsec_l2tp_begin, delim_ipsec_l2tp_end, ipsec_conf_flie)
 
-def apply(data):
-    # Do nothing
-    # StrongSWAN should only be restarted when actual tunnels are configured
-    # Restart ipsec for l2tp
+def restart_ipsec():
     os.system("ipsec restart >&/dev/null")
+    # counter for apply swanctl config
+    counter = 10
+    while counter <= 10:
+        if os.path.exists(charon_pidfile):
+            os.system("swanctl -q >&/dev/null")
+            break
+        counter -=1
+        time.sleep(1)
+        if counter == 0:
+            raise ConfigError('VPN configuration error: IPSec is not running.')
+
+def apply(data):
+    # Restart IPSec daemon
+    restart_ipsec()
 
 if __name__ == '__main__':
     try:
diff --git a/src/conf_mode/syslog.py b/src/conf_mode/system-syslog.py
index c4f3d2c9c..4f0a54962 100755
--- a/src/conf_mode/syslog.py
+++ b/src/conf_mode/system-syslog.py
@@ -53,10 +53,18 @@ $outchannel {{file}},{{files[file]['log-file']}},{{files[file]['max-size']}},{{f
 ## remote logging
 {% for host in hosts %}
 {% if hosts[host]['proto'] == 'tcp' %}
+{% if hosts[host]['port'] %}
+{{hosts[host]['selectors']}} @@{{host}}:{{hosts[host]['port']}}
+{% else %}
 {{hosts[host]['selectors']}} @@{{host}}
+{% endif %}
+{% else %}
+{% if hosts[host]['port'] %}
+{{hosts[host]['selectors']}} @{{host}}:{{hosts[host]['port']}}
 {% else %}
 {{hosts[host]['selectors']}} @{{host}}
 {% endif %}
+{% endif %}
 {% endfor %}
 {% endif %}
 {% if user %}
@@ -177,13 +185,14 @@ def get_config():
 
     # set system syslog host
     if c.exists('host'):
-        proto = 'udp'
         rhosts = c.list_nodes('host')
         for rhost in rhosts:
             for fac in c.list_nodes('host ' + rhost + ' facility'):
                 if c.exists('host ' + rhost + ' facility ' + fac + ' protocol'):
                     proto = c.return_value(
                         'host ' + rhost + ' facility ' + fac + ' protocol')
+                else:
+                    proto = 'udp'
 
             config_data['hosts'].update(
                 {
@@ -193,6 +202,9 @@ def get_config():
                     }
                 }
             )
+            if c.exists('host ' + rhost + ' port'):
+                config_data['hosts'][rhost][
+                    'port'] = c.return_value(['host', rhost, 'port'])
 
     # set system syslog user
     if c.exists('user'):
@@ -261,7 +273,8 @@ def generate(c):
 def verify(c):
     if c == None:
         return None
-    #
+
+    # may be obsolete
     # /etc/rsyslog.conf is generated somewhere and copied over the original (exists in /opt/vyatta/etc/rsyslog.conf)
     # it interferes with the global logging, to make sure we are using a single base, template is enforced here
     #
@@ -273,6 +286,7 @@ def verify(c):
     # /var/log/vyos-rsyslog were the old files, we may want to clean those up, but currently there
     # is a chance that someone still needs it, so I don't automatically remove
     # them
+    #
 
     if c == None:
         return None
diff --git a/src/migration-scripts/system/11-to-12 b/src/migration-scripts/system/11-to-12
new file mode 100755
index 000000000..64425e2b9
--- /dev/null
+++ b/src/migration-scripts/system/11-to-12
@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+
+# converts 'set system syslog host <address>:<port>'
+# to 'set system syslog host <address> port <port>'
+
+import sys
+import re
+
+from vyos.configtree import ConfigTree
+
+if (len(sys.argv) < 1):
+  print("Must specify file name!")
+  sys.exit(1)
+
+file_name = sys.argv[1]
+
+with open(file_name, 'r') as f:
+  config_file = f.read()
+
+config = ConfigTree(config_file)
+cbase = ['system', 'syslog', 'host']
+
+if not config.exists(cbase):
+    sys.exit(0)
+
+for host in config.list_nodes(cbase):
+    if re.search(':[0-9]{1,5}$',host):
+        h = re.search('^[a-zA-Z\-0-9\.]+', host).group(0)
+        p = re.sub(':', '', re.search(':[0-9]+$', host).group(0))
+        config.set(cbase + [h])        
+        config.set(cbase + [h, 'port'], value=p)
+        for fac in config.list_nodes(cbase + [host, 'facility']):
+            config.set(cbase + [h, 'facility', fac])
+            config.set_tag(cbase + [h, 'facility'])
+            if config.exists(cbase + [host, 'facility', fac, 'protocol']):
+                proto = config.return_value(cbase + [host, 'facility', fac, 'protocol'])
+                config.set(cbase + [h, 'facility', fac, 'protocol'], value=proto)
+            if config.exists(cbase + [host, 'facility', fac, 'level']):
+                lvl = config.return_value(cbase + [host, 'facility', fac, 'level'])
+                config.set(cbase + [h, 'facility', fac, 'level'], value=lvl)
+        config.delete(cbase + [host])
+
+    try:
+        open(file_name,'w').write(config.to_string())
+    except OSError as e:
+        print("Failed to save the modified config: {}".format(e))
+        sys.exit(1)
diff --git a/src/op_mode/powerctrl.py b/src/op_mode/powerctrl.py
index 46ebf5ffb..8de25d752 100755
--- a/src/op_mode/powerctrl.py
+++ b/src/op_mode/powerctrl.py
@@ -25,48 +25,55 @@ from subprocess import check_output, CalledProcessError, STDOUT
 from vyos.util import ask_yes_no
 
 
-def valid_time(s):
+def parse_time(s):
   try:
     return datetime.strptime(s, "%H:%M").time()
   except ValueError:
     return None
 
-
-def valid_date(s):
-  try:
-    return datetime.strptime(s, "%d%m%Y").date()
-  except ValueError:
+def parse_date(s):
+  for fmt in ["%d%m%Y", "%d/%m/%Y", "%d.%m.%Y", "%d:%m:%Y", "%Y-%m-%d"]:
     try:
-      return datetime.strptime(s, "%d/%m/%Y").date()
+      return datetime.strptime(s, fmt).date()
     except ValueError:
-      try:
-        return datetime.strptime(s, "%d.%m.%Y").date()
-      except ValueError:
-        try:
-          return datetime.strptime(s, "%d:%m:%Y").date()
-        except ValueError:
-          return None
+      continue
+  # If nothing matched...
+  return None
 
+def get_shutdown_status():
+  try:
+    output = check_output(["/bin/systemctl", "status", "systemd-shutdownd.service"]).decode()
+    return output
+  except CalledProcessError:
+    return None
 
 def check_shutdown():
-  try:
-    cmd = check_output(["/bin/systemctl","status","systemd-shutdownd.service"])
-    #Shutodwn is scheduled
-    r = re.findall(r'Status: \"(.*)\"\n', cmd.decode())[0]
-    print(r)
-  except CalledProcessError as e:
-    #Shutdown is not scheduled
-    print("Shutdown is not scheduled")
+  output = get_shutdown_status()
+  if output:
+    r = re.findall(r'Status: \"(.*)\"\n', output)
+    if r:
+        # When available, that line is like
+        # Status: "Shutting down at Thu 1970-01-01 00:00:00 UTC (poweroff)..."
+        print(r[0])
+    else:
+        # Sometimes status string is not available immediately
+        # after service startup
+        print("Poweroff or reboot is scheduled")
+  else:
+    print("Poweroff or reboot is not scheduled")
 
 def cancel_shutdown():
-  try:
-    timenow = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
-    cmd = check_output(["/sbin/shutdown","-c","--no-wall"])
-    message = "Reboot scheduled has been cancelled %s" % timenow
-    #Generate broadcast message about cancel reboot
-    os.system("wall %s" % message)
-  except CalledProcessError as e:
-    sys.exit("Error aborting shutdown: %s" % e)
+  output = get_shutdown_status()
+  if output:
+    try:
+      timenow = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+      cmd = check_output(["/sbin/shutdown","-c","--no-wall"])
+      message = "Scheduled reboot or poweroff has been cancelled %s" % timenow
+      os.system("wall %s" % message)
+    except CalledProcessError as e:
+      sys.exit("Could not cancel a reboot or poweroff: %s" % e)
+  else:
+    print("Reboot or poweroff is not scheduled")
 
 def execute_shutdown(time, reboot = True, ask=True):
   if not ask:
@@ -84,31 +91,29 @@ def execute_shutdown(time, reboot = True, ask=True):
     cmd = check_output(["/sbin/shutdown",action,"now"],stderr=STDOUT)
     print(cmd.decode().split(",",1)[0])
     return
-
-  # Try to extract date from the first argument
-  if len(time) == 1:
-    time = time[0].split(" ",1)
-
-  if len(time) == 1:
-    ts = valid_time(time[0])
-    if time[0].isdigit() or valid_time(time[0]):
-      cmd = check_output(["/sbin/shutdown",action,time[0]],stderr=STDOUT)
+  elif len(time) == 1:
+    # Assume the argument is just time
+    ts = parse_time(time[0])
+    if ts:
+      cmd = check_output(["/sbin/shutdown", action, time[0]], stderr=STDOUT)
     else:
-      sys.exit("Timestamp needs to be in format of 12:34")
-
+      sys.exit("Invalid time \"{0}\". The valid format is HH:MM".format(time[0]))
   elif len(time) == 2:
-    ts = valid_time(time[0])
-    ds = valid_date(time[1])
+    # Assume it's date and time
+    ts = parse_time(time[0])
+    ds = parse_date(time[1])
     if ts and ds:
       t = datetime.combine(ds, ts)
       td = t - datetime.now()
       t2 = 1 + int(td.total_seconds())//60 # Get total minutes
-      cmd = check_output(["/sbin/shutdown",action,str(t2)],stderr=STDOUT)
+      cmd = check_output(["/sbin/shutdown", action, str(t2)], stderr=STDOUT)
     else:
-      sys.exit("Timestamp needs to be in format of 12:34\nDatestamp in the format of DD.MM.YY")
+      if not ts:
+        sys.exit("Invalid time \"{0}\". The valid format is HH:MM".format(time[0]))
+      else:
+        sys.exit("Invalid time \"{0}\". A valid format is YYYY-MM-DD [HH:MM]".format(time[1]))
   else:
-    sys.exit("Could not decode time and date")
-
+    sys.exit("Could not decode date and time. Valids formats are HH:MM or YYYY-MM-DD HH:MM")
   check_shutdown()
 
 def chk_vyatta_based_reboots():
@@ -117,7 +122,7 @@ def chk_vyatta_based_reboots():
   ### name is the node of scheduled the job, commit-confirm checks for that
 
   f = r'/var/run/confirm.job'
-  if os .path.exists(f):
+  if os.path.exists(f):
     jid = open(f).read().strip()
     if jid != 0:
       subprocess.call(['sudo', 'atrm', jid])
@@ -126,7 +131,7 @@ def chk_vyatta_based_reboots():
 def main():
   parser = argparse.ArgumentParser()
   parser.add_argument("--yes", "-y",
-            help="dont as for shutdown",
+            help="Do not ask for confirmation",
             action="store_true",
             dest="yes")
   action = parser.add_mutually_exclusive_group(required=True)
@@ -164,3 +169,4 @@ def main():
 
 if __name__ == "__main__":
   main()
+