Merge branch 'current' into crux

12 files changed, 175 insertions, 55 deletions

diff --git a/Makefile b/Makefile
index a9926137c..dd3d2d00f 100644
--- a/Makefile
+++ b/Makefile
@@ -44,7 +44,7 @@ clean:
 
 .PHONY: test
 test:
-	PYTHONPATH=python/ python3 -m "nose" --with-xunit src --with-coverage --cover-erase --cover-xml --cover-package src/conf_mode,src/op_mode,src/completion,src/helpers,src/validators --verbose
+	PYTHONPATH=python/ python3 -m "nose" --with-xunit src --with-coverage --cover-erase --cover-xml --cover-package src/conf_mode,src/op_mode,src/completion,src/helpers,src/validators,src/tests --verbose
 
 .PHONY: sonar
 sonar:
diff --git a/debian/changelog b/debian/changelog
index 7666cfd68..415cee0ff 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,22 @@
+vyos-1x (1.2.0-10) unstable; urgency=low
+
+  * T1178: Scheduled script breaks ability to modify configuration
+
+ -- hagbard <vyosdev@derith.de>  Tue, 22 Jan 2019 13:35:09 -0800
+
+vyos-1x (1.2.0-9) unstable; urgency=low
+
+  * T1168: Upgrade: 1,1,7 -> 1.2.0-epa2
+    - keyword change in config
+
+ -- hagbard <vyosdev@derith.de>  Mon, 07 Jan 2019 11:42:49 -0800
+
+vyos-1x (1.2.0-8) unstable; urgency=low
+
+  * T1162: WireGuard: Unable to modify tunnels - KeyError: 'state'
+
+ -- hagbard <vyosdev@derith.de>  Sun, 06 Jan 2019 15:58:40 -0800
+
 vyos-1x (1.2.0-7) unstable; urgency=low
 
   * T1061: Wireguard: Missing option to administrativly shutdown interface
diff --git a/src/conf_mode/task_scheduler.py b/src/conf_mode/task_scheduler.py
index 285afe2b5..b171e9576 100755
--- a/src/conf_mode/task_scheduler.py
+++ b/src/conf_mode/task_scheduler.py
@@ -49,7 +49,7 @@ def make_command(executable, arguments):
     if arguments:
         return("sg vyattacfg \"{0} {1}\"".format(executable, arguments))
     else:
-        return(executable)
+        return("sg vyattacfg \"{0}\"".format(executable))
 
 def get_config():
     conf = Config()
diff --git a/src/conf_mode/wireguard.py b/src/conf_mode/wireguard.py
index f5452579e..c88e9085a 100755
--- a/src/conf_mode/wireguard.py
+++ b/src/conf_mode/wireguard.py
@@ -124,7 +124,6 @@ def get_config():
           if c.exists(cnf + ' peer ' + p + ' preshared-key'):
             config_data['interfaces'][intfc]['peer'][p]['psk'] = c.return_value(cnf + ' peer ' + p + ' preshared-key')
   
-
   return config_data
 
 def verify(c):
@@ -166,12 +165,13 @@ def apply(c):
   ### link status up/down aka interface disable
 
   for intf in c['interfaces']:
-    if c['interfaces'][intf]['state'] == 'disable':
-      sl.syslog(sl.LOG_NOTICE, "disable interface " + intf)
-      subprocess.call(['ip l s dev ' + intf + ' down ' + ' &>/dev/null'], shell=True)
-    else:
-      sl.syslog(sl.LOG_NOTICE, "enable interface " + intf)
-      subprocess.call(['ip l s dev ' + intf + ' up ' + ' &>/dev/null'], shell=True)
+    if not c['interfaces'][intf]['status'] == 'delete':
+      if c['interfaces'][intf]['state'] == 'disable':
+        sl.syslog(sl.LOG_NOTICE, "disable interface " + intf)
+        subprocess.call(['ip l s dev ' + intf + ' down ' + ' &>/dev/null'], shell=True)
+      else:
+        sl.syslog(sl.LOG_NOTICE, "enable interface " + intf)
+        subprocess.call(['ip l s dev ' + intf + ' up ' + ' &>/dev/null'], shell=True)
 
   ### deletion of a specific interface
   for intf in c['interfaces']:
diff --git a/src/helpers/commands-pipe.py b/src/helpers/commands-pipe.py
deleted file mode 100755
index ab68ccade..000000000
--- a/src/helpers/commands-pipe.py
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/usr/bin/python3
-
-import sys
-import re
-
-from signal import signal, SIGPIPE, SIG_DFL
-from vyos.configtree import ConfigTree
-
-signal(SIGPIPE,SIG_DFL)
-
-config_string = sys.stdin.read().strip()
-config_string = config_string.replace("\\", "\\\\")
-
-if not config_string:
-    sys.exit(0)
-
-# When used in conf mode pipe, the config given to the script is likely incomplete
-# and breaks the "all top level nodes are neither tag nor leaf"
-# invariant, so we wrap it into a fake node.
-# Since nodes don't normally start with an underscore,
-# __root__ is hygienic enough.
-config_string = "__root__ {{ {0} \n }}".format(config_string)
-
-config_re = re.compile(r'(set|comment)\s+__root__\s+(.*)')
-
-config = ConfigTree(config_string)
-commands = config.to_commands()
-commands = config_re.sub("\\1 \\2", commands)
-
-print(commands)
diff --git a/src/migration-scripts/ipsec/4-to-5 b/src/migration-scripts/ipsec/4-to-5
new file mode 100755
index 000000000..b64aa8462
--- /dev/null
+++ b/src/migration-scripts/ipsec/4-to-5
@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+
+# log-modes have changed, keyword  all to any
+
+import sys
+
+from vyos.configtree import ConfigTree
+
+if (len(sys.argv) < 1):
+    print("Must specify file name!")
+    sys.exit(1)
+
+file_name = sys.argv[1]
+
+with open(file_name, 'r') as f:
+    config_file = f.read()
+
+ctree = ConfigTree(config_file)
+
+if not ctree.exists(['vpn', 'ipsec', 'logging','log-modes']):
+    # Nothing to do
+    sys.exit(0)
+else:
+  lmodes = ctree.return_values(['vpn', 'ipsec', 'logging','log-modes'])
+  for mode in lmodes:
+    if mode == 'all':
+      ctree.set(['vpn', 'ipsec', 'logging','log-modes'], value='any', replace=True)
+
+  try:
+    open(file_name,'w').write(ctree.to_string())
+  except OSError as e:
+    print("Failed to save the modified config: {}".format(e))
+    sys.exit(1)
diff --git a/src/migration-scripts/pppoe-server/0-to-1 b/src/migration-scripts/pppoe-server/0-to-1
index df816a321..bb24211b6 100755
--- a/src/migration-scripts/pppoe-server/0-to-1
+++ b/src/migration-scripts/pppoe-server/0-to-1
@@ -1,6 +1,8 @@
 #!/usr/bin/env python3
 
-# Delete "service ssh allow-root" option
+# Convert "service pppoe-server authentication radius-server  node key"
+# to:
+# "service pppoe-server authentication radius-server node secret"  
 
 import sys
 
diff --git a/src/migration-scripts/webproxy/1-to-2 b/src/migration-scripts/webproxy/1-to-2
index 4acabba3e..070ff356d 100755
--- a/src/migration-scripts/webproxy/1-to-2
+++ b/src/migration-scripts/webproxy/1-to-2
@@ -19,7 +19,7 @@ with open(file_name, 'r') as f:
 config = ConfigTree(config_file)
 
 cfg_webproxy_base = ['service', 'webproxy']
-if not config.exists(cfg_webproxy_base):
+if not config.exists(cfg_webproxy_base + ['proxy-bypass']):
     # Nothing to do
     sys.exit(0)
 else:
diff --git a/src/op_mode/show_ipsec_sa.py b/src/op_mode/show_ipsec_sa.py
index 4c39aba66..bad4f33f0 100755
--- a/src/op_mode/show_ipsec_sa.py
+++ b/src/op_mode/show_ipsec_sa.py
@@ -14,19 +14,38 @@ def parse_conn_spec(s):
     except AttributeError:
         # No active SAs found, so we have nothing to display
         print("No established security associations found.")
-        print("Use \"show vpn ipsec sa\" to view inactive and connecting tunnels.")
+        print("Use \"show vpn ipsec sa verbose\" to view inactive and connecting tunnels.")
         sys.exit(0)
 
-def parse_ike_line(s):
+def parse_sa_counters(s):
+    bytes_in, bytes_out = None, None
     try:
         # Example with traffic: AES_CBC_256/HMAC_SHA2_256_128/ECP_521, 2382660 bytes_i (1789 pkts, 2s ago), 2382660 bytes_o ...
-        return re.search(r'.*:\s+(.*\/.*(?:\/.*)?),\s+(\d+)\s+bytes_i\s\(.*pkts,.*\),\s+(\d+)\s+bytes_o', s).groups()
+        bytes_in, bytes_out = re.search(r'\s+(\d+)\s+bytes_i\s\(.*pkts,.*\),\s+(\d+)\s+bytes_o', s).groups()
     except AttributeError:
         try:
             # Example without traffic: 3DES_CBC/HMAC_MD5_96/MODP_1024, 0 bytes_i, 0 bytes_o, rekeying in 45 minutes
-            return re.search(r'.*:\s+(.*\/.*(?:\/.*)?),\s+(\d+)\s+bytes_i,\s+(\d+)\s+bytes_o,\s+rekeying', s).groups()
+            bytes_in, bytes_out = re.search(r'\s+(\d+)\s+bytes_i,\s+(\d+)\s+bytes_o,\s+rekeying', s).groups()
         except AttributeError:
-            return (None, None, None, None, None)
+            pass
+
+    if (bytes_in is not None) and (bytes_out is not None):
+        # Convert bytes to human-readable units
+        bytes_in = hurry.filesize.size(int(bytes_in))
+        bytes_out = hurry.filesize.size(int(bytes_out))
+
+        result = "{0}/{1}".format(bytes_in, bytes_out)
+    else:
+        result = "N/A"
+
+    return result
+
+def parse_ike_proposal(s):
+    result = re.search(r'IKE proposal:\s+(.*)\s', s)
+    if result:
+        return result.groups(0)[0]
+    else:
+        return "N/A"
 
 
 # Get a list of all configured connections
@@ -35,24 +54,29 @@ with open('/etc/ipsec.conf', 'r') as f:
     connections = set(re.findall(r'conn\s([^\s]+)\s*\n', config))
     connections = list(filter(lambda s: s != '%default', connections))
 
+try:
+    # DMVPN connections have to be handled separately
+    with open('/etc/swanctl/swanctl.conf', 'r') as f:
+        dmvpn_config = f.read()
+        dmvpn_connections = re.findall(r'\s+(dmvpn-.*)\s+{\n', dmvpn_config)
+    connections += dmvpn_connections
+except:
+    pass
+
 status_data = []
 
 for conn in connections:
     status = subprocess.check_output("ipsec statusall {0}".format(conn), shell=True).decode()
-    if re.search(r'no match', status):
+    if re.search(r'no match|CONNECTING', status):
         status_line = [conn, "down", None, None, None, None, None]
     else:
         try:
             time, _, _, ip, id = parse_conn_spec(status)
             if ip == id:
                 id = None
-            enc, bytes_in, bytes_out = parse_ike_line(status)
-
-            # Convert bytes to human-readable units
-            bytes_in = hurry.filesize.size(int(bytes_in))
-            bytes_out = hurry.filesize.size(int(bytes_out))
-
-            status_line = [conn, "up", time, "{0}/{1}".format(bytes_in, bytes_out), ip, id, enc]
+            counters = parse_sa_counters(status)
+            enc = parse_ike_proposal(status)
+            status_line = [conn, "up", time, counters, ip, id, enc]
         except Exception as e:
             print(status)
             raise e
diff --git a/src/tests/test_config_parser.py b/src/tests/test_config_parser.py
new file mode 100644
index 000000000..f58ff23bf
--- /dev/null
+++ b/src/tests/test_config_parser.py
@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2018 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import os
+import tempfile
+import unittest
+from unittest import TestCase, mock
+
+import vyos.configtree
+
+
+class TestConfigParser(TestCase):
+    def setUp(self):
+        with open('tests/data/config.valid', 'r') as f:
+            config_string = f.read()
+            self.config = vyos.configtree.ConfigTree(config_string)
+
+    def test_top_level_valueless(self):
+        self.assertTrue(self.config.exists(["top-level-valueless-node"]))
+
+    def test_top_level_leaf(self):
+        self.assertTrue(self.config.exists(["top-level-leaf-node"]))
+        self.assertEqual(self.config.return_value(["top-level-leaf-node"]), "foo")
+
+    def test_top_level_tag(self):
+        self.assertTrue(self.config.exists(["top-level-tag-node"]))
+        # No sorting is intentional, child order must be preserved
+        self.assertEqual(self.config.list_nodes(["top-level-tag-node"]), ["foo", "bar"])
diff --git a/src/tests/test_initial_setup.py b/src/tests/test_initial_setup.py
index 023a30723..c4c59b827 100644
--- a/src/tests/test_initial_setup.py
+++ b/src/tests/test_initial_setup.py
@@ -25,7 +25,7 @@ import vyos.configtree
 import vyos.initialsetup as vis
 
 
-class TestHostName(TestCase):
+class TestInitialSetup(TestCase):
     def setUp(self):
         with open('tests/data/config.boot.default', 'r') as f:
             config_string = f.read()
diff --git a/tests/data/config.valid b/tests/data/config.valid
new file mode 100644
index 000000000..353e96df4
--- /dev/null
+++ b/tests/data/config.valid
@@ -0,0 +1,29 @@
+/* top level leaf node */
+top-level-leaf-node foo
+
+top-level-valueless-node
+
+top-level-tag-node foo {
+    top-level-tag-node-child some-value
+}
+
+top-level-tag-node bar {
+    top-level-tag-node-child another-value
+}
+
+normal-node {
+    normal-node-child {
+        valueless-node
+        multi-node value1
+        /* valueless node comment */
+        another-valueless-node
+        multi-node value1
+        tag-node foo {
+        }
+        one-more-valueless-node
+        tag-node bar {
+            some-option some-value
+        }
+    }
+    option-with-quoted-value "some-value"
+}