Merge remote-tracking branch 'upstream/current' into current

* upstream/current:
  Do not try to decode data read from /sys files in the show version script, it's already an str.
  Dependencies on file and pystache, for install and show version scripts..
  Add dependency on hvinfo, too.
  T637, T638: add dependencies on tcpdump and bmon.
  T638: new op mode CLI for the bandwidth monitor commands.
  T637: new op mode for traffic dumps based on tcpdump.
  Correct the logic of generating tag nodes for op mode.
  Add missing vyos.base module
  Fix cron interval regex to allow single digit values.
  Fix misplaces ConfigError exception.
  Some more valueless fixes.
  Mark nodes in SSH and NTP valueless (related to T602).

21 files changed, 182 insertions, 45 deletions

diff --git a/Makefile b/Makefile
index c328d6a8d..8a75a91e4 100644
--- a/Makefile
+++ b/Makefile
@@ -26,6 +26,7 @@ op_mode_definitions:
 	rm -f $(OP_TMPL_DIR)/show/dns/node.def
 	rm -f $(OP_TMPL_DIR)/reset/node.def
 	rm -f $(OP_TMPL_DIR)/restart/node.def
+	rm -f $(OP_TMPL_DIR)/monitor/node.def
 
 .PHONY: all
 all: interface_definitions op_mode_definitions
diff --git a/debian/control b/debian/control
index 351715c07..c31c470e9 100644
--- a/debian/control
+++ b/debian/control
@@ -2,14 +2,27 @@ Source: vyos-1x
 Section: contrib/net
 Priority: extra
 Maintainer: VyOS Package Maintainers <maintainers@vyos.net>
-Build-Depends: debhelper (>= 9), python3, python3-setuptools, quilt,
- python3-lxml
+Build-Depends: debhelper (>= 9),
+  quilt,
+  python3,
+  python3-setuptools,
+  quilt,
+  python3-lxml
 Standards-Version: 3.9.6
 
 Package: vyos-1x
 Architecture: all
-Depends: python3, ${python3:Depends}, python3-netifaces,
- python3-jinja2, ipaddrcheck
- ${shlibs:Depends}, ${misc:Depends}
+Depends: python3,
+  ${python3:Depends},
+  python3-netifaces,
+  python3-jinja2,
+  python3-pystache,
+  ipaddrcheck,
+  tcpdump,
+  bmon,
+  hvinfo,
+  file,
+  ${shlibs:Depends},
+  ${misc:Depends}
 Description: VyOS configuration scripts and data
  VyOS configuration scripts, interface definitions, and everything
diff --git a/interface-definitions/cron.xml b/interface-definitions/cron.xml
index 8e4c1294a..65b95c5a4 100644
--- a/interface-definitions/cron.xml
+++ b/interface-definitions/cron.xml
@@ -45,7 +45,7 @@
                     <description>Execution interval in days</description>
                   </valueHelp>
                   <constraint>
-                    <regex>[1-9]([0-9]+)([mhd]{0,1})</regex>
+                    <regex>[1-9]([0-9]*)([mhd]{0,1})</regex>
                   </constraint>
                 </properties>
               </leafNode>
diff --git a/interface-definitions/dns-forwarding.xml b/interface-definitions/dns-forwarding.xml
index 81aa6063d..60b90c6a7 100644
--- a/interface-definitions/dns-forwarding.xml
+++ b/interface-definitions/dns-forwarding.xml
@@ -62,6 +62,7 @@
               <leafNode name="ignore-hosts-file">
                 <properties>
                   <help>Do not use local /etc/hosts file in name resolution</help>
+                  <valueless/>
                 </properties>
               </leafNode>
               <leafNode name="listen-on">
@@ -94,6 +95,7 @@
               <leafNode name="system">
                 <properties>
                   <help>DNS forwarding to system nameservers</help>
+                  <valueless/>
                 </properties>
               </leafNode>
             </children>
diff --git a/interface-definitions/ntp.xml b/interface-definitions/ntp.xml
index c6fd8aa14..d9930a262 100644
--- a/interface-definitions/ntp.xml
+++ b/interface-definitions/ntp.xml
@@ -19,21 +19,25 @@
               <leafNode name="dynamic">
                 <properties>
                   <help>Allow server to be configured even if not reachable</help>
+                  <valueless/>
                 </properties>
               </leafNode>
               <leafNode name="noselect">
                 <properties>
                   <help>Marks the server as unused</help>
+                  <valueless/>
                 </properties>
               </leafNode>
               <leafNode name="preempt">
                 <properties>
                   <help>Specifies the association as preemptable rather than the default persistent</help>
+                  <valueless/>
                 </properties>
               </leafNode>
               <leafNode name="prefer">
                 <properties>
                   <help>Marks the server as preferred</help>
+                  <valueless/>
                 </properties>
               </leafNode>
             </children>
diff --git a/interface-definitions/ssh.xml b/interface-definitions/ssh.xml
index f898f3934..ba5b887bc 100644
--- a/interface-definitions/ssh.xml
+++ b/interface-definitions/ssh.xml
@@ -41,6 +41,7 @@
           <leafNode name="allow-root">
             <properties>
               <help>Enable root login over ssh</help>
+              <valueless/>
             </properties>
           </leafNode>
           <leafNode name="ciphers">
@@ -54,11 +55,13 @@
           <leafNode name="disable-host-validation">
             <properties>
               <help>Don't validate the remote host name with DNS</help>
+              <valueless/>
             </properties>
           </leafNode>
           <leafNode name="disable-password-authentication">
             <properties>
               <help>Don't allow unknown user to login with password</help>
+              <valueless/>
             </properties>
           </leafNode>
           <leafNode name="key-exchange">
diff --git a/op-mode-definitions/bandwidth-monitor.xml b/op-mode-definitions/bandwidth-monitor.xml
new file mode 100644
index 000000000..a6ddcfd4b
--- /dev/null
+++ b/op-mode-definitions/bandwidth-monitor.xml
@@ -0,0 +1,23 @@
+<?xml version="1.0"?>
+<interfaceDefinition>
+  <node name="monitor">
+    <children>
+      <node name="bandwidth">
+        <properties>
+          <help>Monitor interface bandwidth in real time</help>
+        </properties>
+        <children>
+          <tagNode name="interface">
+            <command>bmon -p $4</command>
+            <properties>
+              <help>Monitor bandwidth usage on specified interface</help>
+              <completionHelp>
+                <script>${vyos_bindir}/vyos-list-interfaces.py</script>
+              </completionHelp>
+            </properties>
+          </tagNode>
+        </children>
+      </node>
+    </children>
+  </node>
+</interfaceDefinition>
diff --git a/op-mode-definitions/traffic-dump.xml b/op-mode-definitions/traffic-dump.xml
new file mode 100644
index 000000000..be53f866b
--- /dev/null
+++ b/op-mode-definitions/traffic-dump.xml
@@ -0,0 +1,45 @@
+<?xml version="1.0"?>
+<interfaceDefinition>
+  <node name="monitor">
+    <children>
+      <node name="traffic">
+        <properties>
+          <help>Monitor traffic dumps</help>
+        </properties>
+        <children>
+          <tagNode name="interface">
+            <command>tcpdump -i $4</command>
+            <properties>
+              <help>Monitor traffic dump from an interface</help>
+              <completionHelp>
+                <script>${vyos_bindir}/vyos-list-dumpable-interfaces.py</script>
+              </completionHelp>
+            </properties>
+            <children>
+              <tagNode name="filter">
+                <command>tcpdump -n -i $4 $6</command>
+                <properties>
+                  <help>Monitor traffic matching filter conditions</help>
+                </properties>
+              </tagNode>
+              <tagNode name="save">
+                <command>tcpdump -n -i $4 -w $6</command>
+                <properties>
+                  <help>Save traffic dump from an interface to a file</help>
+                </properties>
+                <children>
+                  <tagNode name="filter">
+                    <command>tcpdump -n -i $4 -w $6 $8</command>
+                    <properties>
+                      <help>Save a dump of traffic matching filter conditions to a file</help>
+                    </properties>
+                  </tagNode>
+                </children>
+              </tagNode>
+            </children>
+          </tagNode>
+        </children>
+      </node>
+    </children>
+  </node>
+</interfaceDefinition>
diff --git a/python/vyos/__init__.py b/python/vyos/__init__.py
index e69de29bb..9b5ed21c9 100644
--- a/python/vyos/__init__.py
+++ b/python/vyos/__init__.py
@@ -0,0 +1 @@
+from .base import *
diff --git a/python/vyos/base.py b/python/vyos/base.py
new file mode 100644
index 000000000..6197ed074
--- /dev/null
+++ b/python/vyos/base.py
@@ -0,0 +1,22 @@
+# Copyright (c) 2018 VyOS maintainers and contributors
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the Software
+# is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included 
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+# INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
+# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
+#  IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
+class ConfigError(Exception):
+    pass
diff --git a/python/vyos/version.py b/python/vyos/version.py
index b3eff3965..5d32d878d 100644
--- a/python/vyos/version.py
+++ b/python/vyos/version.py
@@ -19,10 +19,6 @@
 
 import json
 
-class ConfigError(Exception):
-    pass
-
-
 def get_version_data(file='/opt/vyatta/etc/version.json'):
     with open(file, 'r') as f:
         version_data = json.load(f)
diff --git a/scripts/build-command-op-templates b/scripts/build-command-op-templates
index 72879fe74..865590c2c 100755
--- a/scripts/build-command-op-templates
+++ b/scripts/build-command-op-templates
@@ -93,7 +93,7 @@ def get_properties(p):
     try:
         props["help"] = p.find("help").text
     except:
-        pass
+        props["help"] = "No help available"
 
 
     # Get the completion help strings
@@ -113,7 +113,7 @@ def get_properties(p):
             for i in paths:
                 comp_exprs.append("/bin/cli-shell-api listNodes {0}".format(i.text))
             for i in scripts:
-                comp_exprs.append("sh -c \"{0}\"".format(i.text))
+                comp_exprs.append("{0}".format(i.text))
             comp_help = " && ".join(comp_exprs)
             props["comp_help"] = comp_help
     except:
@@ -128,14 +128,6 @@ def make_node_def(props, command):
 
     node_def = ""
 
-    if "tag" in props:
-        node_def += "tag:\n"
-
-
-    if "type" in props:
-        node_def += "type: {0}\n".format(props["type"])
-
-
     if "help" in props:
         node_def += "help: {0}\n".format(props["help"])
 
@@ -173,32 +165,45 @@ def process_node(n, tmpl_dir):
 
     props = get_properties(props_elem)
 
-    # Type should not be set for non-tag, non-leaf nodes
-    if node_type != "node":
-        props["type"] = "txt"
+    if node_type == "node":
+        if debug:
+          print("Processing node {}".format(name))
+
+        with open(os.path.join(make_path(my_tmpl_dir), "node.def"), "w") as f:
+            f.write(make_node_def(props, command))
+
+        if children is not None:
+            inner_nodes = children.iterfind("*")
+            for inner_n in inner_nodes:
+                process_node(inner_n, my_tmpl_dir)
     if node_type == "tagNode":
-        props["tag"] = "True"
-    
+        if debug:
+          print("Processing tag node {}".format(name))
 
-    with open(os.path.join(make_path(my_tmpl_dir), "node.def"), "w") as f:
-        f.write(make_node_def(props, command))
+        os.makedirs(make_path(my_tmpl_dir), exist_ok=True)
 
+        with open(os.path.join(make_path(my_tmpl_dir), "node.def"), "w") as f:
+            f.write('help: {0}\0'.format(props['help']))
 
-    if node_type == "node":
-        inner_nodes = children.iterfind("*")
-        for inner_n in inner_nodes:
-            process_node(inner_n, my_tmpl_dir)
-    if node_type == "tagNode":
         my_tmpl_dir.append("node.tag")
-        if debug:
-            print("Created path for the tagNode:", end="")
         os.makedirs(make_path(my_tmpl_dir), exist_ok=True)
-        inner_nodes = children.iterfind("*")
-        for inner_n in inner_nodes:
-            process_node(inner_n, my_tmpl_dir)
+        if debug:
+            print("Created path for the tagNode: {}".format(make_path(my_tmpl_dir)), end="")
+
+        with open(os.path.join(make_path(my_tmpl_dir), "node.def"), "w") as f:
+            f.write(make_node_def(props, command))
+
+        if children is not None:
+            inner_nodes = children.iterfind("*")
+            for inner_n in inner_nodes:
+                process_node(inner_n, my_tmpl_dir)
     else:
         # This is a leaf node
-        pass
+        if debug:
+            print("Processing leaf node {}".format(name))
+
+        with open(os.path.join(make_path(my_tmpl_dir), "node.def"), "w") as f:
+            f.write(make_node_def(props, command))
 
 
 root = xml.getroot()
diff --git a/src/conf-mode/vyos-config-bcast-relay.py b/src/conf-mode/vyos-config-bcast-relay.py
index f60664051..785690d9c 100755
--- a/src/conf-mode/vyos-config-bcast-relay.py
+++ b/src/conf-mode/vyos-config-bcast-relay.py
@@ -23,7 +23,7 @@ import time
 import subprocess
 
 from vyos.config import Config
-from vyos.util import ConfigError
+from vyos import ConfigError
 
 config_file = r'/etc/default/udp-broadcast-relay'
 
diff --git a/src/conf-mode/vyos-config-dns-forwarding.py b/src/conf-mode/vyos-config-dns-forwarding.py
index 9f3bb7aee..5494d07aa 100755
--- a/src/conf-mode/vyos-config-dns-forwarding.py
+++ b/src/conf-mode/vyos-config-dns-forwarding.py
@@ -23,7 +23,7 @@ import netifaces
 import jinja2
 
 from vyos.config import Config
-from vyos.util import ConfigError
+from vyos import ConfigError
 
 config_file = r'/etc/powerdns/recursor.conf'
 
diff --git a/src/conf-mode/vyos-config-mdns-repeater.py b/src/conf-mode/vyos-config-mdns-repeater.py
index 3b8ca26ad..e648fd64f 100755
--- a/src/conf-mode/vyos-config-mdns-repeater.py
+++ b/src/conf-mode/vyos-config-mdns-repeater.py
@@ -22,7 +22,7 @@ import netifaces
 import time
 
 from vyos.config import Config
-from vyos.util import ConfigError
+from vyos import ConfigError
 
 config_file = r'/etc/default/mdns-repeater'
 
diff --git a/src/conf-mode/vyos-config-ntp.py b/src/conf-mode/vyos-config-ntp.py
index 061e8760e..8be12e44e 100755
--- a/src/conf-mode/vyos-config-ntp.py
+++ b/src/conf-mode/vyos-config-ntp.py
@@ -23,7 +23,7 @@ import jinja2
 import ipaddress
 
 from vyos.config import Config
-from vyos.util import ConfigError
+from vyos import ConfigError
 
 config_file = r'/etc/ntp.conf'
 
diff --git a/src/conf-mode/vyos-config-ssh.py b/src/conf-mode/vyos-config-ssh.py
index 1605dcd74..4949d6463 100755
--- a/src/conf-mode/vyos-config-ssh.py
+++ b/src/conf-mode/vyos-config-ssh.py
@@ -22,7 +22,7 @@ import os
 import jinja2
 
 from vyos.config import Config
-from vyos.util import ConfigError
+from vyos import ConfigError
 
 config_file = r'/etc/ssh/sshd_config'
 
diff --git a/src/conf-mode/vyos-update-crontab.py b/src/conf-mode/vyos-update-crontab.py
index 2d15de8ea..c19b88007 100755
--- a/src/conf-mode/vyos-update-crontab.py
+++ b/src/conf-mode/vyos-update-crontab.py
@@ -21,7 +21,7 @@ import re
 import sys
 
 from vyos.config import Config
-from vyos.util import ConfigError
+from vyos import ConfigError
 
 
 crontab_file = "/etc/cron.d/vyos-crontab"
diff --git a/src/op-mode/vyos-list-dumpable-interfaces.py b/src/op-mode/vyos-list-dumpable-interfaces.py
new file mode 100755
index 000000000..53ee89633
--- /dev/null
+++ b/src/op-mode/vyos-list-dumpable-interfaces.py
@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+
+# Extract the list of interfaces available for traffic dumps from tcpdump -D
+
+import re
+import subprocess
+
+if __name__ == '__main__':
+    out = subprocess.check_output(['/usr/sbin/tcpdump', '-D']).decode().strip()
+    out = out.split("\n")
+
+    intfs = " ".join(map(lambda s: re.search(r'\d+\.(\S+)\s', s).group(1), out))
+
+    print(intfs)
diff --git a/src/op-mode/vyos-list-interfaces.py b/src/op-mode/vyos-list-interfaces.py
new file mode 100755
index 000000000..59c9dffad
--- /dev/null
+++ b/src/op-mode/vyos-list-interfaces.py
@@ -0,0 +1,8 @@
+#!/usr/bin/env python3
+
+import netifaces
+
+if __name__ == '__main__':
+    interfaces = netifaces.interfaces()
+
+    print(" ".join(interfaces))
diff --git a/src/op-mode/vyos-show-version.py b/src/op-mode/vyos-show-version.py
index 0990dd648..ce3b3b54f 100755
--- a/src/op-mode/vyos-show-version.py
+++ b/src/op-mode/vyos-show-version.py
@@ -41,7 +41,7 @@ def read_file(name):
     try:
         with open (name, "r") as f:
             data = f.read()
-        return data.decode().strip()
+        return data.strip()
     except:
         # This works since we only read /sys/class/* stuff
         # with this function