Merge branch 'udev' of github.com:c-po/vyos-1x into current

* 'udev' of github.com:c-po/vyos-1x:
  usb: op-mode: T2560: display USB interface information
  pppoe: op-mode: T2488: retrieve log info from journalctl
  wwan: op-mode: T2488: retrieve log info from journalctl
  wwan: T2241: interface is not bond- or bridgeable
  wwan: T2488: remove generation of dedicated logfile
  wwan: T2529: migrate device from ttyUSB to usbXbY.YpZ.Z
  udev: T2490: add persistent USB device files

10 files changed, 208 insertions, 52 deletions

diff --git a/data/templates/wwan/peer.tmpl b/data/templates/wwan/peer.tmpl
index 04ab4f844..0168283fd 100644
--- a/data/templates/wwan/peer.tmpl
+++ b/data/templates/wwan/peer.tmpl
@@ -10,11 +10,10 @@ linkname {{ intf }}
 usepeerdns
 {%- endif %}
 # physical device
-/dev/{{ device }}
+{{ device }}
 lcp-echo-failure 0
 115200
 debug
-logfile {{ logfile }}
 nodefaultroute
 ipcp-max-failure 4
 ipcp-accept-local
diff --git a/debian/control b/debian/control
index 20423aee1..7c1555416 100644
--- a/debian/control
+++ b/debian/control
@@ -34,6 +34,7 @@ Depends: python3,
   python3-zmq,
   python3-jmespath,
   python3-xmltodict,
+  python3-pyudev,
   bsdmainutils,
   cron,
   etherwake,
diff --git a/interface-definitions/interfaces-wirelessmodem.xml.in b/interface-definitions/interfaces-wirelessmodem.xml.in
index 91eee56ab..d5f2e04d7 100644
--- a/interface-definitions/interfaces-wirelessmodem.xml.in
+++ b/interface-definitions/interfaces-wirelessmodem.xml.in
@@ -46,12 +46,16 @@
           #include <include/interface-vrf.xml.i>
           <leafNode name="device">
             <properties>
-              <help>System device name (default: ttyUSB0)</help>
+              <help>Serial device </help>
               <completionHelp>
-                <script>ls -1 /dev | grep ttyUSB</script>
+                <script>ls -1 /dev /dev/serial/by-bus | grep -e ttyS[0-9]* -e usb[0-9]*</script>
               </completionHelp>
               <valueHelp>
-                <format>ttyXXX</format>
+                <format>ttySXX</format>
+                <description>System TTY device name</description>
+              </valueHelp>
+              <valueHelp>
+                <format>usbXbY.YpZ.Z</format>
                 <description>System TTY device name</description>
               </valueHelp>
             </properties>
diff --git a/op-mode-definitions/show-interfaces-pppoe.xml b/op-mode-definitions/show-interfaces-pppoe.xml
index 65363f7e2..01acd4fc6 100644
--- a/op-mode-definitions/show-interfaces-pppoe.xml
+++ b/op-mode-definitions/show-interfaces-pppoe.xml
@@ -13,9 +13,15 @@
             </properties>
             <command>${vyos_op_scripts_dir}/show_interfaces.py --intf="$4"</command>
             <children>
+              <leafNode name="log">
+                <properties>
+                  <help>Show specified PPPoE interface log</help>
+                </properties>
+                <command>/usr/bin/journalctl -u "ppp@$4".service</command>
+              </leafNode>
               <leafNode name="statistics">
                 <properties>
-                  <help>Show PPPoE interface statistics</help>
+                  <help>Show specified PPPoE interface statistics</help>
                   <completionHelp>
                     <path>interfaces pppoe</path>
                   </completionHelp>
diff --git a/op-mode-definitions/show-interfaces-wirelessmodem.xml b/op-mode-definitions/show-interfaces-wirelessmodem.xml
index b21bb520c..1f710b3dc 100644
--- a/op-mode-definitions/show-interfaces-wirelessmodem.xml
+++ b/op-mode-definitions/show-interfaces-wirelessmodem.xml
@@ -13,9 +13,15 @@
             </properties>
             <command>${vyos_op_scripts_dir}/show_interfaces.py --intf="$4"</command>
             <children>
+              <leafNode name="log">
+                <properties>
+                  <help>Show specified WWAN interface log</help>
+                </properties>
+                <command>/usr/bin/journalctl -u "ppp@$4".service</command>
+              </leafNode>
               <leafNode name="statistics">
                 <properties>
-                  <help>Show specified wirelessmodem interface statistics</help>
+                  <help>Show specified WWAN interface statistics</help>
                   <completionHelp>
                     <path>interfaces wirelessmodem</path>
                   </completionHelp>
diff --git a/op-mode-definitions/show-system-info.xml b/op-mode-definitions/show-system.xml
index 61c947bbe..b7f56ae20 100644
--- a/op-mode-definitions/show-system-info.xml
+++ b/op-mode-definitions/show-system.xml
@@ -7,7 +7,6 @@
           <help>Show system information</help>
         </properties>
         <children>
-
           <node name="connections">
             <properties>
               <help>Show active network connections on the system</help>
@@ -50,21 +49,18 @@
               </node>
             </children>
           </node>
-
           <leafNode name= "integrity">
             <properties>
               <help>Checks overall system integrity</help>
             </properties>
             <command>sudo ${vyos_op_scripts_dir}/system_integrity.py</command>
           </leafNode>
-
           <leafNode name="kernel-messages">
             <properties>
               <help>Show messages in kernel ring buffer</help>
             </properties>
             <command>sudo dmesg</command>
           </leafNode>
-
           <node name="login">
             <properties>
               <help>Show user accounts</help>
@@ -104,7 +100,6 @@
               </node>
             </children>
           </node>
-
           <node name="memory">
             <properties>
               <help>Show system memory usage</help>
@@ -125,7 +120,6 @@
               </leafNode>
             </children>
           </node>
-
           <node name="processes">
             <properties>
               <help>Show system processes</help>
@@ -152,21 +146,32 @@
               </leafNode>
             </children>
           </node>
-
           <leafNode name="storage">
             <properties>
               <help>Show filesystem usage</help>
             </properties>
             <command>df -h -x squashfs</command>
           </leafNode>
-
           <leafNode name="uptime">
             <properties>
               <help>Show how long the system has been up</help>
             </properties>
             <command>uptime</command>
           </leafNode>
-
+          <node name="usb">
+            <properties>
+              <help>Show information about Universal Serial Bus (USB)</help>
+            </properties>
+            <command>/usr/bin/lsusb -t</command>
+            <children>
+              <leafNode name="serial">
+                <properties>
+                  <help>Show information about connected USB serial ports</help>
+                </properties>
+                <command>${vyos_op_scripts_dir}/show_usb_serial.py</command>
+              </leafNode>
+            </children>
+          </node>
         </children>
       </node>
     </children>
diff --git a/src/conf_mode/interfaces-wirelessmodem.py b/src/conf_mode/interfaces-wirelessmodem.py
index a13c70990..c05ca684e 100755
--- a/src/conf_mode/interfaces-wirelessmodem.py
+++ b/src/conf_mode/interfaces-wirelessmodem.py
@@ -16,14 +16,15 @@
 
 import os
 
-from sys import exit
 from copy import deepcopy
+from fnmatch import fnmatch
 from netifaces import interfaces
+from sys import exit
 
 from vyos.config import Config
 from vyos.ifconfig import BridgeIf, Section
 from vyos.template import render
-from vyos.util import chown, chmod_755, cmd, call
+from vyos.util import call
 from vyos.validate import is_member
 from vyos import ConfigError
 
@@ -31,16 +32,14 @@ from vyos import airbag
 airbag.enable()
 
 default_config_data = {
-    'address': [],
     'apn': '',
     'chat_script': '',
     'deleted': False,
     'description': '',
-    'device': 'ttyUSB0',
+    'device': '',
     'disable': False,
     'disable_link_detect': 1,
     'on_demand': False,
-    'logfile': '',
     'metric': '10',
     'mtu': '1500',
     'name_server': True,
@@ -56,6 +55,16 @@ def check_kmod():
             if call(f'modprobe {module}') != 0:
                 raise ConfigError(f'Loading Kernel module {module} failed')
 
+def find_device_file(device):
+    """ Recurively search /dev for the given device file and return its full path.
+        If no device file was found 'None' is returned """
+    for root, dirs, files in os.walk('/dev'):
+        for basename in files:
+            if fnmatch(basename, device):
+                return os.path.join(root, basename)
+
+    return None
+
 def get_config():
     wwan = deepcopy(default_config_data)
     conf = Config()
@@ -65,12 +74,8 @@ def get_config():
         raise ConfigError('Interface (VYOS_TAGNODE_VALUE) not specified')
 
     wwan['intf'] = os.environ['VYOS_TAGNODE_VALUE']
-    wwan['logfile'] = f"/var/log/vyatta/ppp_{wwan['intf']}.log"
     wwan['chat_script'] = f"/etc/ppp/peers/chat.{wwan['intf']}"
 
-    # check if interface is member if a bridge
-    wwan['is_bridge_member'] = is_member(conf, wwan['intf'], 'bridge')
-
     # Check if interface has been removed
     if not conf.exists('interfaces wirelessmodem ' + wwan['intf']):
         wwan['deleted'] = True
@@ -93,7 +98,13 @@ def get_config():
 
     # System device name
     if conf.exists(['device']):
-        wwan['device'] = conf.return_value(['device'])
+        tmp = conf.return_value(['device'])
+        wwan['device'] = find_device_file(tmp)
+        # If device file was not found in /dev we will just re-use
+        # the plain device name, thus we can trigger the exception
+        # in verify() as it's a non existent file
+        if wwan['device'] == None:
+            wwan['device'] = tmp
 
     # disable interface
     if conf.exists('disable'):
@@ -123,35 +134,21 @@ def get_config():
 
 def verify(wwan):
     if wwan['deleted']:
-        if wwan['is_bridge_member']:
-            raise ConfigError((
-                f'Cannot delete interface "{wwan["intf"]}" as it is a '
-                f'member of bridge "{wwan["is_bridge_member"]}"!'))
-
         return None
 
     if not wwan['apn']:
-        raise ConfigError(f"APN for {wwan['intf']} not configured")
+        raise ConfigError('No APN configured for "{intf}"'.format(**wwan))
+
+    if not wwan['device']:
+        raise ConfigError('Physical "device" must be configured')
 
     # we can not use isfile() here as Linux device files are no regular files
     # thus the check will return False
-    if not os.path.exists(f"/dev/{wwan['device']}"):
-        raise ConfigError(f"Device {wwan['device']} does not exist")
-
-    if wwan['is_bridge_member'] and wwan['address']:
-        raise ConfigError((
-            f'Cannot assign address to interface "{wwan["intf"]}" '
-            f'as it is a member of bridge "{wwan["is_bridge_member"]}"!'))
-
-    if wwan['vrf']:
-        if wwan['vrf'] not in interfaces():
-            raise ConfigError(f'VRF "{wwan["vrf"]}" does not exist')
+    if not os.path.exists('{device}'.format(**wwan)):
+        raise ConfigError('Device "{device}" does not exist'.format(**wwan))
 
-        if wwan['is_bridge_member']:
-            raise ConfigError((
-                f'Interface "{wwan["intf"]}" cannot be member of VRF '
-                f'"{wwan["vrf"]}" and bridge {wwan["is_bridge_member"]} '
-                f'at the same time!'))
+    if wwan['vrf'] not in interfaces():
+        raise ConfigError('VRF "{vrf}" does not exist'.format(**wwan))
 
     return None
 
@@ -169,7 +166,7 @@ def generate(wwan):
                     script_wwan_ip_up, script_wwan_ip_down]
 
     # Always hang-up WWAN connection prior generating new configuration file
-    cmd(f'systemctl stop ppp@{intf}.service')
+    call(f'systemctl stop ppp@{intf}.service')
 
     if wwan['deleted']:
         # Delete PPP configuration files
@@ -205,9 +202,7 @@ def apply(wwan):
     if not wwan['disable']:
         # "dial" WWAN connection
         intf = wwan['intf']
-        cmd(f'systemctl start ppp@{intf}.service')
-        # make logfile owned by root / vyattacfg
-        chown(wwan['logfile'], 'root', 'vyattacfg')
+        call(f'systemctl start ppp@{intf}.service')
 
         # re-add ourselves to any bridge we might have fallen out of
         # FIXME: wwan isn't under vyos.ifconfig so we can't call
diff --git a/src/etc/udev/rules.d/90-vyos-serial.rules b/src/etc/udev/rules.d/90-vyos-serial.rules
new file mode 100644
index 000000000..3f10f4924
--- /dev/null
+++ b/src/etc/udev/rules.d/90-vyos-serial.rules
@@ -0,0 +1,28 @@
+# do not edit this file, it will be overwritten on update
+
+ACTION=="remove", GOTO="serial_end"
+SUBSYSTEM!="tty", GOTO="serial_end"
+
+SUBSYSTEMS=="pci", ENV{ID_BUS}="pci", ENV{ID_VENDOR_ID}="$attr{vendor}", ENV{ID_MODEL_ID}="$attr{device}"
+SUBSYSTEMS=="pci", IMPORT{builtin}="hwdb --subsystem=pci"
+SUBSYSTEMS=="usb", IMPORT{builtin}="usb_id", IMPORT{builtin}="hwdb --subsystem=usb"
+
+# /dev/serial/by-path/, /dev/serial/by-id/ for USB devices
+KERNEL!="ttyUSB[0-9]*|ttyACM[0-9]*", GOTO="serial_end"
+
+SUBSYSTEMS=="usb-serial", ENV{.ID_PORT}="$attr{port_number}"
+
+IMPORT{builtin}="path_id", IMPORT{builtin}="usb_id"
+
+# Change the name of the usb id to a "more" human redable format.
+#
+# - $env{ID_PATH} usually is a name like: "pci-0000:00:10.0-usb-0:2.3.3.4:1.0-port0" so we strip the "pci-*"
+#   portion and only use the usb part
+# - Transform the USB "speach" to the tree like structure so we start with "usb0" as root-complex 0.
+#   (tr -d -) does the replacement
+# - Replace the first group after ":" to represent the bus relation (sed -e 0,/:/s//b/) indicated by "b"
+# - Replace the next group after ":" to represent the port relation (sed -e 0,/:/s//p/) indicated by "p"
+ENV{ID_PATH}=="?*", ENV{.ID_PORT}=="",   PROGRAM="/bin/sh -c 'D=$env{ID_PATH}; echo ${D:17} | tr -d - | sed -e 0,/:/s//b/ | sed -e 0,/:/s//p/'", SYMLINK+="serial/by-bus/$result"
+ENV{ID_PATH}=="?*", ENV{.ID_PORT}=="?*", PROGRAM="/bin/sh -c 'D=$env{ID_PATH}; echo ${D:17} | tr -d - | sed -e 0,/:/s//b/ | sed -e 0,/:/s//p/'", SYMLINK+="serial/by-bus/$result"
+
+LABEL="serial_end"
diff --git a/src/migration-scripts/interfaces/10-to-11 b/src/migration-scripts/interfaces/10-to-11
new file mode 100755
index 000000000..6b8e49ed9
--- /dev/null
+++ b/src/migration-scripts/interfaces/10-to-11
@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2020 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# rename WWAN (wirelessmodem) serial interface from non persistent ttyUSB2 to
+# a bus like name, e.g. "usb0b1.3p1.3"
+
+import os
+
+from sys import exit, argv
+from vyos.configtree import ConfigTree
+
+if __name__ == '__main__':
+    if (len(argv) < 1):
+        print("Must specify file name!")
+        exit(1)
+
+    file_name = argv[1]
+    with open(file_name, 'r') as f:
+        config_file = f.read()
+
+    config = ConfigTree(config_file)
+    base = ['interfaces', 'wirelessmodem']
+    if not config.exists(base):
+        # Nothing to do
+        exit(0)
+
+    for wwan in config.list_nodes(base):
+        if config.exists(base + [wwan, 'device']):
+            device = config.return_value(base + [wwan, 'device'])
+
+            for root, dirs, files in os.walk('/dev/serial/by-bus'):
+                for file in files:
+                    device_file = os.path.realpath(os.path.join(root, file))
+                    if os.path.basename(device_file) == device:
+                        config.set(base + [wwan, 'device'], value=file, replace=True)
+
+    try:
+        with open(file_name, 'w') as f:
+            f.write(config.to_string())
+    except OSError as e:
+        print("Failed to save the modified config: {}".format(e))
+        exit(1)
diff --git a/src/op_mode/show_usb_serial.py b/src/op_mode/show_usb_serial.py
new file mode 100755
index 000000000..776898c25
--- /dev/null
+++ b/src/op_mode/show_usb_serial.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2018 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+
+from jinja2 import Template
+from pyudev import Context, Devices
+from sys import exit
+
+OUT_TMPL_SRC = """Device           Model               Vendor
+------           ------              ------
+{%- for d in devices %}
+{{ "%-16s" | format(d.device) }} {{ "%-19s" | format(d.model)}} {{d.vendor}}
+{%- endfor %}
+
+"""
+
+data = {
+    'devices': []
+}
+
+
+base_directory = '/dev/serial/by-bus'
+if not os.path.isdir(base_directory):
+    print("No USB to serial converter connected")
+    exit(0)
+
+context = Context()
+for root, dirs, files in os.walk(base_directory):
+    for basename in files:
+        os.path.join(root, basename)
+        device = Devices.from_device_file(context, os.path.join(root, basename))
+        tmp = {
+            'device': basename,
+            'model': device.properties.get('ID_MODEL'),
+            'vendor': device.properties.get('ID_VENDOR_FROM_DATABASE')
+        }
+        data['devices'].append(tmp)
+
+data['devices'] = sorted(data['devices'], key = lambda i: i['device'])
+tmpl = Template(OUT_TMPL_SRC)
+print(tmpl.render(data))
+
+exit(0)