Merge branch 'current' into 2fa

3 files changed, 439 insertions, 4 deletions

diff --git a/src/helpers/strip-private.py b/src/helpers/strip-private.py
index c165d2cba..c74a379aa 100755
--- a/src/helpers/strip-private.py
+++ b/src/helpers/strip-private.py
@@ -47,7 +47,7 @@ ipv4_re = re.compile(r'(\d{1,3}\.){2}(\d{1,3}\.\d{1,3})')
 ipv4_subst = r'xxx.xxx.\2'
 
 # Censor all but the first two fields.
-ipv6_re = re.compile(r'([0-9a-fA-F]{1,4}\:){2}(\S+)')
+ipv6_re = re.compile(r'([0-9a-fA-F]{1,4}\:){2}([0-9a-fA-F:]+)')
 ipv6_subst = r'xxxx:xxxx:\2'
 
 def ip_match(match: re.Match, subst: str) -> str:
@@ -96,12 +96,12 @@ if __name__ == "__main__":
     args = parser.parse_args()
     # Strict mode is the default and the absence of loose mode implies presence of strict mode.
     if not args.loose:
-        for arg in [args.mac, args.domain, args.hostname, args.username, args.dhcp, args.asn, args.snmp, args.lldp]:
-            arg = True
+        args.mac = args.domain = args.hostname = args.username = args.dhcp = args.asn = args.snmp = args.lldp = True
         if not args.public_address and not args.keep_address:
             args.address = True
     elif not args.address and not args.public_address:
         args.keep_address = True
+
     # (condition, precompiled regexp, substitution string)
     stripping_rules = [
         # Strip passwords
@@ -120,7 +120,7 @@ if __name__ == "__main__":
         (True, re.compile(r'private-key \S+'), 'private-key xxxxxx'),
 
         # Strip MAC addresses
-        (args.mac, re.compile(r'([0-9a-fA-F]{2}\:){5}([0-9a-fA-F]{2}((\:{0,1})){3})'), r'XX:XX:XX:XX:XX:\2'),
+        (args.mac, re.compile(r'([0-9a-fA-F]{2}\:){5}([0-9a-fA-F]{2}((\:{0,1})){3})'), r'xx:xx:xx:xx:xx:\2'),
 
         # Strip host-name, domain-name, and domain-search
         (args.hostname, re.compile(r'(host-name|domain-name|domain-search) \S+'), r'\1 xxxxxx'),
diff --git a/src/helpers/vyos-interface-rescan.py b/src/helpers/vyos-interface-rescan.py
new file mode 100755
index 000000000..1ac1810e0
--- /dev/null
+++ b/src/helpers/vyos-interface-rescan.py
@@ -0,0 +1,206 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2021 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import os
+import stat
+import argparse
+import logging
+import netaddr
+
+from vyos.configtree import ConfigTree
+from vyos.defaults import directories
+from vyos.util import get_cfg_group_id
+
+debug = False
+
+vyos_udev_dir = directories['vyos_udev_dir']
+vyos_log_dir = directories['log']
+log_file = os.path.splitext(os.path.basename(__file__))[0]
+vyos_log_file = os.path.join(vyos_log_dir, log_file)
+
+logger = logging.getLogger(__name__)
+handler = logging.FileHandler(vyos_log_file, mode='a')
+formatter = logging.Formatter('%(levelname)s: %(message)s')
+handler.setFormatter(formatter)
+logger.addHandler(handler)
+
+passlist = {
+    '02:07:01' : 'Interlan',
+    '02:60:60' : '3Com',
+    '02:60:8c' : '3Com',
+    '02:a0:c9' : 'Intel',
+    '02:aa:3c' : 'Olivetti',
+    '02:cf:1f' : 'CMC',
+    '02:e0:3b' : 'Prominet',
+    '02:e6:d3' : 'BTI',
+    '52:54:00' : 'Realtek',
+    '52:54:4c' : 'Novell 2000',
+    '52:54:ab' : 'Realtec',
+    'e2:0c:0f' : 'Kingston Technologies'
+}
+
+def is_multicast(addr: netaddr.eui.EUI) -> bool:
+    return bool(addr.words[0] & 0b1)
+
+def is_locally_administered(addr: netaddr.eui.EUI) -> bool:
+    return bool(addr.words[0] & 0b10)
+
+def is_on_passlist(hwid: str) -> bool:
+    top = hwid.rsplit(':', 3)[0]
+    if top in list(passlist):
+        return True
+    return False
+
+def is_persistent(hwid: str) -> bool:
+    addr = netaddr.EUI(hwid)
+    if is_multicast(addr):
+        return False
+    if is_locally_administered(addr) and not is_on_passlist(hwid):
+        return False
+    return True
+
+def get_wireless_physical_device(intf: str) -> str:
+    if 'wlan' not in intf:
+        return ''
+    try:
+        tmp = os.readlink(f'/sys/class/net/{intf}/phy80211')
+    except OSError:
+        logger.critical(f"Failed to read '/sys/class/net/{intf}/phy80211'")
+        return ''
+    phy = os.path.basename(tmp)
+    logger.info(f"wireless phy is {phy}")
+    return phy
+
+def get_interface_type(intf: str) -> str:
+    if 'eth' in intf:
+        intf_type = 'ethernet'
+    elif 'wlan' in intf:
+        intf_type = 'wireless'
+    else:
+        logger.critical('Unrecognized interface type!')
+        intf_type = ''
+    return intf_type
+
+def get_new_interfaces() -> dict:
+    """ Read any new interface data left in /run/udev/vyos by vyos_net_name
+    """
+    interfaces = {}
+
+    for intf in os.listdir(vyos_udev_dir):
+        path = os.path.join(vyos_udev_dir, intf)
+        try:
+            with open(path) as f:
+                hwid = f.read().rstrip()
+        except OSError as e:
+            logger.error(f"OSError {e}")
+            continue
+        interfaces[intf] = hwid
+
+    # reverse sort to simplify insertion in config
+    interfaces = {key: value for key, value in sorted(interfaces.items(),
+                                                      reverse=True)}
+    return interfaces
+
+def filter_interfaces(intfs: dict) -> dict:
+    """ Ignore no longer existing interfaces or non-persistent mac addresses
+    """
+    filtered = {}
+
+    for intf, hwid in intfs.items():
+        if not os.path.isdir(os.path.join('/sys/class/net', intf)):
+            continue
+        if not is_persistent(hwid):
+            continue
+        filtered[intf] = hwid
+
+    return filtered
+
+def interface_rescan(config_path: str):
+    """ Read new data and update config file
+    """
+    interfaces = get_new_interfaces()
+
+    logger.debug(f"interfaces from udev: {interfaces}")
+
+    interfaces = filter_interfaces(interfaces)
+
+    logger.debug(f"filtered interfaces: {interfaces}")
+
+    try:
+        with open(config_path) as f:
+            config_file = f.read()
+    except OSError as e:
+        logger.critical(f"OSError {e}")
+        exit(1)
+
+    config = ConfigTree(config_file)
+
+    for intf, hwid in interfaces.items():
+        logger.info(f"Writing '{intf}' '{hwid}' to config file")
+        intf_type = get_interface_type(intf)
+        if not intf_type:
+            continue
+        if not config.exists(['interfaces', intf_type]):
+            config.set(['interfaces', intf_type])
+            config.set_tag(['interfaces', intf_type])
+        config.set(['interfaces', intf_type, intf, 'hw-id'], value=hwid)
+
+        if intf_type == 'wireless':
+            phy = get_wireless_physical_device(intf)
+            if not phy:
+                continue
+            config.set(['interfaces', intf_type, intf, 'physical-device'],
+                       value=phy)
+
+    try:
+        with open(config_path, 'w') as f:
+            f.write(config.to_string())
+    except OSError as e:
+        logger.critical(f"OSError {e}")
+
+def main():
+    global debug
+
+    argparser = argparse.ArgumentParser(
+        formatter_class=argparse.RawTextHelpFormatter)
+    argparser.add_argument('configfile', type=str)
+    argparser.add_argument('--debug', action='store_true')
+    args = argparser.parse_args()
+
+    if args.debug:
+        debug = True
+        logger.setLevel(logging.DEBUG)
+    else:
+        logger.setLevel(logging.INFO)
+
+    configfile = args.configfile
+
+    # preserve vyattacfg group write access to running config
+    os.setgid(get_cfg_group_id())
+    os.umask(0o002)
+
+    # log file perms are not automatic; this could be cleaner by moving to a
+    # logging config file
+    os.chown(vyos_log_file, 0, get_cfg_group_id())
+    os.chmod(vyos_log_file,
+             stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH)
+
+    interface_rescan(configfile)
+
+if __name__ == '__main__':
+    main()
diff --git a/src/helpers/vyos_net_name b/src/helpers/vyos_net_name
new file mode 100755
index 000000000..0652e98b1
--- /dev/null
+++ b/src/helpers/vyos_net_name
@@ -0,0 +1,229 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2021 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import os
+import re
+import time
+import logging
+import threading
+from sys import argv
+
+from vyos.configtree import ConfigTree
+from vyos.defaults import directories
+from vyos.util import cmd
+
+vyos_udev_dir = directories['vyos_udev_dir']
+vyos_log_dir = '/run/udev/log'
+vyos_log_file = os.path.join(vyos_log_dir, 'vyos-net-name')
+
+config_path = '/opt/vyatta/etc/config/config.boot'
+config_status = '/tmp/vyos-config-status'
+
+lock = threading.Lock()
+
+try:
+    os.mkdir(vyos_log_dir)
+except FileExistsError:
+    pass
+
+logging.basicConfig(filename=vyos_log_file, level=logging.DEBUG)
+
+def boot_configuration_complete() -> bool:
+    """ Check if vyos-router has completed, hence hotplug event
+    """
+    if os.path.isfile(config_status):
+        return True
+    return False
+
+def is_available(intfs: dict, intf_name: str) -> bool:
+    """ Check if interface name is already assigned
+    """
+    if intf_name in list(intfs.values()):
+        return False
+    return True
+
+def find_available(intfs: dict, prefix: str) -> str:
+    """ Find lowest indexed iterface name that is not assigned
+    """
+    index_list = [int(x.replace(prefix, '')) for x in list(intfs.values()) if prefix in x]
+    index_list.sort()
+    # find 'holes' in list, if any
+    missing = sorted(set(range(index_list[0], index_list[-1])) - set(index_list))
+    if missing:
+        return f'{prefix}{missing[0]}'
+
+    return f'{prefix}{len(index_list)}'
+
+def get_biosdevname(ifname: str) -> str:
+    """ Use legacy vyatta-biosdevname to query for name
+
+    This is carried over for compatability only, and will likely be dropped
+    going forward.
+    XXX: This throws an error, and likely has for a long time, unnoticed
+    since vyatta_net_name redirected stderr to /dev/null.
+    """
+    if 'eth' not in ifname:
+        return ifname
+    if os.path.isdir('/proc/xen'):
+        return ifname
+
+    time.sleep(1)
+
+    try:
+        biosname = cmd(f'/sbin/biosdevname --policy all_ethN -i {ifname}')
+    except Exception as e:
+        logging.error(f'biosdevname error: {e}')
+        biosname = ''
+
+    return ifname if biosname == '' else biosname
+
+def leave_rescan_hint(intf_name: str, hwid: str):
+    """Write interface information reported by udev
+
+    This script is called while the root mount is still read-only. Leave
+    information in /run/udev: file name, the interface; contents, the
+    hardware id.
+    """
+    try:
+        os.mkdir(vyos_udev_dir)
+    except FileExistsError:
+        pass
+    except Exception as e:
+        logging.critical(f"Error creating rescan hint directory: {e}")
+        exit(1)
+
+    try:
+        with open(os.path.join(vyos_udev_dir, intf_name), 'w') as f:
+            f.write(hwid)
+    except OSError as e:
+        logging.critical(f"OSError {e}")
+
+def get_configfile_interfaces() -> dict:
+    """Read existing interfaces from config file
+    """
+    interfaces: dict = {}
+
+    if not os.path.isfile(config_path):
+        # If the case, then we are running off of livecd; return empty
+        return interfaces
+
+    try:
+        with open(config_path) as f:
+            config_file = f.read()
+    except OSError as e:
+        logging.critical(f"OSError {e}")
+        exit(1)
+
+    config = ConfigTree(config_file)
+
+    base = ['interfaces', 'ethernet']
+    if config.exists(base):
+        eth_intfs = config.list_nodes(base)
+        for intf in eth_intfs:
+            path = base + [intf, 'hw-id']
+            if not config.exists(path):
+                logging.warning(f"no 'hw-id' entry for {intf}")
+                continue
+            hwid = config.return_value(path)
+            if hwid in list(interfaces):
+                logging.warning(f"multiple entries for {hwid}: {interfaces[hwid]}, {intf}")
+                continue
+            interfaces[hwid] = intf
+
+    base = ['interfaces', 'wireless']
+    if config.exists(base):
+        wlan_intfs = config.list_nodes(base)
+        for intf in wlan_intfs:
+            path = base + [intf, 'hw-id']
+            if not config.exists(path):
+                logging.warning(f"no 'hw-id' entry for {intf}")
+                continue
+            hwid = config.return_value(path)
+            if hwid in list(interfaces):
+                logging.warning(f"multiple entries for {hwid}: {interfaces[hwid]}, {intf}")
+                continue
+            interfaces[hwid] = intf
+
+    logging.debug(f"config file entries: {interfaces}")
+
+    return interfaces
+
+def add_assigned_interfaces(intfs: dict):
+    """Add interfaces found by previous invocation of udev rule
+    """
+    if not os.path.isdir(vyos_udev_dir):
+        return
+
+    for intf in os.listdir(vyos_udev_dir):
+        path = os.path.join(vyos_udev_dir, intf)
+        try:
+            with open(path) as f:
+                hwid = f.read().rstrip()
+        except OSError as e:
+            logging.error(f"OSError {e}")
+            continue
+        intfs[hwid] = intf
+
+def on_boot_event(intf_name: str, hwid: str, predefined: str = '') -> str:
+    """Called on boot by vyos-router: 'coldplug' in vyatta_net_name
+    """
+    logging.info(f"lookup {intf_name}, {hwid}")
+    interfaces = get_configfile_interfaces()
+    logging.debug(f"config file interfaces are {interfaces}")
+
+    if hwid in list(interfaces) and intf_name == interfaces[hwid]:
+        logging.info(f"use mapping from config file: '{hwid}' -> '{intf_name}'")
+        return intf_name
+
+    add_assigned_interfaces(interfaces)
+    logging.debug(f"adding assigned interfaces: {interfaces}")
+
+    if predefined:
+        newname = predefined
+        logging.info(f"predefined interface name for '{intf_name}' is '{newname}'")
+    else:
+        newname = get_biosdevname(intf_name)
+        logging.info(f"biosdevname returned '{newname}' for '{intf_name}'")
+
+    if not is_available(interfaces, newname):
+        prefix = re.sub(r'\d+$', '', newname)
+        newname = find_available(interfaces, prefix)
+
+    logging.info(f"new name for '{intf_name}' is '{newname}'")
+
+    leave_rescan_hint(newname, hwid)
+
+    return newname
+
+def hotplug_event():
+    # Not yet implemented, since interface-rescan will only be run on boot.
+    pass
+
+if len(argv) > 3:
+    predef_name = argv[3]
+else:
+    predef_name = ''
+
+lock.acquire()
+if not boot_configuration_complete():
+    res = on_boot_event(argv[1], argv[2], predefined=predef_name)
+    logging.debug(f"on boot, returned name is {res}")
+else:
+    logging.debug("boot configuration complete")
+lock.release()
+