T5195: vyos.util -> vyos.utils package refactoring (#2093)

* T5195: move run, cmd, call, rc_cmd helper to vyos.utils.process

* T5195: use read_file and write_file implementation from vyos.utils.file

Changed code automatically using:

find . -type f -not -path '*/\.*' -exec sed -i 's/^from vyos.util import read_file$/from vyos.utils.file import read_file/g' {} +
find . -type f -not -path '*/\.*' -exec sed -i 's/^from vyos.util import write_file$/from vyos.utils.file import write_file/g' {} +

* T5195: move chmod* helpers to vyos.utils.permission

* T5195: use colon_separated_to_dict from vyos.utils.dict

* T5195: move is_systemd_service_* to vyos.utils.process

* T5195: fix boot issues with missing imports

* T5195: move dict_search_* helpers to vyos.utils.dict

* T5195: move network helpers to vyos.utils.network

* T5195: move commit_* helpers to vyos.utils.commit

* T5195: move user I/O helpers to vyos.utils.io

6 files changed, 525 insertions, 4 deletions

diff --git a/python/vyos/utils/__init__.py b/python/vyos/utils/__init__.py
index 5c7a9ecb8..abc9af5da 100644
--- a/python/vyos/utils/__init__.py
+++ b/python/vyos/utils/__init__.py
@@ -13,5 +13,13 @@
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library.  If not, see <http://www.gnu.org/licenses/>.
 
-from vyos.utils import network
 from vyos.utils import boot
+from vyos.utils import commit
+from vyos.utils import convert
+from vyos.utils import dict
+from vyos.utils import file
+from vyos.utils import io
+from vyos.utils import network
+from vyos.utils import permission
+from vyos.utils import process
+from vyos.utils import system
diff --git a/python/vyos/utils/commit.py b/python/vyos/utils/commit.py
new file mode 100644
index 000000000..105aed8c2
--- /dev/null
+++ b/python/vyos/utils/commit.py
@@ -0,0 +1,60 @@
+# Copyright 2023 VyOS maintainers and contributors <maintainers@vyos.io>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library.  If not, see <http://www.gnu.org/licenses/>.
+
+def commit_in_progress():
+    """ Not to be used in normal op mode scripts! """
+
+    # The CStore backend locks the config by opening a file
+    # The file is not removed after commit, so just checking
+    # if it exists is insufficient, we need to know if it's open by anyone
+
+    # There are two ways to check if any other process keeps a file open.
+    # The first one is to try opening it and see if the OS objects.
+    # That's faster but prone to race conditions and can be intrusive.
+    # The other one is to actually check if any process keeps it open.
+    # It's non-intrusive but needs root permissions, else you can't check
+    # processes of other users.
+    #
+    # Since this will be used in scripts that modify the config outside of the CLI
+    # framework, those knowingly have root permissions.
+    # For everything else, we add a safeguard.
+    from psutil import process_iter
+    from psutil import NoSuchProcess
+    from getpass import getuser
+    from vyos.defaults import commit_lock
+
+    if getuser() != 'root':
+        raise OSError('This functions needs to be run as root to return correct results!')
+
+    for proc in process_iter():
+        try:
+            files = proc.open_files()
+            if files:
+                for f in files:
+                    if f.path == commit_lock:
+                        return True
+        except NoSuchProcess as err:
+            # Process died before we could examine it
+            pass
+    # Default case
+    return False
+
+
+def wait_for_commit_lock():
+    """ Not to be used in normal op mode scripts! """
+    from time import sleep
+    # Very synchronous approach to multiprocessing
+    while commit_in_progress():
+        sleep(1)
diff --git a/python/vyos/utils/file.py b/python/vyos/utils/file.py
index 2560a35be..667a2464b 100644
--- a/python/vyos/utils/file.py
+++ b/python/vyos/utils/file.py
@@ -14,7 +14,19 @@
 # License along with this library.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
+from vyos.utils.permission import chown
 
+def makedir(path, user=None, group=None):
+    if os.path.exists(path):
+        return
+    os.makedirs(path, mode=0o755)
+    chown(path, user, group)
+
+def file_is_persistent(path):
+    import re
+    location = r'^(/config|/opt/vyatta/etc/config)'
+    absolute = os.path.abspath(os.path.dirname(path))
+    return re.match(location,absolute)
 
 def read_file(fname, defaultonfailure=None):
     """
diff --git a/python/vyos/utils/network.py b/python/vyos/utils/network.py
index 209bc9ecc..8db598f05 100644
--- a/python/vyos/utils/network.py
+++ b/python/vyos/utils/network.py
@@ -13,6 +13,8 @@
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library.  If not, see <http://www.gnu.org/licenses/>.
 
+import os
+
 def get_protocol_by_name(protocol_name):
     """Get protocol number by protocol name
 
@@ -27,7 +29,7 @@ def get_protocol_by_name(protocol_name):
         return protocol_name
 
 def interface_exists_in_netns(interface_name, netns):
-    from vyos.util import rc_cmd
+    from vyos.utils.process import rc_cmd
     rc, out = rc_cmd(f'ip netns exec {netns} ip link show dev {interface_name}')
     if rc == 0:
         return True
@@ -35,9 +37,155 @@ def interface_exists_in_netns(interface_name, netns):
 
 def get_interface_vrf(interface):
     """ Returns VRF of given interface """
-    from vyos.util import dict_search
-    from vyos.util import get_interface_config
+    from vyos.utils.dict import dict_search
+    from vyos.utils.network import get_interface_config
     tmp = get_interface_config(interface)
     if dict_search('linkinfo.info_slave_kind', tmp) == 'vrf':
         return tmp['master']
     return 'default'
+
+def get_interface_config(interface):
+    """ Returns the used encapsulation protocol for given interface.
+        If interface does not exist, None is returned.
+    """
+    if not os.path.exists(f'/sys/class/net/{interface}'):
+        return None
+    from json import loads
+    from vyos.utils.process import cmd
+    tmp = loads(cmd(f'ip --detail --json link show dev {interface}'))[0]
+    return tmp
+
+def get_interface_address(interface):
+    """ Returns the used encapsulation protocol for given interface.
+        If interface does not exist, None is returned.
+    """
+    if not os.path.exists(f'/sys/class/net/{interface}'):
+        return None
+    from json import loads
+    from vyos.utils.process import cmd
+    tmp = loads(cmd(f'ip --detail --json addr show dev {interface}'))[0]
+    return tmp
+
+def get_interface_namespace(iface):
+    """
+       Returns wich netns the interface belongs to
+    """
+    from json import loads
+    from vyos.utils.process import cmd
+    # Check if netns exist
+    tmp = loads(cmd(f'ip --json netns ls'))
+    if len(tmp) == 0:
+        return None
+
+    for ns in tmp:
+        netns = f'{ns["name"]}'
+        # Search interface in each netns
+        data = loads(cmd(f'ip netns exec {netns} ip --json link show'))
+        for tmp in data:
+            if iface == tmp["ifname"]:
+                return netns
+
+
+def is_wwan_connected(interface):
+    """ Determine if a given WWAN interface, e.g. wwan0 is connected to the
+    carrier network or not """
+    import json
+    from vyos.utils.process import cmd
+
+    if not interface.startswith('wwan'):
+        raise ValueError(f'Specified interface "{interface}" is not a WWAN interface')
+
+    # ModemManager is required for connection(s) - if service is not running,
+    # there won't be any connection at all!
+    if not is_systemd_service_active('ModemManager.service'):
+        return False
+
+    modem = interface.lstrip('wwan')
+
+    tmp = cmd(f'mmcli --modem {modem} --output-json')
+    tmp = json.loads(tmp)
+
+    # return True/False if interface is in connected state
+    return dict_search('modem.generic.state', tmp) == 'connected'
+
+def get_bridge_fdb(interface):
+    """ Returns the forwarding database entries for a given interface """
+    if not os.path.exists(f'/sys/class/net/{interface}'):
+        return None
+    from json import loads
+    from vyos.utils.process import cmd
+    tmp = loads(cmd(f'bridge -j fdb show dev {interface}'))
+    return tmp
+
+def get_all_vrfs():
+    """ Return a dictionary of all system wide known VRF instances """
+    from json import loads
+    from vyos.utils.process import cmd
+    tmp = loads(cmd('ip --json vrf list'))
+    # Result is of type [{"name":"red","table":1000},{"name":"blue","table":2000}]
+    # so we will re-arrange it to a more nicer representation:
+    # {'red': {'table': 1000}, 'blue': {'table': 2000}}
+    data = {}
+    for entry in tmp:
+        name = entry.pop('name')
+        data[name] = entry
+    return data
+
+def mac2eui64(mac, prefix=None):
+    """
+    Convert a MAC address to a EUI64 address or, with prefix provided, a full
+    IPv6 address.
+    Thankfully copied from https://gist.github.com/wido/f5e32576bb57b5cc6f934e177a37a0d3
+    """
+    import re
+    from ipaddress import ip_network
+    # http://tools.ietf.org/html/rfc4291#section-2.5.1
+    eui64 = re.sub(r'[.:-]', '', mac).lower()
+    eui64 = eui64[0:6] + 'fffe' + eui64[6:]
+    eui64 = hex(int(eui64[0:2], 16) ^ 2)[2:].zfill(2) + eui64[2:]
+
+    if prefix is None:
+        return ':'.join(re.findall(r'.{4}', eui64))
+    else:
+        try:
+            net = ip_network(prefix, strict=False)
+            euil = int('0x{0}'.format(eui64), 16)
+            return str(net[euil])
+        except:  # pylint: disable=bare-except
+            return
+
+
+def check_port_availability(ipaddress, port, protocol):
+    """
+    Check if port is available and not used by any service
+    Return False if a port is busy or IP address does not exists
+    Should be used carefully for services that can start listening
+    dynamically, because IP address may be dynamic too
+    """
+    from socketserver import TCPServer, UDPServer
+    from ipaddress import ip_address
+
+    # verify arguments
+    try:
+        ipaddress = ip_address(ipaddress).compressed
+    except:
+        raise ValueError(f'The {ipaddress} is not a valid IPv4 or IPv6 address')
+    if port not in range(1, 65536):
+        raise ValueError(f'The port number {port} is not in the 1-65535 range')
+    if protocol not in ['tcp', 'udp']:
+        raise ValueError(f'The protocol {protocol} is not supported. Only tcp and udp are allowed')
+
+    # check port availability
+    try:
+        if protocol == 'tcp':
+            server = TCPServer((ipaddress, port), None, bind_and_activate=True)
+        if protocol == 'udp':
+            server = UDPServer((ipaddress, port), None, bind_and_activate=True)
+        server.server_close()
+    except Exception as e:
+        # errno.h:
+        #define EADDRINUSE  98  /* Address already in use */
+        if e.errno == 98:
+            return False
+
+    return True
diff --git a/python/vyos/utils/permission.py b/python/vyos/utils/permission.py
new file mode 100644
index 000000000..8c2d72b83
--- /dev/null
+++ b/python/vyos/utils/permission.py
@@ -0,0 +1,63 @@
+# Copyright 2023 VyOS maintainers and contributors <maintainers@vyos.io>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+
+def chown(path, user, group):
+    """ change file/directory owner """
+    from pwd import getpwnam
+    from grp import getgrnam
+
+    if user is None or group is None:
+        return False
+
+    # path may also be an open file descriptor
+    if not isinstance(path, int) and not os.path.exists(path):
+        return False
+
+    uid = getpwnam(user).pw_uid
+    gid = getgrnam(group).gr_gid
+    os.chown(path, uid, gid)
+    return True
+
+def chmod(path, bitmask):
+    # path may also be an open file descriptor
+    if not isinstance(path, int) and not os.path.exists(path):
+        return
+    if bitmask is None:
+        return
+    os.chmod(path, bitmask)
+
+def chmod_600(path):
+    """ make file only read/writable by owner """
+    from stat import S_IRUSR, S_IWUSR
+
+    bitmask = S_IRUSR | S_IWUSR
+    chmod(path, bitmask)
+
+def chmod_750(path):
+    """ make file/directory only executable to user and group """
+    from stat import S_IRUSR, S_IWUSR, S_IXUSR, S_IRGRP, S_IXGRP
+
+    bitmask = S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IXGRP
+    chmod(path, bitmask)
+
+def chmod_755(path):
+    """ make file executable by all """
+    from stat import S_IRUSR, S_IWUSR, S_IXUSR, S_IRGRP, S_IXGRP, S_IROTH, S_IXOTH
+
+    bitmask = S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IXGRP | \
+              S_IROTH | S_IXOTH
+    chmod(path, bitmask)
diff --git a/python/vyos/utils/process.py b/python/vyos/utils/process.py
new file mode 100644
index 000000000..15b26f4eb
--- /dev/null
+++ b/python/vyos/utils/process.py
@@ -0,0 +1,230 @@
+# Copyright 2023 VyOS maintainers and contributors <maintainers@vyos.io>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library.  If not, see <http://www.gnu.org/licenses/>.
+
+from subprocess import Popen
+from subprocess import PIPE
+from subprocess import STDOUT
+from subprocess import DEVNULL
+
+def popen(command, flag='', shell=None, input=None, timeout=None, env=None,
+          stdout=PIPE, stderr=PIPE, decode='utf-8'):
+    """
+    popen is a wrapper helper aound subprocess.Popen
+    with it default setting it will return a tuple (out, err)
+    out: the output of the program run
+    err: the error code returned by the program
+
+    it can be affected by the following flags:
+    shell:   do not try to auto-detect if a shell is required
+             for example if a pipe (|) or redirection (>, >>) is used
+    input:   data to sent to the child process via STDIN
+             the data should be bytes but string will be converted
+    timeout: time after which the command will be considered to have failed
+    env:     mapping that defines the environment variables for the new process
+    stdout:  define how the output of the program should be handled
+              - PIPE (default), sends stdout to the output
+              - DEVNULL, discard the output
+    stderr:  define how the output of the program should be handled
+              - None (default), send/merge the data to/with stderr
+              - PIPE, popen will append it to output
+              - STDOUT, send the data to be merged with stdout
+              - DEVNULL, discard the output
+    decode:  specify the expected text encoding (utf-8, ascii, ...)
+             the default is explicitely utf-8 which is python's own default
+
+    usage:
+    get both stdout and stderr: popen('command', stdout=PIPE, stderr=STDOUT)
+    discard stdout and get stderr: popen('command', stdout=DEVNUL, stderr=PIPE)
+    """
+
+    # airbag must be left as an import in the function as otherwise we have a
+    # a circual import dependency
+    from vyos import debug
+    from vyos import airbag
+
+    # log if the flag is set, otherwise log if command is set
+    if not debug.enabled(flag):
+        flag = 'command'
+
+    cmd_msg = f"cmd '{command}'"
+    debug.message(cmd_msg, flag)
+
+    use_shell = shell
+    stdin = None
+    if shell is None:
+        use_shell = False
+        if ' ' in command:
+            use_shell = True
+        if env:
+            use_shell = True
+
+    if input:
+        stdin = PIPE
+        input = input.encode() if type(input) is str else input
+
+    p = Popen(command, stdin=stdin, stdout=stdout, stderr=stderr,
+              env=env, shell=use_shell)
+
+    pipe = p.communicate(input, timeout)
+
+    pipe_out = b''
+    if stdout == PIPE:
+        pipe_out = pipe[0]
+
+    pipe_err = b''
+    if stderr == PIPE:
+        pipe_err = pipe[1]
+
+    str_out = pipe_out.decode(decode).replace('\r\n', '\n').strip()
+    str_err = pipe_err.decode(decode).replace('\r\n', '\n').strip()
+
+    out_msg = f"returned (out):\n{str_out}"
+    if str_out:
+        debug.message(out_msg, flag)
+
+    if str_err:
+        from sys import stderr
+        err_msg = f"returned (err):\n{str_err}"
+        # this message will also be send to syslog via airbag
+        debug.message(err_msg, flag, destination=stderr)
+
+        # should something go wrong, report this too via airbag
+        airbag.noteworthy(cmd_msg)
+        airbag.noteworthy(out_msg)
+        airbag.noteworthy(err_msg)
+
+    return str_out, p.returncode
+
+
+def run(command, flag='', shell=None, input=None, timeout=None, env=None,
+        stdout=DEVNULL, stderr=PIPE, decode='utf-8'):
+    """
+    A wrapper around popen, which discard the stdout and
+    will return the error code of a command
+    """
+    _, code = popen(
+        command, flag,
+        stdout=stdout, stderr=stderr,
+        input=input, timeout=timeout,
+        env=env, shell=shell,
+        decode=decode,
+    )
+    return code
+
+
+def cmd(command, flag='', shell=None, input=None, timeout=None, env=None,
+        stdout=PIPE, stderr=PIPE, decode='utf-8', raising=None, message='',
+        expect=[0]):
+    """
+    A wrapper around popen, which returns the stdout and
+    will raise the error code of a command
+
+    raising: specify which call should be used when raising
+             the class should only require a string as parameter
+             (default is OSError) with the error code
+    expect:  a list of error codes to consider as normal
+    """
+    decoded, code = popen(
+        command, flag,
+        stdout=stdout, stderr=stderr,
+        input=input, timeout=timeout,
+        env=env, shell=shell,
+        decode=decode,
+    )
+    if code not in expect:
+        feedback = message + '\n' if message else ''
+        feedback += f'failed to run command: {command}\n'
+        feedback += f'returned: {decoded}\n'
+        feedback += f'exit code: {code}'
+        if raising is None:
+            # error code can be recovered with .errno
+            raise OSError(code, feedback)
+        else:
+            raise raising(feedback)
+    return decoded
+
+
+def rc_cmd(command, flag='', shell=None, input=None, timeout=None, env=None,
+           stdout=PIPE, stderr=STDOUT, decode='utf-8'):
+    """
+    A wrapper around popen, which returns the return code
+    of a command and stdout
+
+    % rc_cmd('uname')
+    (0, 'Linux')
+    % rc_cmd('ip link show dev eth99')
+    (1, 'Device "eth99" does not exist.')
+    """
+    out, code = popen(
+        command, flag,
+        stdout=stdout, stderr=stderr,
+        input=input, timeout=timeout,
+        env=env, shell=shell,
+        decode=decode,
+    )
+    return code, out
+
+def call(command, flag='', shell=None, input=None, timeout=None, env=None,
+         stdout=PIPE, stderr=PIPE, decode='utf-8'):
+    """
+    A wrapper around popen, which print the stdout and
+    will return the error code of a command
+    """
+    out, code = popen(
+        command, flag,
+        stdout=stdout, stderr=stderr,
+        input=input, timeout=timeout,
+        env=env, shell=shell,
+        decode=decode,
+    )
+    if out:
+        print(out)
+    return code
+
+def process_running(pid_file):
+    """ Checks if a process with PID in pid_file is running """
+    from psutil import pid_exists
+    if not os.path.isfile(pid_file):
+        return False
+    with open(pid_file, 'r') as f:
+        pid = f.read().strip()
+    return pid_exists(int(pid))
+
+def process_named_running(name, cmdline: str=None):
+    """ Checks if process with given name is running and returns its PID.
+    If Process is not running, return None
+    """
+    from psutil import process_iter
+    for p in process_iter(['name', 'pid', 'cmdline']):
+        if cmdline:
+            if p.info['name'] == name and cmdline in p.info['cmdline']:
+                return p.info['pid']
+        elif p.info['name'] == name:
+            return p.info['pid']
+    return None
+
+def is_systemd_service_active(service):
+    """ Test is a specified systemd service is activated.
+    Returns True if service is active, false otherwise.
+    Copied from: https://unix.stackexchange.com/a/435317 """
+    tmp = cmd(f'systemctl show --value -p ActiveState {service}')
+    return bool((tmp == 'active'))
+
+def is_systemd_service_running(service):
+    """ Test is a specified systemd service is actually running.
+    Returns True if service is running, false otherwise.
+    Copied from: https://unix.stackexchange.com/a/435317 """
+    tmp = cmd(f'systemctl show --value -p SubState {service}')
+    return bool((tmp == 'running'))