From 356950579c2b155f9d41c04ed63c7efde561b43a Mon Sep 17 00:00:00 2001 From: Thomas Mangin Date: Sun, 5 Apr 2020 14:02:49 +0100 Subject: util: T2226: improve popen, cmd and add run improves open and cmd adds a new help function run. the extra optional arguments provide all the features of subprocess used by every module of VyOS. popen matches closely subprocess.Popen cmd matches closely subprocess.check_output run matches closely subprocess.call and os.system popen returns the string and errord code cmd returns the string and raise on error (the class can be defined the default is OSError) run returns the return code of the command the options are: shell=None, the code auto-detect is a shell is required decode=None, use .decode() otherwise the encoding will be used env=None, universal_newlines=None: as per subprocess.Popen input=None, timeout=None, stdout=PIPE, stderr=STDOUT, as per Popen.communicate --- python/vyos/util.py | 67 +++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 52 insertions(+), 15 deletions(-) (limited to 'python') diff --git a/python/vyos/util.py b/python/vyos/util.py index c8ab43c11..f63ed057e 100644 --- a/python/vyos/util.py +++ b/python/vyos/util.py @@ -16,7 +16,7 @@ import os import re import sys -from subprocess import Popen, PIPE, STDOUT +from subprocess import Popen, PIPE, STDOUT, DEVNULL # debugging @@ -33,29 +33,63 @@ def debug_msg(message, section=''): # commands - -def popen(command, section=''): - p = Popen(command, stdout=PIPE, stderr=STDOUT, shell=True) - tmp = p.communicate()[0].strip() +# popen does not raise +# it returns the output of the command and the error code +def popen(command, section='', shell=None, input=None, timeout=None, env=None, universal_newlines=None, stdout=PIPE, stderr=STDOUT, decode=None): + use_shell = shell + if shell is None: + use_shell = True if ' ' in command else False + p = Popen( + command, + stdout=stdout, stderr=stderr, + env=env, shell=use_shell, + universal_newlines=universal_newlines, + ) + tmp = p.communicate(input, timeout)[0].strip() debug_msg(f"cmd '{command}'", section) - decoded = tmp.decode() + decoded = tmp.decode(decode) if decode else tmp.decode() if decoded: debug_msg(f"returned:\n{decoded}", section) return decoded, p.returncode - -def cmd(command, section=''): - decoded, code = popen(command, section) +# run does not raise +# it returns the error code +def run(command, section='', shell=None, input=None, timeout=None, env=None, universal_newlines=None, stdout=PIPE, stderr=STDOUT, decode=None): + _, code = popen( + command, section, + stdout=stdout, stderr=stderr, + input=input, timeout=timeout, + env=env, shell=shell, + universal_newlines=universal_newlines, + decode=decode, + ) + return code + +# cmd does raise +# it returns the output +def cmd(command, section='', shell=None, input=None, timeout=None, env=None, universal_newlines=None, stdout=PIPE, stderr=STDOUT, decode=None, raising=None, message=''): + decoded, code = popen( + command, section, + stdout=stdout, stderr=stderr, + input=input, timeout=timeout, + env=env, shell=shell, + universal_newlines=universal_newlines, + decode=decode, + ) if code != 0: - # error code can be recovered with .errno - raise OSError(code, f'{command}\nreturned: {decoded}') + feedback = message + '\n' if message else '' + feedback += f'failed to run command: {command}\n' + feedback += f'returned: {decoded}\n' + feedback += f'exit code: {code}' + if raising is None: + # error code can be recovered with .errno + raise OSError(code, feedback) + else: + raise raising(feedback) return decoded -# file manipulation - - - +# This is now deprecated def subprocess_cmd(command): """ execute arbitrary command via Popen """ from subprocess import Popen, PIPE @@ -63,6 +97,9 @@ def subprocess_cmd(command): p.communicate() +# file manipulation + + def read_file(path): """ Read a file to string """ with open(path, 'r') as f: -- cgit v1.2.3 From 4b6a6304184a91357c4e8260a134bfa9705f7e23 Mon Sep 17 00:00:00 2001 From: Thomas Mangin Date: Sun, 5 Apr 2020 16:23:12 +0100 Subject: util: T2226: rewrite make_password_hash to use cmd --- python/vyos/authutils.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'python') diff --git a/python/vyos/authutils.py b/python/vyos/authutils.py index 234294649..90a46ffb4 100644 --- a/python/vyos/authutils.py +++ b/python/vyos/authutils.py @@ -15,16 +15,14 @@ import re -from subprocess import Popen, PIPE, STDOUT +from vyos.util import cmd def make_password_hash(password): """ Makes a password hash for /etc/shadow using mkpasswd """ - mkpasswd = Popen(['mkpasswd', '--method=sha-512', '--stdin'], stdout=PIPE, stdin=PIPE, stderr=PIPE) - hash = mkpasswd.communicate(input=password.encode(), timeout=5)[0].decode().strip() - - return hash + mkpassword = 'mkpasswd --method=sha-512 --stdin' + return cmd(mkpassword, input=password.encode(), timeout=5) def split_ssh_public_key(key_string, defaultname=""): """ Splits an SSH public key into its components """ -- cgit v1.2.3 From 9673a300f0b75386fa73b85cb7ccac4094ada6bf Mon Sep 17 00:00:00 2001 From: Thomas Mangin Date: Sun, 5 Apr 2020 16:27:37 +0100 Subject: util: T2226: rewrite EthernetIf.feature to use cmd --- python/vyos/ifconfig/ethernet.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'python') diff --git a/python/vyos/ifconfig/ethernet.py b/python/vyos/ifconfig/ethernet.py index c18d2e72f..291b326bf 100644 --- a/python/vyos/ifconfig/ethernet.py +++ b/python/vyos/ifconfig/ethernet.py @@ -18,7 +18,7 @@ import re from vyos.ifconfig.interface import Interface from vyos.ifconfig.vlan import VLAN -from vyos.util import popen +from vyos.util import run from vyos.validate import * @@ -45,7 +45,7 @@ class EthernetIf(Interface): @staticmethod def feature(ifname, option, value): - out, code = popen(f'/sbin/ethtool -K {ifname} {option} {value}','ifconfig') + run(f'/sbin/ethtool -K {ifname} {option} {value}','ifconfig') return False _command_set = {**Interface._command_set, **{ -- cgit v1.2.3 From 3fb9d920d5d62c0464628a957e227de969dd7239 Mon Sep 17 00:00:00 2001 From: Thomas Mangin Date: Sun, 5 Apr 2020 16:29:07 +0100 Subject: util: T2226: rewrite WireGuardIf to use cmd --- python/vyos/ifconfig/wireguard.py | 1 - 1 file changed, 1 deletion(-) (limited to 'python') diff --git a/python/vyos/ifconfig/wireguard.py b/python/vyos/ifconfig/wireguard.py index def5ab7c5..e2b8a5924 100644 --- a/python/vyos/ifconfig/wireguard.py +++ b/python/vyos/ifconfig/wireguard.py @@ -16,7 +16,6 @@ import os import time -import subprocess from datetime import timedelta from vyos.config import Config -- cgit v1.2.3 From 7219d4013814a55c4eeaf36b7a36d5e8076d09f0 Mon Sep 17 00:00:00 2001 From: Thomas Mangin Date: Sun, 5 Apr 2020 16:30:33 +0100 Subject: util: T2226: rewrite remote function helpers to use cmd --- python/vyos/remote.py | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) (limited to 'python') diff --git a/python/vyos/remote.py b/python/vyos/remote.py index f8a21f068..f0bf41cd4 100644 --- a/python/vyos/remote.py +++ b/python/vyos/remote.py @@ -17,7 +17,8 @@ import sys import os import re import fileinput -import subprocess + +from vyos.util import cmd, DEVNULL def check_and_add_host_key(host_name): @@ -33,10 +34,8 @@ def check_and_add_host_key(host_name): keyscan_cmd = 'ssh-keyscan -t rsa {} 2>/dev/null'.format(host_name) try: - host_key = subprocess.check_output(keyscan_cmd, shell=True, - stderr=subprocess.DEVNULL, - universal_newlines=True) - except subprocess.CalledProcessError as err: + host_key = cmd(keyscan_cmd, stderr=DEVNULL, universal_newlines=True) + except OSError: sys.exit("Can not get RSA host key") # libssh2 (jessie; stretch) does not recognize ec host keys, and curl @@ -64,10 +63,8 @@ def check_and_add_host_key(host_name): fingerprint_cmd = 'ssh-keygen -lf /dev/stdin <<< "{}"'.format(host_key) try: - fingerprint = subprocess.check_output(fingerprint_cmd, shell=True, - stderr=subprocess.DEVNULL, - universal_newlines=True) - except subprocess.CalledProcessError as err: + fingerprint = cmd(fingerprint_cmd, stderr=DEVNULL, universal_newlines=True) + except OSError: sys.exit("Can not get RSA host key fingerprint.") print("RSA host key fingerprint is {}".format(fingerprint.split()[1])) @@ -128,9 +125,8 @@ def get_remote_config(remote_file): # Try header first, and look for 'OK' or 'Moved' codes: curl_cmd = 'curl {0} -q -I {1}'.format(redirect_opt, remote_file) try: - curl_output = subprocess.check_output(curl_cmd, shell=True, - universal_newlines=True) - except subprocess.CalledProcessError: + curl_output = cmd(curl_cmd, shell=True, universal_newlines=True) + except OSError: sys.exit(1) return_vals = re.findall(r'^HTTP\/\d+\.?\d\s+(\d+)\s+(.*)$', @@ -146,9 +142,6 @@ def get_remote_config(remote_file): curl_cmd = 'curl {0} -# {1}'.format(redirect_opt, remote_file) try: - config_file = subprocess.check_output(curl_cmd, shell=True, - universal_newlines=True) - except subprocess.CalledProcessError: - config_file = None - - return config_file + return cmd(curl_cmd, universal_newlines=True) + except OSError: + return None -- cgit v1.2.3 From ecf1e3d499d8a825bb675e11c07bbc7b0981c431 Mon Sep 17 00:00:00 2001 From: Thomas Mangin Date: Sun, 5 Apr 2020 22:10:09 +0100 Subject: util: T2226: rewrite util to use cmd --- python/vyos/util.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'python') diff --git a/python/vyos/util.py b/python/vyos/util.py index f63ed057e..e0030d1bd 100644 --- a/python/vyos/util.py +++ b/python/vyos/util.py @@ -261,12 +261,11 @@ def commit_in_progress(): # Since this will be used in scripts that modify the config outside of the CLI # framework, those knowingly have root permissions. # For everything else, we add a safeguard. - from subprocess import check_output from psutil import process_iter, NoSuchProcess from vyos.defaults import commit_lock - id = check_output(['/usr/bin/id', '-u']).decode().strip() - if id != '0': + idu = cmd('/usr/bin/id -u') + if idu != '0': raise OSError("This functions needs root permissions to return correct results") for proc in process_iter(): -- cgit v1.2.3 From 7256810914e6664bf92041dcd7c3daf649ce0001 Mon Sep 17 00:00:00 2001 From: Thomas Mangin Date: Sun, 5 Apr 2020 23:08:09 +0100 Subject: util: T2226: remove all references to subprocess_cmd --- python/vyos/util.py | 8 -------- src/conf_mode/system-syslog.py | 1 - src/conf_mode/vpn_sstp.py | 2 +- 3 files changed, 1 insertion(+), 10 deletions(-) (limited to 'python') diff --git a/python/vyos/util.py b/python/vyos/util.py index e0030d1bd..fa2b4dd99 100644 --- a/python/vyos/util.py +++ b/python/vyos/util.py @@ -89,14 +89,6 @@ def cmd(command, section='', shell=None, input=None, timeout=None, env=None, uni return decoded -# This is now deprecated -def subprocess_cmd(command): - """ execute arbitrary command via Popen """ - from subprocess import Popen, PIPE - p = Popen(command, stdout=PIPE, shell=True) - p.communicate() - - # file manipulation diff --git a/src/conf_mode/system-syslog.py b/src/conf_mode/system-syslog.py index 915a631ec..25b9b5bed 100755 --- a/src/conf_mode/system-syslog.py +++ b/src/conf_mode/system-syslog.py @@ -22,7 +22,6 @@ from sys import exit from vyos.config import Config from vyos.defaults import directories as vyos_data_dir -from vyos.util import subprocess_cmd from vyos import ConfigError from vyos.util import run diff --git a/src/conf_mode/vpn_sstp.py b/src/conf_mode/vpn_sstp.py index d4fd0f4a3..ca0844c50 100755 --- a/src/conf_mode/vpn_sstp.py +++ b/src/conf_mode/vpn_sstp.py @@ -26,7 +26,7 @@ from jinja2 import FileSystemLoader, Environment from vyos.config import Config from vyos import ConfigError from vyos.defaults import directories as vyos_data_dir -from vyos.util import process_running, subprocess_cmd +from vyos.util import process_running from vyos.util import process_running, cmd, run pidfile = r'/var/run/accel_sstp.pid' -- cgit v1.2.3