From 66af3962fa79e86ca2933289f273ccad08748eab Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Sun, 9 Jun 2024 09:29:52 +0200 Subject: vyos.utils: T5195: import vyos.cpu to this package The intention of vyos.utils package is to have a common ground for repeating actions/helpers. This is also true for number of CPUs and their respective core count. Move vyos.cpu to vyos.utils.cpu (cherry picked from commit e318eb33446de47835480d4b8f1646b39fb5c388) --- python/vyos/configdict.py | 2 +- python/vyos/cpu.py | 102 --------------------------- python/vyos/utils/__init__.py | 3 +- python/vyos/utils/cpu.py | 101 ++++++++++++++++++++++++++ smoketest/scripts/cli/base_accel_ppp_test.py | 2 +- src/conf_mode/container.py | 2 +- src/op_mode/cpu.py | 12 ++-- src/op_mode/uptime.py | 4 +- 8 files changed, 114 insertions(+), 114 deletions(-) delete mode 100644 python/vyos/cpu.py create mode 100644 python/vyos/utils/cpu.py diff --git a/python/vyos/configdict.py b/python/vyos/configdict.py index 870d7cfda..5a353b110 100644 --- a/python/vyos/configdict.py +++ b/python/vyos/configdict.py @@ -631,7 +631,7 @@ def get_accel_dict(config, base, chap_secrets, with_pki=False): Return a dictionary with the necessary interface config keys. """ - from vyos.cpu import get_core_count + from vyos.utils.cpu import get_core_count from vyos.template import is_ipv4 dict = config.get_config_dict(base, key_mangling=('-', '_'), diff --git a/python/vyos/cpu.py b/python/vyos/cpu.py deleted file mode 100644 index 12b6285d0..000000000 --- a/python/vyos/cpu.py +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright (C) 2022-2024 maintainers and contributors -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see . - -""" -Retrieves (or at least attempts to retrieve) the total number of real CPU cores -installed in a Linux system. - -The issue of core count is complicated by existence of SMT, e.g. Intel's Hyper Threading. -GNU nproc returns the number of LOGICAL cores, -which is 2x of the real cores if SMT is enabled. - -The idea is to find all physical CPUs and add up their core counts. -It has special cases for x86_64 and MAY work correctly on other architectures, -but nothing is certain. -""" - -import re - - -def _read_cpuinfo(): - with open('/proc/cpuinfo', 'r') as f: - lines = f.read().strip() - return re.split(r'\n+', lines) - -def _split_line(l): - l = l.strip() - parts = re.split(r'\s*:\s*', l) - return (parts[0], ":".join(parts[1:])) - -def _find_cpus(cpuinfo_lines): - # Make a dict because it's more convenient to work with later, - # when we need to find physicall distinct CPUs there. - cpus = {} - - cpu_number = 0 - - for l in cpuinfo_lines: - key, value = _split_line(l) - if key == 'processor': - cpu_number = value - cpus[cpu_number] = {} - else: - cpus[cpu_number][key] = value - - return cpus - -def _find_physical_cpus(): - cpus = _find_cpus(_read_cpuinfo()) - - phys_cpus = {} - - for num in cpus: - if 'physical id' in cpus[num]: - # On at least some architectures, CPUs in different sockets - # have different 'physical id' field, e.g. on x86_64. - phys_id = cpus[num]['physical id'] - if phys_id not in phys_cpus: - phys_cpus[phys_id] = cpus[num] - else: - # On other architectures, e.g. on ARM, there's no such field. - # We just assume they are different CPUs, - # whether single core ones or cores of physical CPUs. - phys_cpus[num] = cpus[num] - - return phys_cpus - -def get_cpus(): - """ Returns a list of /proc/cpuinfo entries that belong to different CPUs. - """ - cpus_dict = _find_physical_cpus() - return list(cpus_dict.values()) - -def get_core_count(): - """ Returns the total number of physical CPU cores - (even if Hyper-Threading or another SMT is enabled and has inflated - the number of cores in /proc/cpuinfo) - """ - physical_cpus = _find_physical_cpus() - - core_count = 0 - - for num in physical_cpus: - # Some architectures, e.g. x86_64, include a field for core count. - # Since we found unique physical CPU entries, we can sum their core counts. - if 'cpu cores' in physical_cpus[num]: - core_count += int(physical_cpus[num]['cpu cores']) - else: - core_count += 1 - - return core_count diff --git a/python/vyos/utils/__init__.py b/python/vyos/utils/__init__.py index 12ef2d3b8..1cd062a11 100644 --- a/python/vyos/utils/__init__.py +++ b/python/vyos/utils/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2023 VyOS maintainers and contributors +# Copyright 2024 VyOS maintainers and contributors # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -18,6 +18,7 @@ from vyos.utils import auth from vyos.utils import boot from vyos.utils import commit from vyos.utils import convert +from vyos.utils import cpu from vyos.utils import dict from vyos.utils import file from vyos.utils import io diff --git a/python/vyos/utils/cpu.py b/python/vyos/utils/cpu.py new file mode 100644 index 000000000..0f74add0e --- /dev/null +++ b/python/vyos/utils/cpu.py @@ -0,0 +1,101 @@ +# Copyright (C) 2022-2024 maintainers and contributors +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see . + +""" +Retrieves (or at least attempts to retrieve) the total number of real CPU cores +installed in a Linux system. + +The issue of core count is complicated by existence of SMT, e.g. Intel's Hyper Threading. +GNU nproc returns the number of LOGICAL cores, +which is 2x of the real cores if SMT is enabled. + +The idea is to find all physical CPUs and add up their core counts. +It has special cases for x86_64 and MAY work correctly on other architectures, +but nothing is certain. +""" + +import re + +def _read_cpuinfo(): + with open('/proc/cpuinfo', 'r') as f: + lines = f.read().strip() + return re.split(r'\n+', lines) + +def _split_line(l): + l = l.strip() + parts = re.split(r'\s*:\s*', l) + return (parts[0], ":".join(parts[1:])) + +def _find_cpus(cpuinfo_lines): + # Make a dict because it's more convenient to work with later, + # when we need to find physicall distinct CPUs there. + cpus = {} + + cpu_number = 0 + + for l in cpuinfo_lines: + key, value = _split_line(l) + if key == 'processor': + cpu_number = value + cpus[cpu_number] = {} + else: + cpus[cpu_number][key] = value + + return cpus + +def _find_physical_cpus(): + cpus = _find_cpus(_read_cpuinfo()) + + phys_cpus = {} + + for num in cpus: + if 'physical id' in cpus[num]: + # On at least some architectures, CPUs in different sockets + # have different 'physical id' field, e.g. on x86_64. + phys_id = cpus[num]['physical id'] + if phys_id not in phys_cpus: + phys_cpus[phys_id] = cpus[num] + else: + # On other architectures, e.g. on ARM, there's no such field. + # We just assume they are different CPUs, + # whether single core ones or cores of physical CPUs. + phys_cpus[num] = cpus[num] + + return phys_cpus + +def get_cpus(): + """ Returns a list of /proc/cpuinfo entries that belong to different CPUs. + """ + cpus_dict = _find_physical_cpus() + return list(cpus_dict.values()) + +def get_core_count(): + """ Returns the total number of physical CPU cores + (even if Hyper-Threading or another SMT is enabled and has inflated + the number of cores in /proc/cpuinfo) + """ + physical_cpus = _find_physical_cpus() + + core_count = 0 + + for num in physical_cpus: + # Some architectures, e.g. x86_64, include a field for core count. + # Since we found unique physical CPU entries, we can sum their core counts. + if 'cpu cores' in physical_cpus[num]: + core_count += int(physical_cpus[num]['cpu cores']) + else: + core_count += 1 + + return core_count diff --git a/smoketest/scripts/cli/base_accel_ppp_test.py b/smoketest/scripts/cli/base_accel_ppp_test.py index 212dc58ab..c6f6cb804 100644 --- a/smoketest/scripts/cli/base_accel_ppp_test.py +++ b/smoketest/scripts/cli/base_accel_ppp_test.py @@ -19,7 +19,7 @@ from configparser import ConfigParser from vyos.configsession import ConfigSessionError from vyos.template import is_ipv4 -from vyos.cpu import get_core_count +from vyos.utils.cpu import get_core_count from vyos.utils.process import process_named_running from vyos.utils.process import cmd diff --git a/src/conf_mode/container.py b/src/conf_mode/container.py index 3efeb9b40..0ce528045 100755 --- a/src/conf_mode/container.py +++ b/src/conf_mode/container.py @@ -29,7 +29,7 @@ from vyos.configdict import node_changed from vyos.configdict import is_node_changed from vyos.configverify import verify_vrf from vyos.ifconfig import Interface -from vyos.cpu import get_core_count +from vyos.utils.cpu import get_core_count from vyos.utils.file import write_file from vyos.utils.process import call from vyos.utils.process import cmd diff --git a/src/op_mode/cpu.py b/src/op_mode/cpu.py index d53663c17..1a0f7392f 100755 --- a/src/op_mode/cpu.py +++ b/src/op_mode/cpu.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2016-2022 VyOS maintainers and contributors +# Copyright (C) 2016-2024 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -16,8 +16,9 @@ import sys -import vyos.cpu import vyos.opmode +from vyos.utils.cpu import get_cpus +from vyos.utils.cpu import get_core_count from jinja2 import Template @@ -37,15 +38,15 @@ CPU model(s): {{models | join(", ")}} """) def _get_raw_data(): - return vyos.cpu.get_cpus() + return get_cpus() def _format_cpus(cpu_data): env = {'cpus': cpu_data} return cpu_template.render(env).strip() def _get_summary_data(): - count = vyos.cpu.get_core_count() - cpu_data = vyos.cpu.get_cpus() + count = get_core_count() + cpu_data = get_cpus() models = [c['model name'] for c in cpu_data] env = {'count': count, "models": models} @@ -79,4 +80,3 @@ if __name__ == '__main__': except (ValueError, vyos.opmode.Error) as e: print(e) sys.exit(1) - diff --git a/src/op_mode/uptime.py b/src/op_mode/uptime.py index 059a4c3f6..559eed24c 100755 --- a/src/op_mode/uptime.py +++ b/src/op_mode/uptime.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2021-2023 VyOS maintainers and contributors +# Copyright (C) 2021-2024 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -29,8 +29,8 @@ def _get_uptime_seconds(): def _get_load_averages(): from re import search + from vyos.utils.cpu import get_core_count from vyos.utils.process import cmd - from vyos.cpu import get_core_count data = cmd("uptime") matches = search(r"load average:\s*(?P[0-9\.]+)\s*,\s*(?P[0-9\.]+)\s*,\s*(?P[0-9\.]+)\s*", data) -- cgit v1.2.3