diff options
33 files changed, 582 insertions, 124 deletions
diff --git a/.github/workflows/chceck-pr-message.yml b/.github/workflows/chceck-pr-message.yml index 460662014..5eb2d840a 100644 --- a/.github/workflows/chceck-pr-message.yml +++ b/.github/workflows/chceck-pr-message.yml @@ -7,6 +7,7 @@ on: - current - crux - equuleus + - sagitta types: [opened, synchronize, edited] permissions: diff --git a/.github/workflows/check-unused-imports.yml b/.github/workflows/check-unused-imports.yml index aada264f7..0f0cff3ec 100644 --- a/.github/workflows/check-unused-imports.yml +++ b/.github/workflows/check-unused-imports.yml @@ -3,6 +3,7 @@ on: pull_request: branches: - current + - equuleus - sagitta workflow_dispatch: diff --git a/.github/workflows/linit-j2.yml b/.github/workflows/linit-j2.yml deleted file mode 100644 index 364a65a14..000000000 --- a/.github/workflows/linit-j2.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- -name: J2 Lint - -on: - pull_request: - branches: - - current - - crux - - equuleus - -permissions: - pull-requests: write - contents: read - -jobs: - j2lint: - uses: vyos/.github/.github/workflows/lint-j2.yml@feature/T6349-reusable-workflows - secrets: inherit diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml new file mode 100644 index 000000000..5fa005631 --- /dev/null +++ b/.github/workflows/sonarcloud.yml @@ -0,0 +1,20 @@ +name: Sonar Checks +on: + push: + branches: + - current + pull_request_target: + types: [opened, synchronize, reopened] +jobs: + sonar-cloud: + name: SonarCloud + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis + - name: SonarCloud Scan + uses: SonarSource/sonarcloud-github-action@master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} @@ -61,12 +61,13 @@ op_mode_definitions: $(op_xml_obj) rm -f $(OP_TMPL_DIR)/clear/node.def rm -f $(OP_TMPL_DIR)/delete/node.def - # XXX: ping, traceroute and mtr must be able to recursivly call themselves as the + # XXX: tcpdump, ping, traceroute and mtr must be able to recursivly call themselves as the # options are provided from the scripts themselves ln -s ../node.tag $(OP_TMPL_DIR)/ping/node.tag/node.tag/ ln -s ../node.tag $(OP_TMPL_DIR)/traceroute/node.tag/node.tag/ ln -s ../node.tag $(OP_TMPL_DIR)/mtr/node.tag/node.tag/ ln -s ../node.tag $(OP_TMPL_DIR)/monitor/traceroute/node.tag/node.tag/ + ln -s ../node.tag $(OP_TMPL_DIR)/monitor/traffic/interface/node.tag/node.tag/ # XXX: test if there are empty node.def files - this is not allowed as these # could mask help strings or mandatory priority statements diff --git a/data/templates/conntrackd/conntrackd.op-mode.j2 b/data/templates/conntrackd/conntrackd.op-mode.j2 deleted file mode 100644 index 82f7e2859..000000000 --- a/data/templates/conntrackd/conntrackd.op-mode.j2 +++ /dev/null @@ -1,13 +0,0 @@ -Source Destination Protocol -{% for parsed in data if parsed.flow.meta is vyos_defined %} -{% for key in parsed.flow.meta %} -{% if key['@direction'] == 'original' %} -{% set saddr = key.layer3.src | bracketize_ipv6 %} -{% set sport = key.layer4.sport %} -{% set daddr = key.layer3.dst | bracketize_ipv6 %} -{% set dport = key.layer4.dport %} -{% set protocol = key.layer4['@protoname'] %} -{{ "%-48s" | format(saddr ~ ':' ~ sport) }} {{ "%-48s" | format(daddr ~ ':' ~ dport) }} {{ protocol }} -{% endif %} -{% endfor %} -{% endfor %} diff --git a/interface-definitions/container.xml.in b/interface-definitions/container.xml.in index 1ad7215e5..6ea44a6d4 100644 --- a/interface-definitions/container.xml.in +++ b/interface-definitions/container.xml.in @@ -71,6 +71,35 @@ <multi/> </properties> </leafNode> + <node name="sysctl"> + <properties> + <help>Configure namespaced kernel parameters of the container</help> + </properties> + <children> + <tagNode name="parameter"> + <properties> + <help>Sysctl key name</help> + <completionHelp> + <script>${vyos_completion_dir}/list_container_sysctl_parameters.sh</script> + </completionHelp> + <valueHelp> + <format>txt</format> + <description>Sysctl key name</description> + </valueHelp> + <constraint> + <validator name="sysctl"/> + </constraint> + </properties> + <children> + <leafNode name="value"> + <properties> + <help>Sysctl configuration value</help> + </properties> + </leafNode> + </children> + </tagNode> + </children> + </node> #include <include/generic-description.xml.i> <tagNode name="device"> <properties> diff --git a/interface-definitions/include/bgp/peer-group.xml.i b/interface-definitions/include/bgp/peer-group.xml.i index 3866fc017..c80d4a394 100644 --- a/interface-definitions/include/bgp/peer-group.xml.i +++ b/interface-definitions/include/bgp/peer-group.xml.i @@ -3,7 +3,7 @@ <properties> <help>Peer group for this peer</help> <completionHelp> - <path>protocols bgp peer-group</path> + <path>${COMP_WORDS[@]:1:${#COMP_WORDS[@]}-5} peer-group</path> </completionHelp> <valueHelp> <format>txt</format> diff --git a/interface-definitions/nat_cgnat.xml.in b/interface-definitions/nat_cgnat.xml.in index fce5e655d..71f4d67b0 100644 --- a/interface-definitions/nat_cgnat.xml.in +++ b/interface-definitions/nat_cgnat.xml.in @@ -8,6 +8,12 @@ <priority>221</priority> </properties> <children> + <leafNode name="log-allocation"> + <properties> + <help>Log IP address and port allocation</help> + <valueless/> + </properties> + </leafNode> <node name="pool"> <properties> <help>External and internal pool parameters</help> diff --git a/op-mode-definitions/lldp.xml.in b/op-mode-definitions/lldp.xml.in index 985262a89..dc1331cc8 100644 --- a/op-mode-definitions/lldp.xml.in +++ b/op-mode-definitions/lldp.xml.in @@ -13,6 +13,12 @@ </properties> <command>${vyos_op_scripts_dir}/lldp.py show_neighbors</command> <children> + <node name="detail"> + <properties> + <help>Show extended detail for LLDP neighbors</help> + </properties> + <command>${vyos_op_scripts_dir}/lldp.py show_neighbors --detail</command> + </node> <tagNode name="interface"> <properties> <help>Show LLDP for specified interface</help> @@ -21,6 +27,17 @@ </completionHelp> </properties> <command>${vyos_op_scripts_dir}/lldp.py show_neighbors --interface $5</command> + <children> + <node name="detail"> + <properties> + <help>Show detailed LLDP for specified interface</help> + <completionHelp> + <script>${vyos_completion_dir}/list_interfaces</script> + </completionHelp> + </properties> + <command>${vyos_op_scripts_dir}/lldp.py show_neighbors --interface $5 --detail</command> + </node> + </children> </tagNode> </children> </node> diff --git a/op-mode-definitions/monitor-log.xml.in b/op-mode-definitions/monitor-log.xml.in index 559952e25..a2d5d924a 100644 --- a/op-mode-definitions/monitor-log.xml.in +++ b/op-mode-definitions/monitor-log.xml.in @@ -359,6 +359,47 @@ </properties> <command>journalctl --no-hostname --boot --follow --unit keepalived.service</command> </leafNode> + <node name="wireless"> + <properties> + <help>Monitor last lines of Wireless interface log</help> + </properties> + <children> + <node name="wpa-supplicant"> + <properties> + <help>Monitor last lines of WPA supplicant</help> + </properties> + <command>if cli-shell-api existsActive interfaces wireless; then journalctl --no-hostname --boot --follow --unit "wpa_supplicant@*.service"; else echo "No wireless interface configured!"; fi</command> + <children> + <tagNode name="interface"> + <properties> + <help>Monitor last lines of specific wireless interface supplicant</help> + <completionHelp> + <path>interfaces wireless</path> + </completionHelp> + </properties> + <command>if [[ $(cli-shell-api returnActiveValue interfaces wireless $6 type) == "station" ]]; then journalctl --no-hostname --boot --follow --unit "wpa_supplicant@$6.service"; else echo "Wireless interface $6 not configured as station!"; fi</command> + </tagNode> + </children> + </node> + <node name="hostapd"> + <properties> + <help>Monitor last lines of host access point daemon</help> + </properties> + <command>if cli-shell-api existsActive interfaces wireless; then journalctl --no-hostname --boot --follow --unit "hostapd@*.service"; else echo "No wireless interface configured!"; fi</command> + <children> + <tagNode name="interface"> + <properties> + <help>Monitor last lines of specific host access point interface</help> + <completionHelp> + <path>interfaces wireless</path> + </completionHelp> + </properties> + <command>if [[ $(cli-shell-api returnActiveValue interfaces wireless $6 type) == "access-point" ]]; then journalctl --no-hostname --boot --follow --unit "hostapd@$6.service"; else echo "Wireless interface $6 not configured as access-point!"; fi</command> + </tagNode> + </children> + </node> + </children> + </node> </children> </node> </children> diff --git a/op-mode-definitions/show-log.xml.in b/op-mode-definitions/show-log.xml.in index c3aa324ba..7ae3b890b 100644 --- a/op-mode-definitions/show-log.xml.in +++ b/op-mode-definitions/show-log.xml.in @@ -762,6 +762,47 @@ </properties> <command>journalctl --no-hostname --boot --unit keepalived.service</command> </leafNode> + <node name="wireless"> + <properties> + <help>Show log for Wireless interface</help> + </properties> + <children> + <node name="wpa-supplicant"> + <properties> + <help>Show log for WPA supplicant</help> + </properties> + <command>if cli-shell-api existsActive interfaces wireless; then journalctl --no-hostname --boot --unit "wpa_supplicant@*.service"; else echo "No wireless interface configured!"; fi</command> + <children> + <tagNode name="interface"> + <properties> + <help>Show log for specific wireless interface supplicant</help> + <completionHelp> + <path>interfaces wireless</path> + </completionHelp> + </properties> + <command>if [[ $(cli-shell-api returnActiveValue interfaces wireless $6 type) == "station" ]]; then journalctl --no-hostname --boot --unit "wpa_supplicant@$6.service"; else echo "Wireless interface $6 not configured as station!"; fi</command> + </tagNode> + </children> + </node> + <node name="hostapd"> + <properties> + <help>Show log for host access point daemon</help> + </properties> + <command>if cli-shell-api existsActive interfaces wireless; then journalctl --no-hostname --boot --unit "hostapd@*.service"; else echo "No wireless interface configured!"; fi</command> + <children> + <tagNode name="interface"> + <properties> + <help>Show log for specific host access point daemon interface</help> + <completionHelp> + <path>interfaces wireless</path> + </completionHelp> + </properties> + <command>if [[ $(cli-shell-api returnActiveValue interfaces wireless $6 type) == "access-point" ]]; then journalctl --no-hostname --boot --unit "hostapd@$6.service"; else echo "Wireless interface $6 not configured as access-point!"; fi</command> + </tagNode> + </children> + </node> + </children> + </node> <leafNode name="webproxy"> <properties> <help>Show log for Webproxy</help> diff --git a/op-mode-definitions/traffic-dump.xml.in b/op-mode-definitions/traffic-dump.xml.in index 76e3ddce5..e86e69736 100644 --- a/op-mode-definitions/traffic-dump.xml.in +++ b/op-mode-definitions/traffic-dump.xml.in @@ -8,7 +8,7 @@ </properties> <children> <tagNode name="interface"> - <command>sudo tcpdump -i $4</command> + <command>${vyos_op_scripts_dir}/tcpdump.py $4</command> <properties> <help>Monitor traffic dump from an interface</help> <completionHelp> @@ -16,54 +16,15 @@ </completionHelp> </properties> <children> - <node name="verbose"> - <command>sudo tcpdump -vvv -ne -i $4</command> + <leafNode name="node.tag"> <properties> - <help>Provide more detailed packets for each monitored traffic</help> + <help>Traffic capture options</help> + <completionHelp> + <script>${vyos_op_scripts_dir}/tcpdump.py --get-options-nested "${COMP_WORDS[@]}"</script> + </completionHelp> </properties> - <children> - <tagNode name="filter"> - <command>sudo tcpdump -vvv -ne -i $4 "${@:6}"</command> - <properties> - <help>Monitor traffic matching filter conditions</help> - </properties> - </tagNode> - <tagNode name="save"> - <command>sudo tcpdump -vvv -ne -i $4 -w $6</command> - <properties> - <help>Save traffic dump from an interface to a file</help> - </properties> - <children> - <tagNode name="filter"> - <command>sudo tcpdump -vvv -ne -i $4 -w $6 "${@:8}"</command> - <properties> - <help>Save a dump of traffic matching filter conditions to a file</help> - </properties> - </tagNode> - </children> - </tagNode> - </children> - </node> - <tagNode name="filter"> - <command>sudo tcpdump -n -i $4 "${@:6}"</command> - <properties> - <help>Monitor traffic matching filter conditions</help> - </properties> - </tagNode> - <tagNode name="save"> - <command>sudo tcpdump -n -i $4 -w $6</command> - <properties> - <help>Save traffic dump from an interface to a file</help> - </properties> - <children> - <tagNode name="filter"> - <command>sudo tcpdump -n -i $4 -w $6 "${@:8}"</command> - <properties> - <help>Save a dump of traffic matching filter conditions to a file</help> - </properties> - </tagNode> - </children> - </tagNode> + <command>${vyos_op_scripts_dir}/tcpdump.py "${@:4}"</command> + </leafNode> </children> </tagNode> </children> diff --git a/python/vyos/configdict.py b/python/vyos/configdict.py index 870d7cfda..5a353b110 100644 --- a/python/vyos/configdict.py +++ b/python/vyos/configdict.py @@ -631,7 +631,7 @@ def get_accel_dict(config, base, chap_secrets, with_pki=False): Return a dictionary with the necessary interface config keys. """ - from vyos.cpu import get_core_count + from vyos.utils.cpu import get_core_count from vyos.template import is_ipv4 dict = config.get_config_dict(base, key_mangling=('-', '_'), diff --git a/python/vyos/configquery.py b/python/vyos/configquery.py index 71ad5b4f0..5d6ca9be9 100644 --- a/python/vyos/configquery.py +++ b/python/vyos/configquery.py @@ -1,4 +1,4 @@ -# Copyright 2021-2023 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2021-2024 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -19,6 +19,8 @@ settings from op mode, and execution of arbitrary op mode commands. ''' import os +import json +import subprocess from vyos.utils.process import STDOUT from vyos.utils.process import popen @@ -27,6 +29,14 @@ from vyos.utils.boot import boot_configuration_complete from vyos.config import Config from vyos.configsource import ConfigSourceSession, ConfigSourceString from vyos.defaults import directories +from vyos.configtree import ConfigTree +from vyos.utils.dict import embed_dict +from vyos.utils.dict import get_sub_dict +from vyos.utils.dict import mangle_dict_keys +from vyos.utils.error import cli_shell_api_err +from vyos.xml_ref import multi_to_list +from vyos.xml_ref import is_tag +from vyos.base import Warning config_file = os.path.join(directories['config'], 'config.boot') @@ -133,4 +143,50 @@ def query_context(config_query_class=CliShellApiConfigQuery, run = op_run_class() return query, run - +def verify_mangling(key_mangling): + if not (isinstance(key_mangling, tuple) and + len(key_mangling) == 2 and + isinstance(key_mangling[0], str) and + isinstance(key_mangling[1], str)): + raise ValueError("key_mangling must be a tuple of two strings") + +def op_mode_run(cmd): + """ low-level to avoid overhead """ + p = subprocess.Popen(cmd, stdout=subprocess.PIPE) + out = p.stdout.read() + p.wait() + return p.returncode, out.decode() + +def op_mode_config_dict(path=None, key_mangling=None, + no_tag_node_value_mangle=False, + no_multi_convert=False, get_first_key=False): + + if path is None: + path = [] + command = ['/bin/cli-shell-api', '--show-active-only', 'showConfig'] + + rc, out = op_mode_run(command + path) + if rc == cli_shell_api_err.VYOS_EMPTY_CONFIG: + out = '' + if rc == cli_shell_api_err.VYOS_INVALID_PATH: + Warning(out) + return {} + + ct = ConfigTree(out) + d = json.loads(ct.to_json()) + # cli-shell-api output includes last path component if tag node + if is_tag(path): + config_dict = embed_dict(path[:-1], d) + else: + config_dict = embed_dict(path, d) + + if not no_multi_convert: + config_dict = multi_to_list([], config_dict) + + if key_mangling is not None: + verify_mangling(key_mangling) + config_dict = mangle_dict_keys(config_dict, + key_mangling[0], key_mangling[1], + no_tag_node_value_mangle=no_tag_node_value_mangle) + + return get_sub_dict(config_dict, path, get_first_key=get_first_key) diff --git a/python/vyos/utils/__init__.py b/python/vyos/utils/__init__.py index 12ef2d3b8..1cd062a11 100644 --- a/python/vyos/utils/__init__.py +++ b/python/vyos/utils/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2023 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2024 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -18,6 +18,7 @@ from vyos.utils import auth from vyos.utils import boot from vyos.utils import commit from vyos.utils import convert +from vyos.utils import cpu from vyos.utils import dict from vyos.utils import file from vyos.utils import io diff --git a/python/vyos/cpu.py b/python/vyos/utils/cpu.py index cae5f5f4d..3bea5ac12 100644 --- a/python/vyos/cpu.py +++ b/python/vyos/utils/cpu.py @@ -28,7 +28,6 @@ but nothing is certain. import re - def _read_cpuinfo(): with open('/proc/cpuinfo', 'r') as f: lines = f.read().strip() diff --git a/python/vyos/utils/dict.py b/python/vyos/utils/dict.py index d36b6fcfb..062ab9c81 100644 --- a/python/vyos/utils/dict.py +++ b/python/vyos/utils/dict.py @@ -307,6 +307,13 @@ def dict_to_paths(d: dict) -> list: for r in func(d, []): yield r +def embed_dict(p: list[str], d: dict) -> dict: + path = p.copy() + ret = d + while path: + ret = {path.pop(): ret} + return ret + def check_mutually_exclusive_options(d, keys, required=False): """ Checks if a dict has at most one or only one of mutually exclusive keys. diff --git a/python/vyos/utils/error.py b/python/vyos/utils/error.py new file mode 100644 index 000000000..8d4709bff --- /dev/null +++ b/python/vyos/utils/error.py @@ -0,0 +1,24 @@ +# Copyright 2024 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see <http://www.gnu.org/licenses/>. + +from enum import IntEnum + +class cli_shell_api_err(IntEnum): + """ vyatta-cfg/src/vyos-errors.h """ + VYOS_SUCCESS = 0 + VYOS_GENERAL_FAILURE = 1 + VYOS_INVALID_PATH = 2 + VYOS_EMPTY_CONFIG = 3 + VYOS_CONFIG_PARSE_ERROR = 4 diff --git a/smoketest/config-tests/container-simple b/smoketest/config-tests/container-simple index cc80ef4cf..5af365cf9 100644 --- a/smoketest/config-tests/container-simple +++ b/smoketest/config-tests/container-simple @@ -11,3 +11,4 @@ set container name c02 allow-host-networks set container name c02 allow-host-pid set container name c02 capability 'sys-time' set container name c02 image 'busybox:stable' +set container name c02 sysctl parameter kernel.msgmax value '8192'
\ No newline at end of file diff --git a/smoketest/config-tests/dialup-router-wireguard-ipv6 b/smoketest/config-tests/dialup-router-wireguard-ipv6 index c054b4650..814a62d55 100644 --- a/smoketest/config-tests/dialup-router-wireguard-ipv6 +++ b/smoketest/config-tests/dialup-router-wireguard-ipv6 @@ -192,10 +192,6 @@ set service snmp location 'CLOUD' set system conntrack expect-table-size '2048' set system conntrack hash-size '32768' set system conntrack table-size '262144' -set system conntrack timeout icmp '30' -set system conntrack timeout other '600' -set system conntrack timeout udp other '300' -set system conntrack timeout udp stream '300' set system domain-name 'vyos.net' set system host-name 'r1' set system login user vyos authentication encrypted-password '$6$2Ta6TWHd/U$NmrX0x9kexCimeOcYK1MfhMpITF9ELxHcaBU/znBq.X2ukQOj61fVI2UYP/xBzP4QtiTcdkgs7WOQMHWsRymO/' @@ -216,6 +212,10 @@ set firewall global-options receive-redirects 'disable' set firewall global-options send-redirects 'enable' set firewall global-options source-validation 'disable' set firewall global-options syn-cookies 'enable' +set firewall global-options timeout icmp '30' +set firewall global-options timeout other '600' +set firewall global-options timeout udp other '300' +set firewall global-options timeout udp stream '300' set firewall global-options twa-hazards-protection 'disable' set firewall group address-group DMZ-RDP-SERVER address '172.16.33.40' set firewall group address-group DMZ-RDP-SERVER description 'Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata' diff --git a/smoketest/configs/container-simple b/smoketest/configs/container-simple index 82983afb7..b98a440b5 100644 --- a/smoketest/configs/container-simple +++ b/smoketest/configs/container-simple @@ -10,6 +10,11 @@ container { allow-host-pid cap-add sys-time image busybox:stable + sysctl { + parameter kernel.msgmax { + value "8192" + } + } } } interfaces { diff --git a/smoketest/scripts/cli/base_accel_ppp_test.py b/smoketest/scripts/cli/base_accel_ppp_test.py index 212dc58ab..c6f6cb804 100644 --- a/smoketest/scripts/cli/base_accel_ppp_test.py +++ b/smoketest/scripts/cli/base_accel_ppp_test.py @@ -19,7 +19,7 @@ from configparser import ConfigParser from vyos.configsession import ConfigSessionError from vyos.template import is_ipv4 -from vyos.cpu import get_core_count +from vyos.utils.cpu import get_core_count from vyos.utils.process import process_named_running from vyos.utils.process import cmd diff --git a/smoketest/scripts/cli/test_container.py b/smoketest/scripts/cli/test_container.py index 90f821c60..3dd97a175 100755 --- a/smoketest/scripts/cli/test_container.py +++ b/smoketest/scripts/cli/test_container.py @@ -80,6 +80,7 @@ class TestContainer(VyOSUnitTestSHIM.TestCase): self.cli_set(base_path + ['name', cont_name, 'image', cont_image]) self.cli_set(base_path + ['name', cont_name, 'allow-host-networks']) + self.cli_set(base_path + ['name', cont_name, 'sysctl', 'parameter', 'kernel.msgmax', 'value', '4096']) # commit changes self.cli_commit() @@ -91,6 +92,10 @@ class TestContainer(VyOSUnitTestSHIM.TestCase): # Check for running process self.assertEqual(process_named_running(PROCESS_NAME), pid) + # verify + tmp = cmd(f'sudo podman exec -it {cont_name} sysctl kernel.msgmax') + self.assertEqual(tmp, 'kernel.msgmax = 4096') + def test_cpu_limit(self): cont_name = 'c2' diff --git a/src/completion/list_container_sysctl_parameters.sh b/src/completion/list_container_sysctl_parameters.sh new file mode 100755 index 000000000..cf8d006e5 --- /dev/null +++ b/src/completion/list_container_sysctl_parameters.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# +# Copyright (C) 2024 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +declare -a vals +eval "vals=($(/sbin/sysctl -N -a|grep -E '^(fs.mqueue|net)\.|^(kernel.msgmax|kernel.msgmnb|kernel.msgmni|kernel.sem|kernel.shmall|kernel.shmmax|kernel.shmmni|kernel.shm_rmid_forced)$'))" +echo ${vals[@]} +exit 0 diff --git a/src/conf_mode/container.py b/src/conf_mode/container.py index 3efeb9b40..ded370a7a 100755 --- a/src/conf_mode/container.py +++ b/src/conf_mode/container.py @@ -29,7 +29,7 @@ from vyos.configdict import node_changed from vyos.configdict import is_node_changed from vyos.configverify import verify_vrf from vyos.ifconfig import Interface -from vyos.cpu import get_core_count +from vyos.utils.cpu import get_core_count from vyos.utils.file import write_file from vyos.utils.process import call from vyos.utils.process import cmd @@ -43,6 +43,7 @@ from vyos.template import render from vyos.xml_ref import default_value from vyos import ConfigError from vyos import airbag + airbag.enable() config_containers = '/etc/containers/containers.conf' @@ -50,16 +51,19 @@ config_registry = '/etc/containers/registries.conf' config_storage = '/etc/containers/storage.conf' systemd_unit_path = '/run/systemd/system' + def _cmd(command): if os.path.exists('/tmp/vyos.container.debug'): print(command) return cmd(command) + def network_exists(name): # Check explicit name for network, returns True if network exists c = _cmd(f'podman network ls --quiet --filter name=^{name}$') return bool(c) + # Common functions def get_config(config=None): if config: @@ -86,21 +90,22 @@ def get_config(config=None): # registry is a tagNode with default values - merge the list from # default_values['registry'] into the tagNode variables if 'registry' not in container: - container.update({'registry' : {}}) + container.update({'registry': {}}) default_values = default_value(base + ['registry']) for registry in default_values: - tmp = {registry : {}} + tmp = {registry: {}} container['registry'] = dict_merge(tmp, container['registry']) # Delete container network, delete containers tmp = node_changed(conf, base + ['network']) - if tmp: container.update({'network_remove' : tmp}) + if tmp: container.update({'network_remove': tmp}) tmp = node_changed(conf, base + ['name']) - if tmp: container.update({'container_remove' : tmp}) + if tmp: container.update({'container_remove': tmp}) return container + def verify(container): # bail out early - looks like removal from running config if not container: @@ -125,8 +130,8 @@ def verify(container): # of image upgrade and deletion. image = container_config['image'] if run(f'podman image exists {image}') != 0: - Warning(f'Image "{image}" used in container "{name}" does not exist '\ - f'locally. Please use "add container image {image}" to add it '\ + Warning(f'Image "{image}" used in container "{name}" does not exist ' \ + f'locally. Please use "add container image {image}" to add it ' \ f'to the system! Container "{name}" will not be started!') if 'cpu_quota' in container_config: @@ -167,11 +172,11 @@ def verify(container): # We can not use the first IP address of a network prefix as this is used by podman if ip_address(address) == ip_network(network)[1]: - raise ConfigError(f'IP address "{address}" can not be used for a container, '\ + raise ConfigError(f'IP address "{address}" can not be used for a container, ' \ 'reserved for the container engine!') if cnt_ipv4 > 1 or cnt_ipv6 > 1: - raise ConfigError(f'Only one IP address per address family can be used for '\ + raise ConfigError(f'Only one IP address per address family can be used for ' \ f'container "{name}". {cnt_ipv4} IPv4 and {cnt_ipv6} IPv6 address(es)!') if 'device' in container_config: @@ -186,6 +191,13 @@ def verify(container): if not os.path.exists(source): raise ConfigError(f'Device "{dev}" source path "{source}" does not exist!') + if 'sysctl' in container_config and 'parameter' in container_config['sysctl']: + for var, cfg in container_config['sysctl']['parameter'].items(): + if 'value' not in cfg: + raise ConfigError(f'sysctl parameter {var} has no value assigned!') + if var.startswith('net.') and 'allow_host_networks' in container_config: + raise ConfigError(f'sysctl parameter {var} cannot be set when using host networking!') + if 'environment' in container_config: for var, cfg in container_config['environment'].items(): if 'value' not in cfg: @@ -219,7 +231,8 @@ def verify(container): # Can not set both allow-host-networks and network at the same time if {'allow_host_networks', 'network'} <= set(container_config): - raise ConfigError(f'"allow-host-networks" and "network" for "{name}" cannot be both configured at the same time!') + raise ConfigError( + f'"allow-host-networks" and "network" for "{name}" cannot be both configured at the same time!') # gid cannot be set without uid if 'gid' in container_config and 'uid' not in container_config: @@ -235,8 +248,10 @@ def verify(container): raise ConfigError(f'prefix for network "{network}" must be defined!') for prefix in network_config['prefix']: - if is_ipv4(prefix): v4_prefix += 1 - elif is_ipv6(prefix): v6_prefix += 1 + if is_ipv4(prefix): + v4_prefix += 1 + elif is_ipv6(prefix): + v6_prefix += 1 if v4_prefix > 1: raise ConfigError(f'Only one IPv4 prefix can be defined for network "{network}"!') @@ -262,6 +277,7 @@ def verify(container): return None + def generate_run_arguments(name, container_config): image = container_config['image'] cpu_quota = container_config['cpu_quota'] @@ -269,6 +285,12 @@ def generate_run_arguments(name, container_config): shared_memory = container_config['shared_memory'] restart = container_config['restart'] + # Add sysctl options + sysctl_opt = '' + if 'sysctl' in container_config and 'parameter' in container_config['sysctl']: + for k, v in container_config['sysctl']['parameter'].items(): + sysctl_opt += f" --sysctl {k}={v['value']}" + # Add capability options. Should be in uppercase capabilities = '' if 'capability' in container_config: @@ -341,7 +363,7 @@ def generate_run_arguments(name, container_config): if 'allow_host_pid' in container_config: host_pid = '--pid host' - container_base_cmd = f'--detach --interactive --tty --replace {capabilities} --cpus {cpu_quota} ' \ + container_base_cmd = f'--detach --interactive --tty --replace {capabilities} --cpus {cpu_quota} {sysctl_opt} ' \ f'--memory {memory}m --shm-size {shared_memory}m --memory-swap 0 --restart {restart} ' \ f'--name {name} {hostname} {device} {port} {volume} {env_opt} {label} {uid} {host_pid}' @@ -375,6 +397,7 @@ def generate_run_arguments(name, container_config): return f'{container_base_cmd} --no-healthcheck --net {networks} {ip_param} {entrypoint} {image} {command} {command_arguments}'.strip() + def generate(container): # bail out early - looks like removal from running config if not container: @@ -387,7 +410,7 @@ def generate(container): for network, network_config in container['network'].items(): tmp = { 'name': network, - 'id' : sha256(f'{network}'.encode()).hexdigest(), + 'id': sha256(f'{network}'.encode()).hexdigest(), 'driver': 'bridge', 'network_interface': f'pod-{network}', 'subnets': [], @@ -399,7 +422,7 @@ def generate(container): } } for prefix in network_config['prefix']: - net = {'subnet' : prefix, 'gateway' : inc_ip(prefix, 1)} + net = {'subnet': prefix, 'gateway': inc_ip(prefix, 1)} tmp['subnets'].append(net) if is_ipv6(prefix): @@ -418,11 +441,12 @@ def generate(container): file_path = os.path.join(systemd_unit_path, f'vyos-container-{name}.service') run_args = generate_run_arguments(name, container_config) - render(file_path, 'container/systemd-unit.j2', {'name': name, 'run_args': run_args,}, + render(file_path, 'container/systemd-unit.j2', {'name': name, 'run_args': run_args, }, formater=lambda _: _.replace(""", '"').replace("'", "'")) return None + def apply(container): # Delete old containers if needed. We can't delete running container # Option "--force" allows to delete containers with any status @@ -485,6 +509,7 @@ def apply(container): return None + if __name__ == '__main__': try: c = get_config() diff --git a/src/conf_mode/nat_cgnat.py b/src/conf_mode/nat_cgnat.py index d429f6e21..cb336a35c 100755 --- a/src/conf_mode/nat_cgnat.py +++ b/src/conf_mode/nat_cgnat.py @@ -16,9 +16,11 @@ import ipaddress import jmespath +import logging import os from sys import exit +from logging.handlers import SysLogHandler from vyos.config import Config from vyos.template import render @@ -32,6 +34,18 @@ airbag.enable() nftables_cgnat_config = '/run/nftables-cgnat.nft' +# Logging +logger = logging.getLogger('cgnat') +logger.setLevel(logging.DEBUG) + +syslog_handler = SysLogHandler(address="/dev/log") +syslog_handler.setLevel(logging.INFO) + +formatter = logging.Formatter('%(name)s: %(message)s') +syslog_handler.setFormatter(formatter) + +logger.addHandler(syslog_handler) + class IPOperations: def __init__(self, ip_prefix: str): @@ -356,6 +370,22 @@ def apply(config): return None cmd(f'nft --file {nftables_cgnat_config}') + # Logging allocations + if 'log_allocation' in config: + allocations = config['proto_map_elements'] + allocations = allocations.split(',') + for allocation in allocations: + try: + # Split based on the delimiters used in the nft data format + internal_host, rest = allocation.split(' : ') + external_host, port_range = rest.split(' . ') + # Log the parsed data + logger.info( + f"Internal host: {internal_host.lstrip()}, external host: {external_host}, Port range: {port_range}") + except ValueError as e: + # Log error message + logger.error(f"Error processing line '{allocation}': {e}") + if __name__ == '__main__': try: diff --git a/src/migration-scripts/firewall/15-to-16 b/src/migration-scripts/firewall/15-to-16 index 7c8d38fe6..28df1256e 100755 --- a/src/migration-scripts/firewall/15-to-16 +++ b/src/migration-scripts/firewall/15-to-16 @@ -42,8 +42,9 @@ if not config.exists(conntrack_base): for protocol in ['icmp', 'tcp', 'udp', 'other']: if config.exists(conntrack_base + [protocol]): - if not config.exists(firewall_base): + if not config.exists(firewall_base + ['timeout']): config.set(firewall_base + ['timeout']) + config.copy(conntrack_base + [protocol], firewall_base + ['timeout', protocol]) config.delete(conntrack_base + [protocol]) @@ -52,4 +53,4 @@ try: f.write(config.to_string()) except OSError as e: print("Failed to save the modified config: {}".format(e)) - exit(1)
\ No newline at end of file + exit(1) diff --git a/src/op_mode/conntrack_sync.py b/src/op_mode/conntrack_sync.py index 6c86ff492..f3b09b452 100755 --- a/src/op_mode/conntrack_sync.py +++ b/src/op_mode/conntrack_sync.py @@ -19,6 +19,8 @@ import sys import syslog import xmltodict +from tabulate import tabulate + import vyos.opmode from vyos.configquery import CliShellApiConfigQuery @@ -27,7 +29,6 @@ from vyos.utils.commit import commit_in_progress from vyos.utils.process import call from vyos.utils.process import cmd from vyos.utils.process import run -from vyos.template import render_to_string conntrackd_bin = '/usr/sbin/conntrackd' conntrackd_config = '/run/conntrackd/conntrackd.conf' @@ -59,6 +60,26 @@ def flush_cache(direction): if tmp > 0: raise vyos.opmode.Error('Failed to clear {direction} cache') +def get_formatted_output(data): + data_entries = [] + for parsed in data: + for meta in parsed.get('flow', {}).get('meta', []): + direction = meta['@direction'] + if direction == 'original': + src = meta['layer3']['src'] + dst = meta['layer3']['dst'] + sport = meta['layer4'].get('sport') + dport = meta['layer4'].get('dport') + protocol = meta['layer4'].get('@protoname') + orig_src = f'{src}:{sport}' if sport else src + orig_dst = f'{dst}:{dport}' if dport else dst + + data_entries.append([orig_src, orig_dst, protocol]) + + headers = ["Source", "Destination", "Protocol"] + output = tabulate(data_entries, headers, tablefmt="simple") + return output + def from_xml(raw, xml): out = [] for line in xml.splitlines(): @@ -70,7 +91,7 @@ def from_xml(raw, xml): if raw: return out else: - return render_to_string('conntrackd/conntrackd.op-mode.j2', {'data' : out}) + return get_formatted_output(out) def restart(): is_configured() diff --git a/src/op_mode/cpu.py b/src/op_mode/cpu.py index d53663c17..1a0f7392f 100755 --- a/src/op_mode/cpu.py +++ b/src/op_mode/cpu.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2016-2022 VyOS maintainers and contributors +# Copyright (C) 2016-2024 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -16,8 +16,9 @@ import sys -import vyos.cpu import vyos.opmode +from vyos.utils.cpu import get_cpus +from vyos.utils.cpu import get_core_count from jinja2 import Template @@ -37,15 +38,15 @@ CPU model(s): {{models | join(", ")}} """) def _get_raw_data(): - return vyos.cpu.get_cpus() + return get_cpus() def _format_cpus(cpu_data): env = {'cpus': cpu_data} return cpu_template.render(env).strip() def _get_summary_data(): - count = vyos.cpu.get_core_count() - cpu_data = vyos.cpu.get_cpus() + count = get_core_count() + cpu_data = get_cpus() models = [c['model name'] for c in cpu_data] env = {'count': count, "models": models} @@ -79,4 +80,3 @@ if __name__ == '__main__': except (ValueError, vyos.opmode.Error) as e: print(e) sys.exit(1) - diff --git a/src/op_mode/lldp.py b/src/op_mode/lldp.py index 58cfce443..fac622b81 100755 --- a/src/op_mode/lldp.py +++ b/src/op_mode/lldp.py @@ -120,7 +120,12 @@ def _get_formatted_output(raw_data): tmp.append('') # Remote interface - interface = jmespath.search('port.descr', values) + interface = None + if jmespath.search('port.id.type', values) == 'ifname': + # Remote peer has explicitly returned the interface name as the PortID + interface = jmespath.search('port.id.value', values) + if not interface: + interface = jmespath.search('port.descr', values) if not interface: interface = jmespath.search('port.id.value', values) if not interface: @@ -136,11 +141,17 @@ def _get_formatted_output(raw_data): @_verify def show_neighbors(raw: bool, interface: typing.Optional[str], detail: typing.Optional[bool]): - lldp_data = _get_raw_data(interface=interface, detail=detail) - if raw: - return lldp_data - else: - return _get_formatted_output(lldp_data) + if raw or not detail: + lldp_data = _get_raw_data(interface=interface, detail=detail) + if raw: + return lldp_data + else: + return _get_formatted_output(lldp_data) + else: # non-raw, detail + tmp = 'lldpcli -f text show neighbors details' + if interface: + tmp += f' ports {interface}' + return cmd(tmp) if __name__ == "__main__": try: diff --git a/src/op_mode/tcpdump.py b/src/op_mode/tcpdump.py new file mode 100644 index 000000000..607b59603 --- /dev/null +++ b/src/op_mode/tcpdump.py @@ -0,0 +1,165 @@ +#! /usr/bin/env python3 + +# Copyright (C) 2024 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import sys + +from vyos.utils.process import call + +options = { + 'dump': { + 'cmd': '{command} -A', + 'type': 'noarg', + 'help': 'Print each packet (minus its link level header) in ASCII.' + }, + 'hexdump': { + 'cmd': '{command} -X', + 'type': 'noarg', + 'help': 'Print each packet (minus its link level header) in both hex and ASCII.' + }, + 'filter': { + 'cmd': '{command} \'{value}\'', + 'type': '<pcap-filter>', + 'help': 'Match traffic for capture and display with a pcap-filter expression.' + }, + 'numeric': { + 'cmd': '{command} -nn', + 'type': 'noarg', + 'help': 'Do not attempt to resolve addresses, protocols or services to names.' + }, + 'save': { + 'cmd': '{command} -w {value}', + 'type': '<file>', + 'help': 'Write captured raw packets to <file> rather than parsing or printing them out.' + }, + 'verbose': { + 'cmd': '{command} -vvv -ne', + 'type': 'noarg', + 'help': 'Parse packets with increased detail output, including link-level headers and extended decoding protocol sanity checks.' + }, +} + +tcpdump = 'sudo /usr/bin/tcpdump' + +class List(list): + def first(self): + return self.pop(0) if self else '' + + def last(self): + return self.pop() if self else '' + + def prepend(self, value): + self.insert(0, value) + + +def completion_failure(option: str) -> None: + """ + Shows failure message after TAB when option is wrong + :param option: failure option + :type str: + """ + sys.stderr.write('\n\n Invalid option: {}\n\n'.format(option)) + sys.stdout.write('<nocomps>') + sys.exit(1) + + +def expansion_failure(option, completions): + reason = 'Ambiguous' if completions else 'Invalid' + sys.stderr.write( + '\n\n {} command: {} [{}]\n\n'.format(reason, ' '.join(sys.argv), + option)) + if completions: + sys.stderr.write(' Possible completions:\n ') + sys.stderr.write('\n '.join(completions)) + sys.stderr.write('\n') + sys.stdout.write('<nocomps>') + sys.exit(1) + + +def complete(prefix): + return [o for o in options if o.startswith(prefix)] + + +def convert(command, args): + while args: + shortname = args.first() + longnames = complete(shortname) + if len(longnames) != 1: + expansion_failure(shortname, longnames) + longname = longnames[0] + if options[longname]['type'] == 'noarg': + command = options[longname]['cmd'].format( + command=command, value='') + elif not args: + sys.exit(f'monitor traffic: missing argument for {longname} option') + else: + command = options[longname]['cmd'].format( + command=command, value=args.first()) + return command + + +if __name__ == '__main__': + args = List(sys.argv[1:]) + ifname = args.first() + + # Slightly simplified & tweaked version of the code from mtr.py - it may be + # worthwhile to combine and centralise this in a common module. + if ifname == '--get-options-nested': + args.first() # pop monitor + args.first() # pop traffic + args.first() # pop interface + args.first() # pop <ifname> + usedoptionslist = [] + while args: + option = args.first() # pop option + matched = complete(option) # get option parameters + usedoptionslist.append(option) # list of used options + # Select options + if not args: + # remove from Possible completions used options + for o in usedoptionslist: + if o in matched: + matched.remove(o) + if not matched: + sys.stdout.write('<nocomps>') + else: + sys.stdout.write(' '.join(matched)) + sys.exit(0) + + if len(matched) > 1: + sys.stdout.write(' '.join(matched)) + sys.exit(0) + # If option doesn't have value + if matched: + if options[matched[0]]['type'] == 'noarg': + continue + else: + # Unexpected option + completion_failure(option) + + value = args.first() # pop option's value + if not args: + matched = complete(option) + helplines = options[matched[0]]['type'] + # Run helpfunction to get list of possible values + if 'helpfunction' in options[matched[0]]: + result = options[matched[0]]['helpfunction']() + if result: + helplines = '\n' + ' '.join(result) + sys.stdout.write(helplines) + sys.exit(0) + + command = convert(tcpdump, args) + call(f'{command} -i {ifname}') diff --git a/src/op_mode/uptime.py b/src/op_mode/uptime.py index 059a4c3f6..559eed24c 100755 --- a/src/op_mode/uptime.py +++ b/src/op_mode/uptime.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2021-2023 VyOS maintainers and contributors +# Copyright (C) 2021-2024 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -29,8 +29,8 @@ def _get_uptime_seconds(): def _get_load_averages(): from re import search + from vyos.utils.cpu import get_core_count from vyos.utils.process import cmd - from vyos.cpu import get_core_count data = cmd("uptime") matches = search(r"load average:\s*(?P<one>[0-9\.]+)\s*,\s*(?P<five>[0-9\.]+)\s*,\s*(?P<fifteen>[0-9\.]+)\s*", data) |