From e364e9813b6833f6b108e7177ef7ea2d9e7bac33 Mon Sep 17 00:00:00 2001 From: zsdc Date: Wed, 4 Oct 2023 01:57:33 +0300 Subject: pmacct: T5232: Fixed pmacct service control via systemctl pmacct daemons have one very important specific - they handle control signals in the same loop as packets. And packets waiting is blocking operation. Because of this, when systemctl sends SIGTERM to uacctd, this signal has no effect until uacct receives at least one packet via nflog. In some cases, this leads to a 90-second timeout, sending SIGKILL, and improperly finished tasks. As a result, a working folder is not cleaned properly. This commit contains several changes to fix service issues: - add a new nftables table for pmacct with a single rule to get the ability to send a packet to nflog and unlock uacctd - remove PID file options from the uacctd and a systemd service file. Systemd can detect proper PID, and PIDfile is created by uacctd too late, which leads to extra errors in systemd logs - KillMode changed to mixed. Without this, SIGTERM is sent to all plugins and the core process exits with status 1 because it loses connection to plugins too early. As a result, we have errors in logs, and the systemd service is in a failed state. - added logging to uacctd - systemctl service modified to send packets to specific address during a service stop which unlocks uacctd and allows systemctl to finish its work properly --- data/templates/pmacct/override.conf.j2 | 4 +- data/templates/pmacct/uacctd.conf.j2 | 2 +- src/conf_mode/flow_accounting_conf.py | 34 ++++++++++++++++- src/system/uacctd_stop.py | 67 ++++++++++++++++++++++++++++++++++ 4 files changed, 103 insertions(+), 4 deletions(-) create mode 100755 src/system/uacctd_stop.py diff --git a/data/templates/pmacct/override.conf.j2 b/data/templates/pmacct/override.conf.j2 index 213569ddc..44a100bb6 100644 --- a/data/templates/pmacct/override.conf.j2 +++ b/data/templates/pmacct/override.conf.j2 @@ -9,9 +9,9 @@ ConditionPathExists=/run/pmacct/uacctd.conf EnvironmentFile= ExecStart= ExecStart={{ vrf_command }}/usr/sbin/uacctd -f /run/pmacct/uacctd.conf +ExecStop=/usr/libexec/vyos/system/uacctd_stop.py $MAINPID 60 WorkingDirectory= WorkingDirectory=/run/pmacct -PIDFile= -PIDFile=/run/pmacct/uacctd.pid Restart=always RestartSec=10 +KillMode=mixed diff --git a/data/templates/pmacct/uacctd.conf.j2 b/data/templates/pmacct/uacctd.conf.j2 index 1370f8121..aae0a0619 100644 --- a/data/templates/pmacct/uacctd.conf.j2 +++ b/data/templates/pmacct/uacctd.conf.j2 @@ -1,7 +1,7 @@ # Genereated from VyOS configuration daemonize: true promisc: false -pidfile: /run/pmacct/uacctd.pid +syslog: daemon uacctd_group: 2 uacctd_nl_size: 2097152 snaplen: {{ packet_length }} diff --git a/src/conf_mode/flow_accounting_conf.py b/src/conf_mode/flow_accounting_conf.py index 81ee39df1..206f513c8 100755 --- a/src/conf_mode/flow_accounting_conf.py +++ b/src/conf_mode/flow_accounting_conf.py @@ -28,6 +28,7 @@ from vyos.ifconfig import Section from vyos.template import render from vyos.utils.process import call from vyos.utils.process import cmd +from vyos.utils.process import run from vyos.utils.network import is_addr_assigned from vyos import ConfigError from vyos import airbag @@ -116,6 +117,30 @@ def _nftables_config(configured_ifaces, direction, length=None): cmd(command, raising=ConfigError) +def _nftables_trigger_setup(operation: str) -> None: + """Add a dummy rule to unlock the main pmacct loop with a packet-trigger + + Args: + operation (str): 'add' or 'delete' a trigger + """ + # check if a chain exists + table_exists = False + if run('nft -snj list table ip pmacct') == 0: + table_exists = True + + if operation == 'delete' and table_exists: + nft_cmd: str = 'nft delete table ip pmacct' + cmd(nft_cmd, raising=ConfigError) + if operation == 'add' and not table_exists: + nft_cmds: list[str] = [ + 'nft add table ip pmacct', + 'nft add chain ip pmacct pmacct_out { type filter hook output priority raw - 50 \\; policy accept \\; }', + 'nft add rule ip pmacct pmacct_out oif lo ip daddr 127.0.254.0 counter log group 2 snaplen 1 queue-threshold 0 comment NFLOG_TRIGGER' + ] + for nft_cmd in nft_cmds: + cmd(nft_cmd, raising=ConfigError) + + def get_config(config=None): if config: conf = config @@ -252,7 +277,6 @@ def generate(flow_config): call('systemctl daemon-reload') def apply(flow_config): - action = 'restart' # Check if flow-accounting was removed and define command if not flow_config: _nftables_config([], 'ingress') @@ -262,6 +286,10 @@ def apply(flow_config): call(f'systemctl stop {systemd_service}') if os.path.exists(uacctd_conf_path): os.unlink(uacctd_conf_path) + + # must be done after systemctl + _nftables_trigger_setup('delete') + return # Start/reload flow-accounting daemon @@ -277,6 +305,10 @@ def apply(flow_config): else: _nftables_config([], 'egress') + # add a trigger for signal processing + _nftables_trigger_setup('add') + + if __name__ == '__main__': try: config = get_config() diff --git a/src/system/uacctd_stop.py b/src/system/uacctd_stop.py new file mode 100755 index 000000000..7fbac0566 --- /dev/null +++ b/src/system/uacctd_stop.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2023 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Control pmacct daemons in a tricky way. +# Pmacct has signal processing in a main loop, together with packet +# processing. Because of this, while it is waiting for packets, it cannot +# handle the control signal. We need to start the systemctl command and then +# send some packets to pmacct to wake it up + +from argparse import ArgumentParser +from socket import socket +from sys import exit +from time import sleep + +from psutil import Process + + +def stop_process(pid: int, timeout: int) -> None: + """Send a signal to uacctd + and then send packets to special address predefined in a firewall + to unlock main loop in uacctd and finish the process properly + + Args: + pid (int): uacctd PID + timeout (int): seconds to wait for a process end + """ + # find a process + uacctd = Process(pid) + uacctd.terminate() + + # create a socket + trigger = socket() + + first_cycle: bool = True + while uacctd.is_running() and timeout: + trigger.sendto(b'WAKEUP', ('127.0.254.0', 0)) + # do not sleep during first attempt + if not first_cycle: + sleep(1) + timeout -= 1 + first_cycle = False + + +if __name__ == '__main__': + parser = ArgumentParser() + parser.add_argument('process_id', + type=int, + help='PID file of uacctd core process') + parser.add_argument('timeout', + type=int, + help='time to wait for process end') + args = parser.parse_args() + stop_process(args.process_id, args.timeout) + exit() -- cgit v1.2.3