From 0197e6e96ea391d6b44e9440f2add84a5dc70645 Mon Sep 17 00:00:00 2001 From: Denys Fedoryshchenko Date: Thu, 14 May 2026 02:24:40 +0300 Subject: metrics: back off accept loop on persistent errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously the only path out of serv_read()'s accept loop was an EAGAIN/EWOULDBLOCK return; every other failure logged once and fell back into `continue`. With a level-readable listening fd, that means EMFILE/ENFILE/ENOBUFS/ENOMEM pin the worker thread spinning on accept() and saturate the log. Detect that class of error and pause the listener: disable MD_MODE_READ on serv_hnd, arm a one-shot triton timer for one second, and on expiry re-enable the handler. EINTR and ECONNABORTED are kept as transient retries — those are normal and short-lived. Verified by running the daemon under `prlimit --nofile=24` and opening enough slow connections to exhaust the limit. The first accept failure logs metrics: accept failed: Too many open files; backing off 1s then the daemon idles at 0% CPU instead of spinning; once fds free up it resumes accepting. Signed-off-by: Denys Fedoryshchenko --- accel-pppd/extra/metrics.c | 40 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/accel-pppd/extra/metrics.c b/accel-pppd/extra/metrics.c index 78774dd0..51800f10 100644 --- a/accel-pppd/extra/metrics.c +++ b/accel-pppd/extra/metrics.c @@ -70,13 +70,16 @@ static LIST_HEAD(conf_allowed); static int conf_read_timeout = METRICS_DEFAULT_READ_TIMEOUT; static int conf_max_clients = METRICS_DEFAULT_MAX_CLIENTS; +#define METRICS_ACCEPT_BACKOFF 1 /* seconds */ #define METRICS_ACCEPT_BATCH 16 /* max accept()s per serv_read tick */ static struct triton_context_t serv_ctx; static struct triton_md_handler_t serv_hnd; +static struct triton_timer_t accept_resume_timer; static LIST_HEAD(clients); static unsigned int client_count; static int serv_running; +static int accept_paused; static int parse_format(const char *opt, enum metrics_format *out) { @@ -833,6 +836,24 @@ static int cln_read(struct triton_md_handler_t *h) return 0; } +static void accept_resume(struct triton_timer_t *t) +{ + triton_timer_del(t); + accept_paused = 0; + triton_md_enable_handler(&serv_hnd, MD_MODE_READ); +} + +static void accept_pause(void) +{ + if (accept_paused) + return; + accept_paused = 1; + triton_md_disable_handler(&serv_hnd, MD_MODE_READ); + accept_resume_timer.expire = accept_resume; + accept_resume_timer.expire_tv.tv_sec = METRICS_ACCEPT_BACKOFF; + triton_timer_add(&serv_ctx, &accept_resume_timer, 0); +} + static int serv_read(struct triton_md_handler_t *h) { struct sockaddr_in addr; @@ -854,10 +875,20 @@ static int serv_read(struct triton_md_handler_t *h) for (batch = 0; batch < METRICS_ACCEPT_BATCH; batch++) { sock = accept(h->fd, (struct sockaddr *)&addr, &size); if (sock < 0) { - if (errno == EAGAIN) + if (errno == EAGAIN || errno == EWOULDBLOCK) return 0; - log_error("metrics: accept failed: %s\n", strerror(errno)); - continue; + if (errno == EINTR || errno == ECONNABORTED) + continue; + /* For persistent resource-exhaustion errors + * (EMFILE, ENFILE, ENOBUFS, ENOMEM) the kernel will + * keep the listening fd readable, so a bare + * `continue` spins the worker. Disable the listener + * briefly and retry via a one-shot timer. + */ + log_error("metrics: accept failed: %s; backing off %ds\n", + strerror(errno), METRICS_ACCEPT_BACKOFF); + accept_pause(); + return 0; } if (!ip_allowed(addr.sin_addr.s_addr)) { @@ -911,6 +942,9 @@ static void serv_close(struct triton_context_t *ctx) { struct metrics_client_t *cln; + if (accept_resume_timer.tpd) + triton_timer_del(&accept_resume_timer); + while (!list_empty(&clients)) { cln = list_entry(clients.next, typeof(*cln), entry); disconnect_client(cln); -- cgit v1.2.3