radius: introduced max-fail option

Before this patch any single fail (no responce for max_try requests) caused radius server to enter "fail" state,
which may be unwanted behaviour, because radius may not respond for interim Accounting-Request by some reasons.
This patch introduces "max-fail" option which specifies number of unreplied requests in a row after which server enters into "fail" state.
(Actual only for multi-server configurations)

6 files changed, 83 insertions, 50 deletions

diff --git a/accel-pppd/radius/acct.c b/accel-pppd/radius/acct.c
index aa8c70b..d6f2b18 100644
--- a/accel-pppd/radius/acct.c
+++ b/accel-pppd/radius/acct.c
@@ -64,13 +64,8 @@ static int req_set_stat(struct rad_req_t *req, struct ap_session *ses)
 
 static void rad_acct_sent(struct rad_req_t *req, int res)
 {
-	if (res) {
-		if (conf_acct_timeout)
-			ap_session_terminate(req->rpd->ses, TERM_NAS_ERROR, 0);
-		else
-			triton_timer_del(&req->timeout);
+	if (res)
 		return;
-	}
 
 	__sync_add_and_fetch(&req->serv->stat_interim_sent, 1);
 	
@@ -125,13 +120,10 @@ static void rad_acct_timeout(struct triton_timer_t *t)
 	dt = ts.tv_sec - req->ts;
 
 	if (dt > conf_acct_timeout) {
-		rad_server_fail(req->serv);
-		if (rad_server_realloc(req)) {
-			log_ppp_warn("radius:acct: no servers available, terminating session...\n");
-			triton_timer_del(t);
-			ap_session_terminate(req->rpd->ses, TERM_NAS_ERROR, 0);
-			return;
-		}
+		log_ppp_warn("radius: server(%i) not responding, terminating session...\n", req->serv->id);
+		triton_timer_del(t);
+		ap_session_terminate(req->rpd->ses, TERM_NAS_ERROR, 0);
+		return;
 	}
 
 	if (dt > conf_acct_timeout / 2)
@@ -259,6 +251,8 @@ static void rad_acct_start_recv(struct rad_req_t *req)
 static void rad_acct_start_timeout(struct triton_timer_t *t)
 {
 	struct rad_req_t *req = container_of(t, typeof(*req), timeout);
+	
+	rad_server_timeout(req->serv);
 
 	__sync_add_and_fetch(&req->serv->stat_acct_lost, 1);
 	stat_accm_add(req->serv->stat_acct_lost_1m, 1);
@@ -358,8 +352,8 @@ static void rad_acct_stop_timeout(struct triton_timer_t *t)
 	    log_switch(triton_context_self(), NULL);
 
 	if (req->active) {
-		rad_server_req_exit(req);
 		rad_server_timeout(req->serv);
+		rad_server_req_exit(req);
 
 		__sync_add_and_fetch(&req->serv->stat_acct_lost, 1);
 		stat_accm_add(req->serv->stat_acct_lost_1m, 1);
diff --git a/accel-pppd/radius/auth.c b/accel-pppd/radius/auth.c
index d166192..2124c4b 100644
--- a/accel-pppd/radius/auth.c
+++ b/accel-pppd/radius/auth.c
@@ -199,6 +199,8 @@ static void rad_auth_recv(struct rad_req_t *req)
 static void rad_auth_timeout(struct triton_timer_t *t)
 {
 	struct rad_req_t *req = container_of(t, typeof(*req), timeout);
+
+	rad_server_timeout(req->serv);
 	
 	__sync_add_and_fetch(&req->serv->stat_auth_lost, 1);
 	stat_accm_add(req->serv->stat_auth_lost_1m, 1);
diff --git a/accel-pppd/radius/radius.c b/accel-pppd/radius/radius.c
index 6cf5ead..045dee8 100644
--- a/accel-pppd/radius/radius.c
+++ b/accel-pppd/radius/radius.c
@@ -670,14 +670,6 @@ static int load_config(void)
 	if (opt)
 		conf_acct_delay_time = atoi(opt);
 
-	opt = conf_get_opt("radius", "fail-time");
-	if (opt)
-		conf_fail_time = atoi(opt);
-
-	opt = conf_get_opt("radius", "req-limit");
-	if (opt)
-		conf_req_limit = atoi(opt);
-	
 	conf_attr_tunnel_type = conf_get_opt("radius", "attr-tunnel-type");
 	
 	conf_default_realm = conf_get_opt("radius", "default-realm");
diff --git a/accel-pppd/radius/radius_p.h b/accel-pppd/radius/radius_p.h
index 7e8b0fd..6f1be67 100644
--- a/accel-pppd/radius/radius_p.h
+++ b/accel-pppd/radius/radius_p.h
@@ -101,10 +101,12 @@ struct rad_server_t {
 	int req_limit;
 	int req_cnt;
 	int queue_cnt;
+	int fail_timeout;
+	int max_fail;
+
 	struct list_head req_queue;
 	int client_cnt[2];
 	time_t fail_time;
-	int conf_fail_time;
 	int timeout_cnt;
 	int weight;
 	pthread_mutex_t lock;
@@ -160,8 +162,6 @@ extern in_addr_t conf_dm_coa_server;
 extern int conf_dm_coa_port;
 extern int conf_acct_interim_interval;
 extern int conf_accounting;
-extern int conf_fail_time;
-extern int conf_req_limit;
 extern const char *conf_attr_tunnel_type;
 
 int rad_check_nas_pack(struct rad_packet_t *pack);
diff --git a/accel-pppd/radius/req.c b/accel-pppd/radius/req.c
index 84a4f3d..32fa44c 100644
--- a/accel-pppd/radius/req.c
+++ b/accel-pppd/radius/req.c
@@ -304,7 +304,8 @@ out_err:
 int __rad_req_send(struct rad_req_t *req, int async)
 {
 	if (async == -1) {
-		req->try = conf_max_try - 1;
+		if (req->active)
+			req->try = conf_max_try;
 		if (rad_req_send(req))
 			req->sent(req, -1);
 		return 0;
@@ -352,7 +353,6 @@ int rad_req_send(struct rad_req_t *req)
 	req->send = __rad_req_send;
 
 	if (req->try++ == conf_max_try) {
-		rad_server_fail(req->serv);
 		rad_server_req_exit(req);
 			
 		if (rad_server_realloc(req)) {
diff --git a/accel-pppd/radius/serv.c b/accel-pppd/radius/serv.c
index f3594b9..935caa4 100644
--- a/accel-pppd/radius/serv.c
+++ b/accel-pppd/radius/serv.c
@@ -24,6 +24,9 @@
 #include "memdebug.h"
 
 static int conf_acct_on;
+static int conf_fail_timeout;
+static int conf_max_fail;
+static int conf_req_limit;
 
 static int num;
 static LIST_HEAD(serv_list);
@@ -285,26 +288,28 @@ void rad_server_fail(struct rad_server_t *s)
 	pthread_mutex_lock(&s->lock);
 
 	if (ts.tv_sec >= s->fail_time) {
-		s->fail_time = ts.tv_sec + s->conf_fail_time + 1;
+		s->fail_time = ts.tv_sec + s->fail_timeout;
 		log_ppp_warn("radius: server(%i) not responding\n", s->id);
 		log_warn("radius: server(%i) not responding\n", s->id);
 	}
 
-	if (s->conf_fail_time) {
-		while (!list_empty(&s->req_queue)) {
-			r = list_entry(s->req_queue.next, typeof(*r), entry);
-			list_del(&r->entry);
-			triton_context_call(r->rpd ? r->rpd->ses->ctrl->ctx : NULL, (triton_event_func)req_wakeup_failed, r);
-		}
-		s->queue_cnt = 0;
+	while (!list_empty(&s->req_queue)) {
+		r = list_entry(s->req_queue.next, typeof(*r), entry);
+		list_del(&r->entry);
+		triton_context_call(r->rpd ? r->rpd->ses->ctrl->ctx : NULL, (triton_event_func)req_wakeup_failed, r);
 	}
+	s->queue_cnt = 0;
+	s->stat_fail_cnt++;
 
 	pthread_mutex_unlock(&s->lock);
 }
 
 void rad_server_timeout(struct rad_server_t *s)
 {
-	if (__sync_add_and_fetch(&s->timeout_cnt, 1) >= conf_max_try * 3)
+	if (!s->fail_timeout)
+		return;
+
+	if (__sync_add_and_fetch(&s->timeout_cnt, 1) >= s->max_fail)
 		rad_server_fail(s);
 }
 
@@ -449,14 +454,12 @@ static void show_stat(struct rad_server_t *s, void *client)
 
 	cli_sendv(client, "radius(%i, %s):\r\n", s->id, addr);
 
-	if (s->conf_fail_time > 1) {
-		if (ts.tv_sec < s->fail_time)
-			cli_send(client, "  state: failed\r\n");
-		else
-			cli_send(client, "  state: active\r\n");
+	if (ts.tv_sec < s->fail_time)
+		cli_send(client, "  state: failed\r\n");
+	else
+		cli_send(client, "  state: active\r\n");
 
-		cli_sendv(client, "  fail count: %lu\r\n", s->stat_fail_cnt);
-	}
+	cli_sendv(client, "  fail count: %lu\r\n", s->stat_fail_cnt);
 		
 	cli_sendv(client, "  request count: %i\r\n", s->req_cnt);
 	cli_sendv(client, "  queue length: %i\r\n", s->queue_cnt);
@@ -500,8 +503,9 @@ static void __add_server(struct rad_server_t *s)
 
 	list_for_each_entry(s1, &serv_list, entry) {
 		if (s1->addr == s->addr && s1->auth_port == s->auth_port && s1->acct_port == s->acct_port) {
-			s1->conf_fail_time = s->conf_fail_time;
+			s1->fail_timeout = s->fail_timeout;
 			s1->req_limit = s->req_limit;
+			s1->max_fail = s->max_fail;
 			s1->need_free = 0;
 			_free(s);
 			return;
@@ -626,7 +630,9 @@ static void add_server_old(void)
 	s->addr = auth_addr;
 	s->secret = auth_secret;
 	s->auth_port = auth_port;
-	s->conf_fail_time = conf_fail_time;
+	s->fail_timeout = conf_fail_timeout;
+	s->req_limit = conf_req_limit;
+	s->max_fail = conf_max_fail;
 
 	if (auth_addr == acct_addr && !strcmp(auth_secret, acct_secret)) {
 		s->acct_port = acct_port;
@@ -642,7 +648,9 @@ static void add_server_old(void)
 		s->addr = acct_addr;
 		s->secret = acct_secret;
 		s->acct_port = acct_port;
-		s->conf_fail_time = conf_fail_time;
+		s->fail_timeout = conf_fail_timeout;
+		s->req_limit = conf_req_limit;
+		s->max_fail = conf_max_fail;
 		__add_server(s);
 	}
 }
@@ -690,7 +698,9 @@ static int parse_server1(const char *_opt, struct rad_server_t *s)
 		s->acct_port = 1813;
 
 	s->secret = _strdup(ptr1 + 1);
-	s->conf_fail_time = conf_fail_time;
+	s->fail_timeout = conf_fail_timeout;
+	s->req_limit = conf_req_limit;
+	s->max_fail = conf_max_fail;
 	
 	return 0;
 
@@ -739,13 +749,28 @@ static int parse_server2(const char *_opt, struct rad_server_t *s)
 	} else
 		s->req_limit = conf_req_limit;
 
-	ptr3 = strstr(ptr2, ",fail-time=");
+	ptr3 = strstr(ptr2, ",fail-timeout=");
+	if (ptr3) {
+		s->fail_timeout = strtol(ptr3 + 14, &endptr, 10);
+		if (*endptr != ',' && *endptr != 0)
+			goto out;
+	} else {
+		ptr3 = strstr(ptr2, ",fail-time=");
+		if (ptr3) {
+			s->fail_timeout = strtol(ptr3 + 11, &endptr, 10);
+			if (*endptr != ',' && *endptr != 0)
+				goto out;
+		} else
+			s->fail_timeout = conf_fail_timeout;
+	}
+	
+	ptr3 = strstr(ptr2, ",max-fail=");
 	if (ptr3) {
-		s->conf_fail_time = strtol(ptr3 + 11, &endptr, 10);
+		s->max_fail = strtol(ptr3 + 10, &endptr, 10);
 		if (*endptr != ',' && *endptr != 0)
 			goto out;
 	} else
-		s->conf_fail_time = conf_fail_time;
+		s->max_fail = conf_max_fail;
 	
 	ptr3 = strstr(ptr2, ",weight=");
 	if (ptr3) {
@@ -815,6 +840,26 @@ static void load_config(void)
 		conf_acct_on = atoi(opt1);
 	else
 		conf_acct_on = 0;
+	
+	opt1 = conf_get_opt("radius", "fail-timeout");
+	if (!opt1)
+		opt1 = conf_get_opt("radius", "fail-time");
+	if (opt1)
+		conf_fail_timeout = atoi(opt1);
+	else
+		conf_fail_timeout = 0;
+
+	opt1 = conf_get_opt("radius", "req-limit");
+	if (opt1)
+		conf_req_limit = atoi(opt1);
+	else
+		conf_req_limit = 0;
+	
+	opt1 = conf_get_opt("radius", "max-fail");
+	if (opt1)
+		conf_max_fail = atoi(opt1);
+	else
+		conf_max_fail = conf_req_limit + conf_max_try;
 
 	list_for_each_entry(opt, &sect->items, entry) {
 		if (strcmp(opt->name, "server"))