radius: better radius server fails handling

8 files changed, 88 insertions, 41 deletions

diff --git a/accel-pppd/radius/acct.c b/accel-pppd/radius/acct.c
index 4f1d9018..04b21fa7 100644
--- a/accel-pppd/radius/acct.c
+++ b/accel-pppd/radius/acct.c
@@ -84,12 +84,13 @@ static int rad_acct_read(struct triton_md_handler_t *h)
 		r = rad_packet_recv(h->fd, &pack, NULL);
 
 		if (pack) {
+			rad_server_reply(req->serv);
 			if (req->reply)
 				rad_packet_free(req->reply);
 			req->reply = pack;
 			if (conf_interim_verbose) {
 				log_ppp_info2("recv ");
-				rad_packet_print(req->reply, log_ppp_info2);
+				rad_packet_print(req->reply, req->serv, log_ppp_info2);
 			}
 		}
 
@@ -119,13 +120,11 @@ static int rad_acct_read(struct triton_md_handler_t *h)
 static void __rad_req_send(struct rad_req_t *req)
 {
 	while (1) {
-
 		if (rad_server_req_enter(req)) {
-			if (rad_server_realloc(req, 1)) {
-				log_ppp_warn("radius:acct: no servers available, terminating session...\n");
+			if (rad_server_realloc(req, RAD_SERV_ACCT)) {
 				if (conf_acct_timeout) {
+					log_ppp_warn("radius:acct: no servers available, terminating session...\n");
 					ppp_terminate(req->rpd->ppp, TERM_NAS_ERROR, 0);
-					return;
 				}
 				break;
 			}
@@ -133,6 +132,10 @@ static void __rad_req_send(struct rad_req_t *req)
 		}
 
 		rad_req_send(req, conf_interim_verbose);
+		if (!req->hnd.tpd) {
+			triton_md_register_handler(req->rpd->ppp->ctrl->ctx, &req->hnd);
+			triton_md_enable_handler(&req->hnd, MD_MODE_READ);
+		}
 
 		rad_server_req_exit(req);
 
@@ -148,14 +151,20 @@ static void rad_acct_timeout(struct triton_timer_t *t)
 	__sync_add_and_fetch(&stat_interim_lost, 1);
 	stat_accm_add(stat_interim_lost_1m, 1);
 	stat_accm_add(stat_interim_lost_5m, 1);
-	
+
+	if (conf_acct_timeout == 0) {
+		rad_server_timeout(req->serv);
+		triton_timer_del(t);
+		return;
+	}
+
 	time(&ts);
 
 	dt = ts - req->rpd->acct_timestamp;
 
 	if (dt > conf_acct_timeout) {
 		rad_server_fail(req->serv);
-		if (rad_server_realloc(req, 1)) {
+		if (rad_server_realloc(req, RAD_SERV_ACCT)) {
 			log_ppp_warn("radius:acct: no servers available, terminating session...\n");
 			ppp_terminate(req->rpd->ppp, TERM_NAS_ERROR, 0);
 			return;
@@ -209,10 +218,8 @@ static void rad_acct_interim_update(struct triton_timer_t *t)
 	__rad_req_send(rpd->acct_req);
 
 	__sync_add_and_fetch(&stat_interim_sent, 1);
-	if (conf_acct_timeout) {
-		rpd->acct_req->timeout.period = conf_timeout * 1000;
-		triton_timer_add(rpd->ppp->ctrl->ctx, &rpd->acct_req->timeout, 0);
-	}
+
+	triton_timer_add(rpd->ppp->ctrl->ctx, &rpd->acct_req->timeout, 0);
 }
 
 int rad_acct_start(struct radius_pd_t *rpd)
@@ -253,7 +260,7 @@ int rad_acct_start(struct radius_pd_t *rpd)
 	while (1) {
 
 		if (rad_server_req_enter(rpd->acct_req)) {
-			if (rad_server_realloc(rpd->acct_req, 1)) {
+			if (rad_server_realloc(rpd->acct_req, RAD_SERV_ACCT)) {
 				log_ppp_warn("radius:acct_start: no servers available\n");
 				goto out_err;
 			}
@@ -308,7 +315,7 @@ int rad_acct_start(struct radius_pd_t *rpd)
 			break;
 
 		rad_server_fail(rpd->acct_req->serv);
-		if (rad_server_realloc(rpd->acct_req, 1)) {
+		if (rad_server_realloc(rpd->acct_req, RAD_SERV_ACCT)) {
 			log_ppp_warn("radius:acct_start: no servers available\n");
 			goto out_err;
 		}
@@ -399,7 +406,7 @@ void rad_acct_stop(struct radius_pd_t *rpd)
 		while (1) {
 
 			if (rad_server_req_enter(rpd->acct_req)) {
-				if (rad_server_realloc(rpd->acct_req, 1)) {
+				if (rad_server_realloc(rpd->acct_req, RAD_SERV_ACCT)) {
 					log_ppp_warn("radius:acct_stop: no servers available\n");
 					break;
 				}
@@ -447,7 +454,7 @@ void rad_acct_stop(struct radius_pd_t *rpd)
 				break;
 
 			rad_server_fail(rpd->acct_req->serv);
-			if (rad_server_realloc(rpd->acct_req, 1)) {
+			if (rad_server_realloc(rpd->acct_req, RAD_SERV_ACCT)) {
 				log_ppp_warn("radius:acct_stop: no servers available\n");
 				break;
 			}
diff --git a/accel-pppd/radius/auth.c b/accel-pppd/radius/auth.c
index e4810fa6..639284a5 100644
--- a/accel-pppd/radius/auth.c
+++ b/accel-pppd/radius/auth.c
@@ -151,7 +151,7 @@ static int rad_auth_send(struct rad_req_t *req)
 
 	while (1) {
 		if (rad_server_req_enter(req)) {
-			if (rad_server_realloc(req, 0)) {
+			if (rad_server_realloc(req, RAD_SERV_AUTH)) {
 				log_ppp_warn("radius: no available servers\n");
 				break;
 			}
@@ -198,7 +198,7 @@ static int rad_auth_send(struct rad_req_t *req)
 
 		if (!req->reply) {
 			rad_server_fail(req->serv);
-			if (rad_server_realloc(req, 0)) {
+			if (rad_server_realloc(req, RAD_SERV_AUTH)) {
 				log_ppp_warn("radius: no available servers\n");
 				break;
 			}
diff --git a/accel-pppd/radius/dm_coa.c b/accel-pppd/radius/dm_coa.c
index 481f2f50..10b3350e 100644
--- a/accel-pppd/radius/dm_coa.c
+++ b/accel-pppd/radius/dm_coa.c
@@ -78,7 +78,7 @@ static int dm_coa_send_ack(int fd, struct rad_packet_t *req, struct sockaddr_in
 
 	if (conf_verbose) {
 		log_ppp_info2("send ");
-		rad_packet_print(reply, log_ppp_info2);
+		rad_packet_print(reply, NULL, log_ppp_info2);
 	}
 
 	rad_packet_send(reply, fd, addr);
@@ -113,7 +113,7 @@ static int dm_coa_send_nak(int fd, struct rad_packet_t *req, struct sockaddr_in
 
 	if (conf_verbose) {
 		log_ppp_info2("send ");
-		rad_packet_print(reply, log_ppp_info2);
+		rad_packet_print(reply, NULL, log_ppp_info2);
 	}
 
 	rad_packet_send(reply, fd, addr);
@@ -128,7 +128,7 @@ static void disconnect_request(struct radius_pd_t *rpd)
 {
 	if (conf_verbose) {
 		log_ppp_info2("recv ");
-		rad_packet_print(rpd->dm_coa_req, log_ppp_info2);
+		rad_packet_print(rpd->dm_coa_req, NULL, log_ppp_info2);
 	}
 
 	dm_coa_send_ack(serv.hnd.fd, rpd->dm_coa_req, &rpd->dm_coa_addr);
@@ -151,7 +151,7 @@ static void coa_request(struct radius_pd_t *rpd)
 
 	if (conf_verbose) {
 		log_ppp_info2("recv ");
-		rad_packet_print(rpd->dm_coa_req, log_ppp_info2);
+		rad_packet_print(rpd->dm_coa_req, NULL, log_ppp_info2);
 	}
 
 	triton_event_fire(EV_RADIUS_COA, &ev);
@@ -201,7 +201,7 @@ static int dm_coa_read(struct triton_md_handler_t *h)
 
 		if (conf_verbose) {
 			log_debug("recv ");
-			rad_packet_print(pack, log_debug);
+			rad_packet_print(pack, NULL, log_debug);
 		}
 
 		if (rad_check_nas_pack(pack)) {
diff --git a/accel-pppd/radius/packet.c b/accel-pppd/radius/packet.c
index 7f90a100..39541f7c 100644
--- a/accel-pppd/radius/packet.c
+++ b/accel-pppd/radius/packet.c
@@ -275,7 +275,7 @@ void rad_packet_free(struct rad_packet_t *pack)
 	mempool_free(pack);
 }
 
-void rad_packet_print(struct rad_packet_t *pack, void (*print)(const char *fmt, ...))
+void rad_packet_print(struct rad_packet_t *pack, struct rad_server_t *s, void (*print)(const char *fmt, ...))
 {
 	struct rad_attr_t *attr;
 	struct rad_dict_value_t *val;
@@ -285,7 +285,10 @@ void rad_packet_print(struct rad_packet_t *pack, void (*print)(const char *fmt,
 		uint16_t u16[4];
 	} ifid_u;
 	
-	print("[RADIUS ");
+	if (s)
+		print("[RADIUS(%i) ", s->id);
+	else
+		print("[RADIUS ");
 	switch(pack->code) {
 		case CODE_ACCESS_REQUEST:
 			print("Access-Request");
diff --git a/accel-pppd/radius/radius_p.h b/accel-pppd/radius/radius_p.h
index 578c7770..1fecd881 100644
--- a/accel-pppd/radius/radius_p.h
+++ b/accel-pppd/radius/radius_p.h
@@ -67,6 +67,7 @@ struct rad_req_t
 struct rad_server_t
 {
 	struct list_head entry;
+	int id;
 	in_addr_t auth_addr;
 	int auth_port;
 	char *auth_secret;
@@ -79,9 +80,12 @@ struct rad_server_t
 	int client_cnt;
 	time_t fail_time;
 	int conf_fail_time;
+	int timeout_cnt;
 	pthread_mutex_t lock;
 };
 
+#define RAD_SERV_AUTH 0
+#define RAD_SERV_ACCT 1
 
 extern int conf_max_try;
 extern int conf_timeout;
@@ -138,7 +142,7 @@ struct rad_packet_t *rad_packet_alloc(int code);
 int rad_packet_build(struct rad_packet_t *pack, uint8_t *RA);
 int rad_packet_recv(int fd, struct rad_packet_t **, struct sockaddr_in *addr);
 void rad_packet_free(struct rad_packet_t *);
-void rad_packet_print(struct rad_packet_t *pack, void (*print)(const char *fmt, ...));
+void rad_packet_print(struct rad_packet_t *pack, struct rad_server_t *s, void (*print)(const char *fmt, ...));
 int rad_packet_send(struct rad_packet_t *pck, int fd, struct sockaddr_in *addr);
 
 void dm_coa_cancel(struct radius_pd_t *pd);
@@ -149,6 +153,8 @@ int rad_server_req_enter(struct rad_req_t *);
 void rad_server_req_exit(struct rad_req_t *);
 int rad_server_realloc(struct rad_req_t *, int);
 void rad_server_fail(struct rad_server_t *);
+void rad_server_timeout(struct rad_server_t *);
+void rad_server_reply(struct rad_server_t *);
 
 struct stat_accm_t;
 struct stat_accm_t *stat_accm_create(unsigned int time);
diff --git a/accel-pppd/radius/req.c b/accel-pppd/radius/req.c
index 9f41ecc6..1f956bf2 100644
--- a/accel-pppd/radius/req.c
+++ b/accel-pppd/radius/req.c
@@ -32,7 +32,7 @@ struct rad_req_t *rad_req_alloc(struct radius_pd_t *rpd, int code, const char *u
 	req->hnd.fd = -1;
 	req->ctx.before_switch = log_switch;
 
-	req->serv = rad_server_get(code == CODE_ACCOUNTING_REQUEST);
+	req->serv = rad_server_get(code == CODE_ACCESS_REQUEST ? RAD_SERV_AUTH : RAD_SERV_ACCT);
 	if (!req->serv)
 		goto out_err;
 	
@@ -213,7 +213,7 @@ int rad_req_send(struct rad_req_t *req, int verbose)
 	
 	if (verbose) {
 		log_ppp_info1("send ");
-		rad_packet_print(req->pack, log_ppp_info1);
+		rad_packet_print(req->pack, req->serv, log_ppp_info1);
 	}
 
 	rad_packet_send(req->pack, req->hnd.fd, NULL);
@@ -283,7 +283,7 @@ int rad_req_wait(struct rad_req_t *req, int timeout)
 
 	if (conf_verbose && req->reply) {
 		log_ppp_info1("recv ");
-		rad_packet_print(req->reply, log_ppp_info1);
+		rad_packet_print(req->reply, req->serv, log_ppp_info1);
 	}
 	return 0;
 }
diff --git a/accel-pppd/radius/serv.c b/accel-pppd/radius/serv.c
index bcdce144..880268bf 100644
--- a/accel-pppd/radius/serv.c
+++ b/accel-pppd/radius/serv.c
@@ -16,9 +16,10 @@
 
 #include "memdebug.h"
 
+static int num;
 static LIST_HEAD(serv_list);
 
-struct rad_server_t *rad_server_get(int type)
+static struct rad_server_t *__rad_server_get(int type, struct rad_server_t *exclude)
 {
 	struct rad_server_t *s, *s0 = NULL;
 	struct timespec ts;
@@ -26,12 +27,15 @@ struct rad_server_t *rad_server_get(int type)
 	clock_gettime(CLOCK_MONOTONIC, &ts);
 
 	list_for_each_entry(s, &serv_list, entry) {
+		if (s == exclude)
+			continue;
+
 		if (s->fail_time && ts.tv_sec < s->fail_time)
 			continue;
 
-		if (type == 0 && !s->auth_addr)
+		if (type == RAD_SERV_AUTH && !s->auth_addr)
 			continue;
-		else if (type == 1 && !s->acct_addr)
+		else if (type == RAD_SERV_ACCT && !s->acct_addr)
 			continue;
 
 		if (!s0) {
@@ -51,6 +55,11 @@ struct rad_server_t *rad_server_get(int type)
 	return s0;
 }
 
+struct rad_server_t *rad_server_get(int type)
+{
+	return __rad_server_get(type, NULL);
+}
+
 void rad_server_put(struct rad_server_t *s)
 {
 	__sync_sub_and_fetch(&s->client_cnt, 1);
@@ -61,7 +70,10 @@ int rad_server_req_enter(struct rad_req_t *req)
 	struct timespec ts;
 	
 	clock_gettime(CLOCK_MONOTONIC, &ts);
-	
+
+	if (ts.tv_sec < req->serv->fail_time)
+		return -1;
+
 	if (!req->serv->max_req_cnt)
 		return 0;
 
@@ -111,18 +123,22 @@ void rad_server_req_exit(struct rad_req_t *req)
 
 int rad_server_realloc(struct rad_req_t *req, int type)
 {
-	if (req->hnd.fd != -1) {
-		close(req->hnd.fd);
-		req->hnd.fd = -1;
-	}
+	struct rad_server_t *s = __rad_server_get(type, req->serv);
+
+	if (!s)
+		return -1;
 
 	if (req->serv)
 		rad_server_put(req->serv);
 
-	req->serv = rad_server_get(type);
+	req->serv = s;
 
-	if (!req->serv)
-		return -1;
+	if (req->hnd.fd != -1) {
+		if (req->hnd.tpd)
+			triton_md_unregister_handler(&req->hnd);
+		close(req->hnd.fd);
+		req->hnd.fd = -1;
+	}
 
 	if (type) {
 		req->server_addr = req->serv->acct_addr;
@@ -146,8 +162,8 @@ void rad_server_fail(struct rad_server_t *s)
 
 	if (ts.tv_sec > s->fail_time) {
 		s->fail_time = ts.tv_sec + s->conf_fail_time;
-		log_ppp_warn("radius: server not responding\n");
-		log_warn("radius: server noy responding\n");
+		log_ppp_warn("radius: server(%i) not responding\n", s->id);
+		log_warn("radius: server(%i) not responding\n", s->id);
 	}
 
 	while (!list_empty(&s->req_queue)) {
@@ -158,8 +174,21 @@ void rad_server_fail(struct rad_server_t *s)
 	pthread_mutex_unlock(&s->lock);
 }
 
+void rad_server_timeout(struct rad_server_t *s)
+{
+	if (__sync_add_and_fetch(&s->timeout_cnt, 1) >= conf_max_try)
+		rad_server_fail(s);
+}
+
+void rad_server_reply(struct rad_server_t *s)
+{
+	__sync_synchronize();
+	s->timeout_cnt = 0;
+}
+
 static void __add_server(struct rad_server_t *s)
 {
+	s->id = ++num;
 	INIT_LIST_HEAD(&s->req_queue);
 	pthread_mutex_init(&s->lock, NULL);
 	s->conf_fail_time = conf_fail_time;
diff --git a/accel-pppd/triton/md.c b/accel-pppd/triton/md.c
index 6c1092d0..c9ec32e6 100644
--- a/accel-pppd/triton/md.c
+++ b/accel-pppd/triton/md.c
@@ -152,6 +152,8 @@ void __export triton_md_unregister_handler(struct triton_md_handler_t *ud)
 	list_add_tail(&h->entry, &freed_list);
 	pthread_mutex_unlock(&freed_list_lock);
 
+	ud->tpd = NULL;
+
 	triton_stat.md_handler_count--;
 }
 int __export triton_md_enable_handler(struct triton_md_handler_t *ud, int mode)