src: add cache statistics via `-s cache'

This patch adds cache statistics that you can check via
`conntrackd -s cache'. This information is useful for
trouble-shooting.

This patch replaces several log messages that can be triggered in
runtime. The idea behind this patch is to avoid log message flooding
under errors.

Signed-off-by: Pablo Neira Ayuso <pablo@netfilter.org>

8 files changed, 119 insertions, 54 deletions

diff --git a/conntrackd.8 b/conntrackd.8
index ab834fb..7d38740 100644
--- a/conntrackd.8
+++ b/conntrackd.8
@@ -44,9 +44,10 @@ option will not flush your internal and external cache).
 .BI "-k "
 Kill the daemon
 .TP
-.BI "-s " "[|network]"
+.BI "-s " "[|network|cache]"
 Dump statistics. If no parameter is passed, it displays the general statistics.
 If "network" is passed as parameter it displays the networking statistics.
+If "cache" is passed as parameter, it shows the extended cache statistics.
 .TP
 .BI "-R "
 Force a resync against the kernel connection tracking table
diff --git a/include/cache.h b/include/cache.h
index 45c3b7e..ebed70a 100644
--- a/include/cache.h
+++ b/include/cache.h
@@ -50,21 +50,28 @@ struct cache {
 	unsigned int extra_offset;
 
         /* statistics */
-	unsigned int active;
-
-	unsigned int add_ok;
-	unsigned int del_ok;
-	unsigned int upd_ok;
-
-	unsigned int add_fail;
-	unsigned int del_fail;
-	unsigned int upd_fail;
-
-	unsigned int commit_ok;
-	unsigned int commit_exist;
-	unsigned int commit_fail;
-
-	unsigned int flush;
+	struct {
+		uint32_t	active;
+	
+		uint32_t	add_ok;
+		uint32_t	del_ok;
+		uint32_t	upd_ok;
+		
+		uint32_t	add_fail;
+		uint32_t	del_fail;
+		uint32_t	upd_fail;
+
+		uint32_t	add_fail_enomem;
+		uint32_t	add_fail_enospc;
+		uint32_t	del_fail_enoent;
+		uint32_t	upd_fail_enoent;
+
+		uint32_t	commit_ok;
+		uint32_t	commit_exist;
+		uint32_t	commit_fail;
+
+		uint32_t	flush;
+	} stats;
 };
 
 struct cache_extra {
@@ -88,6 +95,7 @@ int __cache_del_timer(struct cache *c, struct us_conntrack *u, int timeout);
 struct us_conntrack *cache_find(struct cache *c, struct nf_conntrack *ct);
 int cache_test(struct cache *c, struct nf_conntrack *ct);
 void cache_stats(const struct cache *c, int fd);
+void cache_stats_extended(const struct cache *c, int fd);
 struct us_conntrack *cache_get_conntrack(struct cache *, void *);
 void *cache_get_extra(struct cache *, void *);
 void cache_iterate(struct cache *c, void *data, int (*iterate)(void *data1, void *data2));
diff --git a/include/conntrackd.h b/include/conntrackd.h
index 7199985..98934ce 100644
--- a/include/conntrackd.h
+++ b/include/conntrackd.h
@@ -27,6 +27,7 @@
 #define RESET_TIMERS	26	/* reset kernel timers			*/
 #define DEBUG_INFO	27	/* show debug info (if any)		*/
 #define STATS_NETWORK	28	/* extended network stats		*/
+#define STATS_CACHE	29	/* extended cache stats			*/
 
 #define DEFAULT_CONFIGFILE	"/etc/conntrackd/conntrackd.conf"
 #define DEFAULT_LOCKFILE	"/var/lock/conntrackd.lock"
diff --git a/src/cache.c b/src/cache.c
index 40c7b1d..5e7d738 100644
--- a/src/cache.c
+++ b/src/cache.c
@@ -206,7 +206,7 @@ static struct us_conntrack *__add(struct cache *c, struct nf_conntrack *ct)
 		if (c->extra && c->extra->add)
 			c->extra->add(u, ((char *) u) + c->extra_offset);
 
-		c->active++;
+		c->stats.active++;
 		return u;
 	}
 	free(newct);
@@ -220,11 +220,16 @@ struct us_conntrack *cache_add(struct cache *c, struct nf_conntrack *ct)
 
 	u = __add(c, ct);
 	if (u) {
-		c->add_ok++;
+		c->stats.add_ok++;
 		return u;
 	}
-	if (errno != EEXIST)
-		c->add_fail++;
+	if (errno != EEXIST) {
+		c->stats.add_fail++;
+		if (errno == ENOSPC)
+			c->stats.add_fail_enospc++;
+		if (errno == ENOMEM)
+			c->stats.add_fail_enomem++;
+	}
 
 	return NULL;
 }
@@ -268,10 +273,12 @@ struct us_conntrack *cache_update(struct cache *c, struct nf_conntrack *ct)
 
 	u = __update(c, ct);
 	if (u) {
-		c->upd_ok++;
+		c->stats.upd_ok++;
 		return u;
 	}
-	c->upd_fail++;
+	c->stats.upd_fail++;
+	if (errno == ENOENT)
+		c->stats.upd_fail_enoent++;
 	
 	return NULL;
 }
@@ -302,8 +309,8 @@ static void __cache_del(struct cache *c, struct us_conntrack *u)
 	 * __cache_del_timer.
 	 */
 	if (!alarm_pending(&u->alarm)) {
-		c->del_ok++;
-		c->active--;
+		c->stats.del_ok++;
+		c->stats.active--;
 	}
 	del_alarm(&u->alarm);
 	__del(c, u);
@@ -317,7 +324,7 @@ struct us_conntrack *cache_update_force(struct cache *c,
 	u = cache_find(c, ct);
 	if (u) {
 		if (!alarm_pending(&u->alarm)) {
-			c->upd_ok++;
+			c->stats.upd_ok++;
 			__cache_update(c, u, ct);
 			return u;
 		} else {
@@ -325,10 +332,15 @@ struct us_conntrack *cache_update_force(struct cache *c,
 		}
 	}
 	if ((u = __add(c, ct)) != NULL) {
-		c->add_ok++;
+		c->stats.add_ok++;
 		return u;
 	}
-	c->add_fail++;
+	c->stats.add_fail++;
+	if (errno == ENOSPC)
+		c->stats.add_fail_enospc++;
+	if (errno == ENOMEM)
+		c->stats.add_fail_enomem++;
+
 	return NULL;
 }
 
@@ -359,7 +371,9 @@ int cache_del(struct cache *c, struct nf_conntrack *ct)
 		__cache_del(c, u);
 		return 1;
 	}
-	c->del_fail++;
+	c->stats.del_fail++;
+	if (errno == ENOENT)
+		c->stats.del_fail_enoent++;
 
 	return 0;
 }
@@ -386,8 +400,8 @@ __cache_del_timer(struct cache *c, struct us_conntrack *u, int timeout)
 		 * that the replication protocol does not work 
 		 * properly.
 		 */
-		c->del_ok++;
-		c->active--;
+		c->stats.del_ok++;
+		c->stats.active--;
 		return 1;
 	}
 	return 0;
@@ -425,13 +439,46 @@ void cache_stats(const struct cache *c, int fd)
 			    "connections updated:\t\t%12u\tfailed:\t%12u\n"
 			    "connections destroyed:\t\t%12u\tfailed:\t%12u\n\n",
 			    			 c->name,
-						 c->active,
-			    			 c->add_ok, 
-			    			 c->add_fail,
-						 c->upd_ok,
-						 c->upd_fail,
-						 c->del_ok,
-						 c->del_fail);
+						 c->stats.active,
+			    			 c->stats.add_ok, 
+			    			 c->stats.add_fail,
+						 c->stats.upd_ok,
+						 c->stats.upd_fail,
+						 c->stats.del_ok,
+						 c->stats.del_fail);
+	send(fd, buf, size, 0);
+}
+
+void cache_stats_extended(const struct cache *c, int fd)
+{
+	char buf[512];
+	int size;
+
+	size = snprintf(buf, sizeof(buf),
+			    "cache:%s\tactive connections:\t%12u\n"
+			    "\tcreation OK:\t\t\t%12u\n"
+			    "\tcreation failed:\t\t%12u\n"
+			    "\t\tno memory available:\t%12u\n"
+			    "\t\tno space left in cache:\t%12u\n"
+			    "\tupdate OK:\t\t\t%12u\n"
+			    "\tupdate failed:\t\t\t%12u\n"
+			    "\t\tentry not found:\t%12u\n"
+			    "\tdeletion created:\t\t%12u\n"
+			    "\tdeletion failed:\t\t%12u\n"
+			    "\t\tentry not found:\t%12u\n",
+			    c->name,
+			    c->stats.active,
+			    c->stats.add_ok,
+			    c->stats.add_fail,
+			    c->stats.add_fail_enomem,
+			    c->stats.add_fail_enospc,
+			    c->stats.upd_ok,
+			    c->stats.upd_fail,
+			    c->stats.upd_fail_enoent,
+			    c->stats.del_ok,
+			    c->stats.del_fail,
+			    c->stats.del_fail_enoent);
+
 	send(fd, buf, size, 0);
 }
 
diff --git a/src/cache_iterators.c b/src/cache_iterators.c
index 12ffcff..8ad9612 100644
--- a/src/cache_iterators.c
+++ b/src/cache_iterators.c
@@ -130,12 +130,12 @@ try_again:
 			}
 			dlog(LOG_ERR, "commit-create: %s", strerror(errno));
 			dlog_ct(STATE(log), ct, NFCT_O_PLAIN);
-			tmp->c->commit_fail++;
+			tmp->c->stats.commit_fail++;
 		} else
-			tmp->c->commit_ok++;
+			tmp->c->stats.commit_ok++;
 		break;
 	case 1:
-		tmp->c->commit_exist++;
+		tmp->c->stats.commit_exist++;
 		if (nl_update_conntrack(tmp->h, ct) == -1) {
 			if (errno == ENOMEM || errno == ETIME) {
 				if (retry) {
@@ -154,14 +154,14 @@ try_again:
 				}
 				dlog(LOG_ERR, "commit-rm: %s", strerror(errno));
 				dlog_ct(STATE(log), ct, NFCT_O_PLAIN);
-				tmp->c->commit_fail++;
+				tmp->c->stats.commit_fail++;
 				break;
 			} 
 			dlog(LOG_ERR, "commit-update: %s", strerror(errno));
 			dlog_ct(STATE(log), ct, NFCT_O_PLAIN);
-			tmp->c->commit_fail++;
+			tmp->c->stats.commit_fail++;
 		} else
-			tmp->c->commit_ok++;
+			tmp->c->stats.commit_ok++;
 		break;
 	}
 }
@@ -191,9 +191,9 @@ static int do_commit_master(void *data1, void *data2)
 /* no need to clone, called from child process */
 void cache_commit(struct cache *c)
 {
-	unsigned int commit_ok = c->commit_ok;
-	unsigned int commit_exist = c->commit_exist;
-	unsigned int commit_fail = c->commit_fail;
+	unsigned int commit_ok = c->stats.commit_ok;
+	unsigned int commit_exist = c->stats.commit_exist;
+	unsigned int commit_fail = c->stats.commit_fail;
 	struct __commit_container tmp;
 
 	tmp.h = nfct_open(CONNTRACK, 0);
@@ -208,9 +208,9 @@ void cache_commit(struct cache *c)
 	hashtable_iterate(c->h, &tmp, do_commit_related);
 
 	/* calculate new entries committed */
-	commit_ok = c->commit_ok - commit_ok;
-	commit_fail = c->commit_fail - commit_fail;
-	commit_exist = c->commit_exist - commit_exist;
+	commit_ok = c->stats.commit_ok - commit_ok;
+	commit_fail = c->stats.commit_fail - commit_fail;
+	commit_exist = c->stats.commit_exist - commit_exist;
 
 	/* log results */
 	dlog(LOG_NOTICE, "Committed %u new entries", commit_ok);
@@ -292,5 +292,5 @@ static int do_flush(void *data1, void *data2)
 void cache_flush(struct cache *c)
 {
 	hashtable_iterate(c->h, c, do_flush);
-	c->flush++;
+	c->stats.flush++;
 }
diff --git a/src/main.c b/src/main.c
index 022915d..f621a2e 100644
--- a/src/main.c
+++ b/src/main.c
@@ -43,7 +43,7 @@ static const char usage_client_commands[] =
 	"  -i, display content of the internal cache\n"
 	"  -e, display the content of the external cache\n"
 	"  -k, kill conntrack daemon\n"
-	"  -s  [|network], dump statistics\n"
+	"  -s  [|network|cache], dump statistics\n"
 	"  -R, resync with kernel conntrack table\n"
 	"  -n, request resync with other node (only FT-FW and NOTRACK modes)\n"
 	"  -x, dump cache in XML format (requires -i or -e)"
@@ -161,6 +161,10 @@ int main(int argc, char *argv[])
 					    strlen(argv[i+1])) == 0) {
 					action = STATS_NETWORK;
 					i++;
+				} else if (strncmp(argv[i+1], "cache",
+						 strlen(argv[i+1])) == 0) {
+					action = STATS_CACHE;
+					i++;
 				} else {
 					fprintf(stderr, "ERROR: unknown "
 							"parameter `%s' for "
diff --git a/src/stats-mode.c b/src/stats-mode.c
index ad28008..d340b0d 100644
--- a/src/stats-mode.c
+++ b/src/stats-mode.c
@@ -79,6 +79,9 @@ static int local_handler_stats(int fd, int type, void *data)
 		cache_stats(STATE_STATS(cache), fd);
 		dump_traffic_stats(fd);
 		break;
+	case STATS_CACHE:
+		cache_stats_extended(STATE_STATS(cache), fd);
+		break;
 	default:
 		ret = 0;
 		break;
diff --git a/src/sync-mode.c b/src/sync-mode.c
index e7b9359..6779487 100644
--- a/src/sync-mode.c
+++ b/src/sync-mode.c
@@ -418,6 +418,10 @@ static int local_handler_sync(int fd, int type, void *data)
 		mcast_dump_stats(fd, STATE_SYNC(mcast_client), 
 				     STATE_SYNC(mcast_server));
 		break;
+	case STATS_CACHE:
+		cache_stats_extended(STATE_SYNC(internal), fd);
+		cache_stats_extended(STATE_SYNC(external), fd);
+		break;
 	default:
 		if (STATE_SYNC(sync)->local)
 			ret = STATE_SYNC(sync)->local(fd, type, data);
@@ -519,9 +523,6 @@ retry:
 			cache_del(STATE_SYNC(internal), ct);
 			goto retry;
 		}
-
-		dlog(LOG_ERR, "can't add to internal cache: "
-			      "%s\n", strerror(errno));
 		debug_ct(ct, "can't add");
 	}
 }