conntrackd: support for expectation synchronization

This patch adds support to synchronize expectations between
firewalls. This addition aims to re-use as much as possible
of the existing infrastructure for stability reasons. The
expectation support has been tested with the FTP helper.

This extension requires libnetfilter_conntrack 1.0.0.

If this is the first time you're playing with conntrackd,
I *strongly* recommend you to get working setup of conntrackd
without expectation support before as described in the
documentation. Then, enabling expectation support is rather
easy.

To know more about expectations, if you're not familiar with them,
I suggest you to read:

"Netfilter's Connection Tracking System"
http://people.netfilter.org/pablo/docs/login.pdf

Reprinted from ;login: The Magazine of USENIX, vol. 31, no. 3
(Berkeley, CA: USENIX Association, 2006, pp40-45.)

In short, expectations allow one Linux firewall to filter multi-flow
traffic like FTP, SIP and H.323.

In my testbed, there are two firewalls in a primary-backup configuration
running keepalived. The use a couple of floating cluster IP address
(192.168.0.100 and 192.168.1.100) that are used by the client. These
firewalls protect one FTP server (192.168.1.2) that will be accessed by
one client.

In ASCII art, it looks like this:

     192.168.0.100      192.168.1.100
              eth1      eth2
                   fw-1
                 /      \       FTP
 -- client ------       ------ server --
  192.168.0.2    \      /   192.168.1.2
                   fw-2

This is the rule-set for the firewalls:

-A POSTROUTING -t nat -s 192.168.0.2/32 -d 192.168.1.2/32 -j SNAT --to-source 192.168.1.100

-A INPUT -p tcp -m tcp --dport 22 -j ACCEPT
-A INPUT -m state --state INVALID -j DROP

-A FORWARD -m state --state RELATED -j ACCEPT
-A FORWARD -i eth2 -m state --state ESTABLISHED -j ACCEPT
-A FORWARD -i eth1 -p tcp -m tcp --dport 21 --tcp-flags FIN,SYN,RST,ACK SYN -m state --state NEW -j ACCEPT
-A FORWARD -i eth1 -p tcp -m state --state ESTABLISHED -j ACCEPT
-A FORWARD -m state --state INVALID -j LOG --log-prefix "invalid: "

The following steps detail how to check that the expectation support
works fine for conntrackd:

1) You have to enable the expectation support in the configuration
file with the following option:

 Sync {
        ...
        Options {
                ExpectationSync {
                        ftp
                        sip
                        h323
                }
        }
 }

This enables expectation synchronization for the FTP, SIP and H.323 helpers.
You can alternatively use:

 Sync {
        ...
        Options {
                ExpectationSync On
        }
 }

To enable expectation synchronization for all helpers.

2) Make sure you have loaded the FTP helper in both firewalls.

root@fw1# modprobe nf_conntrack_ftp
root@fw2# modprobe nf_conntrack_ftp

3) Switch to the client. Start one FTP control connection to one
server that is protected by the firewalls, enter passive mode:

(term-1) user@client$ nc 192.168.1.2 21
220 dummy FTP server
USER anonymous
331 Please specify the password.
PASS nothing
230 Login successful.
PASV
227 Entering Passive Mode (192,168,1,2,163,11).

This means that port 163*256+11=41739 will be used for the data
traffic. Read this if you are not familiar with the FTP protocol:
http://www.freefire.org/articles/ftpexample.php

3) Switch to fw-1 (primary) to check that the expectation is in the
   internal cache.

root@fw1# conntrackd -i exp
proto=6 src=192.168.0.2 dst=192.168.1.2 sport=0 dport=41739 mask-src=255.255.255.255 mask-dst=255.255.255.255 sport=0 dport=65535 master-src=192.168.0.2 master-dst=192.168.1.2 sport=36390 dport=21 [active since 5s]

4) Switch to fw-2 (backup) to check that the expectation has been successfully
   replicated.

root@fw2# conntrackd -e exp
proto=6 src=192.168.0.2 dst=192.168.1.2 sport=0 dport=41739 mask-src=255.255.255.255 mask-dst=255.255.255.255 sport=0 dport=65535 master-src=192.168.0.2 master-dst=192.168.1.2 sport=36390 dport=21 [active since 8s]

5) Make the primary firewall fw-1 fail. Now fw-2 becomes primary.

6) Switch to fw-2 (primary) to commit the external cache into the kernel.

root@fw2# conntrackd -c exp

The logs should display that the commit was successful:

root@fw2# tail -100f /var/log/conntrackd.log
[Wed Dec  7 22:16:31 2011] (pid=19195) [notice] committing external cache: expectations
[Wed Dec  7 22:16:31 2011] (pid=19195) [notice] Committed 1 new entries
[Wed Dec  7 22:16:31 2011] (pid=19195) [notice] commit has taken 0.000366 seconds

7) Switch to the client. Open a new terminal and connect to the port that
   has been announced by the server:

(term-2) user@client$ nc -vvv 192.168.1.2 41739
(UNKNOWN) [192.168.1.2] 41739 (?) open

8) Switch to term-1 and ask for the file listing:

[...]
227 Entering Passive Mode (192,168,1,2,163,11).
LIST

9) Switch to term-2, it should display the listing. That means
   everything has worked fine.

You may want to try disabling the expectation support and
repeating the steps to check that *it does not work* without
the state-synchronization.

You can also display expectation statistics by means of:
root@fwX# conntrackd -s exp

This update requires no changes in the primary-backup.sh script
that is used by the HA manager to interact with conntrackd. Thus,
we provide a backward compatible command line interface.

Regarding the Filter clause and expectations, we use the master
conntrack to filter expectation events. The filtering is performed
in user-space. No kernel-space filtering support for expectations
yet (this support should go in libnetfilter_conntrack at some
point).

This patch also includes support to disable caching and to allow
direct injection of expectations.

Signed-off-by: Pablo Neira Ayuso <pablo@netfilter.org>

8 files changed, 105 insertions, 2 deletions

diff --git a/include/cache.h b/include/cache.h
index abebb97..3af2741 100644
--- a/include/cache.h
+++ b/include/cache.h
@@ -52,6 +52,7 @@ extern struct cache_feature timer_feature;
 enum cache_type {
 	CACHE_T_NONE = 0,
 	CACHE_T_CT,
+	CACHE_T_EXP,
 	CACHE_T_MAX
 };
 
@@ -128,6 +129,9 @@ struct cache_ops {
 extern struct cache_ops cache_sync_internal_ct_ops;
 extern struct cache_ops cache_sync_external_ct_ops;
 extern struct cache_ops cache_stats_ct_ops;
+/* templates to configure expectation caching. */
+extern struct cache_ops cache_sync_internal_exp_ops;
+extern struct cache_ops cache_sync_external_exp_ops;
 
 struct nf_conntrack;
 
diff --git a/include/conntrackd.h b/include/conntrackd.h
index 697d3d7..8baa088 100644
--- a/include/conntrackd.h
+++ b/include/conntrackd.h
@@ -37,6 +37,16 @@
 #define CT_FLUSH_EXT_CACHE	34	/* flush external cache		*/
 #define STATS_PROCESS		35	/* child process stats		*/
 #define STATS_QUEUE		36	/* queue stats			*/
+#define EXP_STATS		37	/* dump statistics		*/
+#define EXP_FLUSH_MASTER	38	/* flush kernel expect table    */
+#define EXP_RESYNC_MASTER	39	/* resync with kernel exp table	*/
+#define EXP_DUMP_INTERNAL	40	/* dump internal expect cache	*/
+#define EXP_DUMP_EXTERNAL	41	/* dump external expect cache	*/
+#define EXP_COMMIT		42	/* commit expectations		*/
+#define ALL_FLUSH_MASTER	43	/* flush all kernel tables	*/
+#define ALL_RESYNC_MASTER	44	/* resync w/all kernel tables	*/
+#define ALL_FLUSH_CACHE		45	/* flush all caches		*/
+#define ALL_COMMIT		46	/* commit all tables		*/
 
 #define DEFAULT_CONFIGFILE	"/etc/conntrackd/conntrackd.conf"
 #define DEFAULT_LOCKFILE	"/var/lock/conntrackd.lock"
@@ -56,6 +66,7 @@
 #define CTD_SYNC_ALARM		(1UL << 3)
 #define CTD_SYNC_NOTRACK	(1UL << 4)
 #define CTD_POLL		(1UL << 5)
+#define CTD_EXPECT		(1UL << 6)
 
 /* FILENAME_MAX is 4096 on my system, perhaps too much? */
 #ifndef FILENAME_MAXLEN
@@ -105,6 +116,8 @@ struct ct_conf {
 		int tcp_window_tracking;
 	} sync;
 	struct {
+		int subsys_id;
+		int groups;
 		int events_reliable;
 	} netlink;
 	struct {
@@ -130,6 +143,7 @@ struct ct_general_state {
 	struct local_server		local;
 	struct ct_mode 			*mode;
 	struct ct_filter		*us_filter;
+	struct exp_filter		*exp_filter;
 
 	struct nfct_handle		*event;         /* event handler */
 	struct nfct_filter		*filter;	/* event filter */
@@ -177,6 +191,10 @@ struct ct_general_state {
 	} stats;
 };
 
+struct commit_runqueue {
+	int		(*cb)(struct nfct_handle *h, int step);
+};
+
 #define STATE_SYNC(x) state.sync->x
 
 struct ct_sync_state {
@@ -196,6 +214,7 @@ struct ct_sync_state {
 		struct nfct_handle	*h;
 		struct evfd		*evfd;
 		int			current;
+		struct commit_runqueue  rq[2];
 		struct {
 			int 		ok;
 			int		fail;
diff --git a/include/external.h b/include/external.h
index eef0e42..70f0c5c 100644
--- a/include/external.h
+++ b/include/external.h
@@ -18,6 +18,17 @@ struct external_handler {
 		void	(*stats)(int fd);
 		void	(*stats_ext)(int fd);
 	} ct;
+	struct {
+		void	(*new)(struct nf_expect *exp);
+		void	(*upd)(struct nf_expect *exp);
+		void	(*del)(struct nf_expect *exp);
+
+		void	(*dump)(int fd, int type);
+		void	(*flush)(void);
+		int	(*commit)(struct nfct_handle *h, int fd);
+		void	(*stats)(int fd);
+		void	(*stats_ext)(int fd);
+	} exp;
 };
 
 extern struct external_handler external_cache;
diff --git a/include/filter.h b/include/filter.h
index f19b18b..3c7c8cc 100644
--- a/include/filter.h
+++ b/include/filter.h
@@ -52,4 +52,11 @@ void ct_filter_set_logic(struct ct_filter *f,
 			 enum ct_filter_logic logic);
 int ct_filter_conntrack(const struct nf_conntrack *ct, int userspace);
 
+struct exp_filter;
+struct nf_expect;
+
+struct exp_filter *exp_filter_create(void);
+int exp_filter_add(struct exp_filter *f, const char *helper_name);
+int exp_filter_find(struct exp_filter *f, const struct nf_expect *exp);
+
 #endif
diff --git a/include/internal.h b/include/internal.h
index f50eb79..2ba9714 100644
--- a/include/internal.h
+++ b/include/internal.h
@@ -34,6 +34,23 @@ struct internal_handler {
 		void	(*stats)(int fd);
 		void	(*stats_ext)(int fd);
 	} ct;
+	struct {
+		void	*data;
+
+		void	(*new)(struct nf_expect *exp, int origin_type);
+		void	(*upd)(struct nf_expect *exp, int origin_type);
+		int	(*del)(struct nf_expect *exp, int origin_type);
+
+		void	(*dump)(int fd, int type);
+		void	(*populate)(struct nf_expect *exp);
+		void	(*purge)(void);
+		int	(*resync)(enum nf_conntrack_msg_type type,
+				  struct nf_expect *exp, void *data);
+		void	(*flush)(void);
+
+		void	(*stats)(int fd);
+		void	(*stats_ext)(int fd);
+	} exp;
 };
 
 extern struct internal_handler internal_cache;
diff --git a/include/log.h b/include/log.h
index f5c5b4f..ae58e79 100644
--- a/include/log.h
+++ b/include/log.h
@@ -4,10 +4,12 @@
 #include <stdio.h>
 
 struct nf_conntrack;
+struct nf_expect;
 
 int init_log(void);
 void dlog(int priority, const char *format, ...);
 void dlog_ct(FILE *fd, struct nf_conntrack *ct, unsigned int type);
+void dlog_exp(FILE *fd, struct nf_expect *exp, unsigned int type);
 void close_log(void);
 
 #endif
diff --git a/include/netlink.h b/include/netlink.h
index 0df0cbb..3bde30c 100644
--- a/include/netlink.h
+++ b/include/netlink.h
@@ -30,4 +30,11 @@ static inline int ct_is_related(const struct nf_conntrack *ct)
 		nfct_attr_is_set(ct, ATTR_MASTER_PORT_DST));
 }
 
+int nl_create_expect(struct nfct_handle *h, const struct nf_expect *orig, int timeout);
+int nl_destroy_expect(struct nfct_handle *h, const struct nf_expect *exp);
+int nl_get_expect(struct nfct_handle *h, const struct nf_expect *exp);
+int nl_dump_expect_table(struct nfct_handle *h);
+int nl_flush_expect_table(struct nfct_handle *h);
+int nl_send_expect_resync(struct nfct_handle *h);
+
 #endif
diff --git a/include/network.h b/include/network.h
index d0531b9..ab95499 100644
--- a/include/network.h
+++ b/include/network.h
@@ -4,9 +4,10 @@
 #include <stdint.h>
 #include <sys/types.h>
 
-#define CONNTRACKD_PROTOCOL_VERSION	0
+#define CONNTRACKD_PROTOCOL_VERSION	1
 
 struct nf_conntrack;
+struct nf_expect;
 
 struct nethdr {
 #if __BYTE_ORDER == __LITTLE_ENDIAN
@@ -28,7 +29,10 @@ enum nethdr_type {
 	NET_T_STATE_CT_NEW = 0,
 	NET_T_STATE_CT_UPD,
 	NET_T_STATE_CT_DEL,
-	NET_T_STATE_MAX = NET_T_STATE_CT_DEL,
+	NET_T_STATE_EXP_NEW = 3,
+	NET_T_STATE_EXP_UPD,
+	NET_T_STATE_EXP_DEL,
+	NET_T_STATE_MAX = NET_T_STATE_EXP_DEL,
 	NET_T_CTL = 10,
 };
 
@@ -92,6 +96,17 @@ enum {
 	__hdr;							\
 })
 
+#define BUILD_NETMSG_FROM_EXP(exp, query)			\
+({								\
+	static char __net[4096];				\
+	struct nethdr *__hdr = (struct nethdr *) __net;		\
+	memset(__hdr, 0, NETHDR_SIZ);				\
+	nethdr_set(__hdr, query);				\
+	exp2msg(exp, __hdr);					\
+	HDR_HOST2NETWORK(__hdr);				\
+	__hdr;							\
+})
+
 struct mcast_sock_multi;
 
 enum {
@@ -239,4 +254,25 @@ struct nta_attr_natseqadj {
 void ct2msg(const struct nf_conntrack *ct, struct nethdr *n);
 int msg2ct(struct nf_conntrack *ct, struct nethdr *n, size_t remain);
 
+enum nta_exp_attr {
+	NTA_EXP_MASTER_IPV4 = 0,	/* struct nfct_attr_grp_ipv4 */
+	NTA_EXP_MASTER_IPV6,		/* struct nfct_attr_grp_ipv6 */
+	NTA_EXP_MASTER_L4PROTO,		/* uint8_t */
+	NTA_EXP_MASTER_PORT,		/* struct nfct_attr_grp_port */
+	NTA_EXP_EXPECT_IPV4 = 4,	/* struct nfct_attr_grp_ipv4 */
+	NTA_EXP_EXPECT_IPV6,		/* struct nfct_attr_grp_ipv6 */
+	NTA_EXP_EXPECT_L4PROTO,		/* uint8_t */
+	NTA_EXP_EXPECT_PORT,		/* struct nfct_attr_grp_port */
+	NTA_EXP_MASK_IPV4 = 8,		/* struct nfct_attr_grp_ipv4 */
+	NTA_EXP_MASK_IPV6,		/* struct nfct_attr_grp_ipv6 */
+	NTA_EXP_MASK_L4PROTO,		/* uint8_t */
+	NTA_EXP_MASK_PORT,		/* struct nfct_attr_grp_port */
+	NTA_EXP_TIMEOUT,		/* uint32_t */
+	NTA_EXP_FLAGS,			/* uint32_t */
+	NTA_EXP_MAX
+};
+
+void exp2msg(const struct nf_expect *exp, struct nethdr *n);
+int msg2exp(struct nf_expect *exp, struct nethdr *n, size_t remain);
+
 #endif