bug fixes on mult targets. added packet rate limit option to rules

11 files changed, 346 insertions, 105 deletions

diff --git a/src/lbdata.cc b/src/lbdata.cc
index 9c33f51..4dc8d76 100644
--- a/src/lbdata.cc
+++ b/src/lbdata.cc
@@ -69,9 +69,6 @@ void
 LBHealth::start_new_test_cycle()
 {
   _test_iter = _test_coll.begin();
-  if (_test_iter != _test_coll.end()) {
-    _test_iter->second->init();
-  }
   _test_success = false;
 }
 
@@ -85,6 +82,7 @@ LBHealth::send_test()
   if (_test_success == true || _test_iter == _test_coll.end()) {
     return; //means we are done
   }
+  _test_iter->second->init();
   _test_iter->second->send(*this);
 }
 
@@ -226,6 +224,26 @@ LBData::dump()
     cout << "    " << r_iter->second._d_addr << endl;
     cout << "    " << r_iter->second._d_port << endl;
     
+    if (r_iter->second._limit) {
+      cout << "     limit:" << endl;
+      cout << "       burst: " << r_iter->second._limit_burst << endl;
+      cout << "       rate: " << r_iter->second._limit_burst << endl;
+      if (r_iter->second._limit_mode) {
+	cout << "       thresh: above" << endl;
+      }
+      else { 
+	cout << "       thresh: below" << endl;
+      }
+      if (r_iter->second._limit_period == LBRule::K_SECOND) {
+	cout << "       period: second" << endl;
+      }
+      else if (r_iter->second._limit_period == LBRule::K_MINUTE) {
+	cout << "       period: minute" << endl;
+      }
+      else if (r_iter->second._limit_period == LBRule::K_HOUR) {
+	cout << "       period: hour" << endl;
+      }
+    }
     LBRule::InterfaceDistIter ri_iter = r_iter->second._iface_dist_coll.begin();
     while (ri_iter != r_iter->second._iface_dist_coll.end()) {
       cout << "      interface: " << ri_iter->first << endl;
diff --git a/src/lbdata.hh b/src/lbdata.hh
index 39115a6..4444009 100644
--- a/src/lbdata.hh
+++ b/src/lbdata.hh
@@ -28,12 +28,19 @@ class LBRule {
   typedef map<string, int>::iterator InterfaceDistIter;
 
   typedef enum {ALL,ICMP,UDP,TCP} Protocol;
+  typedef enum {K_SECOND,K_MINUTE,K_HOUR} LimitPeriod;
 
   LBRule() :
     _proto("all"),
     _exclude(false),
     _failover(false),
-    _enable_source_based_routing(false)      {}
+    _enable_source_based_routing(false),
+    _limit(false),
+    _limit_burst("5"),
+    _limit_rate("1"),
+    _limit_mode(false),
+    _limit_period(K_HOUR)
+  {}
 
  public:
   string _proto;
@@ -50,6 +57,12 @@ class LBRule {
 
   bool _enable_source_based_routing;
 
+  bool _limit;
+  string _limit_burst;
+  string _limit_rate;
+  bool _limit_mode; //true above, false below
+  LimitPeriod _limit_period;
+
   string _in_iface;
   InterfaceDistColl _iface_dist_coll;
 };
diff --git a/src/lbdatafactory.cc b/src/lbdatafactory.cc
index f230f4f..f708096 100644
--- a/src/lbdatafactory.cc
+++ b/src/lbdatafactory.cc
@@ -107,6 +107,17 @@ LBDataFactory::load(const string &conf_file)
     return false;
   }
 
+  //insert default test if empty
+  LBData::InterfaceHealthIter iter = _lb_data._iface_health_coll.begin();
+  while (iter != _lb_data._iface_health_coll.end()) {
+    if (iter->second._test_coll.empty()) {
+      LBTestICMP *test = new LBTestICMP(_debug);
+      iter->second._test_coll.insert(pair<int,LBTest*>(1,test));
+      //add default ping test...
+    }
+    ++iter;
+  }
+
   if (_debug) {
     _lb_data.dump();
   }
@@ -184,6 +195,9 @@ LBDataFactory::process(const vector<string> &path, int depth, const string &key,
     else if (depth > 0 && path[1] == "enable-source-based-routing") {
       process_rule_enablesourcebasedrouting(l_key,l_value);
     }
+    else if (depth > 1 && path[1] == "limit") {
+      process_rule_limit(l_key,l_value);
+    }
     else {
       process_rule(l_key,l_value);
     }
@@ -345,6 +359,9 @@ LBDataFactory::process_rule(const string &key, const string &value)
 void
 LBDataFactory::process_rule_protocol(const string &key, const string &value)
 {
+  if (_rule_iter == _lb_data._lb_rule_coll.end()) {
+    return;
+  }
   if (key == "protocol") {
     if (strcasecmp(value.c_str(),"ALL") == 0) {
       _rule_iter->second._proto = "all";
@@ -370,24 +387,36 @@ LBDataFactory::process_rule_protocol(const string &key, const string &value)
 void
 LBDataFactory::process_rule_exclude(const string &key, const string &value)
 {
+  if (_rule_iter == _lb_data._lb_rule_coll.end()) {
+    return;
+  }
   _rule_iter->second._exclude = true;
 }
 
 void
 LBDataFactory::process_rule_failover(const string &key, const string &value)
 {
+  if (_rule_iter == _lb_data._lb_rule_coll.end()) {
+    return;
+  }
   _rule_iter->second._failover = true;
 }
 
 void
 LBDataFactory::process_rule_enablesourcebasedrouting(const string &key, const string &value)
 {
+  if (_rule_iter == _lb_data._lb_rule_coll.end()) {
+    return;
+  }
   _rule_iter->second._enable_source_based_routing = true;
 }
 
 void
 LBDataFactory::process_rule_source(const string &key, const string &value)
 {
+  if (_rule_iter == _lb_data._lb_rule_coll.end()) {
+    return;
+  }
   if (key == "address") {
     _rule_iter->second._s_addr = value;
   }
@@ -402,6 +431,9 @@ LBDataFactory::process_rule_source(const string &key, const string &value)
 void
 LBDataFactory::process_rule_destination(const string &key, const string &value)
 {
+  if (_rule_iter == _lb_data._lb_rule_coll.end()) {
+    return;
+  }
   if (key == "address") {
     _rule_iter->second._d_addr = value;
   }
@@ -416,6 +448,9 @@ LBDataFactory::process_rule_destination(const string &key, const string &value)
 void
 LBDataFactory::process_rule_inbound_interface(const string &key, const string &value)
 {
+  if (_rule_iter == _lb_data._lb_rule_coll.end()) {
+    return;
+  }
   if (_debug) {
     cout << "LBDataFactory::process_rule_inbound_interface(): " << key << ", " << value << endl;
   }
@@ -460,7 +495,41 @@ LBDataFactory::process_rule_interface(const string &key, const string &value)
   }
 }
 
-
+void
+LBDataFactory::process_rule_limit(const string &key, const string &value)
+{
+  if (_rule_iter == _lb_data._lb_rule_coll.end()) {
+    return;
+  }
+  
+  _rule_iter->second._limit = true;
+  
+  if (key == "burst") {
+    _rule_iter->second._limit_burst = value;
+  }
+  else if (key == "rate") {    
+    _rule_iter->second._limit_rate = value;
+  }
+  else if (key == "period") {
+    if (key == "second") {
+      _rule_iter->second._limit_period = LBRule::K_SECOND;
+    }
+    else if (key == "minute") {
+      _rule_iter->second._limit_period = LBRule::K_MINUTE;
+    }
+    else if (key == "hour") {
+      _rule_iter->second._limit_period = LBRule::K_HOUR;
+    }
+  }
+  else if (key == "threshold") {
+    if (value == "true") {
+      _rule_iter->second._limit_mode = true;
+    }
+    else {
+      _rule_iter->second._limit_mode = false;
+    }
+  }
+}
 
 LBData
 LBDataFactory::get()
diff --git a/src/lbdatafactory.hh b/src/lbdatafactory.hh
index 2901f72..f1a0896 100644
--- a/src/lbdatafactory.hh
+++ b/src/lbdatafactory.hh
@@ -97,6 +97,9 @@ private:
   void
   process_rule_enablesourcebasedrouting(const string &key, const string &value);
 
+  void
+  process_rule_limit(const string &key, const string &value);
+
 private:
   bool _debug;
   LBHealth _lb_health;
diff --git a/src/lbdecision.cc b/src/lbdecision.cc
index 261aadd..d7ff65b 100644
--- a/src/lbdecision.cc
+++ b/src/lbdecision.cc
@@ -305,35 +305,79 @@ LBDecision::run(LBData &lb_data)
 	//no rules here!
       }
       else {
+	char rule_str[20];
+	sprintf(rule_str,"%d",iter->first);
+
+	if (iter->second._limit) {
+	  string limit_cmd = get_limit_cmd(iter->second);
+	  execute(string("iptables -t mangle -N WANLOADBALANCE_PRE_LIMIT_") + rule_str, stdout);
+	  execute(string("iptables -t mangle -F WANLOADBALANCE_PRE_LIMIT_") + rule_str, stdout);
+	  execute(string("iptables -t mangle -N WANLOADBALANCE_OUT_LIMIT_") + rule_str, stdout);
+	  execute(string("iptables -t mangle -F WANLOADBALANCE_OUT_LIMIT_") + rule_str, stdout);
+	  
+	  execute(string("iptables -t mangle -A WANLOADBALANCE_PRE ") + app_cmd + " " + limit_cmd + " -j WANLOADBALANCE_PRE_LIMIT_" + rule_str, stdout);
+	  execute(string("iptables -t mangle -A WANLOADBALANCE_OUT ") + app_cmd_local + " " + limit_cmd + " -j WANLOADBALANCE_OUT_LIMIT_" + rule_str, stdout);
+	}
+
 	char fbuf[20],dbuf[80];
 	map<string,float>::iterator w_iter = weights.begin();
 	for (w_iter = weights.begin(); w_iter != (--weights.end()); w_iter++) {
 	  sprintf(fbuf,"%f",w_iter->second);
 	  sprintf(dbuf,"%s",w_iter->first.c_str());
 	  if (iter->second._enable_source_based_routing) {
-	    execute(string("iptables -t mangle -A WANLOADBALANCE_PRE ") + app_cmd + " -m statistic --mode random --probability " + fbuf + " -j ISP_" + dbuf, stdout);
-	    execute(string("iptables -t mangle -A WANLOADBALANCE_OUT ") + app_cmd_local + " -m statistic --mode random --probability " + fbuf + " -j ISP_" + dbuf, stdout);
+	    if (iter->second._limit) {
+	      //fill in limit statement here
+	      execute(string("iptables -t mangle -A WANLOADBALANCE_PRE_LIMIT_") + rule_str + " -m statistic --mode random --probability " + fbuf + " -j ISP_" + dbuf, stdout);
+	      execute(string("iptables -t mangle -A WANLOADBALANCE_OUT_LIMIT_") + rule_str + " -m statistic --mode random --probability " + fbuf + " -j ISP_" + dbuf, stdout);
+	    }
+	    else {
+	      execute(string("iptables -t mangle -A WANLOADBALANCE_PRE ") + app_cmd + " -m statistic --mode random --probability " + fbuf + " -j ISP_" + dbuf, stdout);
+	      execute(string("iptables -t mangle -A WANLOADBALANCE_OUT ") + app_cmd_local + " -m statistic --mode random --probability " + fbuf + " -j ISP_" + dbuf, stdout);
+	    }
 	  }
 	  else {
-	    execute(string("iptables -t mangle -A WANLOADBALANCE_PRE ") + app_cmd + " -m state --state NEW -m statistic --mode random --probability " + fbuf + " -j ISP_" + dbuf, stdout);
-	    execute(string("iptables -t mangle -A WANLOADBALANCE_OUT ") + app_cmd_local + " -m state --state NEW -m statistic --mode random --probability " + fbuf + " -j ISP_" + dbuf, stdout);
+	    if (iter->second._limit) {
+	      //fill in limit statement here
+	      execute(string("iptables -t mangle -A WANLOADBALANCE_PRE_LIMIT_") + rule_str + " -m statistic --mode random --probability " + fbuf + " -j ISP_" + dbuf, stdout);
+	      execute(string("iptables -t mangle -A WANLOADBALANCE_OUT_LIMIT_") + rule_str + " -m statistic --mode random --probability " + fbuf + " -j ISP_" + dbuf, stdout);
+	    }
+	    else {
+	      execute(string("iptables -t mangle -A WANLOADBALANCE_PRE ") + app_cmd + " -m state --state NEW -m statistic --mode random --probability " + fbuf + " -j ISP_" + dbuf, stdout);
+	      execute(string("iptables -t mangle -A WANLOADBALANCE_OUT ") + app_cmd_local + " -m state --state NEW -m statistic --mode random --probability " + fbuf + " -j ISP_" + dbuf, stdout);
+	    }
 	  }
 	}
 	sprintf(dbuf,"%s",(--weights.end())->first.c_str());
 	if (iter->second._enable_source_based_routing) {
-	  execute(string("iptables -t mangle -A WANLOADBALANCE_PRE ") + app_cmd + " -j ISP_" + dbuf, stdout);
-	  execute(string("iptables -t mangle -A WANLOADBALANCE_OUT ") + app_cmd_local + " -j ISP_" + dbuf, stdout);
+	  if (iter->second._limit) {
+	    //fill in limit statement here
+	    execute(string("iptables -t mangle -A WANLOADBALANCE_PRE_LIMIT_") + rule_str + " -j ISP_" + dbuf, stdout);
+	    execute(string("iptables -t mangle -A WANLOADBALANCE_OUT_LIMIT_") + rule_str + " -j ISP_" + dbuf, stdout);
+	    execute(string("iptables -t mangle -A WANLOADBALANCE_PRE_LIMIT_") + rule_str + " -j ACCEPT", stdout);
+	    execute(string("iptables -t mangle -A WANLOADBALANCE_OUT_LIMIT_") + rule_str + " -j ACCEPT", stdout);
+	  }
+	  else {
+	    execute(string("iptables -t mangle -A WANLOADBALANCE_PRE ") + app_cmd + " -j ISP_" + dbuf, stdout);
+	    execute(string("iptables -t mangle -A WANLOADBALANCE_OUT ") + app_cmd_local + " -j ISP_" + dbuf, stdout);
+	  }
+	  
 	}
 	else {
-	  execute(string("iptables -t mangle -A WANLOADBALANCE_PRE ") + app_cmd + " -m state --state NEW -j ISP_" + dbuf, stdout);
-	  execute(string("iptables -t mangle -A WANLOADBALANCE_OUT ") + app_cmd_local + " -m state --state NEW -j ISP_" + dbuf, stdout);
+	  if (iter->second._limit) {
+	    //fill in limit statement here
+	    execute(string("iptables -t mangle -A WANLOADBALANCE_PRE_LIMIT_") + rule_str + " -j ISP_" + dbuf, stdout);
+	    execute(string("iptables -t mangle -A WANLOADBALANCE_OUT_LIMIT_") + rule_str + " -j ISP_" + dbuf, stdout);
+	  }
+	  else {
+	    execute(string("iptables -t mangle -A WANLOADBALANCE_PRE ") + app_cmd + " -m state --state NEW -j ISP_" + dbuf, stdout);
+	    execute(string("iptables -t mangle -A WANLOADBALANCE_OUT ") + app_cmd_local + " -m state --state NEW -j ISP_" + dbuf, stdout);
+	  }
 	}
 	execute(string("iptables -t mangle -A WANLOADBALANCE_PRE ") + app_cmd + " -j CONNMARK --restore-mark", stdout);
 	execute(string("iptables -t mangle -A WANLOADBALANCE_OUT ") + app_cmd_local + " -j CONNMARK --restore-mark", stdout);
       }
     }
     ++iter;
-    continue;
   }
 }
 
@@ -352,6 +396,18 @@ LBDecision::shutdown(LBData &data)
   execute("iptables -t mangle -D PREROUTING -j WANLOADBALANCE_PRE", stdout);
   execute("iptables -t mangle -D OUTPUT -j WANLOADBALANCE_OUT", stdout);
 
+  LBData::LBRuleIter iter = data._lb_rule_coll.begin();
+  while (iter != data._lb_rule_coll.end()) {
+    if (iter->second._limit) {
+      char rule_str[20];
+      sprintf(rule_str,"%d",iter->first);
+      execute(string("iptables -t mangle -D PREROUTING -j WANLOADBALANCE_PRE_LIMIT_") + rule_str,stdout);
+      execute(string("iptables -t mangle -D PREROUTING -j WANLOADBALANCE_OUT_LIMIT_") + rule_str,stdout);
+    }
+    ++iter;
+  }
+
+
   //clear out nat as well
   execute("iptables -t nat -F WANLOADBALANCE", stdout);
   execute("iptables -t nat -D VYATTA_PRE_SNAT_HOOK -j WANLOADBALANCE", stdout);
@@ -631,4 +687,33 @@ LBDecision::fetch_iface_addr(const string &iface)
   return string("");
 }
 
-
+/**
+ * Builds out the limit matching criteria
+ **/
+string
+LBDecision::get_limit_cmd(LBRule &rule)
+{
+  string cmd;
+  if (!rule._limit) {
+    return cmd;
+  }
+  //needs to be of the form:
+  //-m limit [!] --limit 1/second --limit-burst 5 
+  cmd = "-m limit ";
+  if (rule._limit_mode) {
+    cmd += "! ";
+  }
+  cmd += string("--limit ") + rule._limit_rate + "/";
+  if (rule._limit_period == LBRule::K_SECOND) {
+    cmd += "second ";
+  }
+  else if (rule._limit_period == LBRule::K_MINUTE) {
+    cmd += "minute ";
+  }
+  else {
+    cmd += "hour ";
+  }
+  
+  cmd += string("--limit-burst ") + rule._limit_burst;
+  return cmd;
+}
diff --git a/src/lbdecision.hh b/src/lbdecision.hh
index fdfdfb3..d8c9ecf 100644
--- a/src/lbdecision.hh
+++ b/src/lbdecision.hh
@@ -51,6 +51,9 @@ private:
   string
   fetch_iface_nexthop(const string &iface);
 
+  string
+  get_limit_cmd(LBRule &rule);
+
 private:
   bool _debug;
 };
diff --git a/src/lbpathtest.cc b/src/lbpathtest.cc
index 7fd45a5..d9ec242 100644
--- a/src/lbpathtest.cc
+++ b/src/lbpathtest.cc
@@ -63,11 +63,11 @@ LBPathTest::start(LBData &lb_data)
 
   set<LBHealth*> coll;
 
+  //iterate over the health interfaces
   LBData::InterfaceHealthIter iter = lb_data._iface_health_coll.begin();
   while (iter != lb_data._iface_health_coll.end()) {
     iter->second.start_new_test_cycle();
     coll.insert(&(iter->second));
-
     ++iter;
   }
   
diff --git a/src/lbtest_icmp.cc b/src/lbtest_icmp.cc
index 645aeed..278b927 100644
--- a/src/lbtest_icmp.cc
+++ b/src/lbtest_icmp.cc
@@ -43,7 +43,11 @@ ICMPEngine::init()
   if (_debug) {
     cout << "LBTestICMP::init(): initializing test system" << endl;
   }
-  _results.erase(_results.begin(),_results.end());
+  if (_initialized == false) {
+    _results.erase(_results.begin(),_results.end());
+  }
+  _initialized = true;
+  _received = false;
 }
 
 /**
@@ -68,10 +72,10 @@ ICMPEngine::process(LBHealth &health,LBTestICMP *data)
   if (target.empty()) {
     return -1;
   }
+  _packet_id = ++_packet_id % 32767;
   if (_debug) {
-    cout << "LBTestICMP::start(): sending ping test for: " << health._interface << " for " << target << endl;
+    cout << "LBTestICMP::start(): sending ping test for: " << health._interface << " for " << target << " id: " << _packet_id << endl;
   }
-  _packet_id = ++_packet_id % 32767;
   send(data->_send_icmp_sock, health._interface, target, _packet_id);
   _results.insert(pair<int,PktData>(_packet_id,PktData(health._interface,-1)));
 }
@@ -83,63 +87,84 @@ ICMPEngine::process(LBHealth &health,LBTestICMP *data)
 int
 ICMPEngine::recv(LBHealth &health,LBTestICMP *data) 
 {
-  struct timeval send_time;
-  gettimeofday(&send_time,NULL);
-
-  if (_results.empty() == true) {
-    return -1;
-  }
-  
-  //use gettimeofday to calculate time to millisecond
-  //then iterate over recv socket and receive and record
-  //use sysinfo to make sure we don't get stuck in a loop with timechange
-  struct sysinfo si;
-  sysinfo(&si);
-  //for now hardcode to 5 second overall timeout
-  unsigned long timeout = si.uptime + 5; //seconds
-  unsigned long cur_time = si.uptime;
-  while (cur_time < timeout) {
-    int id = receive(data->_recv_icmp_sock);
-    if (_debug) {
-      cout << "LBTestICMP::start(): " << id << endl;
-    }
-
-    //update current time for comparison
+  _initialized = false;
+  if (_received == false) {
+    //use gettimeofday to calculate time to millisecond
+    //then iterate over recv socket and receive and record
+    //use sysinfo to make sure we don't get stuck in a loop with timechange
+    struct timeval send_time;
+    gettimeofday(&send_time,NULL);
     struct sysinfo si;
     sysinfo(&si);
-    timeval recv_time;
-    gettimeofday(&recv_time,NULL);
-    cur_time = si.uptime;
-    map<int,PktData>::iterator r_iter = _results.find(id);
-    if (r_iter != _results.end()) {
-      
-      //calculate time in milliseconds
-      int secs = 0;
-      int msecs = recv_time.tv_usec - send_time.tv_usec;
-      if (msecs < 0) {
-	secs = recv_time.tv_sec - send_time.tv_sec - 1;
+    //for now hardcode to 5 second overall timeout
+    unsigned long timeout = si.uptime + 5; //seconds
+    unsigned long cur_time = si.uptime;
+    
+    int pending_result_ct = _results.size();
+    //let's pull off all of the results...
+    while (cur_time < timeout && pending_result_ct != 0) {
+      int id = receive(data->_recv_icmp_sock);
+      if (_debug) {
+	cout << "LBTestICMP::recv(): " << id << endl;
       }
-      else {
-	secs = recv_time.tv_sec - send_time.tv_sec;
+      
+      //update current time for comparison
+      struct sysinfo si;
+      sysinfo(&si);
+      timeval recv_time;
+      gettimeofday(&recv_time,NULL);
+      cur_time = si.uptime;
+      map<int,PktData>::iterator r_iter = _results.find(id);
+      if (r_iter != _results.end()) {
+	//calculate time in milliseconds
+	int secs = 0;
+	int msecs = recv_time.tv_usec - send_time.tv_usec;
+	if (msecs < 0) {
+	  secs = recv_time.tv_sec - send_time.tv_sec - 1;
+	}
+	else {
+	  secs = recv_time.tv_sec - send_time.tv_sec;
+	}
+	//time in milliseconds below
+	r_iter->second._rtt = abs(msecs) / 1000 + 1000 * secs;
+	--pending_result_ct;
       }
-      //time in milliseconds below
-      int rtt = abs(msecs) / 1000 + 1000 * secs;
-      if (rtt < data->_resp_time) {
+    }
+    if (_debug) {
+      cout << "LBTestICMP::recv(): finished heath test" << endl;
+    }
+    _received = true;
+  }
+
+  //now let's just look the packet up since we are through with the receive option
+  map<int,PktData>::iterator r_iter = _results.begin();
+  data->_state = LBTest::K_FAILURE;
+  while (r_iter != _results.end()) {
+
+    if (r_iter->second._iface == health._interface) {
+      if (r_iter->second._rtt < data->_resp_time) {
 	data->_state = LBTest::K_SUCCESS;
+	if (_debug) {
+	  cout << "LBTestICMP::recv(): success for " << r_iter->second._iface << " : " << r_iter->second._rtt << endl;
+	}
+	int rtt = r_iter->second._rtt;
+	_results.erase(r_iter);
 	return rtt;
       }
       else {
-	data->_state = LBTest::K_FAILURE;
+	if (_debug) {
+	  cout << "LBTestICMP::recv(): failure for " << r_iter->second._iface << " : " << r_iter->second._rtt << endl;
+	}
+	_results.erase(r_iter);
 	return -1;
       }
-      _results.erase(r_iter);
     }
+    ++r_iter;
   }
   
   if (_debug) {
-    cout << "LBTestICMP::start(): finished heath test" << endl;
+    cout << "LBTestICMP::recv(): failure for " << health._interface << " : unable to find interface" << endl;
   }
-  data->_state = LBTest::K_FAILURE;
   return -1;
 }
 
diff --git a/src/lbtest_icmp.hh b/src/lbtest_icmp.hh
index ea8e893..8b55130 100644
--- a/src/lbtest_icmp.hh
+++ b/src/lbtest_icmp.hh
@@ -41,6 +41,7 @@ public:
   ICMPEngine() : 
     _debug(true),
     _initialized(false),
+    _received(false),
     _packet_id(0) 
   {}
 
@@ -66,6 +67,7 @@ private:
 private:
   bool _debug;
   bool _initialized;
+  bool _received;
   int _packet_id;
   map<int,PktData> _results;
 };
diff --git a/src/lbtest_ttl.cc b/src/lbtest_ttl.cc
index 843bc57..d6b067b 100644
--- a/src/lbtest_ttl.cc
+++ b/src/lbtest_ttl.cc
@@ -44,7 +44,11 @@ TTLEngine::init()
   if (_debug) {
     cout << "LBTestICMP::init(): initializing test system" << endl;
   }
-  _results.erase(_results.begin(),_results.end());
+  if (_initialized == false) {
+    _results.erase(_results.begin(),_results.end());
+  }
+  _initialized = true;
+  _received = false;
 }
 
 /**
@@ -88,65 +92,82 @@ TTLEngine::process(LBHealth &health,LBTestTTL *data)
 int
 TTLEngine::recv(LBHealth &health,LBTestTTL *data) 
 {
-  if (_results.empty() == true) {
-    return -1;
-  }
-  
-  if (_debug) {
-    cout << "TTLEngine::recv()" << endl;
-  }
-  struct timeval send_time;
-  gettimeofday(&send_time,NULL);
+  _initialized = false;
+  if (_received == false) {
   
-  //use gettimeofday to calculate time to millisecond
-  //then iterate over recv socket and receive and record
-  //use sysinfo to make sure we don't get stuck in a loop with timechange
-  struct sysinfo si;
-  sysinfo(&si);
-  //for now hardcode to 5 second overall timeout
-  unsigned long timeout = si.uptime + 5; //seconds
-  unsigned long cur_time = si.uptime;
-  while (cur_time < timeout) {
-    int id = receive(data->_recv_icmp_sock);
-    if (_debug) {
-      cout << "TTLEngine::recv(): " << id << endl;
-    }
-    //update current time for comparison
+    //use gettimeofday to calculate time to millisecond
+    //then iterate over recv socket and receive and record
+    //use sysinfo to make sure we don't get stuck in a loop with timechange
+    struct timeval send_time;
+    gettimeofday(&send_time,NULL);
     struct sysinfo si;
     sysinfo(&si);
-    timeval recv_time;
-    gettimeofday(&recv_time,NULL);
-    cur_time = si.uptime;
-    map<int,PktData>::iterator r_iter = _results.find(id);
-    if (r_iter != _results.end()) {
-      
-      //calculate time in milliseconds
-      int secs = 0;
-      int msecs = recv_time.tv_usec - send_time.tv_usec;
-      if (msecs < 0) {
-	secs = recv_time.tv_sec - send_time.tv_sec - 1;
+    //for now hardcode to 5 second overall timeout
+    unsigned long timeout = si.uptime + 5; //seconds
+    unsigned long cur_time = si.uptime;
+
+    int pending_result_ct = _results.size();
+    while (cur_time < timeout) {
+      int id = receive(data->_recv_icmp_sock);
+      if (_debug) {
+	cout << "TTLEngine::recv(): " << id << endl;
       }
-      else {
-	secs = recv_time.tv_sec - send_time.tv_sec;
+      //update current time for comparison
+      struct sysinfo si;
+      sysinfo(&si);
+      timeval recv_time;
+      gettimeofday(&recv_time,NULL);
+      cur_time = si.uptime;
+      map<int,PktData>::iterator r_iter = _results.find(id);
+      if (r_iter != _results.end()) {
+	//calculate time in milliseconds
+	int secs = 0;
+	int msecs = recv_time.tv_usec - send_time.tv_usec;
+	if (msecs < 0) {
+	  secs = recv_time.tv_sec - send_time.tv_sec - 1;
+	}
+	else {
+	  secs = recv_time.tv_sec - send_time.tv_sec;
+	}
+	//time in milliseconds below
+	r_iter->second._rtt = abs(msecs) / 1000 + 1000 * secs;
+	--pending_result_ct;
       }
-      //time in milliseconds below
-      int rtt = abs(msecs) / 1000 + 1000 * secs;
-      if (rtt < data->_resp_time) {
+    }
+    if (_debug) {
+      cout << "TTLEngine::recv(): finished heath test" << endl;
+    }
+    _received = true;
+  }
+  //now let's just look the packet up since we are through with the receive option
+  map<int,PktData>::iterator r_iter = _results.begin();
+  data->_state = LBTest::K_FAILURE;
+  while (r_iter != _results.end()) {
+
+    if (r_iter->second._iface == health._interface) {
+      if (r_iter->second._rtt < data->_resp_time) {
 	data->_state = LBTest::K_SUCCESS;
+	if (_debug) {
+	  cout << "TTLEngine::recv(): success for " << r_iter->second._iface << " : " << r_iter->second._rtt << endl;
+	}
+	int rtt = r_iter->second._rtt;
+	_results.erase(r_iter);
 	return rtt;
       }
       else {
-	data->_state = LBTest::K_FAILURE;
+	if (_debug) {
+	  cout << "TTLEngine::recv(): failure for " << r_iter->second._iface << " : " << r_iter->second._rtt << endl;
+	}
+	_results.erase(r_iter);
 	return -1;
       }
-      _results.erase(r_iter);
     }
+    ++r_iter;
   }
   
   if (_debug) {
-    cout << "TTLEngine::recv(): finished heath test" << endl;
+    cout << "EngineTTL::recv(): failure for " << health._interface << " : unable to find interface" << endl;
   }
-  data->_state = LBTest::K_FAILURE;
   return -1;
 }
 
diff --git a/src/lbtest_ttl.hh b/src/lbtest_ttl.hh
index 8f2035f..ca7f38c 100644
--- a/src/lbtest_ttl.hh
+++ b/src/lbtest_ttl.hh
@@ -36,6 +36,7 @@ public:
   TTLEngine() : 
     _debug(true),
     _initialized(false),
+    _received(false),
     _packet_id(32767),
     _min_port_id(32767),
     _max_port_id(55000)
@@ -69,6 +70,7 @@ private:
 private:
   bool _debug;
   bool _initialized;
+  bool _received;
   unsigned short _packet_id;
   unsigned short _min_port_id;
   unsigned long _max_port_id;