Keep track of basic aliveness for peers regardless if direct or indirect connectivity and use this for multicast propagation. Also consolidate adding of active bridges via the same functor as regular multicast next hops.

8 files changed, 167 insertions, 92 deletions

diff --git a/node/Constants.hpp b/node/Constants.hpp
index 4d720d5d..1745ef6b 100644
--- a/node/Constants.hpp
+++ b/node/Constants.hpp
@@ -314,14 +314,19 @@ error_no_byte_order_defined;
 #define ZT_TCP_TUNNEL_FAILOVER_TIMEOUT (ZT_STARTUP_AGGRO * 5)
 
 /**
+ * Timeout for overall peer activity (measured from last receive)
+ */
+#define ZT_PEER_ACTIVITY_TIMEOUT ((ZT_PEER_DIRECT_PING_DELAY * 2) + ZT_PING_CHECK_DELAY)
+
+/**
  * Path activity timeout (for non-fixed paths)
  */
-#define ZT_PEER_PATH_ACTIVITY_TIMEOUT ((ZT_PEER_DIRECT_PING_DELAY * 2) + ZT_PING_CHECK_DELAY)
+#define ZT_PEER_PATH_ACTIVITY_TIMEOUT ZT_PEER_ACTIVITY_TIMEOUT
 
 /**
  * Close TCP sockets if unused for this long (SocketManager)
  */
-#define ZT_TCP_TUNNEL_ACTIVITY_TIMEOUT ZT_PEER_PATH_ACTIVITY_TIMEOUT
+#define ZT_TCP_TUNNEL_ACTIVITY_TIMEOUT ZT_PEER_ACTIVITY_TIMEOUT
 
 /**
  * Stop relaying via peers that have not responded to direct sends
@@ -387,14 +392,14 @@ error_no_byte_order_defined;
 #define ZT_UPDATE_MIN_INTERVAL 120000
 
 /**
- * Maximum interval between checks for new versions (2 hours)
+ * Maximum interval between checks for new versions
  */
 #define ZT_UPDATE_MAX_INTERVAL 7200000
 
 /**
  * Software update HTTP timeout in seconds
  */
-#define ZT_UPDATE_HTTP_TIMEOUT 30
+#define ZT_UPDATE_HTTP_TIMEOUT 120
 
 /**
  * Sanity limit on maximum bridge routes
diff --git a/node/Multicaster.hpp b/node/Multicaster.hpp
index 7d67aada..262f0304 100644
--- a/node/Multicaster.hpp
+++ b/node/Multicaster.hpp
@@ -40,6 +40,7 @@
 #include "Constants.hpp"
 #include "Mutex.hpp"
 #include "MulticastGroup.hpp"
+#include "Topology.hpp"
 #include "Address.hpp"
 #include "Buffer.hpp"
 
@@ -112,6 +113,7 @@ public:
 	 * @param nwid Network ID
 	 * @param mg Multicast group
 	 * @param nextHopFunc Function to call for each address, search stops if it returns false
+	 * @tparam F Function to receive each next hop address
 	 */
 	template<typename F>
 	inline void getNextHops(uint64_t nwid,const MulticastGroup &mg,F nextHopFunc)
@@ -151,20 +153,40 @@ public:
 		 * @param origin Originating address
 		 * @param prefixBits Number of bits in propagation restriction prefix
 		 * @param prefix Propagation restrition prefix
+		 * @param topology Topology database
+		 * @param now Current time
 		 */
-		AddToPropagationQueue(unsigned char **ptr,unsigned char *end,unsigned char *bloom,uint16_t bloomNonce,const Address &origin,unsigned int prefixBits,uint64_t prefix)
-			throw() :
+		AddToPropagationQueue(
+			unsigned char **ptr,
+			unsigned char *end,
+			unsigned char *bloom,
+			uint16_t bloomNonce,
+			const Address &origin,
+			unsigned int prefixBits,
+			uint64_t prefix,
+			const Topology *topology,
+			uint64_t now)
+		throw() :
 			_origin(origin),
 			_bloomNonce((uint64_t)bloomNonce),
 			_prefix(prefix),
+			_now(now),
 			_ptr(ptr),
 			_end(end),
 			_bloom(bloom),
+			_topology(topology),
 			_prefixBits(prefixBits) {}
 
+		/**
+		 * @param a Address to (possibly) add
+		 * @return True if FIFO still contains room for more possible addresses
+		 */
 		inline bool operator()(const Address &a)
 			throw()
 		{
+			if (*_ptr >= _end)
+				return false;
+
 			// Exclude original sender -- obviously they've already seen it
 			if (a == _origin)
 				return true;
@@ -173,27 +195,36 @@ public:
 			if (!a.withinMulticastPropagationPrefix(_prefix,_prefixBits))
 				return true;
 
-			// Exclude addresses remembered in bloom filter -- or else remember them
+			// Exclude addresses remembered in bloom filter
 			uint64_t aint = a.toInt() + _bloomNonce;
 			const unsigned int bit = (unsigned int)(aint ^ (aint >> 13) ^ (aint >> 26) ^ (aint >> 39)) & 0x1fff;
 			unsigned char *const bbyte = _bloom + (bit >> 3); // note: bloom filter size == 1024 is hard-coded here
 			const unsigned char bmask = 1 << (bit & 7);
 			if ((*bbyte & bmask))
+				return true; // address already visited
+
+			// Exclude peers that don't appear to be online
+			SharedPtr<Peer> p(_topology->getPeer(a));
+			if ((!p)||(!p->alive(_now)))
 				return true;
-			else *bbyte |= bmask;
+
+			// Remember address in bloom filter
+			*bbyte |= bmask;
 
 			a.copyTo(*_ptr,ZT_ADDRESS_LENGTH);
-			return ((*_ptr += ZT_ADDRESS_LENGTH) != _end);
+			return ((*_ptr += ZT_ADDRESS_LENGTH) < _end);
 		}
 
 	private:
 		const Address _origin;
 		const uint64_t _bloomNonce;
 		const uint64_t _prefix;
+		const uint64_t _now;
 		unsigned char **const _ptr;
 		unsigned char *const _end;
 		unsigned char *const _bloom;
-		unsigned int _prefixBits;
+		const Topology *const _topology;
+		const unsigned int _prefixBits;
 	};
 
 private:
diff --git a/node/PacketDecoder.cpp b/node/PacketDecoder.cpp
index 1e62664e..369eda7b 100644
--- a/node/PacketDecoder.cpp
+++ b/node/PacketDecoder.cpp
@@ -530,7 +530,7 @@ bool PacketDecoder::_doMULTICAST_FRAME(const RuntimeEnvironment *_r,const Shared
 			return false;
 		}
 
-		// These fields change
+		// These fields in the packet are changed by each forwarder
 		unsigned int depth = at<uint16_t>(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_PROPAGATION_DEPTH);
 		unsigned char *const fifo = field(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_PROPAGATION_FIFO,ZT_PROTO_VERB_MULTICAST_FRAME_LEN_PROPAGATION_FIFO);
 		unsigned char *const bloom = field(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_PROPAGATION_BLOOM,ZT_PROTO_VERB_MULTICAST_FRAME_LEN_PROPAGATION_BLOOM);
@@ -568,6 +568,9 @@ bool PacketDecoder::_doMULTICAST_FRAME(const RuntimeEnvironment *_r,const Shared
 		*/
 
 		SharedPtr<Network> network(_r->nc->network(nwid));
+		SharedPtr<NetworkConfig> nconf;
+		if (network)
+			nconf = network->config2();
 
 		/* Grab, verify, and learn certificate of network membership if any -- provided we are
 		 * a member of this network. Note: we can do this before verification of the actual
@@ -679,50 +682,30 @@ bool PacketDecoder::_doMULTICAST_FRAME(const RuntimeEnvironment *_r,const Shared
 			// be the case unless we're a supernode), check to see if we should
 			// inject the packet. This also gives us an opportunity to check things
 			// like multicast bandwidth constraints.
-			if (network) {
-				SharedPtr<NetworkConfig> nconf(network->config2());
-				if (nconf) {
-					// Learn real maxDepth from netconf
-					maxDepth = std::min((unsigned int)ZT_MULTICAST_GLOBAL_MAX_DEPTH,nconf->multicastDepth());
-					if (!maxDepth)
-						maxDepth = ZT_MULTICAST_GLOBAL_MAX_DEPTH;
-
-					if (!network->isAllowed(origin)) {
-						// Papers, please...
-						Packet outp(source(),_r->identity.address(),Packet::VERB_ERROR);
-						outp.append((unsigned char)Packet::VERB_MULTICAST_FRAME);
-						outp.append(packetId());
-						outp.append((unsigned char)Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE);
-						outp.append(nwid);
-						outp.armor(peer->key(),true);
-						_fromSock->send(_remoteAddress,outp.data(),outp.size());
-						TRACE("dropped MULTICAST_FRAME from %s(%s) into %.16llx: sender %s not allowed or we don't have a certificate",source().toString().c_str(),_remoteAddress.toString().c_str(),nwid,origin.toString().c_str());
-						return true;
-					}
-
-					if (MAC(origin,network->id()) != sourceMac) {
-						if (!nconf->permitsBridging(origin)) {
-#ifdef ZT_TRACE_MULTICAST
-							Utils::snprintf(mct,sizeof(mct),
-								"%.16llx %.2u %.3u%s %c %s dropped: bridging not allowed",
-								guid,
-								prefix,
-								depth,
-								mctdepth,
-								(_r->topology->amSupernode() ? 'S' : '-'),
-								_r->identity.address().toString().c_str());
-							_r->sm->sendUdp(ZT_DEFAULTS.multicastTraceWatcher,mct,strlen(mct));
-#endif
-							TRACE("dropped MULTICAST_FRAME from %s(%s) into %.16llx: source mac %s doesn't belong to %s, and bridging is not supported on network",source().toString().c_str(),_remoteAddress.toString().c_str(),nwid,sourceMac.toString().c_str(),origin.toString().c_str());
-							return true;
-						}
-						network->learnBridgeRoute(sourceMac,origin);
-					}
+			if ((network)&&(nconf)) {
+				// Learn real maxDepth from netconf
+				maxDepth = std::min((unsigned int)ZT_MULTICAST_GLOBAL_MAX_DEPTH,nconf->multicastDepth());
+				if (!maxDepth)
+					maxDepth = ZT_MULTICAST_GLOBAL_MAX_DEPTH;
+
+				if (!network->isAllowed(origin)) {
+					// Papers, please...
+					Packet outp(source(),_r->identity.address(),Packet::VERB_ERROR);
+					outp.append((unsigned char)Packet::VERB_MULTICAST_FRAME);
+					outp.append(packetId());
+					outp.append((unsigned char)Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE);
+					outp.append(nwid);
+					outp.armor(peer->key(),true);
+					_fromSock->send(_remoteAddress,outp.data(),outp.size());
+					TRACE("dropped MULTICAST_FRAME from %s(%s) into %.16llx: sender %s not allowed or we don't have a certificate",source().toString().c_str(),_remoteAddress.toString().c_str(),nwid,origin.toString().c_str());
+					return true;
+				}
 
-					if (!nconf->permitsEtherType(etherType)) {
+				if (MAC(origin,network->id()) != sourceMac) {
+					if (!nconf->permitsBridging(origin)) {
 #ifdef ZT_TRACE_MULTICAST
 						Utils::snprintf(mct,sizeof(mct),
-							"%.16llx %.2u %.3u%s %c %s dropped: ethertype not allowed",
+							"%.16llx %.2u %.3u%s %c %s dropped: bridging not allowed",
 							guid,
 							prefix,
 							depth,
@@ -731,31 +714,48 @@ bool PacketDecoder::_doMULTICAST_FRAME(const RuntimeEnvironment *_r,const Shared
 							_r->identity.address().toString().c_str());
 						_r->sm->sendUdp(ZT_DEFAULTS.multicastTraceWatcher,mct,strlen(mct));
 #endif
-						TRACE("dropped MULTICAST_FRAME from %s(%s) into %.16llx: ethertype %u is not allowed",source().toString().c_str(),nwid,_remoteAddress.toString().c_str(),etherType);
+						TRACE("dropped MULTICAST_FRAME from %s(%s) into %.16llx: source mac %s doesn't belong to %s, and bridging is not supported on network",source().toString().c_str(),_remoteAddress.toString().c_str(),nwid,sourceMac.toString().c_str(),origin.toString().c_str());
 						return true;
 					}
+					network->learnBridgeRoute(sourceMac,origin);
+				}
 
-					if (!network->updateAndCheckMulticastBalance(origin,dest,frameLen)) {
-						// Rate limits can only be checked by members of this network, but
-						// there should be enough of them that over-limit multicasts get
-						// their propagation aborted.
+				if (!nconf->permitsEtherType(etherType)) {
 #ifdef ZT_TRACE_MULTICAST
-						Utils::snprintf(mct,sizeof(mct),
-							"%.16llx %.2u %.3u%s %c %s dropped: rate limits exceeded",
-							guid,
-							prefix,
-							depth,
-							mctdepth,
-							(_r->topology->amSupernode() ? 'S' : '-'),
-							_r->identity.address().toString().c_str());
-						_r->sm->sendUdp(ZT_DEFAULTS.multicastTraceWatcher,mct,strlen(mct));
+					Utils::snprintf(mct,sizeof(mct),
+						"%.16llx %.2u %.3u%s %c %s dropped: ethertype not allowed",
+						guid,
+						prefix,
+						depth,
+						mctdepth,
+						(_r->topology->amSupernode() ? 'S' : '-'),
+						_r->identity.address().toString().c_str());
+					_r->sm->sendUdp(ZT_DEFAULTS.multicastTraceWatcher,mct,strlen(mct));
 #endif
-						TRACE("dropped MULTICAST_FRAME from %s(%s): rate limits exceeded for sender %s",source().toString().c_str(),_remoteAddress.toString().c_str(),origin.toString().c_str());
-						return true;
-					}
+					TRACE("dropped MULTICAST_FRAME from %s(%s) into %.16llx: ethertype %u is not allowed",source().toString().c_str(),nwid,_remoteAddress.toString().c_str(),etherType);
+					return true;
+				}
 
-					network->tapPut(sourceMac,dest.mac(),etherType,frame,frameLen);
+				if (!network->updateAndCheckMulticastBalance(origin,dest,frameLen)) {
+					// Rate limits can only be checked by members of this network, but
+					// there should be enough of them that over-limit multicasts get
+					// their propagation aborted.
+#ifdef ZT_TRACE_MULTICAST
+					Utils::snprintf(mct,sizeof(mct),
+						"%.16llx %.2u %.3u%s %c %s dropped: rate limits exceeded",
+						guid,
+						prefix,
+						depth,
+						mctdepth,
+						(_r->topology->amSupernode() ? 'S' : '-'),
+						_r->identity.address().toString().c_str());
+					_r->sm->sendUdp(ZT_DEFAULTS.multicastTraceWatcher,mct,strlen(mct));
+#endif
+					TRACE("dropped MULTICAST_FRAME from %s(%s): rate limits exceeded for sender %s",source().toString().c_str(),_remoteAddress.toString().c_str(),origin.toString().c_str());
+					return true;
 				}
+
+				network->tapPut(sourceMac,dest.mac(),etherType,frame,frameLen);
 			}
 		}
 
@@ -816,11 +816,27 @@ bool PacketDecoder::_doMULTICAST_FRAME(const RuntimeEnvironment *_r,const Shared
 			} else break;
 		}
 
-		// Add any next hops we know about to FIFO
+		// Add any other next hops we know about to FIFO
 #ifdef ZT_TRACE_MULTICAST
 		unsigned char *beforeAdd = newFifoPtr;
 #endif
-		_r->mc->getNextHops(nwid,dest,Multicaster::AddToPropagationQueue(&newFifoPtr,newFifoEnd,bloom,bloomNonce,origin,prefixBits,prefix));
+		Multicaster::AddToPropagationQueue appender(
+			&newFifoPtr,
+			newFifoEnd,
+			bloom,
+			bloomNonce,
+			origin,
+			prefixBits,
+			prefix,
+			_r->topology,
+			Utils::now());
+		if (nconf) {
+			for(std::set<Address>::const_iterator ab(nconf->activeBridges().begin());ab!=nconf->activeBridges().end();++ab) {
+				if (!appender(*ab))
+					break;
+			}
+		}
+		_r->mc->getNextHops(nwid,dest,appender);
 #ifdef ZT_TRACE_MULTICAST
 		unsigned int numAdded = (unsigned int)(newFifoPtr - beforeAdd) / ZT_ADDRESS_LENGTH;
 #endif
diff --git a/node/Peer.cpp b/node/Peer.cpp
index 54e30a74..e61631e2 100644
--- a/node/Peer.cpp
+++ b/node/Peer.cpp
@@ -35,6 +35,7 @@ namespace ZeroTier {
 
 Peer::Peer() :
 	_lastUsed(0),
+	_lastReceive(0),
 	_lastUnicastFrame(0),
 	_lastMulticastFrame(0),
 	_lastAnnouncedTo(0),
@@ -47,6 +48,7 @@ Peer::Peer(const Identity &myIdentity,const Identity &peerIdentity)
 	throw(std::runtime_error) :
 	_id(peerIdentity),
 	_lastUsed(0),
+	_lastReceive(0),
 	_lastUnicastFrame(0),
 	_lastMulticastFrame(0),
 	_lastAnnouncedTo(0),
@@ -73,6 +75,9 @@ void Peer::receive(
 	// Update system-wide last packet receive time
 	*((const_cast<uint64_t *>(&(_r->timeOfLastPacketReceived)))) = now;
 
+	// Global last receive time regardless of path
+	_lastReceive = now;
+
 	// Learn paths from direct packets (hops == 0)
 	if (!hops) {
 		{
diff --git a/node/Peer.hpp b/node/Peer.hpp
index a950b2fc..edb66e21 100644
--- a/node/Peer.hpp
+++ b/node/Peer.hpp
@@ -50,7 +50,7 @@
 #include "NonCopyable.hpp"
 #include "Mutex.hpp"
 
-#define ZT_PEER_SERIALIZATION_VERSION 8
+#define ZT_PEER_SERIALIZATION_VERSION 9
 
 namespace ZeroTier {
 
@@ -247,6 +247,11 @@ public:
 	}
 
 	/**
+	 * @return Time of last receive of anything, whether direct or relayed
+	 */
+	inline uint64_t lastReceive() const throw() { return _lastReceive; }
+
+	/**
 	 * @return Time of most recent unicast frame received
 	 */
 	inline uint64_t lastUnicastFrame() const throw() { return _lastUnicastFrame; }
@@ -267,6 +272,16 @@ public:
 	inline uint64_t lastAnnouncedTo() const throw() { return _lastAnnouncedTo; }
 
 	/**
+	 * @param now Current time
+	 * @return True if peer has received something within ZT_PEER_ACTIVITY_TIMEOUT ms
+	 */
+	inline bool alive(uint64_t now) const
+		throw()
+	{
+		return ((now - _lastReceive) < ZT_PEER_ACTIVITY_TIMEOUT);
+	}
+
+	/**
 	 * @return Current latency or 0 if unknown (max: 65535)
 	 */
 	inline unsigned int latency() const
@@ -424,6 +439,7 @@ public:
 		_id.serialize(b,false);
 		b.append(_key,sizeof(_key));
 		b.append(_lastUsed);
+		b.append(_lastReceive);
 		b.append(_lastUnicastFrame);
 		b.append(_lastMulticastFrame);
 		b.append(_lastAnnouncedTo);
@@ -448,6 +464,7 @@ public:
 		p += _id.deserialize(b,p);
 		memcpy(_key,b.field(p,sizeof(_key)),sizeof(_key)); p += sizeof(_key);
 		_lastUsed = b.template at<uint64_t>(p); p += sizeof(uint64_t);
+		_lastReceive = b.template at<uint64_t>(p); p += sizeof(uint64_t);
 		_lastUnicastFrame = b.template at<uint64_t>(p); p += sizeof(uint64_t);
 		_lastMulticastFrame = b.template at<uint64_t>(p); p += sizeof(uint64_t);
 		_lastAnnouncedTo = b.template at<uint64_t>(p); p += sizeof(uint64_t);
@@ -472,6 +489,7 @@ private:
 	std::vector<Path> _paths;
 
 	volatile uint64_t _lastUsed;
+	volatile uint64_t _lastReceive; // direct or indirect
 	volatile uint64_t _lastUnicastFrame;
 	volatile uint64_t _lastMulticastFrame;
 	volatile uint64_t _lastAnnouncedTo;
diff --git a/node/Switch.cpp b/node/Switch.cpp
index 46fb0b49..9b0e3e6e 100644
--- a/node/Switch.cpp
+++ b/node/Switch.cpp
@@ -164,26 +164,26 @@ void Switch::onLocalEthernet(const SharedPtr<Network> &network,const MAC &from,c
 			memset(bloom,0,sizeof(bloom));
 			unsigned char *fifoPtr = fifo;
 
-			// All multicasts visit all active bridges first -- this is one of the two active/passive bridge differences
+			// Add all active bridges and then next hops we know about to propagation queue
+			Multicaster::AddToPropagationQueue appender(
+				&fifoPtr,
+				fifoEnd,
+				bloom,
+				bloomNonce,
+				_r->identity.address(),
+				nconf->multicastPrefixBits(),
+				prefix,
+				_r->topology,
+				now);
 			for(std::set<Address>::const_iterator ab(nconf->activeBridges().begin());ab!=nconf->activeBridges().end();++ab) {
-				if ((*ab != _r->identity.address())&&(ab->withinMulticastPropagationPrefix(prefix,nconf->multicastPrefixBits()))) {
-					SharedPtr<Peer> abPeer(_r->topology->getPeer(*ab));
-					if ((abPeer)&&(abPeer->hasActiveDirectPath(now))) {
-						ab->copyTo(fifoPtr,ZT_ADDRESS_LENGTH);
-						if ((fifoPtr += ZT_ADDRESS_LENGTH) == fifoEnd)
-							break;
-					}
-				}
+				if (!appender(*ab))
+					break;
 			}
+			_r->mc->getNextHops(network->id(),mg,appender);
 
-			// Then visit next hops according to multicaster (if there's room... almost certainly will be)
-			if (fifoPtr != fifoEnd) {
-				_r->mc->getNextHops(network->id(),mg,Multicaster::AddToPropagationQueue(&fifoPtr,fifoEnd,bloom,bloomNonce,_r->identity.address(),nconf->multicastPrefixBits(),prefix));
-
-				// Pad remainder of FIFO with zeroes
-				while (fifoPtr != fifoEnd)
-					*(fifoPtr++) = (unsigned char)0;
-			}
+			// Pad remainder of FIFO with zeroes
+			while (fifoPtr != fifoEnd)
+				*(fifoPtr++) = (unsigned char)0;
 
 			// First element in FIFO is first hop, rest of FIFO is sent in packet *to* first hop
 			Address firstHop(fifo,ZT_ADDRESS_LENGTH);
diff --git a/node/Topology.cpp b/node/Topology.cpp
index d88b64ca..d74179fe 100644
--- a/node/Topology.cpp
+++ b/node/Topology.cpp
@@ -89,7 +89,7 @@ SharedPtr<Peer> Topology::addPeer(const SharedPtr<Peer> &peer)
 	return p;
 }
 
-SharedPtr<Peer> Topology::getPeer(const Address &zta)
+SharedPtr<Peer> Topology::getPeer(const Address &zta) const
 {
 	if (zta == _r->identity.address()) {
 		TRACE("BUG: ignored attempt to getPeer() for self, returned NULL");
diff --git a/node/Topology.hpp b/node/Topology.hpp
index 74db8b50..293ac446 100644
--- a/node/Topology.hpp
+++ b/node/Topology.hpp
@@ -82,7 +82,7 @@ public:
 	 * @param zta ZeroTier address of peer
 	 * @return Peer or NULL if not found
 	 */
-	SharedPtr<Peer> getPeer(const Address &zta);
+	SharedPtr<Peer> getPeer(const Address &zta) const;
 
 	/**
 	 * Get an identity if cached or available in a peer record