Refactor: move network COMs out of Network and into Peer in prep for tightening up multicast lookup and other things.

11 files changed, 289 insertions, 215 deletions

diff --git a/node/Constants.hpp b/node/Constants.hpp
index 4f783550..27c43aa7 100644
--- a/node/Constants.hpp
+++ b/node/Constants.hpp
@@ -325,6 +325,13 @@
 #define ZT_DIRECT_PATH_PUSH_INTERVAL 300000
 
 /**
+ * How long (max) to remember network certificates of membership?
+ *
+ * This only applies to networks we don't belong to.
+ */
+#define ZT_PEER_NETWORK_COM_EXPIRATION 3600000
+
+/**
  * Sanity limit on maximum bridge routes
  *
  * If the number of bridge routes exceeds this, we cull routes from the
diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp
index 3866abf1..e892edc2 100644
--- a/node/IncomingPacket.cpp
+++ b/node/IncomingPacket.cpp
@@ -421,9 +421,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 					// OK(MULTICAST_FRAME) includes certificate of membership update
 					CertificateOfMembership com;
 					offset += com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_COM_AND_GATHER_RESULTS);
-					SharedPtr<Network> network(RR->node->network(nwid));
-					if ((network)&&(com.hasRequiredFields()))
-						network->validateAndAddMembershipCertificate(com);
+					peer->validateAndSetNetworkMembershipCertificate(RR,nwid,com);
 				}
 
 				if ((flags & 0x02) != 0) {
@@ -511,7 +509,7 @@ bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 		const SharedPtr<Network> network(RR->node->network(at<uint64_t>(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID)));
 		if (network) {
 			if (size() > ZT_PROTO_VERB_FRAME_IDX_PAYLOAD) {
-				if (!network->isAllowed(peer->address())) {
+				if (!network->isAllowed(peer)) {
 					TRACE("dropped FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),(unsigned long long)network->id());
 					_sendErrorNeedCertificate(RR,peer,network->id());
 					return true;
@@ -552,13 +550,11 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr<P
 				if ((flags & 0x01) != 0) {
 					CertificateOfMembership com;
 					comLen = com.deserialize(*this,ZT_PROTO_VERB_EXT_FRAME_IDX_COM);
-					if (com.hasRequiredFields()) {
-						if (!network->validateAndAddMembershipCertificate(com))
-							comFailed = true; // technically this check is redundant to isAllowed(), but do it anyway for thoroughness
-					}
+					if (!peer->validateAndSetNetworkMembershipCertificate(RR,network->id(),com))
+						comFailed = true;
 				}
 
-				if ((comFailed)||(!network->isAllowed(peer->address()))) {
+				if ((comFailed)||(!network->isAllowed(peer))) {
 					TRACE("dropped EXT_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),network->id());
 					_sendErrorNeedCertificate(RR,peer,network->id());
 					return true;
@@ -642,11 +638,7 @@ bool IncomingPacket::_doNETWORK_MEMBERSHIP_CERTIFICATE(const RuntimeEnvironment
 		unsigned int ptr = ZT_PACKET_IDX_PAYLOAD;
 		while (ptr < size()) {
 			ptr += com.deserialize(*this,ptr);
-			if (com.hasRequiredFields()) {
-				SharedPtr<Network> network(RR->node->network(com.networkId()));
-				if (network)
-					network->validateAndAddMembershipCertificate(com);
-			}
+			peer->validateAndSetNetworkMembershipCertificate(RR,com.networkId(),com);
 		}
 
 		peer->received(RR,_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_NETWORK_MEMBERSHIP_CERTIFICATE,0,Packet::VERB_NOP);
@@ -809,13 +801,12 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share
 			if ((flags & 0x01) != 0) {
 				CertificateOfMembership com;
 				offset += com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_FRAME_IDX_COM);
-				if (com.hasRequiredFields())
-					network->validateAndAddMembershipCertificate(com);
+				peer->validateAndSetNetworkMembershipCertificate(RR,nwid,com);
 			}
 
 			// Check membership after we've read any included COM, since
 			// that cert might be what we needed.
-			if (!network->isAllowed(peer->address())) {
+			if (!network->isAllowed(peer)) {
 				TRACE("dropped MULTICAST_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),(unsigned long long)network->id());
 				_sendErrorNeedCertificate(RR,peer,network->id());
 				return true;
diff --git a/node/Network.cpp b/node/Network.cpp
index 2b24d5f9..9c8aabfa 100644
--- a/node/Network.cpp
+++ b/node/Network.cpp
@@ -59,6 +59,9 @@ Network::Network(const RuntimeEnvironment *renv,uint64_t nwid) :
 	Utils::snprintf(confn,sizeof(confn),"networks.d/%.16llx.conf",_id);
 	Utils::snprintf(mcdbn,sizeof(mcdbn),"networks.d/%.16llx.mcerts",_id);
 
+	// These files are no longer used, so clean them.
+	RR->node->dataStoreDelete(mcdbn);
+
 	if (_id == ZT_TEST_NETWORK_ID) {
 		applyConfiguration(NetworkConfig::createTestNetworkConfig(RR->identity.address()));
 
@@ -79,24 +82,6 @@ Network::Network(const RuntimeEnvironment *renv,uint64_t nwid) :
 			// Save a one-byte CR to persist membership while we request a real netconf
 			RR->node->dataStorePut(confn,"\n",1,false);
 		}
-
-		try {
-			std::string mcdb(RR->node->dataStoreGet(mcdbn));
-			if (mcdb.length() > 6) {
-				const char *p = mcdb.data();
-				const char *e = p + mcdb.length();
-				if (!memcmp("ZTMCD0",p,6)) {
-					p += 6;
-					while (p != e) {
-						CertificateOfMembership com;
-						com.deserialize2(p,e);
-						if (!com)
-							break;
-						_certInfo[com.issuedTo()].com = com;
-					}
-				}
-			}
-		} catch ( ... ) {} // ignore invalid MCDB, we'll re-learn from peers
 	}
 
 	if (!_portInitialized) {
@@ -115,32 +100,10 @@ Network::~Network()
 	char n[128];
 	if (_destroyed) {
 		RR->node->configureVirtualNetworkPort(_id,ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY,&ctmp);
-
 		Utils::snprintf(n,sizeof(n),"networks.d/%.16llx.conf",_id);
 		RR->node->dataStoreDelete(n);
-		Utils::snprintf(n,sizeof(n),"networks.d/%.16llx.mcerts",_id);
-		RR->node->dataStoreDelete(n);
 	} else {
 		RR->node->configureVirtualNetworkPort(_id,ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DOWN,&ctmp);
-
-		clean();
-
-		Utils::snprintf(n,sizeof(n),"networks.d/%.16llx.mcerts",_id);
-
-		Mutex::Lock _l(_lock);
-		if ((!_config)||(_config->isPublic())||(_certInfo.empty())) {
-			RR->node->dataStoreDelete(n);
-		} else {
-			std::string buf("ZTMCD0");
-			Hashtable< Address,_RemoteMemberCertificateInfo >::Iterator i(_certInfo);
-			Address *a = (Address *)0;
-			_RemoteMemberCertificateInfo *ci = (_RemoteMemberCertificateInfo *)0;
-			while (i.next(a,ci)) {
-				if (ci->com)
-					ci->com.serialize2(buf);
-			}
-			RR->node->dataStorePut(n,buf,true);
-		}
 	}
 }
 
@@ -281,70 +244,6 @@ void Network::requestConfiguration()
 	RR->sw->send(outp,true,0);
 }
 
-bool Network::validateAndAddMembershipCertificate(const CertificateOfMembership &cert)
-{
-	if (!cert) // sanity check
-		return false;
-
-	Mutex::Lock _l(_lock);
-
-	{
-		const _RemoteMemberCertificateInfo *ci = _certInfo.get(cert.issuedTo());
-		if ((ci)&&(ci->com == cert))
-			return true; // we already have it
-	}
-
-	// Check signature, log and return if cert is invalid
-	if (cert.signedBy() != controller()) {
-		TRACE("rejected network membership certificate for %.16llx signed by %s: signer not a controller of this network",(unsigned long long)_id,cert.signedBy().toString().c_str());
-		return false; // invalid signer
-	}
-
-	if (cert.signedBy() == RR->identity.address()) {
-
-		// We are the controller: RR->identity.address() == controller() == cert.signedBy()
-		// So, verify that we signed th cert ourself
-		if (!cert.verify(RR->identity)) {
-			TRACE("rejected network membership certificate for %.16llx self signed by %s: signature check failed",(unsigned long long)_id,cert.signedBy().toString().c_str());
-			return false; // invalid signature
-		}
-
-	} else {
-
-		SharedPtr<Peer> signer(RR->topology->getPeer(cert.signedBy()));
-
-		if (!signer) {
-			// This would be rather odd, since this is our controller... could happen
-			// if we get packets before we've gotten config.
-			RR->sw->requestWhois(cert.signedBy());
-			return false; // signer unknown
-		}
-
-		if (!cert.verify(signer->identity())) {
-			TRACE("rejected network membership certificate for %.16llx signed by %s: signature check failed",(unsigned long long)_id,cert.signedBy().toString().c_str());
-			return false; // invalid signature
-		}
-	}
-
-	// If we made it past authentication, add or update cert in our cert info store
-	_certInfo[cert.issuedTo()].com = cert;
-
-	return true;
-}
-
-bool Network::peerNeedsOurMembershipCertificate(const Address &to,uint64_t now)
-{
-	Mutex::Lock _l(_lock);
-	if ((_config)&&(!_config->isPublic())&&(_config->com())) {
-		_RemoteMemberCertificateInfo &ci = _certInfo[to];
-		if ((now - ci.lastPushed) > (ZT_NETWORK_AUTOCONF_DELAY / 2)) {
-			ci.lastPushed = now;
-			return true;
-		}
-	}
-	return false;
-}
-
 void Network::clean()
 {
 	const uint64_t now = RR->node->now();
@@ -353,22 +252,6 @@ void Network::clean()
 	if (_destroyed)
 		return;
 
-	if ((_config)&&(_config->isPublic())) {
-		// Open (public) networks do not track certs or cert pushes at all.
-		_certInfo.clear();
-	} else if (_config) {
-		// Clean obsolete entries from private network cert info table
-		Hashtable< Address,_RemoteMemberCertificateInfo >::Iterator i(_certInfo);
-		Address *a = (Address *)0;
-		_RemoteMemberCertificateInfo *ci = (_RemoteMemberCertificateInfo *)0;
-		const uint64_t forgetIfBefore = now - (ZT_PEER_ACTIVITY_TIMEOUT * 16); // arbitrary reasonable cutoff
-		while (i.next(a,ci)) {
-			if ((ci->lastPushed < forgetIfBefore)&&(!ci->com.agreesWith(_config->com())))
-				_certInfo.erase(*a);
-		}
-	}
-
-	// Clean learned multicast groups if we haven't heard from them in a while
 	{
 		Hashtable< MulticastGroup,uint64_t >::Iterator i(_multicastGroupsBehindMe);
 		MulticastGroup *mg = (MulticastGroup *)0;
@@ -494,7 +377,7 @@ void Network::_externalConfig(ZT_VirtualNetworkConfig *ec) const
 	} else ec->assignedAddressCount = 0;
 }
 
-bool Network::_isAllowed(const Address &peer) const
+bool Network::_isAllowed(const SharedPtr<Peer> &peer) const
 {
 	// Assumes _lock is locked
 	try {
@@ -502,10 +385,7 @@ bool Network::_isAllowed(const Address &peer) const
 			return false;
 		if (_config->isPublic())
 			return true;
-		const _RemoteMemberCertificateInfo *ci = _certInfo.get(peer);
-		if (!ci)
-			return false;
-		return _config->com().agreesWith(ci->com);
+		return ((_config->com())&&(peer->networkMembershipCertificatesAgree(_id,_config->com())));
 	} catch (std::exception &exc) {
 		TRACE("isAllowed() check failed for peer %s: unexpected exception: %s",peer.toString().c_str(),exc.what());
 	} catch ( ... ) {
@@ -542,7 +422,7 @@ public:
 
 	inline void operator()(Topology &t,const SharedPtr<Peer> &p)
 	{
-		if ( ( (p->hasActiveDirectPath(_now)) && ( (_network->_isAllowed(p->address())) || (p->address() == _network->controller()) ) ) || (std::find(_rootAddresses.begin(),_rootAddresses.end(),p->address()) != _rootAddresses.end()) ) {
+		if ( ( (p->hasActiveDirectPath(_now)) && ( (_network->_isAllowed(p)) || (p->address() == _network->controller()) ) ) || (std::find(_rootAddresses.begin(),_rootAddresses.end(),p->address()) != _rootAddresses.end()) ) {
 			Packet outp(p->address(),RR->identity.address(),Packet::VERB_MULTICAST_LIKE);
 
 			for(std::vector<MulticastGroup>::iterator mg(_allMulticastGroups.begin());mg!=_allMulticastGroups.end();++mg) {
diff --git a/node/Network.hpp b/node/Network.hpp
index ad9f18de..37077650 100644
--- a/node/Network.hpp
+++ b/node/Network.hpp
@@ -56,6 +56,7 @@ namespace ZeroTier {
 
 class RuntimeEnvironment;
 class _AnnounceMulticastGroupsToPeersWithActiveDirectPaths;
+class Peer;
 
 /**
  * A virtual LAN
@@ -95,6 +96,12 @@ public:
 	inline Address controller() throw() { return Address(_id >> 24); }
 
 	/**
+	 * @param nwid Network ID
+	 * @return Address of network's controller
+	 */
+	static inline Address controllerFor(uint64_t nwid) throw() { return Address(nwid >> 24); }
+
+	/**
 	 * @return Multicast group memberships for this network's port (local, not learned via bridging)
 	 */
 	inline std::vector<MulticastGroup> multicastGroups() const
@@ -177,33 +184,10 @@ public:
 	void requestConfiguration();
 
 	/**
-	 * Add or update a membership certificate
-	 *
-	 * @param cert Certificate of membership
-	 * @return True if certificate was accepted as valid
-	 */
-	bool validateAndAddMembershipCertificate(const CertificateOfMembership &cert);
-
-	/**
-	 * Check if we should push membership certificate to a peer, AND update last pushed
-	 *
-	 * If we haven't pushed a cert to this peer in a long enough time, this returns
-	 * true and updates the last pushed time. Otherwise it returns false.
-	 *
-	 * This doesn't actually send anything, since COMs can hitch a ride with several
-	 * different kinds of packets.
-	 *
-	 * @param to Destination peer
-	 * @param now Current time
-	 * @return True if we should include a COM with whatever we're currently sending
-	 */
-	bool peerNeedsOurMembershipCertificate(const Address &to,uint64_t now);
-
-	/**
-	 * @param peer Peer address to check
+	 * @param peer Peer to check
 	 * @return True if peer is allowed to communicate on this network
 	 */
-	inline bool isAllowed(const Address &peer) const
+	inline bool isAllowed(const SharedPtr<Peer> &peer) const
 	{
 		Mutex::Lock _l(_lock);
 		return _isAllowed(peer);
@@ -347,16 +331,9 @@ public:
 	inline bool operator>=(const Network &n) const throw() { return (_id >= n._id); }
 
 private:
-	struct _RemoteMemberCertificateInfo
-	{
-		_RemoteMemberCertificateInfo() : com(),lastPushed(0) {}
-		CertificateOfMembership com; // remote member's COM
-		uint64_t lastPushed; // when did we last push ours to them?
-	};
-
 	ZT_VirtualNetworkStatus _status() const;
 	void _externalConfig(ZT_VirtualNetworkConfig *ec) const; // assumes _lock is locked
-	bool _isAllowed(const Address &peer) const;
+	bool _isAllowed(const SharedPtr<Peer> &peer) const;
 	void _announceMulticastGroups();
 	std::vector<MulticastGroup> _allMulticastGroups() const;
 
@@ -370,8 +347,6 @@ private:
 	Hashtable< MulticastGroup,uint64_t > _multicastGroupsBehindMe; // multicast groups that seem to be behind us and when we last saw them (if we are a bridge)
 	Hashtable< MAC,Address > _remoteBridgeRoutes; // remote addresses where given MACs are reachable (for tracking devices behind remote bridges)
 
-	Hashtable< Address,_RemoteMemberCertificateInfo > _certInfo;
-
 	SharedPtr<NetworkConfig> _config; // Most recent network configuration, which is an immutable value-object
 	volatile uint64_t _lastConfigUpdate;
 
diff --git a/node/Node.hpp b/node/Node.hpp
index b81c1943..0f659f47 100644
--- a/node/Node.hpp
+++ b/node/Node.hpp
@@ -168,6 +168,16 @@ public:
 		return _network(nwid);
 	}
 
+	inline bool belongsToNetwork(uint64_t nwid) const
+	{
+		Mutex::Lock _l(_networks_m);
+		for(std::vector< std::pair< uint64_t, SharedPtr<Network> > >::const_iterator i=_networks.begin();i!=_networks.end();++i) {
+			if (i->first == nwid)
+				return true;
+		}
+		return false;
+	}
+
 	inline std::vector< SharedPtr<Network> > allNetworks() const
 	{
 		std::vector< SharedPtr<Network> > nw;
diff --git a/node/OutboundMulticast.cpp b/node/OutboundMulticast.cpp
index 46116c07..c26372cb 100644
--- a/node/OutboundMulticast.cpp
+++ b/node/OutboundMulticast.cpp
@@ -103,11 +103,11 @@ void OutboundMulticast::init(
 void OutboundMulticast::sendOnly(const RuntimeEnvironment *RR,const Address &toAddr)
 {
 	if (_haveCom) {
-		SharedPtr<Network> network(RR->node->network(_nwid));
-		if ((network)&&(network->peerNeedsOurMembershipCertificate(toAddr,RR->node->now()))) {
+		SharedPtr<Peer> peer(RR->topology->getPeer(toAddr));
+		if ( (!peer) || (peer->needsOurNetworkMembershipCertificate(_nwid,RR->node->now(),true)) ) {
+			//TRACE(">>MC %.16llx -> %s (with COM)",(unsigned long long)this,toAddr.toString().c_str());
 			_packetWithCom.newInitializationVector();
 			_packetWithCom.setDestination(toAddr);
-			//TRACE(">>MC %.16llx -> %s (with COM)",(unsigned long long)this,toAddr.toString().c_str());
 			RR->sw->send(_packetWithCom,true,_nwid);
 			return;
 		}
diff --git a/node/Path.hpp b/node/Path.hpp
index 9d97d2df..8d662ff7 100644
--- a/node/Path.hpp
+++ b/node/Path.hpp
@@ -122,14 +122,6 @@ public:
 	 */
 	inline operator bool() const throw() { return (_addr); }
 
-	// Comparisons are by address only
-	inline bool operator==(const Path &p) const throw() { return (_addr == p._addr); }
-	inline bool operator!=(const Path &p) const throw() { return (_addr != p._addr); }
-	inline bool operator<(const Path &p) const throw() { return (_addr < p._addr); }
-	inline bool operator>(const Path &p) const throw() { return (_addr > p._addr); }
-	inline bool operator<=(const Path &p) const throw() { return (_addr <= p._addr); }
-	inline bool operator>=(const Path &p) const throw() { return (_addr >= p._addr); }
-
 	/**
 	 * Check whether this address is valid for a ZeroTier path
 	 *
diff --git a/node/Peer.cpp b/node/Peer.cpp
index 48b77c85..6203d0b4 100644
--- a/node/Peer.cpp
+++ b/node/Peer.cpp
@@ -37,6 +37,8 @@
 
 #include <algorithm>
 
+#define ZT_PEER_PATH_SORT_INTERVAL 5000
+
 namespace ZeroTier {
 
 // Used to send varying values for NAT keepalive
@@ -51,17 +53,38 @@ Peer::Peer(const Identity &myIdentity,const Identity &peerIdentity)
 	_lastAnnouncedTo(0),
 	_lastPathConfirmationSent(0),
 	_lastDirectPathPush(0),
+	_lastPathSort(0),
 	_vMajor(0),
 	_vMinor(0),
 	_vRevision(0),
 	_id(peerIdentity),
 	_numPaths(0),
-	_latency(0)
+	_latency(0),
+	_networkComs(4),
+	_lastPushedComs(4)
 {
 	if (!myIdentity.agree(peerIdentity,_key,ZT_PEER_SECRET_KEY_LENGTH))
 		throw std::runtime_error("new peer identity key agreement failed");
 }
 
+struct _SortPathsByQuality
+{
+	uint64_t _now;
+	_SortPathsByQuality(const uint64_t now) : _now(now) {}
+	inline bool operator()(const RemotePath &a,const RemotePath &b) const
+	{
+		const uint64_t qa = (
+			((uint64_t)a.active(_now) << 63) |
+			(((uint64_t)(a.preferenceRank() & 0xfff)) << 51) |
+			((uint64_t)a.lastReceived() & 0x7ffffffffffffULL) );
+		const uint64_t qb = (
+			((uint64_t)b.active(_now) << 63) |
+			(((uint64_t)(b.preferenceRank() & 0xfff)) << 51) |
+			((uint64_t)b.lastReceived() & 0x7ffffffffffffULL) );
+		return (qb < qa); // invert sense to sort in descending order
+	}
+};
+
 void Peer::received(
 	const RuntimeEnvironment *RR,
 	const InetAddress &localAddr,
@@ -73,6 +96,8 @@ void Peer::received(
 	Packet::Verb inReVerb)
 {
 	const uint64_t now = RR->node->now();
+	Mutex::Lock _l(_lock);
+
 	_lastReceive = now;
 
 	if (!hops) {
@@ -91,6 +116,7 @@ void Peer::received(
 
 			if (!pathIsConfirmed) {
 				if ((verb == Packet::VERB_OK)&&(inReVerb == Packet::VERB_HELLO)) {
+
 					// Learn paths if they've been confirmed via a HELLO
 					RemotePath *slot = (RemotePath *)0;
 					if (np < ZT_MAX_PEER_NETWORK_PATHS) {
@@ -111,8 +137,11 @@ void Peer::received(
 						slot->received(now);
 						_numPaths = np;
 						pathIsConfirmed = true;
+						_sortPaths(now);
 					}
+
 				} else {
+
 					/* If this path is not known, send a HELLO. We don't learn
 					 * paths without confirming that a bidirectional link is in
 					 * fact present, but any packet that decodes and authenticates
@@ -122,6 +151,7 @@ void Peer::received(
 						TRACE("got %s via unknown path %s(%s), confirming...",Packet::verbString(verb),_id.address().toString().c_str(),remoteAddr.toString().c_str());
 						attemptToContactAt(RR,localAddr,remoteAddr,now);
 					}
+
 				}
 			}
 		}
@@ -129,6 +159,7 @@ void Peer::received(
 		/* Announce multicast groups of interest to direct peers if they are
 		 * considered authorized members of a given network. Also announce to
 		 * root servers and network controllers. */
+		/*
 		if ((pathIsConfirmed)&&((now - _lastAnnouncedTo) >= ((ZT_MULTICAST_LIKE_EXPIRE / 2) - 1000))) {
 			_lastAnnouncedTo = now;
 
@@ -158,6 +189,7 @@ void Peer::received(
 				RR->node->putPacket(localAddr,remoteAddr,outp.data(),outp.size());
 			}
 		}
+		*/
 	}
 
 	if ((verb == Packet::VERB_FRAME)||(verb == Packet::VERB_EXT_FRAME))
@@ -166,23 +198,10 @@ void Peer::received(
 		_lastMulticastFrame = now;
 }
 
-RemotePath *Peer::getBestPath(uint64_t now)
-{
-	RemotePath *bestPath = (RemotePath *)0;
-	uint64_t lrMax = 0;
-	int rank = 0;
-	for(unsigned int p=0,np=_numPaths;p<np;++p) {
-		if ( (_paths[p].active(now)) && ((_paths[p].lastReceived() >= lrMax)||(_paths[p].preferenceRank() >= rank)) ) {
-			lrMax = _paths[p].lastReceived();
-			rank = _paths[p].preferenceRank();
-			bestPath = &(_paths[p]);
-		}
-	}
-	return bestPath;
-}
-
 void Peer::attemptToContactAt(const RuntimeEnvironment *RR,const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now)
 {
+	// _lock not required here since _id is immutable and nothing else is accessed
+
 	Packet outp(_id.address(),RR->identity.address(),Packet::VERB_HELLO);
 	outp.append((unsigned char)ZT_PROTO_VERSION);
 	outp.append((unsigned char)ZEROTIER_ONE_VERSION_MAJOR);
@@ -214,7 +233,8 @@ void Peer::attemptToContactAt(const RuntimeEnvironment *RR,const InetAddress &lo
 
 void Peer::doPingAndKeepalive(const RuntimeEnvironment *RR,uint64_t now)
 {
-	RemotePath *const bestPath = getBestPath(now);
+	Mutex::Lock _l(_lock);
+	RemotePath *const bestPath = _getBestPath(now);
 	if (bestPath) {
 		if ((now - bestPath->lastReceived()) >= ZT_PEER_DIRECT_PING_DELAY) {
 			TRACE("PING %s(%s)",_id.address().toString().c_str(),bestPath->address().toString().c_str());
@@ -231,6 +251,8 @@ void Peer::doPingAndKeepalive(const RuntimeEnvironment *RR,uint64_t now)
 
 void Peer::pushDirectPaths(const RuntimeEnvironment *RR,RemotePath *path,uint64_t now,bool force)
 {
+	Mutex::Lock _l(_lock);
+
 	if (((now - _lastDirectPathPush) >= ZT_DIRECT_PATH_PUSH_INTERVAL)||(force)) {
 		_lastDirectPathPush = now;
 
@@ -299,13 +321,16 @@ void Peer::pushDirectPaths(const RuntimeEnvironment *RR,RemotePath *path,uint64_
 	}
 }
 
-void Peer::addPath(const RemotePath &newp)
+void Peer::addPath(const RemotePath &newp,uint64_t now)
 {
+	Mutex::Lock _l(_lock);
+
 	unsigned int np = _numPaths;
 
 	for(unsigned int p=0;p<np;++p) {
 		if (_paths[p].address() == newp.address()) {
 			_paths[p].setFixed(newp.fixed());
+			_sortPaths(now);
 			return;
 		}
 	}
@@ -328,6 +353,8 @@ void Peer::addPath(const RemotePath &newp)
 		*slot = newp;
 		_numPaths = np;
 	}
+
+	_sortPaths(now);
 }
 
 void Peer::clearPaths(bool fixedToo)
@@ -349,6 +376,7 @@ void Peer::clearPaths(bool fixedToo)
 
 bool Peer::resetWithinScope(const RuntimeEnvironment *RR,InetAddress::IpScope scope,uint64_t now)
 {
+	Mutex::Lock _l(_lock);
 	unsigned int np = _numPaths;
 	unsigned int x = 0;
 	unsigned int y = 0;
@@ -364,11 +392,13 @@ bool Peer::resetWithinScope(const RuntimeEnvironment *RR,InetAddress::IpScope sc
 		++x;
 	}
 	_numPaths = y;
+	_sortPaths(now);
 	return (y < np);
 }
 
 void Peer::getBestActiveAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) const
 {
+	Mutex::Lock _l(_lock);
 	uint64_t bestV4 = 0,bestV6 = 0;
 	for(unsigned int p=0,np=_numPaths;p<np;++p) {
 		if (_paths[p].active(now)) {
@@ -390,4 +420,137 @@ void Peer::getBestActiveAddresses(uint64_t now,InetAddress &v4,InetAddress &v6)
 	}
 }
 
+bool Peer::networkMembershipCertificatesAgree(uint64_t nwid,const CertificateOfMembership &com) const
+{
+	Mutex::Lock _l(_lock);
+	const _NetworkCom *ourCom = _networkComs.get(nwid);
+	if (ourCom)
+		return ourCom->com.agreesWith(com);
+	return false;
+}
+
+bool Peer::validateAndSetNetworkMembershipCertificate(const RuntimeEnvironment *RR,uint64_t nwid,const CertificateOfMembership &com)
+{
+	// Sanity checks
+	if ((!com)||(com.issuedTo() != _id.address()))
+		return false;
+
+	// Return true if we already have this *exact* COM
+	{
+		Mutex::Lock _l(_lock);
+		_NetworkCom *ourCom = _networkComs.get(nwid);
+		if ((ourCom)&&(ourCom->com == com))
+			return true;
+	}
+
+	// Check signature, log and return if cert is invalid
+	if (com.signedBy() != Network::controllerFor(nwid)) {
+		TRACE("rejected network membership certificate for %.16llx signed by %s: signer not a controller of this network",(unsigned long long)_id,cert.signedBy().toString().c_str());
+		return false; // invalid signer
+	}
+
+	if (com.signedBy() == RR->identity.address()) {
+
+		// We are the controller: RR->identity.address() == controller() == cert.signedBy()
+		// So, verify that we signed th cert ourself
+		if (!com.verify(RR->identity)) {
+			TRACE("rejected network membership certificate for %.16llx self signed by %s: signature check failed",(unsigned long long)_id,cert.signedBy().toString().c_str());
+			return false; // invalid signature
+		}
+
+	} else {
+
+		SharedPtr<Peer> signer(RR->topology->getPeer(com.signedBy()));
+
+		if (!signer) {
+			// This would be rather odd, since this is our controller... could happen
+			// if we get packets before we've gotten config.
+			RR->sw->requestWhois(com.signedBy());
+			return false; // signer unknown
+		}
+
+		if (!com.verify(signer->identity())) {
+			TRACE("rejected network membership certificate for %.16llx signed by %s: signature check failed",(unsigned long long)_id,cert.signedBy().toString().c_str());
+			return false; // invalid signature
+		}
+	}
+
+	// If we made it past all those checks, add or update cert in our cert info store
+	{
+		Mutex::Lock _l(_lock);
+		_networkComs.set(nwid,_NetworkCom(RR->node->now(),com));
+	}
+
+	return true;
+}
+
+bool Peer::needsOurNetworkMembershipCertificate(uint64_t nwid,uint64_t now,bool updateLastPushedTime)
+{
+	Mutex::Lock _l(_lock);
+	uint64_t &lastPushed = _lastPushedComs[nwid];
+	const uint64_t tmp = lastPushed;
+	if (updateLastPushedTime)
+		lastPushed = now;
+	return ((now - tmp) < (ZT_NETWORK_AUTOCONF_DELAY / 2));
+}
+
+void Peer::clean(const RuntimeEnvironment *RR,uint64_t now)
+{
+	Mutex::Lock _l(_lock);
+
+	{
+		unsigned int np = _numPaths;
+		unsigned int x = 0;
+		unsigned int y = 0;
+		while (x < np) {
+			if (_paths[x].active(now))
+				_paths[y++] = _paths[x];
+			++x;
+		}
+		_numPaths = y;
+	}
+
+	{
+		uint64_t *k = (uint64_t *)0;
+		_NetworkCom *v = (_NetworkCom *)0;
+		Hashtable< uint64_t,_NetworkCom >::Iterator i(_networkComs);
+		while (i.next(k,v)) {
+			if ( (!RR->node->belongsToNetwork(*k)) && ((now - v->ts) >= ZT_PEER_NETWORK_COM_EXPIRATION) )
+				_networkComs.erase(*k);
+		}
+	}
+
+	{
+		uint64_t *k = (uint64_t *)0;
+		uint64_t *v = (uint64_t *)0;
+		Hashtable< uint64_t,uint64_t >::Iterator i(_lastPushedComs);
+		while (i.next(k,v)) {
+			if ((now - *v) > (ZT_NETWORK_AUTOCONF_DELAY * 2))
+				_lastPushedComs.erase(*k);
+		}
+	}
+}
+
+void Peer::_sortPaths(const uint64_t now)
+{
+	// assumes _lock is locked
+	_lastPathSort = now;
+	std::sort(&(_paths[0]),&(_paths[_numPaths]),_SortPathsByQuality(now));
+}
+
+RemotePath *Peer::_getBestPath(const uint64_t now)
+{
+	// assumes _lock is locked
+	if ((now - _lastPathSort) >= ZT_PEER_PATH_SORT_INTERVAL)
+		_sortPaths(now);
+	if (_paths[0].active(now)) {
+		return &(_paths[0]);
+	} else {
+		_sortPaths(now);
+		if (_paths[0].active(now))
+			return &(_paths[0]);
+	}
+	return (RemotePath *)0;
+}
+
 } // namespace ZeroTier
diff --git a/node/Peer.hpp b/node/Peer.hpp
index 4d1031b8..c1692846 100644
--- a/node/Peer.hpp
+++ b/node/Peer.hpp
@@ -49,6 +49,8 @@
 #include "Packet.hpp"
 #include "SharedPtr.hpp"
 #include "AtomicCounter.hpp"
+#include "Hashtable.hpp"
+#include "Mutex.hpp"
 #include "NonCopyable.hpp"
 
 namespace ZeroTier {
@@ -129,7 +131,11 @@ public:
 	 * @param now Current time
 	 * @return Best path or NULL if there are no active (or fixed) direct paths
 	 */
-	RemotePath *getBestPath(uint64_t now);
+	inline RemotePath *getBestPath(uint64_t now)
+	{
+		Mutex::Lock _l(_lock);
+		return _getBestPath(now);
+	}
 
 	/**
 	 * Send via best path
@@ -293,8 +299,9 @@ public:
 	 * Add a path (if we don't already have it)
 	 *
 	 * @param p New path to add
+	 * @param now Current time
 	 */
-	void addPath(const RemotePath &newp);
+	void addPath(const RemotePath &newp,uint64_t now);
 
 	/**
 	 * Clear paths
@@ -382,6 +389,37 @@ public:
 	void getBestActiveAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) const;
 
 	/**
+	 * Check network COM agreement with this peer
+	 *
+	 * @param nwid Network ID
+	 * @param com Another certificate of membership
+	 * @return True if supplied COM agrees with ours, false if not or if we don't have one
+	 */
+	bool networkMembershipCertificatesAgree(uint64_t nwid,const CertificateOfMembership &com) const;
+
+	/**
+	 * Check the validity of the COM and add/update if valid and new
+	 *
+	 * @param RR Runtime Environment
+	 * @param nwid Network ID
+	 * @param com Externally supplied COM
+	 */
+	bool validateAndSetNetworkMembershipCertificate(const RuntimeEnvironment *RR,uint64_t nwid,const CertificateOfMembership &com);
+
+	/**
+	 * @param nwid Network ID
+	 * @param now Current time
+	 * @param updateLastPushedTime If true, go ahead and update the last pushed time regardless of return value
+	 * @return Whether or not this peer needs another COM push from us
+	 */
+	bool needsOurNetworkMembershipCertificate(uint64_t nwid,uint64_t now,bool updateLastPushedTime);
+
+	/**
+	 * Perform periodic cleaning operations
+	 */
+	void clean(const RuntimeEnvironment *RR,uint64_t now);
+
+	/**
 	 * Find a common set of addresses by which two peers can link, if any
 	 *
 	 * @param a Peer A
@@ -402,7 +440,8 @@ public:
 	}
 
 private:
-	void _announceMulticastGroups(const RuntimeEnvironment *RR,uint64_t now);
+	void _sortPaths(const uint64_t now);
+	RemotePath *_getBestPath(const uint64_t now);
 
 	unsigned char _key[ZT_PEER_SECRET_KEY_LENGTH];
 	uint64_t _lastUsed;
@@ -412,6 +451,7 @@ private:
 	uint64_t _lastAnnouncedTo;
 	uint64_t _lastPathConfirmationSent;
 	uint64_t _lastDirectPathPush;
+	uint64_t _lastPathSort;
 	uint16_t _vProto;
 	uint16_t _vMajor;
 	uint16_t _vMinor;
@@ -421,6 +461,17 @@ private:
 	unsigned int _numPaths;
 	unsigned int _latency;
 
+	struct _NetworkCom
+	{
+		_NetworkCom() {}
+		_NetworkCom(uint64_t t,const CertificateOfMembership &c) : ts(t),com(c) {}
+		uint64_t ts;
+		CertificateOfMembership com;
+	};
+	Hashtable<uint64_t,_NetworkCom> _networkComs;
+	Hashtable<uint64_t,uint64_t> _lastPushedComs;
+
+	Mutex _lock;
 	AtomicCounter __refCount;
 };
 
diff --git a/node/Switch.cpp b/node/Switch.cpp
index ecae9b76..9ea8ac49 100644
--- a/node/Switch.cpp
+++ b/node/Switch.cpp
@@ -202,7 +202,8 @@ void Switch::onLocalEthernet(const SharedPtr<Network> &network,const MAC &from,c
 		// Destination is another ZeroTier peer on the same network
 
 		Address toZT(to.toAddress(network->id())); // since in-network MACs are derived from addresses and network IDs, we can reverse this
-		const bool includeCom = network->peerNeedsOurMembershipCertificate(toZT,RR->node->now());
+		SharedPtr<Peer> toPeer(RR->topology->getPeer(toZT));
+		const bool includeCom = ((!toPeer)||(toPeer->needsOurNetworkMembershipCertificate(network->id(),RR->node->now(),true)));;
 		if ((fromBridged)||(includeCom)) {
 			Packet outp(toZT,RR->identity.address(),Packet::VERB_EXT_FRAME);
 			outp.append(network->id());
@@ -267,9 +268,10 @@ void Switch::onLocalEthernet(const SharedPtr<Network> &network,const MAC &from,c
 		}
 
 		for(unsigned int b=0;b<numBridges;++b) {
+			SharedPtr<Peer> bridgePeer(RR->topology->getPeer(bridges[b]));
 			Packet outp(bridges[b],RR->identity.address(),Packet::VERB_EXT_FRAME);
 			outp.append(network->id());
-			if (network->peerNeedsOurMembershipCertificate(bridges[b],RR->node->now())) {
+			if ((!bridgePeer)||(bridgePeer->needsOurNetworkMembershipCertificate(network->id(),RR->node->now(),true))) {
 				outp.append((unsigned char)0x01); // 0x01 -- COM included
 				nconf->com().serialize(outp);
 			} else {
@@ -747,7 +749,7 @@ bool Switch::_trySend(const Packet &packet,bool encrypt,uint64_t nwid)
 				return false; // no paths, no root servers?
 		}
 
-		if ((network)&&(relay)&&(network->isAllowed(peer->address()))) {
+		if ((network)&&(relay)&&(network->isAllowed(peer))) {
 			// Push hints for direct connectivity to this peer if we are relaying
 			peer->pushDirectPaths(RR,viaPath,now,false);
 		}
diff --git a/node/Topology.cpp b/node/Topology.cpp
index e931df1e..b2b4ebbd 100644
--- a/node/Topology.cpp
+++ b/node/Topology.cpp
@@ -62,7 +62,7 @@ void Topology::setRootServers(const std::map< Identity,std::vector<InetAddress>
 			if (!p)
 				p = SharedPtr<Peer>(new Peer(RR->identity,i->first));
 			for(std::vector<InetAddress>::const_iterator j(i->second.begin());j!=i->second.end();++j)
-				p->addPath(RemotePath(InetAddress(),*j,true));
+				p->addPath(RemotePath(InetAddress(),*j,true),now);
 			p->use(now);
 			_rootPeers.push_back(p);
 		}
@@ -252,9 +252,12 @@ void Topology::clean(uint64_t now)
 	Hashtable< Address,SharedPtr<Peer> >::Iterator i(_activePeers);
 	Address *a = (Address *)0;
 	SharedPtr<Peer> *p = (SharedPtr<Peer> *)0;
-	while (i.next(a,p))
+	while (i.next(a,p)) {
 		if (((now - (*p)->lastUsed()) >= ZT_PEER_IN_MEMORY_EXPIRATION)&&(std::find(_rootAddresses.begin(),_rootAddresses.end(),*a) == _rootAddresses.end())) {
 			_activePeers.erase(*a);
+		} else {
+			(*p)->clean(RR,now);
+		}
 	}
 }