.

10 files changed, 251 insertions, 85 deletions

diff --git a/node/Constants.hpp b/node/Constants.hpp
index fbbba76e..88549937 100644
--- a/node/Constants.hpp
+++ b/node/Constants.hpp
@@ -216,7 +216,7 @@
 /**
  * How often Topology::clean() and Network::clean() and similar are called, in ms
  */
-#define ZT_HOUSEKEEPING_PERIOD 120000
+#define ZT_HOUSEKEEPING_PERIOD 10000
 
 /**
  * How long to remember peer records in RAM if they haven't been used
diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp
index 1d55c9f3..4d99e87d 100644
--- a/node/IncomingPacket.cpp
+++ b/node/IncomingPacket.cpp
@@ -1211,8 +1211,8 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt
 							(!( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) == 0) && (peer->hasActivePathTo(now,a)) )) && // not already known
 							(RR->node->shouldUsePathForZeroTierTraffic(tPtr,peer->address(),_path->localAddress(),a)) ) // should use path
 					{
-						if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0)
-							peer->setClusterPreferred(a);
+						//if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0)
+						//	peer->setClusterPreferred(a);
 						if (++countPerScope[(int)a.ipScope()][0] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) {
 							TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str());
 							peer->attemptToContactAt(tPtr,InetAddress(),a,now,false,0);
@@ -1228,8 +1228,8 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt
 							(!( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) == 0) && (peer->hasActivePathTo(now,a)) )) && // not already known
 							(RR->node->shouldUsePathForZeroTierTraffic(tPtr,peer->address(),_path->localAddress(),a)) ) // should use path
 					{
-						if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0)
-							peer->setClusterPreferred(a);
+						//if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0)
+						//	peer->setClusterPreferred(a);
 						if (++countPerScope[(int)a.ipScope()][1] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) {
 							TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str());
 							peer->attemptToContactAt(tPtr,InetAddress(),a,now,false,0);
diff --git a/node/Network.cpp b/node/Network.cpp
index 8c6f2ce8..0a16ded8 100644
--- a/node/Network.cpp
+++ b/node/Network.cpp
@@ -700,10 +700,13 @@ Network::Network(const RuntimeEnvironment *renv,void *tPtr,uint64_t nwid,void *u
 		this->setConfiguration(tPtr,*nconf,false);
 		_lastConfigUpdate = 0; // still want to re-request since it's likely outdated
 	} else {
+		uint64_t tmp[2];
+		tmp[0] = nwid; tmp[1] = 0;
+
 		bool got = false;
 		Dictionary<ZT_NETWORKCONFIG_DICT_CAPACITY> *dict = new Dictionary<ZT_NETWORKCONFIG_DICT_CAPACITY>();
 		try {
-			int n = RR->node->stateObjectGet(tPtr,ZT_STATE_OBJECT_NETWORK_CONFIG,nwid,dict->unsafeData(),ZT_NETWORKCONFIG_DICT_CAPACITY - 1);
+			int n = RR->node->stateObjectGet(tPtr,ZT_STATE_OBJECT_NETWORK_CONFIG,tmp,dict->unsafeData(),ZT_NETWORKCONFIG_DICT_CAPACITY - 1);
 			if (n > 1) {
 				NetworkConfig *nconf = new NetworkConfig();
 				try {
@@ -719,7 +722,7 @@ Network::Network(const RuntimeEnvironment *renv,void *tPtr,uint64_t nwid,void *u
 		delete dict;
 
 		if (!got)
-			RR->node->stateObjectPut(tPtr,ZT_STATE_OBJECT_NETWORK_CONFIG,nwid,"\n",1);
+			RR->node->stateObjectPut(tPtr,ZT_STATE_OBJECT_NETWORK_CONFIG,tmp,"\n",1);
 	}
 
 	if (!_portInitialized) {
@@ -1185,8 +1188,11 @@ int Network::setConfiguration(void *tPtr,const NetworkConfig &nconf,bool saveToD
 		if (saveToDisk) {
 			Dictionary<ZT_NETWORKCONFIG_DICT_CAPACITY> *d = new Dictionary<ZT_NETWORKCONFIG_DICT_CAPACITY>();
 			try {
-				if (nconf.toDictionary(*d,false))
-					RR->node->stateObjectPut(tPtr,ZT_STATE_OBJECT_NETWORK_CONFIG,_id,d->data(),d->sizeBytes());
+				if (nconf.toDictionary(*d,false)) {
+					uint64_t tmp[2];
+					tmp[0] = _id; tmp[1] = 0;
+					RR->node->stateObjectPut(tPtr,ZT_STATE_OBJECT_NETWORK_CONFIG,tmp,d->data(),d->sizeBytes());
+				}
 			} catch ( ... ) {}
 			delete d;
 		}
diff --git a/node/Node.cpp b/node/Node.cpp
index ab49e63b..1112c0f2 100644
--- a/node/Node.cpp
+++ b/node/Node.cpp
@@ -76,22 +76,26 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint6
 	memset(_expectingRepliesTo,0,sizeof(_expectingRepliesTo));
 	memset(_lastIdentityVerification,0,sizeof(_lastIdentityVerification));
 
+	uint64_t idtmp[2];
+	idtmp[0] = 0; idtmp[1] = 0;
 	char tmp[512];
 	std::string tmp2;
-	int n = stateObjectGet(tptr,ZT_STATE_OBJECT_IDENTITY_SECRET,0,tmp,sizeof(tmp) - 1);
+	int n = stateObjectGet(tptr,ZT_STATE_OBJECT_IDENTITY_SECRET,idtmp,tmp,sizeof(tmp) - 1);
 	if (n > 0) {
 		tmp[n] = (char)0;
 		if (!RR->identity.fromString(tmp))
 			n = -1;
 	}
+
+	idtmp[0] = RR->identity.address().toInt(); idtmp[1] = 0;
 	if (n <= 0) {
 		RR->identity.generate();
 		tmp2 = RR->identity.toString(true);
-		stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_SECRET,RR->identity.address().toInt(),tmp2.data(),(unsigned int)tmp2.length());
+		stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_SECRET,idtmp,tmp2.data(),(unsigned int)tmp2.length());
 		tmp2 = RR->identity.toString(false);
-		stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,RR->identity.address().toInt(),tmp2.data(),(unsigned int)tmp2.length());
+		stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,idtmp,tmp2.data(),(unsigned int)tmp2.length());
 	} else {
-		n = stateObjectGet(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,RR->identity.address().toInt(),tmp,sizeof(tmp) - 1);
+		n = stateObjectGet(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,idtmp,tmp,sizeof(tmp) - 1);
 		if (n > 0) {
 			tmp[n] = (char)0;
 			if (RR->identity.toString(false) != tmp)
@@ -99,7 +103,7 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint6
 		}
 		if (n <= 0) {
 			tmp2 = RR->identity.toString(false);
-			stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,RR->identity.address().toInt(),tmp2.data(),(unsigned int)tmp2.length());
+			stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,idtmp,tmp2.data(),(unsigned int)tmp2.length());
 		}
 	}
 
@@ -145,7 +149,7 @@ ZT_ResultCode Node::processStateUpdate(
 	ZT_ResultCode r = ZT_RESULT_OK_IGNORED;
 	switch(type) {
 
-		case ZT_STATE_OBJECT_PEER:
+		case ZT_STATE_OBJECT_PEER_STATE:
 			if (len) {
 			}
 			break;
@@ -380,9 +384,9 @@ ZT_ResultCode Node::processBackgroundTasks(void *tptr,uint64_t now,volatile uint
 	}
 
 	if ((now - _lastHousekeepingRun) >= ZT_HOUSEKEEPING_PERIOD) {
+		_lastHousekeepingRun = now;
 		try {
-			_lastHousekeepingRun = now;
-			RR->topology->clean(now);
+			RR->topology->doPeriodicTasks(tptr,now);
 			RR->sa->clean(now);
 			RR->mc->clean(now);
 		} catch ( ... ) {
@@ -443,7 +447,9 @@ ZT_ResultCode Node::leave(uint64_t nwid,void **uptr,void *tptr)
 		_networks.erase(nwid);
 	}
 
-	RR->node->stateObjectDelete(tptr,ZT_STATE_OBJECT_NETWORK_CONFIG,nwid);
+	uint64_t tmp[2];
+	tmp[0] = nwid; tmp[1] = 0;
+	RR->node->stateObjectDelete(tptr,ZT_STATE_OBJECT_NETWORK_CONFIG,tmp);
 
 	return ZT_RESULT_OK;
 }
diff --git a/node/Node.hpp b/node/Node.hpp
index f407c60c..f1209d00 100644
--- a/node/Node.hpp
+++ b/node/Node.hpp
@@ -197,9 +197,9 @@ public:
 
 	inline bool online() const throw() { return _online; }
 
-	inline int stateObjectGet(void *const tPtr,ZT_StateObjectType type,const uint64_t id,void *const data,const unsigned int maxlen) { return _cb.stateGetFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,tPtr,type,id,data,maxlen); }
-	inline void stateObjectPut(void *const tPtr,ZT_StateObjectType type,const uint64_t id,const void *const data,const unsigned int len) { _cb.statePutFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,tPtr,type,id,data,(int)len); }
-	inline void stateObjectDelete(void *const tPtr,ZT_StateObjectType type,const uint64_t id) { _cb.statePutFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,tPtr,type,id,(const void *)0,-1); }
+	inline int stateObjectGet(void *const tPtr,ZT_StateObjectType type,const uint64_t id[2],void *const data,const unsigned int maxlen) { return _cb.stateGetFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,tPtr,type,id,data,maxlen); }
+	inline void stateObjectPut(void *const tPtr,ZT_StateObjectType type,const uint64_t id[2],const void *const data,const unsigned int len) { _cb.statePutFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,tPtr,type,id,data,(int)len); }
+	inline void stateObjectDelete(void *const tPtr,ZT_StateObjectType type,const uint64_t id[2]) { _cb.statePutFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,tPtr,type,id,(const void *)0,-1); }
 
 #ifdef ZT_TRACE
 	void postTrace(const char *module,unsigned int line,const char *fmt,...);
diff --git a/node/Path.hpp b/node/Path.hpp
index 32bceae0..74b31d8d 100644
--- a/node/Path.hpp
+++ b/node/Path.hpp
@@ -46,6 +46,11 @@
  */
 #define ZT_PATH_MAX_PREFERENCE_RANK ((ZT_INETADDRESS_MAX_SCOPE << 1) | 1)
 
+/**
+ * Maximum distance for a path
+ */
+#define ZT_PATH_DISTANCE_MAX 0xffff
+
 namespace ZeroTier {
 
 class RuntimeEnvironment;
@@ -120,6 +125,7 @@ public:
 		_incomingLinkQualitySlowLogCounter(-64), // discard first fast log
 		_incomingLinkQualityPreviousPacketCounter(0),
 		_outgoingPacketCounter(0),
+		_distance(ZT_PATH_DISTANCE_MAX),
 		_addr(),
 		_localAddress(),
 		_ipScope(InetAddress::IP_SCOPE_NONE)
@@ -137,6 +143,7 @@ public:
 		_incomingLinkQualitySlowLogCounter(-64), // discard first fast log
 		_incomingLinkQualityPreviousPacketCounter(0),
 		_outgoingPacketCounter(0),
+		_distance(ZT_PATH_DISTANCE_MAX),
 		_addr(addr),
 		_localAddress(localAddress),
 		_ipScope(addr.ipScope())
@@ -300,6 +307,28 @@ public:
 	inline uint64_t lastIn() const { return _lastIn; }
 
 	/**
+	 * @return Time last trust-established packet was received
+	 */
+	inline uint64_t lastTrustEstablishedPacketReceived() const { return _lastTrustEstablishedPacketReceived; }
+
+	/**
+	 * @return Distance (higher is further)
+	 */
+	inline unsigned int distance() const { return _distance; }
+
+	/**
+	 * @param lo Last out send
+	 * @param li Last in send
+	 * @param lt Last trust established packet received
+	 */
+	inline void updateFromRemoteState(const uint64_t lo,const uint64_t li,const uint64_t lt)
+	{
+		_lastOut = lo;
+		_lastIn = li;
+		_lastTrustEstablishedPacketReceived = lt;
+	}
+
+	/**
 	 * Return and increment outgoing packet counter (used with Packet::armor())
 	 *
 	 * @return Next value that should be used for outgoing packet counter (only least significant 3 bits are used)
@@ -315,6 +344,7 @@ private:
 	volatile signed int _incomingLinkQualitySlowLogCounter;
 	volatile unsigned int _incomingLinkQualityPreviousPacketCounter;
 	volatile unsigned int _outgoingPacketCounter;
+	volatile unsigned int _distance;
 	InetAddress _addr;
 	InetAddress _localAddress;
 	InetAddress::IpScope _ipScope; // memoize this since it's a computed value checked often
diff --git a/node/Peer.cpp b/node/Peer.cpp
index 84086048..a7466296 100644
--- a/node/Peer.cpp
+++ b/node/Peer.cpp
@@ -38,6 +38,8 @@ namespace ZeroTier {
 
 Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Identity &peerIdentity) :
 	RR(renv),
+	_lastWroteState(0),
+	_lastReceivedStateTimestamp(0),
 	_lastReceive(0),
 	_lastNontrivialReceive(0),
 	_lastTriedMemorizedPath(0),
@@ -75,6 +77,7 @@ void Peer::received(
 {
 	const uint64_t now = RR->node->now();
 
+/*
 #ifdef ZT_ENABLE_CLUSTER
 	bool isClusterSuboptimalPath = false;
 	if ((RR->cluster)&&(hops == 0)) {
@@ -120,6 +123,7 @@ void Peer::received(
 		}
 	}
 #endif
+*/
 
 	_lastReceive = now;
 	switch (verb) {
@@ -143,6 +147,8 @@ void Peer::received(
 
 	if (hops == 0) {
 		bool pathAlreadyKnown = false;
+		bool newPathLearned = false;
+
 		{
 			Mutex::Lock _l(_paths_m);
 			if ((path->address().ss_family == AF_INET)&&(_v4Path.p)) {
@@ -152,9 +158,6 @@ void Peer::received(
 				const struct sockaddr_in *const ll = reinterpret_cast<const struct sockaddr_in *>(&(_v4Path.p->localAddress()));
 				if ((r->sin_addr.s_addr == l->sin_addr.s_addr)&&(r->sin_port == l->sin_port)&&(rl->sin_addr.s_addr == ll->sin_addr.s_addr)&&(rl->sin_port == ll->sin_port)) {
 					_v4Path.lr = now;
-#ifdef ZT_ENABLE_CLUSTER
-					_v4Path.localClusterSuboptimal = isClusterSuboptimalPath;
-#endif
 					pathAlreadyKnown = true;
 				}
 			} else if ((path->address().ss_family == AF_INET6)&&(_v6Path.p)) {
@@ -164,9 +167,6 @@ void Peer::received(
 				const struct sockaddr_in6 *const ll = reinterpret_cast<const struct sockaddr_in6 *>(&(_v6Path.p->localAddress()));
 				if ((!memcmp(r->sin6_addr.s6_addr,l->sin6_addr.s6_addr,16))&&(r->sin6_port == l->sin6_port)&&(!memcmp(rl->sin6_addr.s6_addr,ll->sin6_addr.s6_addr,16))&&(rl->sin6_port == ll->sin6_port)) {
 					_v6Path.lr = now;
-#ifdef ZT_ENABLE_CLUSTER
-					_v6Path.localClusterSuboptimal = isClusterSuboptimalPath;
-#endif
 					pathAlreadyKnown = true;
 				}
 			}
@@ -176,11 +176,11 @@ void Peer::received(
 			Mutex::Lock _l(_paths_m);
 			_PeerPath *potentialNewPeerPath = (_PeerPath *)0;
 			if (path->address().ss_family == AF_INET) {
-				if ( (!_v4Path.p) || (!_v4Path.p->alive(now)) || ((_v4Path.p->address() != _v4ClusterPreferred)&&(path->preferenceRank() >= _v4Path.p->preferenceRank())) ) {
+				if ( (!_v4Path.p) || (!_v4Path.p->alive(now)) || (path->preferenceRank() >= _v4Path.p->preferenceRank()) ) {
 					potentialNewPeerPath = &_v4Path;
 				}
 			} else if (path->address().ss_family == AF_INET6) {
-				if ( (!_v6Path.p) || (!_v6Path.p->alive(now)) || ((_v6Path.p->address() != _v6ClusterPreferred)&&(path->preferenceRank() >= _v6Path.p->preferenceRank())) ) {
+				if ( (!_v6Path.p) || (!_v6Path.p->alive(now)) || (path->preferenceRank() >= _v6Path.p->preferenceRank()) ) {
 					potentialNewPeerPath = &_v6Path;
 				}
 			}
@@ -188,11 +188,7 @@ void Peer::received(
 				if (verb == Packet::VERB_OK) {
 					potentialNewPeerPath->lr = now;
 					potentialNewPeerPath->p = path;
-#ifdef ZT_ENABLE_CLUSTER
-					potentialNewPeerPath->localClusterSuboptimal = isClusterSuboptimalPath;
-					if (RR->cluster)
-						RR->cluster->broadcastHavePeer(_id);
-#endif
+					newPathLearned = true;
 				} else {
 					TRACE("got %s via unknown path %s(%s), confirming...",Packet::verbString(verb),_id.address().toString().c_str(),path->address().toString().c_str());
 					attemptToContactAt(tPtr,path->localAddress(),path->address(),now,true,path->nextOutgoingCounter());
@@ -200,15 +196,12 @@ void Peer::received(
 				}
 			}
 		}
+
+		if (newPathLearned)
+			writeState(tPtr,now);
 	} else if (this->trustEstablished(now)) {
 		// Send PUSH_DIRECT_PATHS if hops>0 (relayed) and we have a trust relationship (common network membership)
-#ifdef ZT_ENABLE_CLUSTER
-		// Cluster mode disables normal PUSH_DIRECT_PATHS in favor of cluster-based peer redirection
-		const bool haveCluster = (RR->cluster);
-#else
-		const bool haveCluster = false;
-#endif
-		if ( ((now - _lastDirectPathPushSent) >= ZT_DIRECT_PATH_PUSH_INTERVAL) && (!haveCluster) ) {
+		if ((now - _lastDirectPathPushSent) >= ZT_DIRECT_PATH_PUSH_INTERVAL) {
 			_lastDirectPathPushSent = now;
 
 			std::vector<InetAddress> pathsToPush;
@@ -439,4 +432,135 @@ bool Peer::doPingAndKeepalive(void *tPtr,uint64_t now,int inetAddressFamily)
 	return false;
 }
 
+void Peer::writeState(void *tPtr,const uint64_t now)
+{
+	try {
+		Buffer<sizeof(Peer) + 32 + (sizeof(Path) * 2)> b;
+
+		b.append((uint8_t)1); // version
+		b.append(now);
+
+		_id.serialize(b);
+
+		{
+			Mutex::Lock _l(_paths_m);
+			unsigned int count = 0;
+			if (_v4Path.lr)
+				++count;
+			if (_v6Path.lr)
+				++count;
+			b.append((uint8_t)count);
+			if (_v4Path.lr) {
+				b.append(_v4Path.lr);
+				b.append(_v4Path.p->lastOut());
+				b.append(_v4Path.p->lastIn());
+				b.append(_v4Path.p->lastTrustEstablishedPacketReceived());
+				b.append((uint16_t)_v4Path.p->distance());
+				_v4Path.p->address().serialize(b);
+				_v4Path.p->localAddress().serialize(b);
+			}
+			if (_v6Path.lr) {
+				b.append(_v6Path.lr);
+				b.append(_v6Path.p->lastOut());
+				b.append(_v6Path.p->lastIn());
+				b.append(_v6Path.p->lastTrustEstablishedPacketReceived());
+				b.append((uint16_t)_v6Path.p->distance());
+				_v6Path.p->address().serialize(b);
+				_v6Path.p->localAddress().serialize(b);
+			}
+		}
+
+		b.append(_lastReceive);
+		b.append(_lastNontrivialReceive);
+		b.append(_lastTriedMemorizedPath);
+		b.append(_lastDirectPathPushSent);
+		b.append(_lastDirectPathPushReceive);
+		b.append(_lastCredentialRequestSent);
+		b.append(_lastWhoisRequestReceived);
+		b.append(_lastEchoRequestReceived);
+		b.append(_lastComRequestReceived);
+		b.append(_lastComRequestSent);
+		b.append(_lastCredentialsReceived);
+		b.append(_lastTrustEstablishedPacketReceived);
+
+		b.append(_vProto);
+		b.append(_vMajor);
+		b.append(_vMinor);
+		b.append(_vRevision);
+
+		b.append((uint16_t)0); // length of additional fields
+
+		uint64_t tmp[2];
+		tmp[0] = _id.address().toInt(); tmp[1] = 0;
+		RR->node->stateObjectPut(tPtr,ZT_STATE_OBJECT_PEER_STATE,tmp,b.data(),b.size());
+
+		_lastWroteState = now;
+	} catch ( ... ) {} // sanity check, should not be possible
+}
+
+bool Peer::applyStateUpdate(const void *data,unsigned int len)
+{
+	try {
+		Buffer<sizeof(Peer) + 32 + (sizeof(Path) * 2)> b(data,len);
+		unsigned int ptr = 0;
+
+		if (b[ptr++] != 1)
+			return false;
+		const uint64_t ts = b.at<uint64_t>(ptr); ptr += 8;
+		if (ts <= _lastReceivedStateTimestamp)
+			return false;
+
+		const unsigned int pathCount = (unsigned int)b[ptr++];
+		{
+			Mutex::Lock _l(_paths_m);
+			for(unsigned int i=0;i<pathCount;++i) {
+				const uint64_t lr = b.at<uint64_t>(ptr); ptr += 8;
+				const uint64_t lastOut = b.at<uint64_t>(ptr); ptr += 8;
+				const uint64_t lastIn = b.at<uint64_t>(ptr); ptr += 8;
+				const uint64_t lastTrustEstablishedPacketReceived = b.at<uint64_t>(ptr); ptr += 8;
+				const unsigned int distance = b.at<uint16_t>(ptr); ptr += 2;
+				InetAddress addr,localAddr;
+				ptr += addr.deserialize(b,ptr);
+				ptr += localAddr.deserialize(b,ptr);
+				if (addr.ss_family == localAddr.ss_family) {
+					_PeerPath *p = (_PeerPath *)0;
+					switch(addr.ss_family) {
+						case AF_INET: p = &_v4Path; break;
+						case AF_INET6: p = &_v6Path; break;
+					}
+					if (p) {
+						if ( ((p->p->address() != addr)||(p->p->localAddress() != localAddr)) && (p->p->distance() > distance) )
+							p->p = RR->topology->getPath(localAddr,addr);
+						p->lr = lr;
+						p->p->updateFromRemoteState(lastOut,lastIn,lastTrustEstablishedPacketReceived);
+					}
+				}
+			}
+		}
+
+		_lastReceive = std::max(_lastReceive,b.at<uint64_t>(ptr)); ptr += 8;
+		_lastNontrivialReceive = std::max(_lastNontrivialReceive,b.at<uint64_t>(ptr)); ptr += 8;
+		_lastTriedMemorizedPath = std::max(_lastTriedMemorizedPath,b.at<uint64_t>(ptr)); ptr += 8;
+		_lastDirectPathPushSent = std::max(_lastDirectPathPushSent,b.at<uint64_t>(ptr)); ptr += 8;
+		_lastDirectPathPushReceive = std::max(_lastDirectPathPushReceive,b.at<uint64_t>(ptr)); ptr += 8;
+		_lastCredentialRequestSent = std::max(_lastCredentialRequestSent,b.at<uint64_t>(ptr)); ptr += 8;
+		_lastWhoisRequestReceived = std::max(_lastWhoisRequestReceived,b.at<uint64_t>(ptr)); ptr += 8;
+		_lastEchoRequestReceived = std::max(_lastEchoRequestReceived,b.at<uint64_t>(ptr)); ptr += 8;
+		_lastComRequestReceived = std::max(_lastComRequestReceived,b.at<uint64_t>(ptr)); ptr += 8;
+		_lastComRequestSent = std::max(_lastComRequestSent,b.at<uint64_t>(ptr)); ptr += 8;
+		_lastCredentialsReceived = std::max(_lastCredentialsReceived,b.at<uint64_t>(ptr)); ptr += 8;
+		_lastTrustEstablishedPacketReceived = std::max(_lastTrustEstablishedPacketReceived,b.at<uint64_t>(ptr)); ptr += 8;
+
+		_vProto = b.at<uint16_t>(ptr); ptr += 2;
+		_vMajor = b.at<uint16_t>(ptr); ptr += 2;
+		_vMinor = b.at<uint16_t>(ptr); ptr += 2;
+		_vRevision = b.at<uint16_t>(ptr); ptr += 2;
+
+		_lastReceivedStateTimestamp = ts;
+
+		return true;
+	} catch ( ... ) {} // ignore invalid state updates
+	return false;
+}
+
 } // namespace ZeroTier
diff --git a/node/Peer.hpp b/node/Peer.hpp
index 9b57f23e..d6b7dad9 100644
--- a/node/Peer.hpp
+++ b/node/Peer.hpp
@@ -194,6 +194,22 @@ public:
 	bool doPingAndKeepalive(void *tPtr,uint64_t now,int inetAddressFamily);
 
 	/**
+	 * Write current peer state to external storage / cluster network
+	 *
+	 * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call
+	 */
+	void writeState(void *tPtr,const uint64_t now);
+
+	/**
+	 * Apply a state update received from e.g. a remote cluster member
+	 *
+	 * @param data State update data
+	 * @param len Length of state update
+	 * @return True if state update was applied, false if ignored or invalid
+	 */
+	bool applyStateUpdate(const void *data,unsigned int len);
+
+	/**
 	 * Reset paths within a given IP scope and address family
 	 *
 	 * Resetting a path involves sending an ECHO to it and then deactivating
@@ -219,19 +235,6 @@ public:
 	}
 
 	/**
-	 * Indicate that the given address was provided by a cluster as a preferred destination
-	 *
-	 * @param addr Address cluster prefers that we use
-	 */
-	inline void setClusterPreferred(const InetAddress &addr)
-	{
-		if (addr.ss_family == AF_INET)
-			_v4ClusterPreferred = addr;
-		else if (addr.ss_family == AF_INET6)
-			_v6ClusterPreferred = addr;
-	}
-
-	/**
 	 * Fill parameters with V4 and V6 addresses if known and alive
 	 *
 	 * @param now Current time
@@ -317,18 +320,6 @@ public:
 		else _latency = std::min(l,(unsigned int)65535);
 	}
 
-#ifdef ZT_ENABLE_CLUSTER
-	/**
-	 * @param now Current time
-	 * @return True if this peer has at least one active direct path that is not cluster-suboptimal
-	 */
-	inline bool hasLocalClusterOptimalPath(uint64_t now) const
-	{
-		Mutex::Lock _l(_paths_m);
-		return ( ((_v4Path.p)&&(_v4Path.p->alive(now))&&(!_v4Path.localClusterSuboptimal)) || ((_v6Path.p)&&(_v6Path.p->alive(now))&&(!_v6Path.localClusterSuboptimal)) );
-	}
-#endif
-
 	/**
 	 * @return 256-bit secret symmetric encryption key
 	 */
@@ -449,22 +440,18 @@ public:
 private:
 	struct _PeerPath
 	{
-#ifdef ZT_ENABLE_CLUSTER
-		_PeerPath() : lr(0),p(),localClusterSuboptimal(false) {}
-#else
 		_PeerPath() : lr(0),p() {}
-#endif
 		uint64_t lr; // time of last valid ZeroTier packet
 		SharedPtr<Path> p;
-#ifdef ZT_ENABLE_CLUSTER
-		bool localClusterSuboptimal; // true if our cluster has determined that we should not be serving this peer
-#endif
 	};
 
 	uint8_t _key[ZT_PEER_SECRET_KEY_LENGTH];
 
 	const RuntimeEnvironment *RR;
 
+	uint64_t _lastWroteState;
+	uint64_t _lastReceivedStateTimestamp;
+
 	uint64_t _lastReceive; // direct or indirect
 	uint64_t _lastNontrivialReceive; // frames, things like netconf, etc.
 	uint64_t _lastTriedMemorizedPath;
@@ -483,9 +470,6 @@ private:
 	uint16_t _vMinor;
 	uint16_t _vRevision;
 
-	InetAddress _v4ClusterPreferred;
-	InetAddress _v6ClusterPreferred;
-
 	_PeerPath _v4Path; // IPv4 direct path
 	_PeerPath _v6Path; // IPv6 direct path
 	Mutex _paths_m;
diff --git a/node/Topology.cpp b/node/Topology.cpp
index d4b424ff..be116b28 100644
--- a/node/Topology.cpp
+++ b/node/Topology.cpp
@@ -69,7 +69,9 @@ Topology::Topology(const RuntimeEnvironment *renv,void *tPtr) :
 	_amRoot(false)
 {
 	uint8_t tmp[ZT_WORLD_MAX_SERIALIZED_LENGTH];
-	int n = RR->node->stateObjectGet(tPtr,ZT_STATE_OBJECT_PLANET,0,tmp,sizeof(tmp));
+	uint64_t idtmp[2];
+	idtmp[0] = 0; idtmp[1] = 0;
+	int n = RR->node->stateObjectGet(tPtr,ZT_STATE_OBJECT_PLANET,idtmp,tmp,sizeof(tmp));
 	if (n > 0) {
 		try {
 			World cachedPlanet;
@@ -159,7 +161,9 @@ void Topology::saveIdentity(void *tPtr,const Identity &id)
 {
 	if (id) {
 		const std::string tmp(id.toString(false));
-		RR->node->stateObjectPut(tPtr,ZT_STATE_OBJECT_PEER_IDENTITY,id.address().toInt(),tmp.data(),(unsigned int)tmp.length());
+		uint64_t idtmp[2];
+		idtmp[0] = id.address().toInt(); idtmp[1] = 0;
+		RR->node->stateObjectPut(tPtr,ZT_STATE_OBJECT_PEER_IDENTITY,idtmp,tmp.data(),(unsigned int)tmp.length());
 	}
 }
 
@@ -329,7 +333,9 @@ bool Topology::addWorld(void *tPtr,const World &newWorld,bool alwaysAcceptNew)
 	try {
 		Buffer<ZT_WORLD_MAX_SERIALIZED_LENGTH> sbuf;
 		existing->serialize(sbuf,false);
-		RR->node->stateObjectPut(tPtr,(existing->type() == World::TYPE_PLANET) ? ZT_STATE_OBJECT_PLANET : ZT_STATE_OBJECT_MOON,existing->id(),sbuf.data(),sbuf.size());
+		uint64_t idtmp[2];
+		idtmp[0] = existing->id(); idtmp[1] = 0;
+		RR->node->stateObjectPut(tPtr,(existing->type() == World::TYPE_PLANET) ? ZT_STATE_OBJECT_PLANET : ZT_STATE_OBJECT_MOON,idtmp,sbuf.data(),sbuf.size());
 	} catch ( ... ) {}
 
 	_memoizeUpstreams(tPtr);
@@ -340,7 +346,9 @@ bool Topology::addWorld(void *tPtr,const World &newWorld,bool alwaysAcceptNew)
 void Topology::addMoon(void *tPtr,const uint64_t id,const Address &seed)
 {
 	char tmp[ZT_WORLD_MAX_SERIALIZED_LENGTH];
-	int n = RR->node->stateObjectGet(tPtr,ZT_STATE_OBJECT_MOON,id,tmp,sizeof(tmp));
+	uint64_t idtmp[2];
+	idtmp[0] = id; idtmp[1] = 0;
+	int n = RR->node->stateObjectGet(tPtr,ZT_STATE_OBJECT_MOON,idtmp,tmp,sizeof(tmp));
 	if (n > 0) {
 		try {
 			World w;
@@ -369,7 +377,9 @@ void Topology::removeMoon(void *tPtr,const uint64_t id)
 		if (m->id() != id) {
 			nm.push_back(*m);
 		} else {
-			RR->node->stateObjectDelete(tPtr,ZT_STATE_OBJECT_MOON,id);
+			uint64_t idtmp[2];
+			idtmp[0] = id; idtmp[1] = 0;
+			RR->node->stateObjectDelete(tPtr,ZT_STATE_OBJECT_MOON,idtmp);
 		}
 	}
 	_moons.swap(nm);
@@ -384,7 +394,7 @@ void Topology::removeMoon(void *tPtr,const uint64_t id)
 	_memoizeUpstreams(tPtr);
 }
 
-void Topology::clean(uint64_t now)
+void Topology::doPeriodicTasks(void *tPtr,uint64_t now)
 {
 	{
 		Mutex::Lock _l1(_peers_m);
@@ -393,10 +403,14 @@ void Topology::clean(uint64_t now)
 		Address *a = (Address *)0;
 		SharedPtr<Peer> *p = (SharedPtr<Peer> *)0;
 		while (i.next(a,p)) {
-			if ( (!(*p)->isAlive(now)) && (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),*a) == _upstreamAddresses.end()) )
+			if ( (!(*p)->isAlive(now)) && (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),*a) == _upstreamAddresses.end()) ) {
 				_peers.erase(*a);
+			} else {
+				(*p)->writeState(tPtr,now);
+			}
 		}
 	}
+
 	{
 		Mutex::Lock _l(_paths_m);
 		Hashtable< Path::HashKey,SharedPtr<Path> >::Iterator i(_paths);
@@ -412,7 +426,9 @@ void Topology::clean(uint64_t now)
 Identity Topology::_getIdentity(void *tPtr,const Address &zta)
 {
 	char tmp[512];
-	int n = RR->node->stateObjectGet(tPtr,ZT_STATE_OBJECT_PEER_IDENTITY,zta.toInt(),tmp,sizeof(tmp) - 1);
+	uint64_t idtmp[2];
+	idtmp[0] = zta.toInt(); idtmp[1] = 0;
+	int n = RR->node->stateObjectGet(tPtr,ZT_STATE_OBJECT_PEER_IDENTITY,idtmp,tmp,sizeof(tmp) - 1);
 	if (n > 0) {
 		tmp[n] = (char)0;
 		try {
diff --git a/node/Topology.hpp b/node/Topology.hpp
index d06ba94b..9bc7c0d8 100644
--- a/node/Topology.hpp
+++ b/node/Topology.hpp
@@ -308,7 +308,7 @@ public:
 	/**
 	 * Clean and flush database
 	 */
-	void clean(uint64_t now);
+	void doPeriodicTasks(void *tPtr,uint64_t now);
 
 	/**
 	 * @param now Current time