.

12 files changed, 326 insertions, 148 deletions

diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h
index 9c295cee..40cae3b4 100644
--- a/include/ZeroTierOne.h
+++ b/include/ZeroTierOne.h
@@ -1115,14 +1115,14 @@ enum ZT_StateObjectType
 	 * Canonical path: <HOME>/peers.d/<ADDRESS> (10-digit hex address)
 	 * Persistence: optional, can be purged at any time
 	 */
-	ZT_STATE_OBJECT_PEER = 3,
+	ZT_STATE_OBJECT_PEER_STATE = 3,
 
 	/**
 	 * The identity of a known peer
 	 *
 	 * Object ID: peer address
 	 * Canonical path: <HOME>/iddb.d/<ADDRESS> (10-digit hex address)
-	 * Persistence: optional, can be purged at any time, recommended ttl 30-60 days
+	 * Persistence: recommended, can be purged at any time, recommended ttl 30-60 days
 	 */
 	ZT_STATE_OBJECT_PEER_IDENTITY = 4,
 
@@ -1248,7 +1248,7 @@ typedef void (*ZT_StatePutFunction)(
 	void *,                                /* User ptr */
 	void *,                                /* Thread ptr */
 	enum ZT_StateObjectType,               /* State object type */
-	uint64_t,                              /* State object ID (if applicable) */
+	const uint64_t [2],                    /* State object ID (if applicable) */
 	const void *,                          /* State object data */
 	int);                                  /* Length of data or -1 to delete */
 
@@ -1264,7 +1264,7 @@ typedef int (*ZT_StateGetFunction)(
 	void *,                                /* User ptr */
 	void *,                                /* Thread ptr */
 	enum ZT_StateObjectType,               /* State object type */
-	uint64_t,                              /* State object ID (if applicable) */
+	const uint64_t [2],                    /* State object ID (if applicable) */
 	void *,                                /* Buffer to store state object data */
 	unsigned int);                         /* Length of data buffer in bytes */
 
diff --git a/node/Constants.hpp b/node/Constants.hpp
index fbbba76e..88549937 100644
--- a/node/Constants.hpp
+++ b/node/Constants.hpp
@@ -216,7 +216,7 @@
 /**
  * How often Topology::clean() and Network::clean() and similar are called, in ms
  */
-#define ZT_HOUSEKEEPING_PERIOD 120000
+#define ZT_HOUSEKEEPING_PERIOD 10000
 
 /**
  * How long to remember peer records in RAM if they haven't been used
diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp
index 1d55c9f3..4d99e87d 100644
--- a/node/IncomingPacket.cpp
+++ b/node/IncomingPacket.cpp
@@ -1211,8 +1211,8 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt
 							(!( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) == 0) && (peer->hasActivePathTo(now,a)) )) && // not already known
 							(RR->node->shouldUsePathForZeroTierTraffic(tPtr,peer->address(),_path->localAddress(),a)) ) // should use path
 					{
-						if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0)
-							peer->setClusterPreferred(a);
+						//if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0)
+						//	peer->setClusterPreferred(a);
 						if (++countPerScope[(int)a.ipScope()][0] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) {
 							TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str());
 							peer->attemptToContactAt(tPtr,InetAddress(),a,now,false,0);
@@ -1228,8 +1228,8 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt
 							(!( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) == 0) && (peer->hasActivePathTo(now,a)) )) && // not already known
 							(RR->node->shouldUsePathForZeroTierTraffic(tPtr,peer->address(),_path->localAddress(),a)) ) // should use path
 					{
-						if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0)
-							peer->setClusterPreferred(a);
+						//if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0)
+						//	peer->setClusterPreferred(a);
 						if (++countPerScope[(int)a.ipScope()][1] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) {
 							TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str());
 							peer->attemptToContactAt(tPtr,InetAddress(),a,now,false,0);
diff --git a/node/Network.cpp b/node/Network.cpp
index 8c6f2ce8..0a16ded8 100644
--- a/node/Network.cpp
+++ b/node/Network.cpp
@@ -700,10 +700,13 @@ Network::Network(const RuntimeEnvironment *renv,void *tPtr,uint64_t nwid,void *u
 		this->setConfiguration(tPtr,*nconf,false);
 		_lastConfigUpdate = 0; // still want to re-request since it's likely outdated
 	} else {
+		uint64_t tmp[2];
+		tmp[0] = nwid; tmp[1] = 0;
+
 		bool got = false;
 		Dictionary<ZT_NETWORKCONFIG_DICT_CAPACITY> *dict = new Dictionary<ZT_NETWORKCONFIG_DICT_CAPACITY>();
 		try {
-			int n = RR->node->stateObjectGet(tPtr,ZT_STATE_OBJECT_NETWORK_CONFIG,nwid,dict->unsafeData(),ZT_NETWORKCONFIG_DICT_CAPACITY - 1);
+			int n = RR->node->stateObjectGet(tPtr,ZT_STATE_OBJECT_NETWORK_CONFIG,tmp,dict->unsafeData(),ZT_NETWORKCONFIG_DICT_CAPACITY - 1);
 			if (n > 1) {
 				NetworkConfig *nconf = new NetworkConfig();
 				try {
@@ -719,7 +722,7 @@ Network::Network(const RuntimeEnvironment *renv,void *tPtr,uint64_t nwid,void *u
 		delete dict;
 
 		if (!got)
-			RR->node->stateObjectPut(tPtr,ZT_STATE_OBJECT_NETWORK_CONFIG,nwid,"\n",1);
+			RR->node->stateObjectPut(tPtr,ZT_STATE_OBJECT_NETWORK_CONFIG,tmp,"\n",1);
 	}
 
 	if (!_portInitialized) {
@@ -1185,8 +1188,11 @@ int Network::setConfiguration(void *tPtr,const NetworkConfig &nconf,bool saveToD
 		if (saveToDisk) {
 			Dictionary<ZT_NETWORKCONFIG_DICT_CAPACITY> *d = new Dictionary<ZT_NETWORKCONFIG_DICT_CAPACITY>();
 			try {
-				if (nconf.toDictionary(*d,false))
-					RR->node->stateObjectPut(tPtr,ZT_STATE_OBJECT_NETWORK_CONFIG,_id,d->data(),d->sizeBytes());
+				if (nconf.toDictionary(*d,false)) {
+					uint64_t tmp[2];
+					tmp[0] = _id; tmp[1] = 0;
+					RR->node->stateObjectPut(tPtr,ZT_STATE_OBJECT_NETWORK_CONFIG,tmp,d->data(),d->sizeBytes());
+				}
 			} catch ( ... ) {}
 			delete d;
 		}
diff --git a/node/Node.cpp b/node/Node.cpp
index ab49e63b..1112c0f2 100644
--- a/node/Node.cpp
+++ b/node/Node.cpp
@@ -76,22 +76,26 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint6
 	memset(_expectingRepliesTo,0,sizeof(_expectingRepliesTo));
 	memset(_lastIdentityVerification,0,sizeof(_lastIdentityVerification));
 
+	uint64_t idtmp[2];
+	idtmp[0] = 0; idtmp[1] = 0;
 	char tmp[512];
 	std::string tmp2;
-	int n = stateObjectGet(tptr,ZT_STATE_OBJECT_IDENTITY_SECRET,0,tmp,sizeof(tmp) - 1);
+	int n = stateObjectGet(tptr,ZT_STATE_OBJECT_IDENTITY_SECRET,idtmp,tmp,sizeof(tmp) - 1);
 	if (n > 0) {
 		tmp[n] = (char)0;
 		if (!RR->identity.fromString(tmp))
 			n = -1;
 	}
+
+	idtmp[0] = RR->identity.address().toInt(); idtmp[1] = 0;
 	if (n <= 0) {
 		RR->identity.generate();
 		tmp2 = RR->identity.toString(true);
-		stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_SECRET,RR->identity.address().toInt(),tmp2.data(),(unsigned int)tmp2.length());
+		stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_SECRET,idtmp,tmp2.data(),(unsigned int)tmp2.length());
 		tmp2 = RR->identity.toString(false);
-		stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,RR->identity.address().toInt(),tmp2.data(),(unsigned int)tmp2.length());
+		stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,idtmp,tmp2.data(),(unsigned int)tmp2.length());
 	} else {
-		n = stateObjectGet(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,RR->identity.address().toInt(),tmp,sizeof(tmp) - 1);
+		n = stateObjectGet(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,idtmp,tmp,sizeof(tmp) - 1);
 		if (n > 0) {
 			tmp[n] = (char)0;
 			if (RR->identity.toString(false) != tmp)
@@ -99,7 +103,7 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint6
 		}
 		if (n <= 0) {
 			tmp2 = RR->identity.toString(false);
-			stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,RR->identity.address().toInt(),tmp2.data(),(unsigned int)tmp2.length());
+			stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,idtmp,tmp2.data(),(unsigned int)tmp2.length());
 		}
 	}
 
@@ -145,7 +149,7 @@ ZT_ResultCode Node::processStateUpdate(
 	ZT_ResultCode r = ZT_RESULT_OK_IGNORED;
 	switch(type) {
 
-		case ZT_STATE_OBJECT_PEER:
+		case ZT_STATE_OBJECT_PEER_STATE:
 			if (len) {
 			}
 			break;
@@ -380,9 +384,9 @@ ZT_ResultCode Node::processBackgroundTasks(void *tptr,uint64_t now,volatile uint
 	}
 
 	if ((now - _lastHousekeepingRun) >= ZT_HOUSEKEEPING_PERIOD) {
+		_lastHousekeepingRun = now;
 		try {
-			_lastHousekeepingRun = now;
-			RR->topology->clean(now);
+			RR->topology->doPeriodicTasks(tptr,now);
 			RR->sa->clean(now);
 			RR->mc->clean(now);
 		} catch ( ... ) {
@@ -443,7 +447,9 @@ ZT_ResultCode Node::leave(uint64_t nwid,void **uptr,void *tptr)
 		_networks.erase(nwid);
 	}
 
-	RR->node->stateObjectDelete(tptr,ZT_STATE_OBJECT_NETWORK_CONFIG,nwid);
+	uint64_t tmp[2];
+	tmp[0] = nwid; tmp[1] = 0;
+	RR->node->stateObjectDelete(tptr,ZT_STATE_OBJECT_NETWORK_CONFIG,tmp);
 
 	return ZT_RESULT_OK;
 }
diff --git a/node/Node.hpp b/node/Node.hpp
index f407c60c..f1209d00 100644
--- a/node/Node.hpp
+++ b/node/Node.hpp
@@ -197,9 +197,9 @@ public:
 
 	inline bool online() const throw() { return _online; }
 
-	inline int stateObjectGet(void *const tPtr,ZT_StateObjectType type,const uint64_t id,void *const data,const unsigned int maxlen) { return _cb.stateGetFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,tPtr,type,id,data,maxlen); }
-	inline void stateObjectPut(void *const tPtr,ZT_StateObjectType type,const uint64_t id,const void *const data,const unsigned int len) { _cb.statePutFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,tPtr,type,id,data,(int)len); }
-	inline void stateObjectDelete(void *const tPtr,ZT_StateObjectType type,const uint64_t id) { _cb.statePutFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,tPtr,type,id,(const void *)0,-1); }
+	inline int stateObjectGet(void *const tPtr,ZT_StateObjectType type,const uint64_t id[2],void *const data,const unsigned int maxlen) { return _cb.stateGetFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,tPtr,type,id,data,maxlen); }
+	inline void stateObjectPut(void *const tPtr,ZT_StateObjectType type,const uint64_t id[2],const void *const data,const unsigned int len) { _cb.statePutFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,tPtr,type,id,data,(int)len); }
+	inline void stateObjectDelete(void *const tPtr,ZT_StateObjectType type,const uint64_t id[2]) { _cb.statePutFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,tPtr,type,id,(const void *)0,-1); }
 
 #ifdef ZT_TRACE
 	void postTrace(const char *module,unsigned int line,const char *fmt,...);
diff --git a/node/Path.hpp b/node/Path.hpp
index 32bceae0..74b31d8d 100644
--- a/node/Path.hpp
+++ b/node/Path.hpp
@@ -46,6 +46,11 @@
  */
 #define ZT_PATH_MAX_PREFERENCE_RANK ((ZT_INETADDRESS_MAX_SCOPE << 1) | 1)
 
+/**
+ * Maximum distance for a path
+ */
+#define ZT_PATH_DISTANCE_MAX 0xffff
+
 namespace ZeroTier {
 
 class RuntimeEnvironment;
@@ -120,6 +125,7 @@ public:
 		_incomingLinkQualitySlowLogCounter(-64), // discard first fast log
 		_incomingLinkQualityPreviousPacketCounter(0),
 		_outgoingPacketCounter(0),
+		_distance(ZT_PATH_DISTANCE_MAX),
 		_addr(),
 		_localAddress(),
 		_ipScope(InetAddress::IP_SCOPE_NONE)
@@ -137,6 +143,7 @@ public:
 		_incomingLinkQualitySlowLogCounter(-64), // discard first fast log
 		_incomingLinkQualityPreviousPacketCounter(0),
 		_outgoingPacketCounter(0),
+		_distance(ZT_PATH_DISTANCE_MAX),
 		_addr(addr),
 		_localAddress(localAddress),
 		_ipScope(addr.ipScope())
@@ -300,6 +307,28 @@ public:
 	inline uint64_t lastIn() const { return _lastIn; }
 
 	/**
+	 * @return Time last trust-established packet was received
+	 */
+	inline uint64_t lastTrustEstablishedPacketReceived() const { return _lastTrustEstablishedPacketReceived; }
+
+	/**
+	 * @return Distance (higher is further)
+	 */
+	inline unsigned int distance() const { return _distance; }
+
+	/**
+	 * @param lo Last out send
+	 * @param li Last in send
+	 * @param lt Last trust established packet received
+	 */
+	inline void updateFromRemoteState(const uint64_t lo,const uint64_t li,const uint64_t lt)
+	{
+		_lastOut = lo;
+		_lastIn = li;
+		_lastTrustEstablishedPacketReceived = lt;
+	}
+
+	/**
 	 * Return and increment outgoing packet counter (used with Packet::armor())
 	 *
 	 * @return Next value that should be used for outgoing packet counter (only least significant 3 bits are used)
@@ -315,6 +344,7 @@ private:
 	volatile signed int _incomingLinkQualitySlowLogCounter;
 	volatile unsigned int _incomingLinkQualityPreviousPacketCounter;
 	volatile unsigned int _outgoingPacketCounter;
+	volatile unsigned int _distance;
 	InetAddress _addr;
 	InetAddress _localAddress;
 	InetAddress::IpScope _ipScope; // memoize this since it's a computed value checked often
diff --git a/node/Peer.cpp b/node/Peer.cpp
index 84086048..a7466296 100644
--- a/node/Peer.cpp
+++ b/node/Peer.cpp
@@ -38,6 +38,8 @@ namespace ZeroTier {
 
 Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Identity &peerIdentity) :
 	RR(renv),
+	_lastWroteState(0),
+	_lastReceivedStateTimestamp(0),
 	_lastReceive(0),
 	_lastNontrivialReceive(0),
 	_lastTriedMemorizedPath(0),
@@ -75,6 +77,7 @@ void Peer::received(
 {
 	const uint64_t now = RR->node->now();
 
+/*
 #ifdef ZT_ENABLE_CLUSTER
 	bool isClusterSuboptimalPath = false;
 	if ((RR->cluster)&&(hops == 0)) {
@@ -120,6 +123,7 @@ void Peer::received(
 		}
 	}
 #endif
+*/
 
 	_lastReceive = now;
 	switch (verb) {
@@ -143,6 +147,8 @@ void Peer::received(
 
 	if (hops == 0) {
 		bool pathAlreadyKnown = false;
+		bool newPathLearned = false;
+
 		{
 			Mutex::Lock _l(_paths_m);
 			if ((path->address().ss_family == AF_INET)&&(_v4Path.p)) {
@@ -152,9 +158,6 @@ void Peer::received(
 				const struct sockaddr_in *const ll = reinterpret_cast<const struct sockaddr_in *>(&(_v4Path.p->localAddress()));
 				if ((r->sin_addr.s_addr == l->sin_addr.s_addr)&&(r->sin_port == l->sin_port)&&(rl->sin_addr.s_addr == ll->sin_addr.s_addr)&&(rl->sin_port == ll->sin_port)) {
 					_v4Path.lr = now;
-#ifdef ZT_ENABLE_CLUSTER
-					_v4Path.localClusterSuboptimal = isClusterSuboptimalPath;
-#endif
 					pathAlreadyKnown = true;
 				}
 			} else if ((path->address().ss_family == AF_INET6)&&(_v6Path.p)) {
@@ -164,9 +167,6 @@ void Peer::received(
 				const struct sockaddr_in6 *const ll = reinterpret_cast<const struct sockaddr_in6 *>(&(_v6Path.p->localAddress()));
 				if ((!memcmp(r->sin6_addr.s6_addr,l->sin6_addr.s6_addr,16))&&(r->sin6_port == l->sin6_port)&&(!memcmp(rl->sin6_addr.s6_addr,ll->sin6_addr.s6_addr,16))&&(rl->sin6_port == ll->sin6_port)) {
 					_v6Path.lr = now;
-#ifdef ZT_ENABLE_CLUSTER
-					_v6Path.localClusterSuboptimal = isClusterSuboptimalPath;
-#endif
 					pathAlreadyKnown = true;
 				}
 			}
@@ -176,11 +176,11 @@ void Peer::received(
 			Mutex::Lock _l(_paths_m);
 			_PeerPath *potentialNewPeerPath = (_PeerPath *)0;
 			if (path->address().ss_family == AF_INET) {
-				if ( (!_v4Path.p) || (!_v4Path.p->alive(now)) || ((_v4Path.p->address() != _v4ClusterPreferred)&&(path->preferenceRank() >= _v4Path.p->preferenceRank())) ) {
+				if ( (!_v4Path.p) || (!_v4Path.p->alive(now)) || (path->preferenceRank() >= _v4Path.p->preferenceRank()) ) {
 					potentialNewPeerPath = &_v4Path;
 				}
 			} else if (path->address().ss_family == AF_INET6) {
-				if ( (!_v6Path.p) || (!_v6Path.p->alive(now)) || ((_v6Path.p->address() != _v6ClusterPreferred)&&(path->preferenceRank() >= _v6Path.p->preferenceRank())) ) {
+				if ( (!_v6Path.p) || (!_v6Path.p->alive(now)) || (path->preferenceRank() >= _v6Path.p->preferenceRank()) ) {
 					potentialNewPeerPath = &_v6Path;
 				}
 			}
@@ -188,11 +188,7 @@ void Peer::received(
 				if (verb == Packet::VERB_OK) {
 					potentialNewPeerPath->lr = now;
 					potentialNewPeerPath->p = path;
-#ifdef ZT_ENABLE_CLUSTER
-					potentialNewPeerPath->localClusterSuboptimal = isClusterSuboptimalPath;
-					if (RR->cluster)
-						RR->cluster->broadcastHavePeer(_id);
-#endif
+					newPathLearned = true;
 				} else {
 					TRACE("got %s via unknown path %s(%s), confirming...",Packet::verbString(verb),_id.address().toString().c_str(),path->address().toString().c_str());
 					attemptToContactAt(tPtr,path->localAddress(),path->address(),now,true,path->nextOutgoingCounter());
@@ -200,15 +196,12 @@ void Peer::received(
 				}
 			}
 		}
+
+		if (newPathLearned)
+			writeState(tPtr,now);
 	} else if (this->trustEstablished(now)) {
 		// Send PUSH_DIRECT_PATHS if hops>0 (relayed) and we have a trust relationship (common network membership)
-#ifdef ZT_ENABLE_CLUSTER
-		// Cluster mode disables normal PUSH_DIRECT_PATHS in favor of cluster-based peer redirection
-		const bool haveCluster = (RR->cluster);
-#else
-		const bool haveCluster = false;
-#endif
-		if ( ((now - _lastDirectPathPushSent) >= ZT_DIRECT_PATH_PUSH_INTERVAL) && (!haveCluster) ) {
+		if ((now - _lastDirectPathPushSent) >= ZT_DIRECT_PATH_PUSH_INTERVAL) {
 			_lastDirectPathPushSent = now;
 
 			std::vector<InetAddress> pathsToPush;
@@ -439,4 +432,135 @@ bool Peer::doPingAndKeepalive(void *tPtr,uint64_t now,int inetAddressFamily)
 	return false;
 }
 
+void Peer::writeState(void *tPtr,const uint64_t now)
+{
+	try {
+		Buffer<sizeof(Peer) + 32 + (sizeof(Path) * 2)> b;
+
+		b.append((uint8_t)1); // version
+		b.append(now);
+
+		_id.serialize(b);
+
+		{
+			Mutex::Lock _l(_paths_m);
+			unsigned int count = 0;
+			if (_v4Path.lr)
+				++count;
+			if (_v6Path.lr)
+				++count;
+			b.append((uint8_t)count);
+			if (_v4Path.lr) {
+				b.append(_v4Path.lr);
+				b.append(_v4Path.p->lastOut());
+				b.append(_v4Path.p->lastIn());
+				b.append(_v4Path.p->lastTrustEstablishedPacketReceived());
+				b.append((uint16_t)_v4Path.p->distance());
+				_v4Path.p->address().serialize(b);
+				_v4Path.p->localAddress().serialize(b);
+			}
+			if (_v6Path.lr) {
+				b.append(_v6Path.lr);
+				b.append(_v6Path.p->lastOut());
+				b.append(_v6Path.p->lastIn());
+				b.append(_v6Path.p->lastTrustEstablishedPacketReceived());
+				b.append((uint16_t)_v6Path.p->distance());
+				_v6Path.p->address().serialize(b);
+				_v6Path.p->localAddress().serialize(b);
+			}
+		}
+
+		b.append(_lastReceive);
+		b.append(_lastNontrivialReceive);
+		b.append(_lastTriedMemorizedPath);
+		b.append(_lastDirectPathPushSent);
+		b.append(_lastDirectPathPushReceive);
+		b.append(_lastCredentialRequestSent);
+		b.append(_lastWhoisRequestReceived);
+		b.append(_lastEchoRequestReceived);
+		b.append(_lastComRequestReceived);
+		b.append(_lastComRequestSent);
+		b.append(_lastCredentialsReceived);
+		b.append(_lastTrustEstablishedPacketReceived);
+
+		b.append(_vProto);
+		b.append(_vMajor);
+		b.append(_vMinor);
+		b.append(_vRevision);
+
+		b.append((uint16_t)0); // length of additional fields
+
+		uint64_t tmp[2];
+		tmp[0] = _id.address().toInt(); tmp[1] = 0;
+		RR->node->stateObjectPut(tPtr,ZT_STATE_OBJECT_PEER_STATE,tmp,b.data(),b.size());
+
+		_lastWroteState = now;
+	} catch ( ... ) {} // sanity check, should not be possible
+}
+
+bool Peer::applyStateUpdate(const void *data,unsigned int len)
+{
+	try {
+		Buffer<sizeof(Peer) + 32 + (sizeof(Path) * 2)> b(data,len);
+		unsigned int ptr = 0;
+
+		if (b[ptr++] != 1)
+			return false;
+		const uint64_t ts = b.at<uint64_t>(ptr); ptr += 8;
+		if (ts <= _lastReceivedStateTimestamp)
+			return false;
+
+		const unsigned int pathCount = (unsigned int)b[ptr++];
+		{
+			Mutex::Lock _l(_paths_m);
+			for(unsigned int i=0;i<pathCount;++i) {
+				const uint64_t lr = b.at<uint64_t>(ptr); ptr += 8;
+				const uint64_t lastOut = b.at<uint64_t>(ptr); ptr += 8;
+				const uint64_t lastIn = b.at<uint64_t>(ptr); ptr += 8;
+				const uint64_t lastTrustEstablishedPacketReceived = b.at<uint64_t>(ptr); ptr += 8;
+				const unsigned int distance = b.at<uint16_t>(ptr); ptr += 2;
+				InetAddress addr,localAddr;
+				ptr += addr.deserialize(b,ptr);
+				ptr += localAddr.deserialize(b,ptr);
+				if (addr.ss_family == localAddr.ss_family) {
+					_PeerPath *p = (_PeerPath *)0;
+					switch(addr.ss_family) {
+						case AF_INET: p = &_v4Path; break;
+						case AF_INET6: p = &_v6Path; break;
+					}
+					if (p) {
+						if ( ((p->p->address() != addr)||(p->p->localAddress() != localAddr)) && (p->p->distance() > distance) )
+							p->p = RR->topology->getPath(localAddr,addr);
+						p->lr = lr;
+						p->p->updateFromRemoteState(lastOut,lastIn,lastTrustEstablishedPacketReceived);
+					}
+				}
+			}
+		}
+
+		_lastReceive = std::max(_lastReceive,b.at<uint64_t>(ptr)); ptr += 8;
+		_lastNontrivialReceive = std::max(_lastNontrivialReceive,b.at<uint64_t>(ptr)); ptr += 8;
+		_lastTriedMemorizedPath = std::max(_lastTriedMemorizedPath,b.at<uint64_t>(ptr)); ptr += 8;
+		_lastDirectPathPushSent = std::max(_lastDirectPathPushSent,b.at<uint64_t>(ptr)); ptr += 8;
+		_lastDirectPathPushReceive = std::max(_lastDirectPathPushReceive,b.at<uint64_t>(ptr)); ptr += 8;
+		_lastCredentialRequestSent = std::max(_lastCredentialRequestSent,b.at<uint64_t>(ptr)); ptr += 8;
+		_lastWhoisRequestReceived = std::max(_lastWhoisRequestReceived,b.at<uint64_t>(ptr)); ptr += 8;
+		_lastEchoRequestReceived = std::max(_lastEchoRequestReceived,b.at<uint64_t>(ptr)); ptr += 8;
+		_lastComRequestReceived = std::max(_lastComRequestReceived,b.at<uint64_t>(ptr)); ptr += 8;
+		_lastComRequestSent = std::max(_lastComRequestSent,b.at<uint64_t>(ptr)); ptr += 8;
+		_lastCredentialsReceived = std::max(_lastCredentialsReceived,b.at<uint64_t>(ptr)); ptr += 8;
+		_lastTrustEstablishedPacketReceived = std::max(_lastTrustEstablishedPacketReceived,b.at<uint64_t>(ptr)); ptr += 8;
+
+		_vProto = b.at<uint16_t>(ptr); ptr += 2;
+		_vMajor = b.at<uint16_t>(ptr); ptr += 2;
+		_vMinor = b.at<uint16_t>(ptr); ptr += 2;
+		_vRevision = b.at<uint16_t>(ptr); ptr += 2;
+
+		_lastReceivedStateTimestamp = ts;
+
+		return true;
+	} catch ( ... ) {} // ignore invalid state updates
+	return false;
+}
+
 } // namespace ZeroTier
diff --git a/node/Peer.hpp b/node/Peer.hpp
index 9b57f23e..d6b7dad9 100644
--- a/node/Peer.hpp
+++ b/node/Peer.hpp
@@ -194,6 +194,22 @@ public:
 	bool doPingAndKeepalive(void *tPtr,uint64_t now,int inetAddressFamily);
 
 	/**
+	 * Write current peer state to external storage / cluster network
+	 *
+	 * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call
+	 */
+	void writeState(void *tPtr,const uint64_t now);
+
+	/**
+	 * Apply a state update received from e.g. a remote cluster member
+	 *
+	 * @param data State update data
+	 * @param len Length of state update
+	 * @return True if state update was applied, false if ignored or invalid
+	 */
+	bool applyStateUpdate(const void *data,unsigned int len);
+
+	/**
 	 * Reset paths within a given IP scope and address family
 	 *
 	 * Resetting a path involves sending an ECHO to it and then deactivating
@@ -219,19 +235,6 @@ public:
 	}
 
 	/**
-	 * Indicate that the given address was provided by a cluster as a preferred destination
-	 *
-	 * @param addr Address cluster prefers that we use
-	 */
-	inline void setClusterPreferred(const InetAddress &addr)
-	{
-		if (addr.ss_family == AF_INET)
-			_v4ClusterPreferred = addr;
-		else if (addr.ss_family == AF_INET6)
-			_v6ClusterPreferred = addr;
-	}
-
-	/**
 	 * Fill parameters with V4 and V6 addresses if known and alive
 	 *
 	 * @param now Current time
@@ -317,18 +320,6 @@ public:
 		else _latency = std::min(l,(unsigned int)65535);
 	}
 
-#ifdef ZT_ENABLE_CLUSTER
-	/**
-	 * @param now Current time
-	 * @return True if this peer has at least one active direct path that is not cluster-suboptimal
-	 */
-	inline bool hasLocalClusterOptimalPath(uint64_t now) const
-	{
-		Mutex::Lock _l(_paths_m);
-		return ( ((_v4Path.p)&&(_v4Path.p->alive(now))&&(!_v4Path.localClusterSuboptimal)) || ((_v6Path.p)&&(_v6Path.p->alive(now))&&(!_v6Path.localClusterSuboptimal)) );
-	}
-#endif
-
 	/**
 	 * @return 256-bit secret symmetric encryption key
 	 */
@@ -449,22 +440,18 @@ public:
 private:
 	struct _PeerPath
 	{
-#ifdef ZT_ENABLE_CLUSTER
-		_PeerPath() : lr(0),p(),localClusterSuboptimal(false) {}
-#else
 		_PeerPath() : lr(0),p() {}
-#endif
 		uint64_t lr; // time of last valid ZeroTier packet
 		SharedPtr<Path> p;
-#ifdef ZT_ENABLE_CLUSTER
-		bool localClusterSuboptimal; // true if our cluster has determined that we should not be serving this peer
-#endif
 	};
 
 	uint8_t _key[ZT_PEER_SECRET_KEY_LENGTH];
 
 	const RuntimeEnvironment *RR;
 
+	uint64_t _lastWroteState;
+	uint64_t _lastReceivedStateTimestamp;
+
 	uint64_t _lastReceive; // direct or indirect
 	uint64_t _lastNontrivialReceive; // frames, things like netconf, etc.
 	uint64_t _lastTriedMemorizedPath;
@@ -483,9 +470,6 @@ private:
 	uint16_t _vMinor;
 	uint16_t _vRevision;
 
-	InetAddress _v4ClusterPreferred;
-	InetAddress _v6ClusterPreferred;
-
 	_PeerPath _v4Path; // IPv4 direct path
 	_PeerPath _v6Path; // IPv6 direct path
 	Mutex _paths_m;
diff --git a/node/Topology.cpp b/node/Topology.cpp
index d4b424ff..be116b28 100644
--- a/node/Topology.cpp
+++ b/node/Topology.cpp
@@ -69,7 +69,9 @@ Topology::Topology(const RuntimeEnvironment *renv,void *tPtr) :
 	_amRoot(false)
 {
 	uint8_t tmp[ZT_WORLD_MAX_SERIALIZED_LENGTH];
-	int n = RR->node->stateObjectGet(tPtr,ZT_STATE_OBJECT_PLANET,0,tmp,sizeof(tmp));
+	uint64_t idtmp[2];
+	idtmp[0] = 0; idtmp[1] = 0;
+	int n = RR->node->stateObjectGet(tPtr,ZT_STATE_OBJECT_PLANET,idtmp,tmp,sizeof(tmp));
 	if (n > 0) {
 		try {
 			World cachedPlanet;
@@ -159,7 +161,9 @@ void Topology::saveIdentity(void *tPtr,const Identity &id)
 {
 	if (id) {
 		const std::string tmp(id.toString(false));
-		RR->node->stateObjectPut(tPtr,ZT_STATE_OBJECT_PEER_IDENTITY,id.address().toInt(),tmp.data(),(unsigned int)tmp.length());
+		uint64_t idtmp[2];
+		idtmp[0] = id.address().toInt(); idtmp[1] = 0;
+		RR->node->stateObjectPut(tPtr,ZT_STATE_OBJECT_PEER_IDENTITY,idtmp,tmp.data(),(unsigned int)tmp.length());
 	}
 }
 
@@ -329,7 +333,9 @@ bool Topology::addWorld(void *tPtr,const World &newWorld,bool alwaysAcceptNew)
 	try {
 		Buffer<ZT_WORLD_MAX_SERIALIZED_LENGTH> sbuf;
 		existing->serialize(sbuf,false);
-		RR->node->stateObjectPut(tPtr,(existing->type() == World::TYPE_PLANET) ? ZT_STATE_OBJECT_PLANET : ZT_STATE_OBJECT_MOON,existing->id(),sbuf.data(),sbuf.size());
+		uint64_t idtmp[2];
+		idtmp[0] = existing->id(); idtmp[1] = 0;
+		RR->node->stateObjectPut(tPtr,(existing->type() == World::TYPE_PLANET) ? ZT_STATE_OBJECT_PLANET : ZT_STATE_OBJECT_MOON,idtmp,sbuf.data(),sbuf.size());
 	} catch ( ... ) {}
 
 	_memoizeUpstreams(tPtr);
@@ -340,7 +346,9 @@ bool Topology::addWorld(void *tPtr,const World &newWorld,bool alwaysAcceptNew)
 void Topology::addMoon(void *tPtr,const uint64_t id,const Address &seed)
 {
 	char tmp[ZT_WORLD_MAX_SERIALIZED_LENGTH];
-	int n = RR->node->stateObjectGet(tPtr,ZT_STATE_OBJECT_MOON,id,tmp,sizeof(tmp));
+	uint64_t idtmp[2];
+	idtmp[0] = id; idtmp[1] = 0;
+	int n = RR->node->stateObjectGet(tPtr,ZT_STATE_OBJECT_MOON,idtmp,tmp,sizeof(tmp));
 	if (n > 0) {
 		try {
 			World w;
@@ -369,7 +377,9 @@ void Topology::removeMoon(void *tPtr,const uint64_t id)
 		if (m->id() != id) {
 			nm.push_back(*m);
 		} else {
-			RR->node->stateObjectDelete(tPtr,ZT_STATE_OBJECT_MOON,id);
+			uint64_t idtmp[2];
+			idtmp[0] = id; idtmp[1] = 0;
+			RR->node->stateObjectDelete(tPtr,ZT_STATE_OBJECT_MOON,idtmp);
 		}
 	}
 	_moons.swap(nm);
@@ -384,7 +394,7 @@ void Topology::removeMoon(void *tPtr,const uint64_t id)
 	_memoizeUpstreams(tPtr);
 }
 
-void Topology::clean(uint64_t now)
+void Topology::doPeriodicTasks(void *tPtr,uint64_t now)
 {
 	{
 		Mutex::Lock _l1(_peers_m);
@@ -393,10 +403,14 @@ void Topology::clean(uint64_t now)
 		Address *a = (Address *)0;
 		SharedPtr<Peer> *p = (SharedPtr<Peer> *)0;
 		while (i.next(a,p)) {
-			if ( (!(*p)->isAlive(now)) && (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),*a) == _upstreamAddresses.end()) )
+			if ( (!(*p)->isAlive(now)) && (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),*a) == _upstreamAddresses.end()) ) {
 				_peers.erase(*a);
+			} else {
+				(*p)->writeState(tPtr,now);
+			}
 		}
 	}
+
 	{
 		Mutex::Lock _l(_paths_m);
 		Hashtable< Path::HashKey,SharedPtr<Path> >::Iterator i(_paths);
@@ -412,7 +426,9 @@ void Topology::clean(uint64_t now)
 Identity Topology::_getIdentity(void *tPtr,const Address &zta)
 {
 	char tmp[512];
-	int n = RR->node->stateObjectGet(tPtr,ZT_STATE_OBJECT_PEER_IDENTITY,zta.toInt(),tmp,sizeof(tmp) - 1);
+	uint64_t idtmp[2];
+	idtmp[0] = zta.toInt(); idtmp[1] = 0;
+	int n = RR->node->stateObjectGet(tPtr,ZT_STATE_OBJECT_PEER_IDENTITY,idtmp,tmp,sizeof(tmp) - 1);
 	if (n > 0) {
 		tmp[n] = (char)0;
 		try {
diff --git a/node/Topology.hpp b/node/Topology.hpp
index d06ba94b..9bc7c0d8 100644
--- a/node/Topology.hpp
+++ b/node/Topology.hpp
@@ -308,7 +308,7 @@ public:
 	/**
 	 * Clean and flush database
 	 */
-	void clean(uint64_t now);
+	void doPeriodicTasks(void *tPtr,uint64_t now);
 
 	/**
 	 * @param now Current time
diff --git a/service/OneService.cpp b/service/OneService.cpp
index 993fb116..f949f348 100644
--- a/service/OneService.cpp
+++ b/service/OneService.cpp
@@ -312,8 +312,8 @@ class OneServiceImpl;
 
 static int SnodeVirtualNetworkConfigFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t nwid,void **nuptr,enum ZT_VirtualNetworkConfigOperation op,const ZT_VirtualNetworkConfig *nwconf);
 static void SnodeEventCallback(ZT_Node *node,void *uptr,void *tptr,enum ZT_Event event,const void *metaData);
-static void SnodeStatePutFunction(ZT_Node *node,void *uptr,void *tptr,enum ZT_StateObjectType type,uint64_t id,const void *data,int len);
-static int SnodeStateGetFunction(ZT_Node *node,void *uptr,void *tptr,enum ZT_StateObjectType type,uint64_t id,void *data,unsigned int maxlen);
+static void SnodeStatePutFunction(ZT_Node *node,void *uptr,void *tptr,enum ZT_StateObjectType type,const uint64_t id[2],const void *data,int len);
+static int SnodeStateGetFunction(ZT_Node *node,void *uptr,void *tptr,enum ZT_StateObjectType type,const uint64_t id[2],void *data,unsigned int maxlen);
 static int SnodeWirePacketSendFunction(ZT_Node *node,void *uptr,void *tptr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *addr,const void *data,unsigned int len,unsigned int ttl);
 static void SnodeVirtualNetworkFrameFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t nwid,void **nuptr,uint64_t sourceMac,uint64_t destMac,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len);
 static int SnodePathCheckFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t ztaddr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *remoteAddr);
@@ -1220,34 +1220,20 @@ public:
 					res["planetWorldId"] = planet.id();
 					res["planetWorldTimestamp"] = planet.timestamp();
 
-/*
-#ifdef ZT_ENABLE_CLUSTER
-					json cj;
-					ZT_ClusterStatus cs;
-					_node->clusterStatus(&cs);
-					if (cs.clusterSize >= 1) {
-						json cja = json::array();
-						for(unsigned int i=0;i<cs.clusterSize;++i) {
-							json cjm;
-							cjm["id"] = (int)cs.members[i].id;
-							cjm["msSinceLastHeartbeat"] = cs.members[i].msSinceLastHeartbeat;
-							cjm["alive"] = (bool)(cs.members[i].alive != 0);
-							cjm["x"] = cs.members[i].x;
-							cjm["y"] = cs.members[i].y;
-							cjm["z"] = cs.members[i].z;
-							cjm["load"] = cs.members[i].load;
-							cjm["peers"] = cs.members[i].peers;
-							cja.push_back(cjm);
+					{
+						json cj(json::object());
+						Mutex::Lock _l(_tcpConnections_m);
+						Mutex::Lock _l2(_localConfig_m);
+						for(std::vector<InetAddress>::const_iterator ca(_clusterBackplaneAddresses.begin());ca!=_clusterBackplaneAddresses.end();++ca) {
+							uint64_t up = 0;
+							for(std::vector<TcpConnection *>::const_iterator c(_tcpConnections.begin());c!=_tcpConnections.end();++c) {
+								if (((*c)->remoteAddr == *ca)&&((*c)->clusterMemberId)&&((*c)->lastReceive > up))
+									up = (*c)->lastReceive;
+							}
+							cj[ca->toString()] = up;
 						}
-						cj["members"] = cja;
-						cj["myId"] = (int)cs.myId;
-						cj["clusterSize"] = cs.clusterSize;
+						res["cluster"] = cj;
 					}
-					res["cluster"] = cj;
-#else
-					res["cluster"] = json();
-#endif
-*/
 
 					scode = 200;
 				} else if (ps[0] == "moon") {
@@ -1877,16 +1863,15 @@ public:
 		return false;
 	}
 
-	void replicateStateObject(const ZT_StateObjectType type,const uint64_t id,const void *const data,const unsigned int len,TcpConnection *tc)
+	void replicateStateObject(const ZT_StateObjectType type,const uint64_t id[2],const void *const data,const unsigned int len,TcpConnection *tc)
 	{
-		char buf[34];
-
+		char buf[42];
 		Mutex::Lock _l2(tc->writeq_m);
 
 		if (tc->writeq.length() == 0)
 			_phy.setNotifyWritable(tc->sock,true);
 
-		const unsigned int mlen = len + 34;
+		const unsigned int mlen = len + 42;
 
 		tc->writeq.push_back((char)((mlen >> 16) & 0xff));
 		tc->writeq.push_back((char)((mlen >> 8) & 0xff));
@@ -1895,24 +1880,32 @@ public:
 		Utils::getSecureRandom(buf,16);
 		buf[24] = (char)CLUSTER_MESSAGE_STATE_OBJECT;
 		buf[25] = (char)type;
-		buf[26] = (char)((id >> 56) & 0xff);
-		buf[27] = (char)((id >> 48) & 0xff);
-		buf[28] = (char)((id >> 40) & 0xff);
-		buf[29] = (char)((id >> 32) & 0xff);
-		buf[30] = (char)((id >> 24) & 0xff);
-		buf[31] = (char)((id >> 16) & 0xff);
-		buf[32] = (char)((id >> 8) & 0xff);
-		buf[33] = (char)(id & 0xff);
+		buf[26] = (char)((id[0] >> 56) & 0xff);
+		buf[27] = (char)((id[0] >> 48) & 0xff);
+		buf[28] = (char)((id[0] >> 40) & 0xff);
+		buf[29] = (char)((id[0] >> 32) & 0xff);
+		buf[30] = (char)((id[0] >> 24) & 0xff);
+		buf[31] = (char)((id[0] >> 16) & 0xff);
+		buf[32] = (char)((id[0] >> 8) & 0xff);
+		buf[33] = (char)(id[0] & 0xff);
+		buf[34] = (char)((id[1] >> 56) & 0xff);
+		buf[35] = (char)((id[1] >> 48) & 0xff);
+		buf[36] = (char)((id[1] >> 40) & 0xff);
+		buf[37] = (char)((id[1] >> 32) & 0xff);
+		buf[38] = (char)((id[1] >> 24) & 0xff);
+		buf[39] = (char)((id[1] >> 16) & 0xff);
+		buf[40] = (char)((id[1] >> 8) & 0xff);
+		buf[41] = (char)(id[1] & 0xff);
 
 		const unsigned long startpos = (unsigned long)tc->writeq.length();
-		tc->writeq.append(buf,34);
+		tc->writeq.append(buf,42);
 		tc->writeq.append(reinterpret_cast<const char *>(data),len);
 
 		char *const outdata = const_cast<char *>(tc->writeq.data()) + startpos;
 		encryptClusterMessage(outdata,mlen);
 	}
 
-	void replicateStateObjectToCluster(const ZT_StateObjectType type,const uint64_t id,const void *const data,const unsigned int len,const uint64_t everyoneBut)
+	void replicateStateObjectToCluster(const ZT_StateObjectType type,const uint64_t id[2],const void *const data,const unsigned int len,const uint64_t everyoneBut)
 	{
 		std::vector<uint64_t> sentTo;
 		if (everyoneBut)
@@ -1927,7 +1920,7 @@ public:
 		}
 	}
 
-	void writeStateObject(enum ZT_StateObjectType type,uint64_t id,const void *data,int len)
+	void writeStateObject(enum ZT_StateObjectType type,const uint64_t id[2],const void *data,int len)
 	{
 		char p[4096];
 		bool secure = false;
@@ -1940,17 +1933,17 @@ public:
 				secure = true;
 				break;
 			case ZT_STATE_OBJECT_PEER_IDENTITY:
-				Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "iddb.d/%.10llx",_homePath.c_str(),(unsigned long long)id);
+				Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "iddb.d/%.10llx",_homePath.c_str(),(unsigned long long)id[0]);
 				break;
 			case ZT_STATE_OBJECT_NETWORK_CONFIG:
-				Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "networks.d/%.16llx.conf",_homePath.c_str(),(unsigned long long)id);
+				Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "networks.d/%.16llx.conf",_homePath.c_str(),(unsigned long long)id[0]);
 				secure = true;
 				break;
 			case ZT_STATE_OBJECT_PLANET:
 				Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "planet",_homePath.c_str());
 				break;
 			case ZT_STATE_OBJECT_MOON:
-				Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "moons.d/%.16llx.moon",_homePath.c_str(),(unsigned long long)id);
+				Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "moons.d/%.16llx.moon",_homePath.c_str(),(unsigned long long)id[0]);
 				break;
 			default:
 				p[0] = (char)0;
@@ -1985,8 +1978,12 @@ public:
 			if (OSUtils::readFile((_homePath + ZT_PATH_SEPARATOR_S + *f).c_str(),buf)) {
 				if (f->length() == 21) {
 					const uint64_t nwid = Utils::hexStrToU64(f->substr(0,16).c_str());
-					if (nwid)
-						replicateStateObject(ZT_STATE_OBJECT_NETWORK_CONFIG,nwid,buf.data(),(int)buf.length(),tc);
+					if (nwid) {
+						uint64_t tmp[2];
+						tmp[0] = nwid;
+						tmp[1] = 0;
+						replicateStateObject(ZT_STATE_OBJECT_NETWORK_CONFIG,tmp,buf.data(),(int)buf.length(),tc);
+					}
 				}
 			}
 		}
@@ -1996,8 +1993,12 @@ public:
 			if (OSUtils::readFile((_homePath + ZT_PATH_SEPARATOR_S + *f).c_str(),buf)) {
 				if (f->length() == 21) {
 					const uint64_t moonId = Utils::hexStrToU64(f->substr(0,16).c_str());
-					if (moonId)
-						replicateStateObject(ZT_STATE_OBJECT_MOON,moonId,buf.data(),(int)buf.length(),tc);
+					if (moonId) {
+						uint64_t tmp[2];
+						tmp[0] = moonId;
+						tmp[1] = 0;
+						replicateStateObject(ZT_STATE_OBJECT_MOON,tmp,buf.data(),(int)buf.length(),tc);
+					}
 				}
 			}
 		}
@@ -2313,8 +2314,9 @@ public:
 									break;
 
 								case CLUSTER_MESSAGE_STATE_OBJECT:
-									if (mlen >= (25 + 9)) { // type + object ID + [data]
-										const uint64_t objId = (
+									if (mlen >= 42) { // type + object ID + [data]
+										uint64_t objId[2];
+										objId[0] = (
 											((uint64_t)data[26] << 56) |
 											((uint64_t)data[27] << 48) |
 											((uint64_t)data[28] << 40) |
@@ -2324,9 +2326,19 @@ public:
 											((uint64_t)data[32] << 8) |
 											(uint64_t)data[33]
 										);
-										if (_node->processStateUpdate((void *)0,(ZT_StateObjectType)data[25],objId,data + 34,(unsigned int)(mlen - 34)) == ZT_RESULT_OK) {
-											writeStateObject((ZT_StateObjectType)data[25],objId,data + 34,(unsigned int)(mlen - 34));
-											replicateStateObjectToCluster((ZT_StateObjectType)data[25],objId,data + 34,(unsigned int)(mlen - 34),tc->clusterMemberId);
+										objId[1] = (
+											((uint64_t)data[34] << 56) |
+											((uint64_t)data[35] << 48) |
+											((uint64_t)data[36] << 40) |
+											((uint64_t)data[37] << 32) |
+											((uint64_t)data[38] << 24) |
+											((uint64_t)data[39] << 16) |
+											((uint64_t)data[40] << 8) |
+											(uint64_t)data[41]
+										);
+										if (_node->processStateUpdate((void *)0,(ZT_StateObjectType)data[25],objId[0],data + 42,(unsigned int)(mlen - 42)) == ZT_RESULT_OK) {
+											writeStateObject((ZT_StateObjectType)data[25],objId,data + 42,(unsigned int)(mlen - 42));
+											replicateStateObjectToCluster((ZT_StateObjectType)data[25],objId,data + 42,(unsigned int)(mlen - 42),tc->clusterMemberId);
 										}
 									}
 									break;
@@ -2543,13 +2555,13 @@ public:
 		}
 	}
 
-	inline void nodeStatePutFunction(enum ZT_StateObjectType type,uint64_t id,const void *data,int len)
+	inline void nodeStatePutFunction(enum ZT_StateObjectType type,const uint64_t id[2],const void *data,int len)
 	{
 		writeStateObject(type,id,data,len);
 		replicateStateObjectToCluster(type,id,data,len,0);
 	}
 
-	inline int nodeStateGetFunction(enum ZT_StateObjectType type,uint64_t id,void *data,unsigned int maxlen)
+	inline int nodeStateGetFunction(enum ZT_StateObjectType type,const uint64_t id[2],void *data,unsigned int maxlen)
 	{
 		char p[4096];
 		switch(type) {
@@ -2866,9 +2878,9 @@ static int SnodeVirtualNetworkConfigFunction(ZT_Node *node,void *uptr,void *tptr
 { return reinterpret_cast<OneServiceImpl *>(uptr)->nodeVirtualNetworkConfigFunction(nwid,nuptr,op,nwconf); }
 static void SnodeEventCallback(ZT_Node *node,void *uptr,void *tptr,enum ZT_Event event,const void *metaData)
 { reinterpret_cast<OneServiceImpl *>(uptr)->nodeEventCallback(event,metaData); }
-static void SnodeStatePutFunction(ZT_Node *node,void *uptr,void *tptr,enum ZT_StateObjectType type,uint64_t id,const void *data,int len)
+static void SnodeStatePutFunction(ZT_Node *node,void *uptr,void *tptr,enum ZT_StateObjectType type,const uint64_t id[2],const void *data,int len)
 { reinterpret_cast<OneServiceImpl *>(uptr)->nodeStatePutFunction(type,id,data,len); }
-static int SnodeStateGetFunction(ZT_Node *node,void *uptr,void *tptr,enum ZT_StateObjectType type,uint64_t id,void *data,unsigned int maxlen)
+static int SnodeStateGetFunction(ZT_Node *node,void *uptr,void *tptr,enum ZT_StateObjectType type,const uint64_t id[2],void *data,unsigned int maxlen)
 { return reinterpret_cast<OneServiceImpl *>(uptr)->nodeStateGetFunction(type,id,data,maxlen); }
 static int SnodeWirePacketSendFunction(ZT_Node *node,void *uptr,void *tptr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *addr,const void *data,unsigned int len,unsigned int ttl)
 { return reinterpret_cast<OneServiceImpl *>(uptr)->nodeWirePacketSendFunction(localAddr,addr,data,len,ttl); }