MERGE current "dev" into "netcon" -- should not affect netcon itself but will retest -- brings ZeroTier core up to 1.1.0

1 files changed, 279 insertions, 99 deletions

diff --git a/node/Node.cpp b/node/Node.cpp
index d5cc50b9..f077424b 100644
--- a/node/Node.cpp
+++ b/node/Node.cpp
@@ -46,7 +46,8 @@
 #include "Address.hpp"
 #include "Identity.hpp"
 #include "SelfAwareness.hpp"
-#include "Defaults.hpp"
+#include "Cluster.hpp"
+#include "DeferredPackets.hpp"
 
 const struct sockaddr_storage ZT_SOCKADDR_NULL = {0};
 
@@ -64,8 +65,7 @@ Node::Node(
 	ZT_WirePacketSendFunction wirePacketSendFunction,
 	ZT_VirtualNetworkFrameFunction virtualNetworkFrameFunction,
 	ZT_VirtualNetworkConfigFunction virtualNetworkConfigFunction,
-	ZT_EventCallback eventCallback,
-	const char *overrideRootTopology) :
+	ZT_EventCallback eventCallback) :
 	_RR(this),
 	RR(&_RR),
 	_uPtr(uptr),
@@ -82,35 +82,33 @@ Node::Node(
 	_lastPingCheck(0),
 	_lastHousekeepingRun(0)
 {
-	_newestVersionSeen[0] = ZEROTIER_ONE_VERSION_MAJOR;
-	_newestVersionSeen[1] = ZEROTIER_ONE_VERSION_MINOR;
-	_newestVersionSeen[2] = ZEROTIER_ONE_VERSION_REVISION;
 	_online = false;
 
 	// Use Salsa20 alone as a high-quality non-crypto PRNG
 	{
 		char foo[32];
 		Utils::getSecureRandom(foo,32);
-		_prng.init(foo,256,foo,8);
+		_prng.init(foo,256,foo);
 		memset(_prngStream,0,sizeof(_prngStream));
-		_prng.encrypt(_prngStream,_prngStream,sizeof(_prngStream));
+		_prng.encrypt12(_prngStream,_prngStream,sizeof(_prngStream));
 	}
 
-	std::string idtmp(dataStoreGet("identity.secret"));
-	if ((!idtmp.length())||(!RR->identity.fromString(idtmp))||(!RR->identity.hasPrivate())) {
-		TRACE("identity.secret not found, generating...");
-		RR->identity.generate();
-		idtmp = RR->identity.toString(true);
-		if (!dataStorePut("identity.secret",idtmp,true))
-			throw std::runtime_error("unable to write identity.secret");
-	}
-	RR->publicIdentityStr = RR->identity.toString(false);
-	RR->secretIdentityStr = RR->identity.toString(true);
-
-	idtmp = dataStoreGet("identity.public");
-	if (idtmp != RR->publicIdentityStr) {
-		if (!dataStorePut("identity.public",RR->publicIdentityStr,false))
-			throw std::runtime_error("unable to write identity.public");
+	{
+		std::string idtmp(dataStoreGet("identity.secret"));
+		if ((!idtmp.length())||(!RR->identity.fromString(idtmp))||(!RR->identity.hasPrivate())) {
+			TRACE("identity.secret not found, generating...");
+			RR->identity.generate();
+			idtmp = RR->identity.toString(true);
+			if (!dataStorePut("identity.secret",idtmp,true))
+				throw std::runtime_error("unable to write identity.secret");
+		}
+		RR->publicIdentityStr = RR->identity.toString(false);
+		RR->secretIdentityStr = RR->identity.toString(true);
+		idtmp = dataStoreGet("identity.public");
+		if (idtmp != RR->publicIdentityStr) {
+			if (!dataStorePut("identity.public",RR->publicIdentityStr,false))
+				throw std::runtime_error("unable to write identity.public");
+		}
 	}
 
 	try {
@@ -119,7 +117,9 @@ Node::Node(
 		RR->antiRec = new AntiRecursion();
 		RR->topology = new Topology(RR);
 		RR->sa = new SelfAwareness(RR);
+		RR->dp = new DeferredPackets(RR);
 	} catch ( ... ) {
+		delete RR->dp;
 		delete RR->sa;
 		delete RR->topology;
 		delete RR->antiRec;
@@ -128,33 +128,25 @@ Node::Node(
 		throw;
 	}
 
-	Dictionary rt;
-	if (overrideRootTopology) {
-		rt.fromString(std::string(overrideRootTopology));
-	} else {
-		std::string rttmp(dataStoreGet("root-topology"));
-		if (rttmp.length() > 0) {
-			rt.fromString(rttmp);
-			if (!Topology::authenticateRootTopology(rt))
-				rt.clear();
-		}
-		if ((!rt.size())||(!rt.contains("rootservers")))
-			rt.fromString(ZT_DEFAULTS.defaultRootTopology);
-	}
-	RR->topology->setRootServers(Dictionary(rt.get("rootservers","")));
-
 	postEvent(ZT_EVENT_UP);
 }
 
 Node::~Node()
 {
 	Mutex::Lock _l(_networks_m);
-	_networks.clear(); // ensure that networks are destroyed before shutdown
+
+	_networks.clear(); // ensure that networks are destroyed before shutdow
+
+	RR->dpEnabled = 0;
+	delete RR->dp;
 	delete RR->sa;
 	delete RR->topology;
 	delete RR->antiRec;
 	delete RR->mc;
 	delete RR->sw;
+#ifdef ZT_ENABLE_CLUSTER
+	delete RR->cluster;
+#endif
 }
 
 ZT_ResultCode Node::processWirePacket(
@@ -197,29 +189,91 @@ public:
 		RR(renv),
 		_now(now),
 		_relays(relays),
-		_rootAddresses(RR->topology->rootAddresses())
+		_world(RR->topology->world())
 	{
 	}
 
-	uint64_t lastReceiveFromUpstream;
+	uint64_t lastReceiveFromUpstream; // tracks last time we got a packet from an 'upstream' peer like a root or a relay
 
 	inline void operator()(Topology &t,const SharedPtr<Peer> &p)
 	{
-		bool isRelay = false;
-		for(std::vector< std::pair<Address,InetAddress> >::const_iterator r(_relays.begin());r!=_relays.end();++r) {
-			if (r->first == p->address()) {
-				isRelay = true;
+		bool upstream = false;
+		InetAddress stableEndpoint4,stableEndpoint6;
+
+		// If this is a world root, pick (if possible) both an IPv4 and an IPv6 stable endpoint to use if link isn't currently alive.
+		for(std::vector<World::Root>::const_iterator r(_world.roots().begin());r!=_world.roots().end();++r) {
+			if (r->identity.address() == p->address()) {
+				upstream = true;
+				for(unsigned long k=0,ptr=(unsigned long)RR->node->prng();k<(unsigned long)r->stableEndpoints.size();++k) {
+					const InetAddress &addr = r->stableEndpoints[ptr++ % r->stableEndpoints.size()];
+					if (!stableEndpoint4) {
+						if (addr.ss_family == AF_INET)
+							stableEndpoint4 = addr;
+					}
+					if (!stableEndpoint6) {
+						if (addr.ss_family == AF_INET6)
+							stableEndpoint6 = addr;
+					}
+				}
 				break;
 			}
 		}
 
-		if ((isRelay)||(std::find(_rootAddresses.begin(),_rootAddresses.end(),p->address()) != _rootAddresses.end())) {
-			p->doPingAndKeepalive(RR,_now);
-			if (p->lastReceive() > lastReceiveFromUpstream)
-				lastReceiveFromUpstream = p->lastReceive();
-		} else {
-			if (p->alive(_now))
-				p->doPingAndKeepalive(RR,_now);
+		if (!upstream) {
+			// If I am a root server, only ping other root servers -- roots don't ping "down"
+			// since that would just be a waste of bandwidth and could potentially cause route
+			// flapping in Cluster mode.
+			if (RR->topology->amRoot())
+				return;
+
+			// Check for network preferred relays, also considered 'upstream' and thus always
+			// pinged to keep links up. If they have stable addresses we will try them there.
+			for(std::vector< std::pair<Address,InetAddress> >::const_iterator r(_relays.begin());r!=_relays.end();++r) {
+				if (r->first == p->address()) {
+					if (r->second.ss_family == AF_INET)
+						stableEndpoint4 = r->second;
+					else if (r->second.ss_family == AF_INET6)
+						stableEndpoint6 = r->second;
+					upstream = true;
+					break;
+				}
+			}
+		}
+
+		if (upstream) {
+			// "Upstream" devices are roots and relays and get special treatment -- they stay alive
+			// forever and we try to keep (if available) both IPv4 and IPv6 channels open to them.
+			bool needToContactIndirect = true;
+			if (p->doPingAndKeepalive(RR,_now,AF_INET)) {
+				needToContactIndirect = false;
+			} else {
+				if (stableEndpoint4) {
+					needToContactIndirect = false;
+					p->sendHELLO(RR,InetAddress(),stableEndpoint4,_now);
+				}
+			}
+			if (p->doPingAndKeepalive(RR,_now,AF_INET6)) {
+				needToContactIndirect = false;
+			} else {
+				if (stableEndpoint6) {
+					needToContactIndirect = false;
+					p->sendHELLO(RR,InetAddress(),stableEndpoint6,_now);
+				}
+			}
+
+			if (needToContactIndirect) {
+				// If this is an upstream and we have no stable endpoint for either IPv4 or IPv6,
+				// send a NOP indirectly if possible to see if we can get to this peer in any
+				// way whatsoever. This will e.g. find network preferred relays that lack
+				// stable endpoints by using root servers.
+				Packet outp(p->address(),RR->identity.address(),Packet::VERB_NOP);
+				RR->sw->send(outp,true,0);
+			}
+
+			lastReceiveFromUpstream = std::max(p->lastReceive(),lastReceiveFromUpstream);
+		} else if (p->activelyTransferringFrames(_now)) {
+			// Normal nodes get their preferred link kept alive if the node has generated frame traffic recently
+			p->doPingAndKeepalive(RR,_now,0);
 		}
 	}
 
@@ -227,7 +281,7 @@ private:
 	const RuntimeEnvironment *RR;
 	uint64_t _now;
 	const std::vector< std::pair<Address,InetAddress> > &_relays;
-	std::vector<Address> _rootAddresses;
+	World _world;
 };
 
 ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline)
@@ -259,24 +313,13 @@ ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextB
 			for(std::vector< SharedPtr<Network> >::const_iterator n(needConfig.begin());n!=needConfig.end();++n)
 				(*n)->requestConfiguration();
 
-			// Attempt to contact network preferred relays that we don't have direct links to
-			std::sort(networkRelays.begin(),networkRelays.end());
-			networkRelays.erase(std::unique(networkRelays.begin(),networkRelays.end()),networkRelays.end());
-			for(std::vector< std::pair<Address,InetAddress> >::const_iterator nr(networkRelays.begin());nr!=networkRelays.end();++nr) {
-				if (nr->second) {
-					SharedPtr<Peer> rp(RR->topology->getPeer(nr->first));
-					if ((rp)&&(!rp->hasActiveDirectPath(now)))
-						rp->attemptToContactAt(RR,InetAddress(),nr->second,now);
-				}
-			}
-
-			// Ping living or root server/relay peers
+			// Do pings and keepalives
 			_PingPeersThatNeedPing pfunc(RR,now,networkRelays);
 			RR->topology->eachPeer<_PingPeersThatNeedPing &>(pfunc);
 
 			// Update online status, post status change as event
-			bool oldOnline = _online;
-			_online = ((now - pfunc.lastReceiveFromUpstream) < ZT_PEER_ACTIVITY_TIMEOUT);
+			const bool oldOnline = _online;
+			_online = (((now - pfunc.lastReceiveFromUpstream) < ZT_PEER_ACTIVITY_TIMEOUT)||(RR->topology->amRoot()));
 			if (oldOnline != _online)
 				postEvent(_online ? ZT_EVENT_ONLINE : ZT_EVENT_OFFLINE);
 		} catch ( ... ) {
@@ -298,7 +341,18 @@ ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextB
 	}
 
 	try {
-		*nextBackgroundTaskDeadline = now + (uint64_t)std::max(std::min(timeUntilNextPingCheck,RR->sw->doTimerTasks(now)),(unsigned long)ZT_CORE_TIMER_TASK_GRANULARITY);
+#ifdef ZT_ENABLE_CLUSTER
+		// If clustering is enabled we have to call cluster->doPeriodicTasks() very often, so we override normal timer deadline behavior
+		if (RR->cluster) {
+			RR->sw->doTimerTasks(now);
+			RR->cluster->doPeriodicTasks();
+			*nextBackgroundTaskDeadline = now + ZT_CLUSTER_PERIODIC_TASK_PERIOD; // this is really short so just tick at this rate
+		} else {
+#endif
+			*nextBackgroundTaskDeadline = now + (uint64_t)std::max(std::min(timeUntilNextPingCheck,RR->sw->doTimerTasks(now)),(unsigned long)ZT_CORE_TIMER_TASK_GRANULARITY);
+#ifdef ZT_ENABLE_CLUSTER
+		}
+#endif
 	} catch ( ... ) {
 		return ZT_RESULT_FATAL_ERROR_INTERNAL;
 	}
@@ -355,6 +409,8 @@ uint64_t Node::address() const
 void Node::status(ZT_NodeStatus *status) const
 {
 	status->address = RR->identity.address().toInt();
+	status->worldId = RR->topology->worldId();
+	status->worldTimestamp = RR->topology->worldTimestamp();
 	status->publicIdentity = RR->publicIdentityStr.c_str();
 	status->secretIdentity = RR->secretIdentityStr.c_str();
 	status->online = _online ? 1 : 0;
@@ -389,14 +445,13 @@ ZT_PeerList *Node::peers() const
 		p->latency = pi->second->latency();
 		p->role = RR->topology->isRoot(pi->second->identity()) ? ZT_PEER_ROLE_ROOT : ZT_PEER_ROLE_LEAF;
 
-		std::vector<RemotePath> paths(pi->second->paths());
-		RemotePath *bestPath = pi->second->getBestPath(_now);
+		std::vector<Path> paths(pi->second->paths());
+		Path *bestPath = pi->second->getBestPath(_now);
 		p->pathCount = 0;
-		for(std::vector<RemotePath>::iterator path(paths.begin());path!=paths.end();++path) {
+		for(std::vector<Path>::iterator path(paths.begin());path!=paths.end();++path) {
 			memcpy(&(p->paths[p->pathCount].address),&(path->address()),sizeof(struct sockaddr_storage));
 			p->paths[p->pathCount].lastSend = path->lastSend();
 			p->paths[p->pathCount].lastReceive = path->lastReceived();
-			p->paths[p->pathCount].fixed = path->fixed() ? 1 : 0;
 			p->paths[p->pathCount].active = path->active(_now) ? 1 : 0;
 			p->paths[p->pathCount].preferred = ((bestPath)&&(*path == *bestPath)) ? 1 : 0;
 			++p->pathCount;
@@ -441,11 +496,11 @@ void Node::freeQueryResult(void *qr)
 		::free(qr);
 }
 
-int Node::addLocalInterfaceAddress(const struct sockaddr_storage *addr,int metric,ZT_LocalInterfaceAddressTrust trust)
+int Node::addLocalInterfaceAddress(const struct sockaddr_storage *addr)
 {
 	if (Path::isAddressValidForPath(*(reinterpret_cast<const InetAddress *>(addr)))) {
 		Mutex::Lock _l(_directPaths_m);
-		_directPaths.push_back(Path(*(reinterpret_cast<const InetAddress *>(addr)),metric,(Path::Trust)trust));
+		_directPaths.push_back(*(reinterpret_cast<const InetAddress *>(addr)));
 		std::sort(_directPaths.begin(),_directPaths.end());
 		_directPaths.erase(std::unique(_directPaths.begin(),_directPaths.end()),_directPaths.end());
 		return 1;
@@ -482,7 +537,7 @@ ZT_ResultCode Node::circuitTestBegin(ZT_CircuitTest *test,void (*reportCallback)
 			outp.append((uint16_t)0);
 			C25519::Signature sig(RR->identity.sign(reinterpret_cast<const char *>(outp.data()) + ZT_PACKET_IDX_PAYLOAD,outp.size() - ZT_PACKET_IDX_PAYLOAD));
 			outp.append((uint16_t)sig.size());
-			outp.append(sig.data,sig.size());
+			outp.append(sig.data,(unsigned int)sig.size());
 			outp.append((uint16_t)0); // originator doesn't need an extra credential, since it's the originator
 			for(unsigned int h=1;h<test->hopCount;++h) {
 				outp.append((uint8_t)0);
@@ -494,7 +549,7 @@ ZT_ResultCode Node::circuitTestBegin(ZT_CircuitTest *test,void (*reportCallback)
 			for(unsigned int a=0;a<test->hops[0].breadth;++a) {
 				outp.newInitializationVector();
 				outp.setDestination(Address(test->hops[0].addresses[a]));
-				RR->sw->send(outp,true,test->credentialNetworkId);
+				RR->sw->send(outp,true,0);
 			}
 		} catch ( ... ) {
 			return ZT_RESULT_FATAL_ERROR_INTERNAL; // probably indicates FIFO too big for packet
@@ -522,13 +577,93 @@ void Node::circuitTestEnd(ZT_CircuitTest *test)
 	}
 }
 
+ZT_ResultCode Node::clusterInit(
+	unsigned int myId,
+	const struct sockaddr_storage *zeroTierPhysicalEndpoints,
+	unsigned int numZeroTierPhysicalEndpoints,
+	int x,
+	int y,
+	int z,
+	void (*sendFunction)(void *,unsigned int,const void *,unsigned int),
+	void *sendFunctionArg,
+	int (*addressToLocationFunction)(void *,const struct sockaddr_storage *,int *,int *,int *),
+	void *addressToLocationFunctionArg)
+{
+#ifdef ZT_ENABLE_CLUSTER
+	if (RR->cluster)
+		return ZT_RESULT_ERROR_BAD_PARAMETER;
+
+	std::vector<InetAddress> eps;
+	for(unsigned int i=0;i<numZeroTierPhysicalEndpoints;++i)
+		eps.push_back(InetAddress(zeroTierPhysicalEndpoints[i]));
+	std::sort(eps.begin(),eps.end());
+	RR->cluster = new Cluster(RR,myId,eps,x,y,z,sendFunction,sendFunctionArg,addressToLocationFunction,addressToLocationFunctionArg);
+
+	return ZT_RESULT_OK;
+#else
+	return ZT_RESULT_ERROR_UNSUPPORTED_OPERATION;
+#endif
+}
+
+ZT_ResultCode Node::clusterAddMember(unsigned int memberId)
+{
+#ifdef ZT_ENABLE_CLUSTER
+	if (!RR->cluster)
+		return ZT_RESULT_ERROR_BAD_PARAMETER;
+	RR->cluster->addMember((uint16_t)memberId);
+	return ZT_RESULT_OK;
+#else
+	return ZT_RESULT_ERROR_UNSUPPORTED_OPERATION;
+#endif
+}
+
+void Node::clusterRemoveMember(unsigned int memberId)
+{
+#ifdef ZT_ENABLE_CLUSTER
+	if (RR->cluster)
+		RR->cluster->removeMember((uint16_t)memberId);
+#endif
+}
+
+void Node::clusterHandleIncomingMessage(const void *msg,unsigned int len)
+{
+#ifdef ZT_ENABLE_CLUSTER
+	if (RR->cluster)
+		RR->cluster->handleIncomingStateMessage(msg,len);
+#endif
+}
+
+void Node::clusterStatus(ZT_ClusterStatus *cs)
+{
+	if (!cs)
+		return;
+#ifdef ZT_ENABLE_CLUSTER
+	if (RR->cluster)
+		RR->cluster->status(*cs);
+	else
+#endif
+	memset(cs,0,sizeof(ZT_ClusterStatus));
+}
+
+void Node::backgroundThreadMain()
+{
+	++RR->dpEnabled;
+	for(;;) {
+		try {
+			if (RR->dp->process() < 0)
+				break;
+		} catch ( ... ) {} // sanity check -- should not throw
+	}
+	--RR->dpEnabled;
+}
+
 /****************************************************************************/
 /* Node methods used only within node/                                      */
 /****************************************************************************/
 
 std::string Node::dataStoreGet(const char *name)
 {
-	char buf[16384];
+	char buf[1024];
 	std::string r;
 	unsigned long olen = 0;
 	do {
@@ -540,16 +675,6 @@ std::string Node::dataStoreGet(const char *name)
 	return r;
 }
 
-void Node::postNewerVersionIfNewer(unsigned int major,unsigned int minor,unsigned int rev)
-{
-	if (Utils::compareVersion(major,minor,rev,_newestVersionSeen[0],_newestVersionSeen[1],_newestVersionSeen[2]) > 0) {
-		_newestVersionSeen[0] = major;
-		_newestVersionSeen[1] = minor;
-		_newestVersionSeen[2] = rev;
-		this->postEvent(ZT_EVENT_SAW_MORE_RECENT_VERSION,(const void *)_newestVersionSeen);
-	}
-}
-
 #ifdef ZT_TRACE
 void Node::postTrace(const char *module,unsigned int line,const char *fmt,...)
 {
@@ -587,7 +712,7 @@ uint64_t Node::prng()
 {
 	unsigned int p = (++_prngStreamPtr % (sizeof(_prngStream) / sizeof(uint64_t)));
 	if (!p)
-		_prng.encrypt(_prngStream,_prngStream,sizeof(_prngStream));
+		_prng.encrypt12(_prngStream,_prngStream,sizeof(_prngStream));
 	return _prngStream[p];
 }
 
@@ -622,12 +747,11 @@ enum ZT_ResultCode ZT_Node_new(
 	ZT_WirePacketSendFunction wirePacketSendFunction,
 	ZT_VirtualNetworkFrameFunction virtualNetworkFrameFunction,
 	ZT_VirtualNetworkConfigFunction virtualNetworkConfigFunction,
-	ZT_EventCallback eventCallback,
-	const char *overrideRootTopology)
+	ZT_EventCallback eventCallback)
 {
 	*node = (ZT_Node *)0;
 	try {
-		*node = reinterpret_cast<ZT_Node *>(new ZeroTier::Node(now,uptr,dataStoreGetFunction,dataStorePutFunction,wirePacketSendFunction,virtualNetworkFrameFunction,virtualNetworkConfigFunction,eventCallback,overrideRootTopology));
+		*node = reinterpret_cast<ZT_Node *>(new ZeroTier::Node(now,uptr,dataStoreGetFunction,dataStorePutFunction,wirePacketSendFunction,virtualNetworkFrameFunction,virtualNetworkConfigFunction,eventCallback));
 		return ZT_RESULT_OK;
 	} catch (std::bad_alloc &exc) {
 		return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY;
@@ -659,8 +783,7 @@ enum ZT_ResultCode ZT_Node_processWirePacket(
 	} catch (std::bad_alloc &exc) {
 		return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY;
 	} catch ( ... ) {
-		reinterpret_cast<ZeroTier::Node *>(node)->postEvent(ZT_EVENT_INVALID_PACKET,(const void *)remoteAddress);
-		return ZT_RESULT_OK;
+		return ZT_RESULT_OK; // "OK" since invalid packets are simply dropped, but the system is still up
 	}
 }
 
@@ -786,6 +909,22 @@ void ZT_Node_freeQueryResult(ZT_Node *node,void *qr)
 	} catch ( ... ) {}
 }
 
+int ZT_Node_addLocalInterfaceAddress(ZT_Node *node,const struct sockaddr_storage *addr)
+{
+	try {
+		return reinterpret_cast<ZeroTier::Node *>(node)->addLocalInterfaceAddress(addr);
+	} catch ( ... ) {
+		return 0;
+	}
+}
+
+void ZT_Node_clearLocalInterfaceAddresses(ZT_Node *node)
+{
+	try {
+		reinterpret_cast<ZeroTier::Node *>(node)->clearLocalInterfaceAddresses();
+	} catch ( ... ) {}
+}
+
 void ZT_Node_setNetconfMaster(ZT_Node *node,void *networkControllerInstance)
 {
 	try {
@@ -793,7 +932,7 @@ void ZT_Node_setNetconfMaster(ZT_Node *node,void *networkControllerInstance)
 	} catch ( ... ) {}
 }
 
-ZT_ResultCode ZT_Node_circuitTestBegin(ZT_Node *node,ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *))
+enum ZT_ResultCode ZT_Node_circuitTestBegin(ZT_Node *node,ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *))
 {
 	try {
 		return reinterpret_cast<ZeroTier::Node *>(node)->circuitTestBegin(test,reportCallback);
@@ -809,19 +948,60 @@ void ZT_Node_circuitTestEnd(ZT_Node *node,ZT_CircuitTest *test)
 	} catch ( ... ) {}
 }
 
-int ZT_Node_addLocalInterfaceAddress(ZT_Node *node,const struct sockaddr_storage *addr,int metric,ZT_LocalInterfaceAddressTrust trust)
+enum ZT_ResultCode ZT_Node_clusterInit(
+	ZT_Node *node,
+	unsigned int myId,
+	const struct sockaddr_storage *zeroTierPhysicalEndpoints,
+	unsigned int numZeroTierPhysicalEndpoints,
+	int x,
+	int y,
+	int z,
+	void (*sendFunction)(void *,unsigned int,const void *,unsigned int),
+	void *sendFunctionArg,
+	int (*addressToLocationFunction)(void *,const struct sockaddr_storage *,int *,int *,int *),
+	void *addressToLocationFunctionArg)
 {
 	try {
-		return reinterpret_cast<ZeroTier::Node *>(node)->addLocalInterfaceAddress(addr,metric,trust);
+		return reinterpret_cast<ZeroTier::Node *>(node)->clusterInit(myId,zeroTierPhysicalEndpoints,numZeroTierPhysicalEndpoints,x,y,z,sendFunction,sendFunctionArg,addressToLocationFunction,addressToLocationFunctionArg);
 	} catch ( ... ) {
-		return 0;
+		return ZT_RESULT_FATAL_ERROR_INTERNAL;
 	}
 }
 
-void ZT_Node_clearLocalInterfaceAddresses(ZT_Node *node)
+enum ZT_ResultCode ZT_Node_clusterAddMember(ZT_Node *node,unsigned int memberId)
 {
 	try {
-		reinterpret_cast<ZeroTier::Node *>(node)->clearLocalInterfaceAddresses();
+		return reinterpret_cast<ZeroTier::Node *>(node)->clusterAddMember(memberId);
+	} catch ( ... ) {
+		return ZT_RESULT_FATAL_ERROR_INTERNAL;
+	}
+}
+
+void ZT_Node_clusterRemoveMember(ZT_Node *node,unsigned int memberId)
+{
+	try {
+		reinterpret_cast<ZeroTier::Node *>(node)->clusterRemoveMember(memberId);
+	} catch ( ... ) {}
+}
+
+void ZT_Node_clusterHandleIncomingMessage(ZT_Node *node,const void *msg,unsigned int len)
+{
+	try {
+		reinterpret_cast<ZeroTier::Node *>(node)->clusterHandleIncomingMessage(msg,len);
+	} catch ( ... ) {}
+}
+
+void ZT_Node_clusterStatus(ZT_Node *node,ZT_ClusterStatus *cs)
+{
+	try {
+		reinterpret_cast<ZeroTier::Node *>(node)->clusterStatus(cs);
+	} catch ( ... ) {}
+}
+
+void ZT_Node_backgroundThreadMain(ZT_Node *node)
+{
+	try {
+		reinterpret_cast<ZeroTier::Node *>(node)->backgroundThreadMain();
 	} catch ( ... ) {}
 }