From 5076c49210542243075556aa1ab74f33d4d50ba3 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Thu, 1 Oct 2015 15:40:54 -0700
Subject: Peer serialization and related changes.

---
 include/ZeroTierOne.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'include')

diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h
index e8a19e33..8eacc993 100644
--- a/include/ZeroTierOne.h
+++ b/include/ZeroTierOne.h
@@ -627,8 +627,8 @@ typedef struct
  */
 typedef enum {
 	ZT_LOCAL_INTERFACE_ADDRESS_TRUST_NORMAL = 0,
-	ZT_LOCAL_INTERFACE_ADDRESS_TRUST_PRIVACY = 1,
-	ZT_LOCAL_INTERFACE_ADDRESS_TRUST_ULTIMATE = 2
+	ZT_LOCAL_INTERFACE_ADDRESS_TRUST_PRIVACY = 10,
+	ZT_LOCAL_INTERFACE_ADDRESS_TRUST_ULTIMATE = 20
 } ZT_LocalInterfaceAddressTrust;
 
 /**
-- 
cgit v1.2.3


From d3f29d09e8eb7fdbbbed682b383da73bc44928d6 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Tue, 6 Oct 2015 14:42:51 -0700
Subject: Plumbing through circuit test stuff.

---
 include/ZeroTierOne.h   | 271 ++++++++++++++++++++++++++++++++++++++++++++++++
 node/IncomingPacket.cpp |   7 +-
 node/Node.cpp           |  52 ++++++++++
 node/Node.hpp           |  12 +++
 node/Packet.hpp         |  38 -------
 5 files changed, 339 insertions(+), 41 deletions(-)

(limited to 'include')

diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h
index 8eacc993..43c8fc0b 100644
--- a/include/ZeroTierOne.h
+++ b/include/ZeroTierOne.h
@@ -115,6 +115,19 @@ extern "C" {
  */
 #define ZT_FEATURE_FLAG_FIPS 0x00000002
 
+/**
+ * Maximum number of hops in a ZeroTier circuit test
+ *
+ * This is more or less the max that can be fit in a given packet (with
+ * fragmentation) and only one address per hop.
+ */
+#define ZT_CIRCUIT_TEST_MAX_HOPS 512
+
+/**
+ * Maximum number of addresses per hop in a circuit test
+ */
+#define ZT_CIRCUIT_TEST_MAX_HOP_BREADTH 256
+
 /**
  * A null/empty sockaddr (all zero) to signify an unspecified socket address
  */
@@ -631,6 +644,231 @@ typedef enum {
 	ZT_LOCAL_INTERFACE_ADDRESS_TRUST_ULTIMATE = 20
 } ZT_LocalInterfaceAddressTrust;
 
+/**
+ * Vendor ID
+ */
+typedef enum {
+	ZT_VENDOR_UNSPECIFIED = 0,
+	ZT_VENDOR_ZEROTIER = 1
+} ZT_Vendor;
+
+/**
+ * Platform type
+ */
+typedef enum {
+	ZT_PLATFORM_UNSPECIFIED = 0,
+	ZT_PLATFORM_LINUX = 1,
+	ZT_PLATFORM_WINDOWS = 2,
+	ZT_PLATFORM_MACOS = 3,
+	ZT_PLATFORM_ANDROID = 4,
+	ZT_PLATFORM_IOS = 5,
+	ZT_PLATFORM_SOLARIS_SMARTOS = 6,
+	ZT_PLATFORM_FREEBSD = 7,
+	ZT_PLATFORM_NETBSD = 8,
+	ZT_PLATFORM_OPENBSD = 9,
+	ZT_PLATFORM_RISCOS = 10,
+	ZT_PLATFORM_VXWORKS = 11,
+	ZT_PLATFORM_FREERTOS = 12,
+	ZT_PLATFORM_SYSBIOS = 13,
+	ZT_PLATFORM_HURD = 14
+} ZT_Platform;
+
+/**
+ * Architecture type
+ */
+typedef enum {
+	ZT_ARCHITECTURE_UNSPECIFIED = 0,
+	ZT_ARCHITECTURE_X86 = 1,
+	ZT_ARCHITECTURE_X64 = 2,
+	ZT_ARCHITECTURE_ARM32 = 3,
+	ZT_ARCHITECTURE_ARM64 = 4,
+	ZT_ARCHITECTURE_MIPS32 = 5,
+	ZT_ARCHITECTURE_MIPS64 = 6,
+	ZT_ARCHITECTURE_POWER32 = 7,
+	ZT_ARCHITECTURE_POWER64 = 8
+} ZT_Architecture;
+
+/**
+ * ZeroTier circuit test configuration and path
+ */
+typedef struct {
+	/**
+	 * Test ID -- an arbitrary 64-bit identifier
+	 */
+	uint64_t testId;
+
+	/**
+	 * Timestamp -- sent with test and echoed back by each reporter
+	 */
+	uint64_t timestamp;
+
+	/**
+	 * Originator credential: network ID
+	 *
+	 * If this is nonzero, a network ID will be set for this test and
+	 * the originator must be its primary network controller. This is
+	 * currently the only authorization method available, so it must
+	 * be set to run a test.
+	 */
+	uint64_t credentialNetworkId;
+
+	/**
+	 * Hops in circuit test (a.k.a. FIFO for graph traversal)
+	 */
+	struct {
+		/**
+		 * Hop flags (currently unused, must be zero)
+		 */
+		unsigned int flags;
+
+		/**
+		 * Number of addresses in this hop (max: ZT_CIRCUIT_TEST_MAX_HOP_BREADTH)
+		 */
+		unsigned int breadth;
+
+		/**
+		 * 40-bit ZeroTier addresses (most significant 24 bits ignored)
+		 */
+		uint64_t addresses[ZT_CIRCUIT_TEST_MAX_HOP_BREADTH];
+	} hops[ZT_CIRCUIT_TEST_MAX_HOPS];
+
+	/**
+	 * Number of hops (max: ZT_CIRCUIT_TEST_MAX_HOPS)
+	 */
+	unsigned int hopCount;
+
+	/**
+	 * If non-zero, circuit test will report back at every hop
+	 */
+	int reportAtEveryHop;
+
+	/**
+	 * An arbitrary user-settable pointer
+	 */
+	void *ptr;
+
+	/**
+	 * Pointer for internal use -- initialize to zero and do not modify
+	 */
+	void *_internalPtr;
+} ZT_CircuitTest;
+
+/**
+ * Circuit test result report
+ */
+typedef struct {
+	/**
+	 * 64-bit test ID
+	 */
+	uint64_t testId;
+
+	/**
+	 * Timestamp from original test (echoed back at each hop)
+	 */
+	uint64_t timestamp;
+
+	/**
+	 * Timestamp on remote device
+	 */
+	uint64_t remoteTimestamp;
+
+	/**
+	 * 64-bit packet ID of packet received by the reporting device
+	 */
+	uint64_t sourcePacketId;
+
+	/**
+	 * Flags (currently unused, will be zero)
+	 */
+	uint64_t flags;
+
+	/**
+	 * ZeroTier protocol-level hop count of packet received by reporting device (>0 indicates relayed)
+	 */
+	unsigned int sourcePacketHopCount;
+
+	/**
+	 * Error code (currently unused, will be zero)
+	 */
+	unsigned int errorCode;
+
+	/**
+	 * Remote device vendor ID
+	 */
+	ZT_Vendor vendor;
+
+	/**
+	 * Remote device protocol compliance version
+	 */
+	unsigned int protocolVersion;
+
+	/**
+	 * Software major version
+	 */
+	unsigned int majorVersion;
+
+	/**
+	 * Software minor version
+	 */
+	unsigned int minorVersion;
+
+	/**
+	 * Software revision
+	 */
+	unsigned int revision;
+
+	/**
+	 * Platform / OS
+	 */
+	ZT_Platform platform;
+
+	/**
+	 * System architecture
+	 */
+	ZT_Architecture architecture;
+
+	/**
+	 * Local device address on which packet was received by reporting device
+	 *
+	 * This may have ss_family equal to zero (null address) if unspecified.
+	 */
+	struct sockaddr_storage receivedOnLocalAddress;
+
+	/**
+	 * Remote address from which reporter received the test packet
+	 *
+	 * This may have ss_family set to zero (null address) if unspecified.
+	 */
+	struct sockaddr_storage receivedFromAddress;
+
+	/**
+	 * Next hops to which packets are being or will be sent by the reporter
+	 *
+	 * In addition to reporting back, the reporter may send the test on if
+	 * there are more recipients in the FIFO. If it does this, it can report
+	 * back the address(es) that make up the next hop and the physical address
+	 * for each if it has one. The physical address being null/unspecified
+	 * typically indicates that no direct path exists and the next packet
+	 * will be relayed.
+	 */
+	struct {
+		/**
+		 * 40-bit ZeroTier address
+		 */
+		uint64_t address;
+
+		/**
+		 * Physical address or null address (ss_family == 0) if unspecified or unknown
+		 */
+		struct sockaddr_storage physicalAddress;
+	} nextHops[ZT_CIRCUIT_TEST_MAX_HOP_BREADTH];
+
+	/**
+	 * Number of next hops reported in nextHops[]
+	 */
+	unsigned int nextHopCount;
+} ZT_CircuitTestReport;
+
 /**
  * An instance of a ZeroTier One node (opaque)
  */
@@ -1061,6 +1299,39 @@ void ZT_Node_clearLocalInterfaceAddresses(ZT_Node *node);
  */
 void ZT_Node_setNetconfMaster(ZT_Node *node,void *networkConfigMasterInstance);
 
+/**
+ * Initiate a VL1 circuit test
+ *
+ * This sends an initial VERB_CIRCUIT_TEST and reports results back to the
+ * supplied callback until circuitTestEnd() is called. The supplied
+ * ZT_CircuitTest structure should be initially zeroed and then filled
+ * in with settings and hops.
+ *
+ * It is the caller's responsibility to call circuitTestEnd() and then
+ * to dispose of the test structure. Otherwise this node will listen
+ * for results forever.
+ *
+ * @param node Node instance
+ * @param test Test configuration
+ * @param reportCallback Function to call each time a report is received
+ * @return OK or error if, for example, test is too big for a packet or support isn't compiled in
+ */
+ZT_ResultCode ZT_Node_circuitTestBegin(ZT_Node *node,ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *));
+
+/**
+ * Stop listening for results to a given circuit test
+ *
+ * This does not free the 'test' structure. The caller may do that
+ * after calling this method to unregister it.
+ *
+ * Any reports that are received for a given test ID after it is
+ * terminated are ignored.
+ *
+ * @param node Node instance
+ * @param test Test configuration to unregister
+ */
+void ZT_Node_circuitTestEnd(ZT_Node *node,ZT_CircuitTest *test);
+
 /**
  * Get ZeroTier One version
  *
diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp
index 49a5981b..c8e4cf5f 100644
--- a/node/IncomingPacket.cpp
+++ b/node/IncomingPacket.cpp
@@ -30,6 +30,7 @@
 #include <stdlib.h>
 
 #include "../version.h"
+#include "../include/ZeroTierOne.h"
 
 #include "Constants.hpp"
 #include "Defaults.hpp"
@@ -1033,13 +1034,13 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt
 			outp.append((uint64_t)timestamp);
 			outp.append((uint64_t)testId);
 			outp.append((uint64_t)now);
-			outp.append((uint8_t)0); // vendor ID, currently unused
+			outp.append((uint8_t)ZT_VENDOR_ZEROTIER);
 			outp.append((uint8_t)ZT_PROTO_VERSION);
 			outp.append((uint8_t)ZEROTIER_ONE_VERSION_MAJOR);
 			outp.append((uint8_t)ZEROTIER_ONE_VERSION_MINOR);
 			outp.append((uint16_t)ZEROTIER_ONE_VERSION_REVISION);
-			outp.append((uint16_t)CIRCUIT_TEST_REPORT_PLATFORM_UNSPECIFIED);
-			outp.append((uint16_t)CIRCUIT_TEST_REPORT_ARCH_UNSPECIFIED);
+			outp.append((uint16_t)ZT_PLATFORM_UNSPECIFIED);
+			outp.append((uint16_t)ZT_ARCHITECTURE_UNSPECIFIED);
 			outp.append((uint16_t)0); // error code, currently unused
 			outp.append((uint64_t)0); // flags, currently unused
 			outp.append((uint64_t)packetId());
diff --git a/node/Node.cpp b/node/Node.cpp
index 6dc83d4e..cd20972b 100644
--- a/node/Node.cpp
+++ b/node/Node.cpp
@@ -464,6 +464,28 @@ void Node::setNetconfMaster(void *networkControllerInstance)
 	RR->localNetworkController = reinterpret_cast<NetworkController *>(networkControllerInstance);
 }
 
+ZT_ResultCode Node::circuitTestBegin(ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *))
+{
+	{
+		test->_internalPtr = reinterpret_cast<void *>(reportCallback);
+		Mutex::Lock _l(_circuitTests_m);
+		if (std::find(_circuitTests.begin(),_circuitTests.end(),test) == _circuitTests.end())
+			_circuitTests.push_back(test);
+	}
+	return ZT_RESULT_OK;
+}
+
+void Node::circuitTestEnd(ZT_CircuitTest *test)
+{
+	Mutex::Lock _l(_circuitTests_m);
+	for(;;) {
+		std::vector< ZT_CircuitTest * >::iterator ct(std::find(_circuitTests.begin(),_circuitTests.end(),test));
+		if (ct == _circuitTests.end())
+			break;
+		else _circuitTests.erase(ct);
+	}
+}
+
 /****************************************************************************/
 /* Node methods used only within node/                                      */
 /****************************************************************************/
@@ -533,6 +555,20 @@ uint64_t Node::prng()
 	return _prngStream[p];
 }
 
+void Node::postCircuitTestReport(const ZT_CircuitTestReport *report)
+{
+	std::vector< ZT_CircuitTest * > toNotify;
+	{
+		Mutex::Lock _l(_circuitTests_m);
+		for(std::vector< ZT_CircuitTest * >::iterator i(_circuitTests.begin());i!=_circuitTests.end();++i) {
+			if ((*i)->testId == report->testId)
+				toNotify.push_back(*i);
+		}
+	}
+	for(std::vector< ZT_CircuitTest * >::iterator i(toNotify.begin());i!=toNotify.end();++i)
+		(reinterpret_cast<void (*)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *)>((*i)->_internalPtr))(reinterpret_cast<ZT_Node *>(this),*i,report);
+}
+
 } // namespace ZeroTier
 
 /****************************************************************************/
@@ -721,6 +757,22 @@ void ZT_Node_setNetconfMaster(ZT_Node *node,void *networkControllerInstance)
 	} catch ( ... ) {}
 }
 
+ZT_ResultCode ZT_Node_circuitTestBegin(ZT_Node *node,ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *))
+{
+	try {
+		return reinterpret_cast<ZeroTier::Node *>(node)->circuitTestBegin(test,reportCallback);
+	} catch ( ... ) {
+		return ZT_RESULT_FATAL_ERROR_INTERNAL;
+	}
+}
+
+void ZT_Node_circuitTestEnd(ZT_Node *node,ZT_CircuitTest *test)
+{
+	try {
+		reinterpret_cast<ZeroTier::Node *>(node)->circuitTestEnd(test);
+	} catch ( ... ) {}
+}
+
 int ZT_Node_addLocalInterfaceAddress(ZT_Node *node,const struct sockaddr_storage *addr,int metric,ZT_LocalInterfaceAddressTrust trust)
 {
 	try {
diff --git a/node/Node.hpp b/node/Node.hpp
index 0f659f47..20c54471 100644
--- a/node/Node.hpp
+++ b/node/Node.hpp
@@ -109,6 +109,8 @@ public:
 	int addLocalInterfaceAddress(const struct sockaddr_storage *addr,int metric,ZT_LocalInterfaceAddressTrust trust);
 	void clearLocalInterfaceAddresses();
 	void setNetconfMaster(void *networkControllerInstance);
+	ZT_ResultCode circuitTestBegin(ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *));
+	void circuitTestEnd(ZT_CircuitTest *test);
 
 	// Internal functions ------------------------------------------------------
 
@@ -238,6 +240,13 @@ public:
 	 */
 	uint64_t prng();
 
+	/**
+	 * Post a circuit test report to any listeners for a given test ID
+	 *
+	 * @param report Report (includes test ID)
+	 */
+	void postCircuitTestReport(const ZT_CircuitTestReport *report);
+
 private:
 	inline SharedPtr<Network> _network(uint64_t nwid) const
 	{
@@ -264,6 +273,9 @@ private:
 	std::vector< std::pair< uint64_t, SharedPtr<Network> > > _networks;
 	Mutex _networks_m;
 
+	std::vector< ZT_CircuitTest * > _circuitTests;
+	Mutex _circuitTests_m;
+
 	std::vector<Path> _directPaths;
 	Mutex _directPaths_m;
 
diff --git a/node/Packet.hpp b/node/Packet.hpp
index eaffb922..409762c7 100644
--- a/node/Packet.hpp
+++ b/node/Packet.hpp
@@ -1020,44 +1020,6 @@ public:
 		VERB_CIRCUIT_TEST_REPORT = 18
 	};
 
-	/**
-	 * Platforms reported in circuit tests
-	 */
-	enum CircuitTestReportPlatform
-	{
-		CIRCUIT_TEST_REPORT_PLATFORM_UNSPECIFIED = 0,
-		CIRCUIT_TEST_REPORT_PLATFORM_LINUX = 1,
-		CIRCUIT_TEST_REPORT_PLATFORM_WINDOWS = 2,
-		CIRCUIT_TEST_REPORT_PLATFORM_MACOS = 3,
-		CIRCUIT_TEST_REPORT_PLATFORM_ANDROID = 4,
-		CIRCUIT_TEST_REPORT_PLATFORM_IOS = 5,
-		CIRCUIT_TEST_REPORT_PLATFORM_SOLARIS_SMARTOS = 6,
-		CIRCUIT_TEST_REPORT_PLATFORM_FREEBSD = 7,
-		CIRCUIT_TEST_REPORT_PLATFORM_NETBSD = 8,
-		CIRCUIT_TEST_REPORT_PLATFORM_OPENBSD = 9,
-		CIRCUIT_TEST_REPORT_PLATFORM_RISCOS = 10,
-		CIRCUIT_TEST_REPORT_PLATFORM_VXWORKS = 11,
-		CIRCUIT_TEST_REPORT_PLATFORM_FREERTOS = 12,
-		CIRCUIT_TEST_REPORT_PLATFORM_SYSBIOS = 13,
-		CIRCUIT_TEST_REPORT_PLATFORM_HURD = 14
-	};
-
-	/**
-	 * Architectures reported in circuit tests
-	 */
-	enum CircuitTestReportArchitecture
-	{
-		CIRCUIT_TEST_REPORT_ARCH_UNSPECIFIED = 0,
-		CIRCUIT_TEST_REPORT_ARCH_X86 = 1,
-		CIRCUIT_TEST_REPORT_ARCH_X64 = 2,
-		CIRCUIT_TEST_REPORT_ARCH_ARM32 = 3,
-		CIRCUIT_TEST_REPORT_ARCH_ARM64 = 4,
-		CIRCUIT_TEST_REPORT_ARCH_MIPS32 = 5,
-		CIRCUIT_TEST_REPORT_ARCH_MIPS64 = 6,
-		CIRCUIT_TEST_REPORT_ARCH_POWER32 = 7,
-		CIRCUIT_TEST_REPORT_ARCH_POWER64 = 8
-	};
-
 	/**
 	 * Error codes for VERB_ERROR
 	 */
-- 
cgit v1.2.3


From 477feee8a3fbc84d00c2939b5fc8a9bbf19af2ca Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Tue, 6 Oct 2015 17:55:57 -0700
Subject: Some work on CIRCUIT_TEST, and a significant speedup to Poly1305.

---
 include/ZeroTierOne.h |   7 +-
 node/Poly1305.cpp     | 279 +++++++++++++++++++++++++++++++++++++++++++++++++-
 selftest.cpp          |  16 +++
 3 files changed, 296 insertions(+), 6 deletions(-)

(limited to 'include')

diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h
index 43c8fc0b..341bb767 100644
--- a/include/ZeroTierOne.h
+++ b/include/ZeroTierOne.h
@@ -757,6 +757,11 @@ typedef struct {
  * Circuit test result report
  */
 typedef struct {
+	/**
+	 * Sender of report
+	 */
+	uint64_t address;
+
 	/**
 	 * 64-bit test ID
 	 */
@@ -839,7 +844,7 @@ typedef struct {
 	 *
 	 * This may have ss_family set to zero (null address) if unspecified.
 	 */
-	struct sockaddr_storage receivedFromAddress;
+	struct sockaddr_storage receivedFromRemoteAddress;
 
 	/**
 	 * Next hops to which packets are being or will be sent by the reporter
diff --git a/node/Poly1305.cpp b/node/Poly1305.cpp
index 230b2eff..77b32a80 100644
--- a/node/Poly1305.cpp
+++ b/node/Poly1305.cpp
@@ -7,14 +7,18 @@ Public domain.
 #include "Constants.hpp"
 #include "Poly1305.hpp"
 
+#include <stdio.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
 #ifdef __WINDOWS__
 #pragma warning(disable: 4146)
 #endif
 
 namespace ZeroTier {
 
-//////////////////////////////////////////////////////////////////////////////
-//////////////////////////////////////////////////////////////////////////////
+#if 0
 
 static inline void add(unsigned int h[17],const unsigned int c[17])
 {
@@ -113,13 +117,278 @@ static inline int crypto_onetimeauth(unsigned char *out,const unsigned char *in,
   return 0;
 }
 
-//////////////////////////////////////////////////////////////////////////////
-//////////////////////////////////////////////////////////////////////////////
-
 void Poly1305::compute(void *auth,const void *data,unsigned int len,const void *key)
 	throw()
 {
 	crypto_onetimeauth((unsigned char *)auth,(const unsigned char *)data,len,(const unsigned char *)key);
 }
 
+#endif
+
+namespace {
+
+typedef struct poly1305_context {
+  size_t aligner;
+  unsigned char opaque[136];
+} poly1305_context;
+
+/*
+  poly1305 implementation using 32 bit * 32 bit = 64 bit multiplication and 64 bit addition
+*/
+
+#define poly1305_block_size 16
+
+/* 17 + sizeof(size_t) + 14*sizeof(unsigned long) */
+typedef struct poly1305_state_internal_t {
+  unsigned long r[5];
+  unsigned long h[5];
+  unsigned long pad[4];
+  size_t leftover;
+  unsigned char buffer[poly1305_block_size];
+  unsigned char final;
+} poly1305_state_internal_t;
+
+/* interpret four 8 bit unsigned integers as a 32 bit unsigned integer in little endian */
+static unsigned long
+U8TO32(const unsigned char *p) {
+  return
+    (((unsigned long)(p[0] & 0xff)      ) |
+       ((unsigned long)(p[1] & 0xff) <<  8) |
+         ((unsigned long)(p[2] & 0xff) << 16) |
+         ((unsigned long)(p[3] & 0xff) << 24));
+}
+
+/* store a 32 bit unsigned integer as four 8 bit unsigned integers in little endian */
+static void
+U32TO8(unsigned char *p, unsigned long v) {
+  p[0] = (v      ) & 0xff;
+  p[1] = (v >>  8) & 0xff;
+  p[2] = (v >> 16) & 0xff;
+  p[3] = (v >> 24) & 0xff;
+}
+
+static inline void
+poly1305_init(poly1305_context *ctx, const unsigned char key[32]) {
+  poly1305_state_internal_t *st = (poly1305_state_internal_t *)ctx;
+
+  /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */
+  st->r[0] = (U8TO32(&key[ 0])     ) & 0x3ffffff;
+  st->r[1] = (U8TO32(&key[ 3]) >> 2) & 0x3ffff03;
+  st->r[2] = (U8TO32(&key[ 6]) >> 4) & 0x3ffc0ff;
+  st->r[3] = (U8TO32(&key[ 9]) >> 6) & 0x3f03fff;
+  st->r[4] = (U8TO32(&key[12]) >> 8) & 0x00fffff;
+
+  /* h = 0 */
+  st->h[0] = 0;
+  st->h[1] = 0;
+  st->h[2] = 0;
+  st->h[3] = 0;
+  st->h[4] = 0;
+
+  /* save pad for later */
+  st->pad[0] = U8TO32(&key[16]);
+  st->pad[1] = U8TO32(&key[20]);
+  st->pad[2] = U8TO32(&key[24]);
+  st->pad[3] = U8TO32(&key[28]);
+
+  st->leftover = 0;
+  st->final = 0;
+}
+
+static inline void
+poly1305_blocks(poly1305_state_internal_t *st, const unsigned char *m, size_t bytes) {
+  const unsigned long hibit = (st->final) ? 0 : (1 << 24); /* 1 << 128 */
+  unsigned long r0,r1,r2,r3,r4;
+  unsigned long s1,s2,s3,s4;
+  unsigned long h0,h1,h2,h3,h4;
+  unsigned long long d0,d1,d2,d3,d4;
+  unsigned long c;
+
+  r0 = st->r[0];
+  r1 = st->r[1];
+  r2 = st->r[2];
+  r3 = st->r[3];
+  r4 = st->r[4];
+
+  s1 = r1 * 5;
+  s2 = r2 * 5;
+  s3 = r3 * 5;
+  s4 = r4 * 5;
+
+  h0 = st->h[0];
+  h1 = st->h[1];
+  h2 = st->h[2];
+  h3 = st->h[3];
+  h4 = st->h[4];
+
+  while (bytes >= poly1305_block_size) {
+    /* h += m[i] */
+    h0 += (U8TO32(m+ 0)     ) & 0x3ffffff;
+    h1 += (U8TO32(m+ 3) >> 2) & 0x3ffffff;
+    h2 += (U8TO32(m+ 6) >> 4) & 0x3ffffff;
+    h3 += (U8TO32(m+ 9) >> 6) & 0x3ffffff;
+    h4 += (U8TO32(m+12) >> 8) | hibit;
+
+    /* h *= r */
+    d0 = ((unsigned long long)h0 * r0) + ((unsigned long long)h1 * s4) + ((unsigned long long)h2 * s3) + ((unsigned long long)h3 * s2) + ((unsigned long long)h4 * s1);
+    d1 = ((unsigned long long)h0 * r1) + ((unsigned long long)h1 * r0) + ((unsigned long long)h2 * s4) + ((unsigned long long)h3 * s3) + ((unsigned long long)h4 * s2);
+    d2 = ((unsigned long long)h0 * r2) + ((unsigned long long)h1 * r1) + ((unsigned long long)h2 * r0) + ((unsigned long long)h3 * s4) + ((unsigned long long)h4 * s3);
+    d3 = ((unsigned long long)h0 * r3) + ((unsigned long long)h1 * r2) + ((unsigned long long)h2 * r1) + ((unsigned long long)h3 * r0) + ((unsigned long long)h4 * s4);
+    d4 = ((unsigned long long)h0 * r4) + ((unsigned long long)h1 * r3) + ((unsigned long long)h2 * r2) + ((unsigned long long)h3 * r1) + ((unsigned long long)h4 * r0);
+
+    /* (partial) h %= p */
+                  c = (unsigned long)(d0 >> 26); h0 = (unsigned long)d0 & 0x3ffffff;
+    d1 += c;      c = (unsigned long)(d1 >> 26); h1 = (unsigned long)d1 & 0x3ffffff;
+    d2 += c;      c = (unsigned long)(d2 >> 26); h2 = (unsigned long)d2 & 0x3ffffff;
+    d3 += c;      c = (unsigned long)(d3 >> 26); h3 = (unsigned long)d3 & 0x3ffffff;
+    d4 += c;      c = (unsigned long)(d4 >> 26); h4 = (unsigned long)d4 & 0x3ffffff;
+    h0 += c * 5;  c =                (h0 >> 26); h0 =                h0 & 0x3ffffff;
+    h1 += c;
+
+    m += poly1305_block_size;
+    bytes -= poly1305_block_size;
+  }
+
+  st->h[0] = h0;
+  st->h[1] = h1;
+  st->h[2] = h2;
+  st->h[3] = h3;
+  st->h[4] = h4;
+}
+
+static inline void
+poly1305_update(poly1305_context *ctx, const unsigned char *m, size_t bytes) {
+  poly1305_state_internal_t *st = (poly1305_state_internal_t *)ctx;
+  size_t i;
+
+  /* handle leftover */
+  if (st->leftover) {
+    size_t want = (poly1305_block_size - st->leftover);
+    if (want > bytes)
+      want = bytes;
+    for (i = 0; i < want; i++)
+      st->buffer[st->leftover + i] = m[i];
+    bytes -= want;
+    m += want;
+    st->leftover += want;
+    if (st->leftover < poly1305_block_size)
+      return;
+    poly1305_blocks(st, st->buffer, poly1305_block_size);
+    st->leftover = 0;
+  }
+
+  /* process full blocks */
+  if (bytes >= poly1305_block_size) {
+    size_t want = (bytes & ~(poly1305_block_size - 1));
+    poly1305_blocks(st, m, want);
+    m += want;
+    bytes -= want;
+  }
+
+  /* store leftover */
+  if (bytes) {
+    for (i = 0; i < bytes; i++)
+      st->buffer[st->leftover + i] = m[i];
+    st->leftover += bytes;
+  }
+}
+
+static inline void
+poly1305_finish(poly1305_context *ctx, unsigned char mac[16]) {
+  poly1305_state_internal_t *st = (poly1305_state_internal_t *)ctx;
+  unsigned long h0,h1,h2,h3,h4,c;
+  unsigned long g0,g1,g2,g3,g4;
+  unsigned long long f;
+  unsigned long mask;
+
+  /* process the remaining block */
+  if (st->leftover) {
+    size_t i = st->leftover;
+    st->buffer[i++] = 1;
+    for (; i < poly1305_block_size; i++)
+      st->buffer[i] = 0;
+    st->final = 1;
+    poly1305_blocks(st, st->buffer, poly1305_block_size);
+  }
+
+  /* fully carry h */
+  h0 = st->h[0];
+  h1 = st->h[1];
+  h2 = st->h[2];
+  h3 = st->h[3];
+  h4 = st->h[4];
+
+               c = h1 >> 26; h1 = h1 & 0x3ffffff;
+  h2 +=     c; c = h2 >> 26; h2 = h2 & 0x3ffffff;
+  h3 +=     c; c = h3 >> 26; h3 = h3 & 0x3ffffff;
+  h4 +=     c; c = h4 >> 26; h4 = h4 & 0x3ffffff;
+  h0 += c * 5; c = h0 >> 26; h0 = h0 & 0x3ffffff;
+  h1 +=     c;
+
+  /* compute h + -p */
+  g0 = h0 + 5; c = g0 >> 26; g0 &= 0x3ffffff;
+  g1 = h1 + c; c = g1 >> 26; g1 &= 0x3ffffff;
+  g2 = h2 + c; c = g2 >> 26; g2 &= 0x3ffffff;
+  g3 = h3 + c; c = g3 >> 26; g3 &= 0x3ffffff;
+  g4 = h4 + c - (1 << 26);
+
+  /* select h if h < p, or h + -p if h >= p */
+  mask = (g4 >> ((sizeof(unsigned long) * 8) - 1)) - 1;
+  g0 &= mask;
+  g1 &= mask;
+  g2 &= mask;
+  g3 &= mask;
+  g4 &= mask;
+  mask = ~mask;
+  h0 = (h0 & mask) | g0;
+  h1 = (h1 & mask) | g1;
+  h2 = (h2 & mask) | g2;
+  h3 = (h3 & mask) | g3;
+  h4 = (h4 & mask) | g4;
+
+  /* h = h % (2^128) */
+  h0 = ((h0      ) | (h1 << 26)) & 0xffffffff;
+  h1 = ((h1 >>  6) | (h2 << 20)) & 0xffffffff;
+  h2 = ((h2 >> 12) | (h3 << 14)) & 0xffffffff;
+  h3 = ((h3 >> 18) | (h4 <<  8)) & 0xffffffff;
+
+  /* mac = (h + pad) % (2^128) */
+  f = (unsigned long long)h0 + st->pad[0]            ; h0 = (unsigned long)f;
+  f = (unsigned long long)h1 + st->pad[1] + (f >> 32); h1 = (unsigned long)f;
+  f = (unsigned long long)h2 + st->pad[2] + (f >> 32); h2 = (unsigned long)f;
+  f = (unsigned long long)h3 + st->pad[3] + (f >> 32); h3 = (unsigned long)f;
+
+  U32TO8(mac +  0, h0);
+  U32TO8(mac +  4, h1);
+  U32TO8(mac +  8, h2);
+  U32TO8(mac + 12, h3);
+
+  /* zero out the state */
+  st->h[0] = 0;
+  st->h[1] = 0;
+  st->h[2] = 0;
+  st->h[3] = 0;
+  st->h[4] = 0;
+  st->r[0] = 0;
+  st->r[1] = 0;
+  st->r[2] = 0;
+  st->r[3] = 0;
+  st->r[4] = 0;
+  st->pad[0] = 0;
+  st->pad[1] = 0;
+  st->pad[2] = 0;
+  st->pad[3] = 0;
+}
+
+} // anonymous namespace
+
+void Poly1305::compute(void *auth,const void *data,unsigned int len,const void *key)
+  throw()
+{
+  poly1305_context ctx;
+  poly1305_init(&ctx,reinterpret_cast<const unsigned char *>(key));
+  poly1305_update(&ctx,reinterpret_cast<const unsigned char *>(data),(size_t)len);
+  poly1305_finish(&ctx,reinterpret_cast<unsigned char *>(auth));
+}
+
 } // namespace ZeroTier
diff --git a/selftest.cpp b/selftest.cpp
index b899ee5a..e938cf4d 100644
--- a/selftest.cpp
+++ b/selftest.cpp
@@ -269,6 +269,22 @@ static int testCrypto()
 	}
 	std::cout << "PASS" << std::endl;
 
+	std::cout << "[crypto] Benchmarking Poly1305... "; std::cout.flush();
+	{
+		unsigned char *bb = (unsigned char *)::malloc(1234567);
+		for(unsigned int i=0;i<1234567;++i)
+			bb[i] = (unsigned char)i;
+		double bytes = 0.0;
+		uint64_t start = OSUtils::now();
+		for(unsigned int i=0;i<200;++i) {
+			Poly1305::compute(buf1,bb,1234567,poly1305TV0Key);
+			bytes += 1234567.0;
+		}
+		uint64_t end = OSUtils::now();
+		std::cout << ((bytes / 1048576.0) / ((double)(end - start) / 1000.0)) << " MiB/second" << std::endl;
+		::free((void *)bb);
+	}
+
 	std::cout << "[crypto] Testing C25519 and Ed25519 against test vectors... "; std::cout.flush();
 	for(int k=0;k<ZT_NUM_C25519_TEST_VECTORS;++k) {
 		C25519::Pair p1,p2;
-- 
cgit v1.2.3


From 1b2cac0cc59790620262f447588274440c7fbefa Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Wed, 7 Oct 2015 09:38:33 -0700
Subject: Trim some cruft that is not used and probably never would be.

---
 include/ZeroTierOne.h   | 27 +--------------------------
 node/IncomingPacket.cpp | 18 ++----------------
 node/Node.cpp           | 16 +---------------
 node/Node.hpp           |  6 ------
 4 files changed, 4 insertions(+), 63 deletions(-)

(limited to 'include')

diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h
index 341bb767..611e3699 100644
--- a/include/ZeroTierOne.h
+++ b/include/ZeroTierOne.h
@@ -255,31 +255,6 @@ enum ZT_Event
 	 */
 	ZT_EVENT_FATAL_ERROR_IDENTITY_COLLISION = 4,
 
-	/**
-	 * A more recent version was observed on the network
-	 *
-	 * Right now this is only triggered if a hub or rootserver reports a
-	 * more recent version, and only once. It can be used to trigger a
-	 * software update check.
-	 *
-	 * Meta-data: unsigned int[3], more recent version number
-	 */
-	ZT_EVENT_SAW_MORE_RECENT_VERSION = 5,
-
-	/**
-	 * A packet failed authentication
-	 *
-	 * Meta-data: struct sockaddr_storage containing origin address of packet
-	 */
-	ZT_EVENT_AUTHENTICATION_FAILURE = 6,
-
-	/**
-	 * A received packet was not valid
-	 *
-	 * Meta-data: struct sockaddr_storage containing origin address of packet
-	 */
-	ZT_EVENT_INVALID_PACKET = 7,
-
 	/**
 	 * Trace (debugging) message
 	 *
@@ -287,7 +262,7 @@ enum ZT_Event
 	 *
 	 * Meta-data: C string, TRACE message
 	 */
-	ZT_EVENT_TRACE = 8
+	ZT_EVENT_TRACE = 5
 };
 
 /**
diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp
index 305232ee..26d4bda8 100644
--- a/node/IncomingPacket.cpp
+++ b/node/IncomingPacket.cpp
@@ -227,7 +227,6 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR)
 				unsigned char key[ZT_PEER_SECRET_KEY_LENGTH];
 				if (RR->identity.agree(id,key,ZT_PEER_SECRET_KEY_LENGTH)) {
 					if (dearmor(key)) { // ensure packet is authentic, otherwise drop
-						RR->node->postEvent(ZT_EVENT_AUTHENTICATION_FAILURE,(const void *)&_remoteAddress);
 						TRACE("rejected HELLO from %s(%s): address already claimed",id.address().toString().c_str(),_remoteAddress.toString().c_str());
 						Packet outp(id.address(),RR->identity.address(),Packet::VERB_ERROR);
 						outp.append((unsigned char)Packet::VERB_HELLO);
@@ -236,11 +235,9 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR)
 						outp.armor(key,true);
 						RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 					} else {
-						RR->node->postEvent(ZT_EVENT_AUTHENTICATION_FAILURE,(const void *)&_remoteAddress);
 						TRACE("rejected HELLO from %s(%s): packet failed authentication",id.address().toString().c_str(),_remoteAddress.toString().c_str());
 					}
 				} else {
-					RR->node->postEvent(ZT_EVENT_AUTHENTICATION_FAILURE,(const void *)&_remoteAddress);
 					TRACE("rejected HELLO from %s(%s): key agreement failed",id.address().toString().c_str(),_remoteAddress.toString().c_str());
 				}
 
@@ -249,7 +246,6 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR)
 				// Identity is the same as the one we already have -- check packet integrity
 
 				if (!dearmor(peer->key())) {
-					RR->node->postEvent(ZT_EVENT_AUTHENTICATION_FAILURE,(const void *)&_remoteAddress);
 					TRACE("rejected HELLO from %s(%s): packet failed authentication",id.address().toString().c_str(),_remoteAddress.toString().c_str());
 					return true;
 				}
@@ -261,7 +257,6 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR)
 
 			// Check identity proof of work
 			if (!id.locallyValidate()) {
-				RR->node->postEvent(ZT_EVENT_AUTHENTICATION_FAILURE,(const void *)&_remoteAddress);
 				TRACE("dropped HELLO from %s(%s): identity invalid",id.address().toString().c_str(),_remoteAddress.toString().c_str());
 				return true;
 			}
@@ -269,7 +264,6 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR)
 			// Check packet integrity and authentication
 			SharedPtr<Peer> newPeer(new Peer(RR->identity,id));
 			if (!dearmor(newPeer->key())) {
-				RR->node->postEvent(ZT_EVENT_AUTHENTICATION_FAILURE,(const void *)&_remoteAddress);
 				TRACE("rejected HELLO from %s(%s): packet failed authentication",id.address().toString().c_str(),_remoteAddress.toString().c_str());
 				return true;
 			}
@@ -284,11 +278,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR)
 		peer->received(RR,_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_HELLO,0,Packet::VERB_NOP);
 		peer->setRemoteVersion(protoVersion,vMajor,vMinor,vRevision);
 
-		bool trusted = false;
-		if (RR->topology->isRoot(id)) {
-			RR->node->postNewerVersionIfNewer(vMajor,vMinor,vRevision);
-			trusted = true;
-		}
+		bool trusted = RR->topology->isRoot(id);
 		if (destAddr)
 			RR->sa->iam(id.address(),_remoteAddress,destAddr,trusted,RR->node->now());
 
@@ -369,11 +359,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 				peer->addDirectLatencyMeasurment(latency);
 				peer->setRemoteVersion(vProto,vMajor,vMinor,vRevision);
 
-				bool trusted = false;
-				if (RR->topology->isRoot(peer->identity())) {
-					RR->node->postNewerVersionIfNewer(vMajor,vMinor,vRevision);
-					trusted = true;
-				}
+				bool trusted = RR->topology->isRoot(peer->identity());
 				if (destAddr)
 					RR->sa->iam(peer->address(),_remoteAddress,destAddr,trusted,RR->node->now());
 			}	break;
diff --git a/node/Node.cpp b/node/Node.cpp
index d5cc50b9..d5ca147c 100644
--- a/node/Node.cpp
+++ b/node/Node.cpp
@@ -82,9 +82,6 @@ Node::Node(
 	_lastPingCheck(0),
 	_lastHousekeepingRun(0)
 {
-	_newestVersionSeen[0] = ZEROTIER_ONE_VERSION_MAJOR;
-	_newestVersionSeen[1] = ZEROTIER_ONE_VERSION_MINOR;
-	_newestVersionSeen[2] = ZEROTIER_ONE_VERSION_REVISION;
 	_online = false;
 
 	// Use Salsa20 alone as a high-quality non-crypto PRNG
@@ -540,16 +537,6 @@ std::string Node::dataStoreGet(const char *name)
 	return r;
 }
 
-void Node::postNewerVersionIfNewer(unsigned int major,unsigned int minor,unsigned int rev)
-{
-	if (Utils::compareVersion(major,minor,rev,_newestVersionSeen[0],_newestVersionSeen[1],_newestVersionSeen[2]) > 0) {
-		_newestVersionSeen[0] = major;
-		_newestVersionSeen[1] = minor;
-		_newestVersionSeen[2] = rev;
-		this->postEvent(ZT_EVENT_SAW_MORE_RECENT_VERSION,(const void *)_newestVersionSeen);
-	}
-}
-
 #ifdef ZT_TRACE
 void Node::postTrace(const char *module,unsigned int line,const char *fmt,...)
 {
@@ -659,8 +646,7 @@ enum ZT_ResultCode ZT_Node_processWirePacket(
 	} catch (std::bad_alloc &exc) {
 		return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY;
 	} catch ( ... ) {
-		reinterpret_cast<ZeroTier::Node *>(node)->postEvent(ZT_EVENT_INVALID_PACKET,(const void *)remoteAddress);
-		return ZT_RESULT_OK;
+		return ZT_RESULT_OK; // "OK" since invalid packets are simply dropped, but the system is still up
 	}
 }
 
diff --git a/node/Node.hpp b/node/Node.hpp
index 20c54471..0ae176c0 100644
--- a/node/Node.hpp
+++ b/node/Node.hpp
@@ -226,11 +226,6 @@ public:
 	 */
 	inline bool online() const throw() { return _online; }
 
-	/**
-	 * If this version is newer than the newest we've seen, post a new version seen event
-	 */
-	void postNewerVersionIfNewer(unsigned int major,unsigned int minor,unsigned int rev);
-
 #ifdef ZT_TRACE
 	void postTrace(const char *module,unsigned int line,const char *fmt,...);
 #endif
@@ -288,7 +283,6 @@ private:
 	uint64_t _now;
 	uint64_t _lastPingCheck;
 	uint64_t _lastHousekeepingRun;
-	unsigned int _newestVersionSeen[3]; // major, minor, revision
 	bool _online;
 };
 
-- 
cgit v1.2.3


From 6c7ce79c8960cd2360657f9247788ff5640ae974 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Wed, 7 Oct 2015 09:51:35 -0700
Subject: Be consistent in how enums are defined in the main .h file.

---
 include/ZeroTierOne.h | 130 +++++++++++++++++++++++++-------------------------
 1 file changed, 65 insertions(+), 65 deletions(-)

(limited to 'include')

diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h
index 611e3699..990f682d 100644
--- a/include/ZeroTierOne.h
+++ b/include/ZeroTierOne.h
@@ -389,6 +389,68 @@ enum ZT_VirtualNetworkConfigOperation
 	ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY = 4
 };
 
+/**
+ * Local interface trust levels
+ */
+enum ZT_LocalInterfaceAddressTrust {
+	ZT_LOCAL_INTERFACE_ADDRESS_TRUST_NORMAL = 0,
+	ZT_LOCAL_INTERFACE_ADDRESS_TRUST_PRIVACY = 10,
+	ZT_LOCAL_INTERFACE_ADDRESS_TRUST_ULTIMATE = 20
+};
+
+/**
+ * What trust hierarchy role does this peer have?
+ */
+enum ZT_PeerRole {
+	ZT_PEER_ROLE_LEAF = 0,     // ordinary node
+	ZT_PEER_ROLE_RELAY = 1,    // relay node
+	ZT_PEER_ROLE_ROOT = 2      // root server
+};
+
+/**
+ * Vendor ID
+ */
+enum ZT_Vendor {
+	ZT_VENDOR_UNSPECIFIED = 0,
+	ZT_VENDOR_ZEROTIER = 1
+};
+
+/**
+ * Platform type
+ */
+enum ZT_Platform {
+	ZT_PLATFORM_UNSPECIFIED = 0,
+	ZT_PLATFORM_LINUX = 1,
+	ZT_PLATFORM_WINDOWS = 2,
+	ZT_PLATFORM_MACOS = 3,
+	ZT_PLATFORM_ANDROID = 4,
+	ZT_PLATFORM_IOS = 5,
+	ZT_PLATFORM_SOLARIS_SMARTOS = 6,
+	ZT_PLATFORM_FREEBSD = 7,
+	ZT_PLATFORM_NETBSD = 8,
+	ZT_PLATFORM_OPENBSD = 9,
+	ZT_PLATFORM_RISCOS = 10,
+	ZT_PLATFORM_VXWORKS = 11,
+	ZT_PLATFORM_FREERTOS = 12,
+	ZT_PLATFORM_SYSBIOS = 13,
+	ZT_PLATFORM_HURD = 14
+};
+
+/**
+ * Architecture type
+ */
+enum ZT_Architecture {
+	ZT_ARCHITECTURE_UNSPECIFIED = 0,
+	ZT_ARCHITECTURE_X86 = 1,
+	ZT_ARCHITECTURE_X64 = 2,
+	ZT_ARCHITECTURE_ARM32 = 3,
+	ZT_ARCHITECTURE_ARM64 = 4,
+	ZT_ARCHITECTURE_MIPS32 = 5,
+	ZT_ARCHITECTURE_MIPS64 = 6,
+	ZT_ARCHITECTURE_POWER32 = 7,
+	ZT_ARCHITECTURE_POWER64 = 8
+};
+
 /**
  * Virtual network configuration
  */
@@ -536,15 +598,6 @@ typedef struct
 	int preferred;
 } ZT_PeerPhysicalPath;
 
-/**
- * What trust hierarchy role does this peer have?
- */
-enum ZT_PeerRole {
-	ZT_PEER_ROLE_LEAF = 0,     // ordinary node
-	ZT_PEER_ROLE_RELAY = 1,    // relay node
-	ZT_PEER_ROLE_ROOT = 2      // root server
-};
-
 /**
  * Peer status result buffer
  */
@@ -610,59 +663,6 @@ typedef struct
 	unsigned long peerCount;
 } ZT_PeerList;
 
-/**
- * Local interface trust levels
- */
-typedef enum {
-	ZT_LOCAL_INTERFACE_ADDRESS_TRUST_NORMAL = 0,
-	ZT_LOCAL_INTERFACE_ADDRESS_TRUST_PRIVACY = 10,
-	ZT_LOCAL_INTERFACE_ADDRESS_TRUST_ULTIMATE = 20
-} ZT_LocalInterfaceAddressTrust;
-
-/**
- * Vendor ID
- */
-typedef enum {
-	ZT_VENDOR_UNSPECIFIED = 0,
-	ZT_VENDOR_ZEROTIER = 1
-} ZT_Vendor;
-
-/**
- * Platform type
- */
-typedef enum {
-	ZT_PLATFORM_UNSPECIFIED = 0,
-	ZT_PLATFORM_LINUX = 1,
-	ZT_PLATFORM_WINDOWS = 2,
-	ZT_PLATFORM_MACOS = 3,
-	ZT_PLATFORM_ANDROID = 4,
-	ZT_PLATFORM_IOS = 5,
-	ZT_PLATFORM_SOLARIS_SMARTOS = 6,
-	ZT_PLATFORM_FREEBSD = 7,
-	ZT_PLATFORM_NETBSD = 8,
-	ZT_PLATFORM_OPENBSD = 9,
-	ZT_PLATFORM_RISCOS = 10,
-	ZT_PLATFORM_VXWORKS = 11,
-	ZT_PLATFORM_FREERTOS = 12,
-	ZT_PLATFORM_SYSBIOS = 13,
-	ZT_PLATFORM_HURD = 14
-} ZT_Platform;
-
-/**
- * Architecture type
- */
-typedef enum {
-	ZT_ARCHITECTURE_UNSPECIFIED = 0,
-	ZT_ARCHITECTURE_X86 = 1,
-	ZT_ARCHITECTURE_X64 = 2,
-	ZT_ARCHITECTURE_ARM32 = 3,
-	ZT_ARCHITECTURE_ARM64 = 4,
-	ZT_ARCHITECTURE_MIPS32 = 5,
-	ZT_ARCHITECTURE_MIPS64 = 6,
-	ZT_ARCHITECTURE_POWER32 = 7,
-	ZT_ARCHITECTURE_POWER64 = 8
-} ZT_Architecture;
-
 /**
  * ZeroTier circuit test configuration and path
  */
@@ -775,7 +775,7 @@ typedef struct {
 	/**
 	 * Remote device vendor ID
 	 */
-	ZT_Vendor vendor;
+	enum ZT_Vendor vendor;
 
 	/**
 	 * Remote device protocol compliance version
@@ -800,12 +800,12 @@ typedef struct {
 	/**
 	 * Platform / OS
 	 */
-	ZT_Platform platform;
+	enum ZT_Platform platform;
 
 	/**
 	 * System architecture
 	 */
-	ZT_Architecture architecture;
+	enum ZT_Architecture architecture;
 
 	/**
 	 * Local device address on which packet was received by reporting device
-- 
cgit v1.2.3


From 9347d6c866f6cae357448762f064a481fb765c00 Mon Sep 17 00:00:00 2001
From: Grant Limberg <glimberg@gmail.com>
Date: Wed, 7 Oct 2015 18:04:40 -0700
Subject: Make it so ZeroTierOne.h can be used with a C compiler again.

---
 include/ZeroTierOne.h | 4 ++--
 node/Node.cpp         | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

(limited to 'include')

diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h
index 990f682d..175cedc5 100644
--- a/include/ZeroTierOne.h
+++ b/include/ZeroTierOne.h
@@ -1255,7 +1255,7 @@ void ZT_Node_freeQueryResult(ZT_Node *node,void *qr);
  * @param trust How much do you trust the local network under this interface?
  * @return Boolean: non-zero if address was accepted and added
  */
-int ZT_Node_addLocalInterfaceAddress(ZT_Node *node,const struct sockaddr_storage *addr,int metric,ZT_LocalInterfaceAddressTrust trust);
+int ZT_Node_addLocalInterfaceAddress(ZT_Node *node,const struct sockaddr_storage *addr,int metric, enum ZT_LocalInterfaceAddressTrust trust);
 
 /**
  * Clear local interface addresses
@@ -1296,7 +1296,7 @@ void ZT_Node_setNetconfMaster(ZT_Node *node,void *networkConfigMasterInstance);
  * @param reportCallback Function to call each time a report is received
  * @return OK or error if, for example, test is too big for a packet or support isn't compiled in
  */
-ZT_ResultCode ZT_Node_circuitTestBegin(ZT_Node *node,ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *));
+enum ZT_ResultCode ZT_Node_circuitTestBegin(ZT_Node *node,ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *, ZT_CircuitTest *,const ZT_CircuitTestReport *));
 
 /**
  * Stop listening for results to a given circuit test
diff --git a/node/Node.cpp b/node/Node.cpp
index d5ca147c..7f469b97 100644
--- a/node/Node.cpp
+++ b/node/Node.cpp
@@ -779,7 +779,7 @@ void ZT_Node_setNetconfMaster(ZT_Node *node,void *networkControllerInstance)
 	} catch ( ... ) {}
 }
 
-ZT_ResultCode ZT_Node_circuitTestBegin(ZT_Node *node,ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *))
+enum ZT_ResultCode ZT_Node_circuitTestBegin(ZT_Node *node,ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *))
 {
 	try {
 		return reinterpret_cast<ZeroTier::Node *>(node)->circuitTestBegin(test,reportCallback);
@@ -795,7 +795,7 @@ void ZT_Node_circuitTestEnd(ZT_Node *node,ZT_CircuitTest *test)
 	} catch ( ... ) {}
 }
 
-int ZT_Node_addLocalInterfaceAddress(ZT_Node *node,const struct sockaddr_storage *addr,int metric,ZT_LocalInterfaceAddressTrust trust)
+int ZT_Node_addLocalInterfaceAddress(ZT_Node *node,const struct sockaddr_storage *addr,int metric, enum ZT_LocalInterfaceAddressTrust trust)
 {
 	try {
 		return reinterpret_cast<ZeroTier::Node *>(node)->addLocalInterfaceAddress(addr,metric,trust);
-- 
cgit v1.2.3


From aec13b50fdbb210e25c9bcfcb8f902da842ac65f Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Fri, 9 Oct 2015 15:05:26 -0700
Subject: Be a bit more verbose in circuit test reports to more clearly track
 current and upstream hop in graph traversal history.

---
 controller/SqliteNetworkController.cpp |  8 +++++---
 include/ZeroTierOne.h                  |  9 +++++++--
 node/IncomingPacket.cpp                | 12 +++++++-----
 node/Packet.hpp                        |  1 +
 4 files changed, 20 insertions(+), 10 deletions(-)

(limited to 'include')

diff --git a/controller/SqliteNetworkController.cpp b/controller/SqliteNetworkController.cpp
index d87e5624..08bd3a15 100644
--- a/controller/SqliteNetworkController.cpp
+++ b/controller/SqliteNetworkController.cpp
@@ -1881,7 +1881,7 @@ void SqliteNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTest
 	Utils::snprintf(tmp,sizeof(tmp),ZT_PATH_SEPARATOR_S"%.16llx_%.16llx",test->timestamp,test->testId);
 	reportSavePath.append(tmp);
 	OSUtils::mkdir(reportSavePath);
-	Utils::snprintf(tmp,sizeof(tmp),ZT_PATH_SEPARATOR_S"%.10llx",report->address);
+	Utils::snprintf(tmp,sizeof(tmp),ZT_PATH_SEPARATOR_S"%.10llx_%.10llx",report->upstream,report->current);
 	reportSavePath.append(tmp);
 
 	{
@@ -1891,7 +1891,8 @@ void SqliteNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTest
 			return;
 		fseek(f,0,SEEK_END);
 		fprintf(f,"%s{\n"
-			"\t\"address\": \"%.10llx\","ZT_EOL_S
+			"\t\"current\": \"%.10llx\","ZT_EOL_S
+			"\t\"upstream\": \"%.10llx\","ZT_EOL_S
 			"\t\"testId\": \"%.16llx\","ZT_EOL_S
 			"\t\"timestamp\": %llu,"ZT_EOL_S
 			"\t\"receivedTimestamp\": %llu,"ZT_EOL_S
@@ -1911,7 +1912,8 @@ void SqliteNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTest
 			"\t\"receivedFromRemoteAddress\": \"%s\""ZT_EOL_S
 			"}",
 			((ftell(f) > 0) ? ",\n" : ""),
-			(unsigned long long)report->address,
+			(unsigned long long)report->current,
+			(unsigned long long)report->upstream,
 			(unsigned long long)test->testId,
 			(unsigned long long)report->timestamp,
 			(unsigned long long)now,
diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h
index 175cedc5..80091f62 100644
--- a/include/ZeroTierOne.h
+++ b/include/ZeroTierOne.h
@@ -733,9 +733,14 @@ typedef struct {
  */
 typedef struct {
 	/**
-	 * Sender of report
+	 * Sender of report (current hop)
 	 */
-	uint64_t address;
+	uint64_t current;
+
+	/**
+	 * Previous hop
+	 */
+	uint64_t upstream;
 
 	/**
 	 * 64-bit test ID
diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp
index 5d31a5d4..443ffeeb 100644
--- a/node/IncomingPacket.cpp
+++ b/node/IncomingPacket.cpp
@@ -993,6 +993,7 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt
 			outp.append((uint16_t)0); // error code, currently unused
 			outp.append((uint64_t)0); // flags, currently unused
 			outp.append((uint64_t)packetId());
+			peer->address().appendTo(outp);
 			outp.append((uint8_t)hops());
 			_localAddress.serialize(outp);
 			_remoteAddress.serialize(outp);
@@ -1039,13 +1040,14 @@ bool IncomingPacket::_doCIRCUIT_TEST_REPORT(const RuntimeEnvironment *RR,const S
 		ZT_CircuitTestReport report;
 		memset(&report,0,sizeof(report));
 
-		report.address = peer->address().toInt();
+		report.current = peer->address().toInt();
+		report.upstream = Address(field(ZT_PACKET_IDX_PAYLOAD + 52,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH).toInt();
 		report.testId = at<uint64_t>(ZT_PACKET_IDX_PAYLOAD + 8);
 		report.timestamp = at<uint64_t>(ZT_PACKET_IDX_PAYLOAD);
 		report.remoteTimestamp = at<uint64_t>(ZT_PACKET_IDX_PAYLOAD + 16);
 		report.sourcePacketId = at<uint64_t>(ZT_PACKET_IDX_PAYLOAD + 44);
 		report.flags = at<uint64_t>(ZT_PACKET_IDX_PAYLOAD + 36);
-		report.sourcePacketHopCount = (*this)[ZT_PACKET_IDX_PAYLOAD + 52];
+		report.sourcePacketHopCount = (*this)[ZT_PACKET_IDX_PAYLOAD + 57]; // end of fixed length headers: 58
 		report.errorCode = at<uint16_t>(ZT_PACKET_IDX_PAYLOAD + 34);
 		report.vendor = (enum ZT_Vendor)((*this)[ZT_PACKET_IDX_PAYLOAD + 24]);
 		report.protocolVersion = (*this)[ZT_PACKET_IDX_PAYLOAD + 25];
@@ -1055,10 +1057,10 @@ bool IncomingPacket::_doCIRCUIT_TEST_REPORT(const RuntimeEnvironment *RR,const S
 		report.platform = (enum ZT_Platform)at<uint16_t>(ZT_PACKET_IDX_PAYLOAD + 30);
 		report.architecture = (enum ZT_Architecture)at<uint16_t>(ZT_PACKET_IDX_PAYLOAD + 32);
 
-		const unsigned int receivedOnLocalAddressLen = reinterpret_cast<InetAddress *>(&(report.receivedOnLocalAddress))->deserialize(*this,ZT_PACKET_IDX_PAYLOAD + 53);
-		const unsigned int receivedFromRemoteAddressLen = reinterpret_cast<InetAddress *>(&(report.receivedFromRemoteAddress))->deserialize(*this,ZT_PACKET_IDX_PAYLOAD + 53 + receivedOnLocalAddressLen);
+		const unsigned int receivedOnLocalAddressLen = reinterpret_cast<InetAddress *>(&(report.receivedOnLocalAddress))->deserialize(*this,ZT_PACKET_IDX_PAYLOAD + 58);
+		const unsigned int receivedFromRemoteAddressLen = reinterpret_cast<InetAddress *>(&(report.receivedFromRemoteAddress))->deserialize(*this,ZT_PACKET_IDX_PAYLOAD + 58 + receivedOnLocalAddressLen);
 
-		unsigned int nhptr = ZT_PACKET_IDX_PAYLOAD + 53 + receivedOnLocalAddressLen + receivedFromRemoteAddressLen;
+		unsigned int nhptr = ZT_PACKET_IDX_PAYLOAD + 58 + receivedOnLocalAddressLen + receivedFromRemoteAddressLen;
 		nhptr += at<uint16_t>(nhptr) + 2; // add "additional field" length, which right now will be zero
 
 		report.nextHopCount = (*this)[nhptr++];
diff --git a/node/Packet.hpp b/node/Packet.hpp
index 5c7253bf..6c178e66 100644
--- a/node/Packet.hpp
+++ b/node/Packet.hpp
@@ -1031,6 +1031,7 @@ public:
 		 *   <[2] 16-bit error code (set to 0, currently unused)>
 		 *   <[8] 64-bit report flags (set to 0, currently unused)>
 		 *   <[8] 64-bit source packet ID>
+		 *   <[5] upstream ZeroTier address from which test was received>
 		 *   <[1] 8-bit source packet hop count (ZeroTier hop count)>
 		 *   <[...] local wire address on which packet was received>
 		 *   <[...] remote wire address from which packet was received>
-- 
cgit v1.2.3


From cae58f43f1b18017b90499811772d107ea2f65b9 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Tue, 13 Oct 2015 08:49:36 -0700
Subject: More World stuff, and mkworld.

---
 include/ZeroTierOne.h   |   4 +-
 make-mac.mk             |   6 +-
 mkworld.cpp             | 153 ++++++++++++++++++++++++++++++++++++++++++++++++
 node/IncomingPacket.cpp |   1 -
 node/Node.cpp           |  24 +-------
 node/Node.hpp           |   3 +-
 node/Packet.hpp         |   6 +-
 node/Topology.cpp       |  78 +++++++++++++++---------
 node/Topology.hpp       |  22 ++++++-
 node/World.hpp          |  60 ++++++++++++-------
 one.cpp                 |  16 +----
 service/OneService.cpp  |   9 +--
 service/OneService.hpp  |   4 +-
 13 files changed, 281 insertions(+), 105 deletions(-)
 create mode 100644 mkworld.cpp

(limited to 'include')

diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h
index 80091f62..f69ab54c 100644
--- a/include/ZeroTierOne.h
+++ b/include/ZeroTierOne.h
@@ -1023,7 +1023,6 @@ typedef int (*ZT_WirePacketSendFunction)(
  * @param dataStorePutFunction Function called to put objects in persistent storage
  * @param virtualNetworkConfigFunction Function to be called when virtual LANs are created, deleted, or their config parameters change
  * @param eventCallback Function to receive status updates and non-fatal error notices
- * @param overrideRootTopology Alternative root server topology or NULL for default (mostly for test/debug use)
  * @return OK (0) or error code if a fatal error condition has occurred
  */
 enum ZT_ResultCode ZT_Node_new(
@@ -1035,8 +1034,7 @@ enum ZT_ResultCode ZT_Node_new(
 	ZT_WirePacketSendFunction wirePacketSendFunction,
 	ZT_VirtualNetworkFrameFunction virtualNetworkFrameFunction,
 	ZT_VirtualNetworkConfigFunction virtualNetworkConfigFunction,
-	ZT_EventCallback eventCallback,
-	const char *overrideRootTopology);
+	ZT_EventCallback eventCallback);
 
 /**
  * Delete a node and free all resources it consumes
diff --git a/make-mac.mk b/make-mac.mk
index 6daa6aa0..9fb613d8 100644
--- a/make-mac.mk
+++ b/make-mac.mk
@@ -79,6 +79,10 @@ selftest: $(OBJS) selftest.o
 	$(CXX) $(CXXFLAGS) -o zerotier-selftest selftest.o $(OBJS) $(LIBS)
 	$(STRIP) zerotier-selftest
 
+mkworld: $(OBJS)
+	rm -f mkworld
+	$(CXX) $(CXXFLAGS) -o mkworld mkworld.cpp $(OBJS) $(LIBS)
+
 # Requires Packages: http://s.sudre.free.fr/Software/Packages/about.html
 mac-dist-pkg: FORCE
 	packagesbuild "ext/installfiles/mac/ZeroTier One.pkgproj"
@@ -93,7 +97,7 @@ official: FORCE
 	make ZT_OFFICIAL_RELEASE=1 mac-dist-pkg
 
 clean:
-	rm -rf *.dSYM build-* *.pkg *.dmg *.o node/*.o controller/*.o service/*.o osdep/*.o ext/http-parser/*.o ext/lz4/*.o ext/json-parser/*.o zerotier-one zerotier-idtool zerotier-selftest zerotier-cli ZeroTierOneInstaller-*
+	rm -rf *.dSYM build-* *.pkg *.dmg *.o node/*.o controller/*.o service/*.o osdep/*.o ext/http-parser/*.o ext/lz4/*.o ext/json-parser/*.o zerotier-one zerotier-idtool zerotier-selftest zerotier-cli ZeroTierOneInstaller-* mkworld
 
 # For those building from source -- installs signed binary tap driver in system ZT home
 install-mac-tap: FORCE
diff --git a/mkworld.cpp b/mkworld.cpp
new file mode 100644
index 00000000..2b41d735
--- /dev/null
+++ b/mkworld.cpp
@@ -0,0 +1,153 @@
+/*
+ * ZeroTier One - Network Virtualization Everywhere
+ * Copyright (C) 2011-2015  ZeroTier, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * ZeroTier may be used and distributed under the terms of the GPLv3, which
+ * are available at: http://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * If you would like to embed ZeroTier into a commercial application or
+ * redistribute it in a modified binary form, please contact ZeroTier Networks
+ * LLC. Start here: http://www.zerotier.com/
+ */
+
+/*
+ * This utility makes the World from the configuration specified below.
+ * It probably won't be much use to anyone outside ZeroTier, Inc. except
+ * for testing and experimentation purposes.
+ *
+ * If you want to make your own World you must edit this file.
+ *
+ * When run, it expects two files in the current directory:
+ *
+ * previous.c25519 - key pair to sign this world (key from previous world)
+ * current.c25519 - key pair whose public key should be embedded in this world
+ *
+ * If these files do not exist, they are both created with the same key pair
+ * and a self-signed initial World is born.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+
+#include <string>
+#include <vector>
+#include <algorithm>
+
+#include "node/Constants.hpp"
+#include "node/World.hpp"
+#include "node/C25519.hpp"
+#include "node/Identity.hpp"
+#include "node/InetAddress.hpp"
+#include "osdep/OSUtils.hpp"
+
+using namespace ZeroTier;
+
+class WorldMaker : public World
+{
+public:
+	static inline World make(uint64_t id,uint64_t ts,const C25519::Public &sk,const std::vector<World::Root> &roots,const C25519::Pair &signWith)
+	{
+		WorldMaker w;
+		w._id = id;
+		w._ts = ts;
+		w._updateSigningKey = sk;
+		w._roots = roots;
+
+		Buffer<ZT_WORLD_MAX_SERIALIZED_LENGTH> tmp;
+		w.serialize(tmp,true);
+		w._signature = C25519::sign(signWith,tmp.data(),tmp.size());
+
+		return w;
+	}
+};
+
+int main(int argc,char **argv)
+{
+	std::string previous,current;
+	if ((!OSUtils::readFile("previous.c25519",previous))||(!OSUtils::readFile("current.c25519",current))) {
+		C25519::Pair np(C25519::generate());
+		previous = std::string();
+		previous.append((const char *)np.pub.data,ZT_C25519_PUBLIC_KEY_LEN);
+		previous.append((const char *)np.priv.data,ZT_C25519_PRIVATE_KEY_LEN);
+		current = previous;
+		OSUtils::writeFile("previous.c25519",previous);
+		OSUtils::writeFile("current.c25519",current);
+		fprintf(stderr,"INFO: created initial world keys: previous.c25519, current.c25519"ZT_EOL_S);
+	}
+
+	if ((previous.length() != (ZT_C25519_PUBLIC_KEY_LEN + ZT_C25519_PRIVATE_KEY_LEN))||(current.length() != (ZT_C25519_PUBLIC_KEY_LEN + ZT_C25519_PRIVATE_KEY_LEN))) {
+		fprintf(stderr,"FATAL: previous.c25519 or current.c25519 empty or invalid"ZT_EOL_S);
+		return 1;
+	}
+	C25519::Pair previousKP;
+	memcpy(previousKP.pub.data,previous.data(),ZT_C25519_PUBLIC_KEY_LEN);
+	memcpy(previousKP.priv.data,previous.data() + ZT_C25519_PUBLIC_KEY_LEN,ZT_C25519_PRIVATE_KEY_LEN);
+	C25519::Pair currentKP;
+	memcpy(currentKP.pub.data,current.data(),ZT_C25519_PUBLIC_KEY_LEN);
+	memcpy(currentKP.priv.data,current.data() + ZT_C25519_PUBLIC_KEY_LEN,ZT_C25519_PRIVATE_KEY_LEN);
+
+	// EDIT BELOW HERE ---------------------------------------------------------
+
+	std::vector<World::Root> roots;
+
+	// old US-SFO
+	roots.push_back(World::Root());
+	roots.back().identity = Identity("7e19876aba:0:2a6e2b2318930f60eb097f70d0f4b028b2cd6d3d0c63c014b9039ff35390e41181f216fb2e6fa8d95c1ee9667156411905c3dccfea78d8c6dfafba688170b3fa");
+	roots.back().stableEndpoints.push_back(InetAddress("198.199.97.220/9993"));
+	std::sort(roots.back().stableEndpoints.begin(),roots.back().stableEndpoints.end());
+
+	// old EU-PARIS
+	roots.push_back(World::Root());
+	roots.back().identity = Identity("8841408a2e:0:bb1d31f2c323e264e9e64172c1a74f77899555ed10751cd56e86405cde118d02dffe555d462ccf6a85b5631c12350c8d5dc409ba10b9025d0f445cf449d92b1c");
+	roots.back().stableEndpoints.push_back(InetAddress("107.191.46.210/9993"));
+	std::sort(roots.back().stableEndpoints.begin(),roots.back().stableEndpoints.end());
+
+	// old US-NYC
+	roots.push_back(World::Root());
+	roots.back().identity = Identity("8acf059fe3:0:482f6ee5dfe902319b419de5bdc765209c0ecda38c4d6e4fcf0d33658398b4527dcd22f93112fb9befd02fd78bf7261b333fc105d192a623ca9e50fc60b374a5");
+	roots.back().stableEndpoints.push_back(InetAddress("162.243.77.111/9993"));
+	std::sort(roots.back().stableEndpoints.begin(),roots.back().stableEndpoints.end());
+
+	// old AP-SNG
+	roots.push_back(World::Root());
+	roots.back().identity = Identity("9d219039f3:0:01f0922a98e3b34ebcbff333269dc265d7a020aab69d72be4d4acc9c8c9294785771256cd1d942a90d1bd1d2dca3ea84ef7d85afe6611fb43ff0b74126d90a6e");
+	roots.back().stableEndpoints.push_back(InetAddress("128.199.197.217/9993"));
+	std::sort(roots.back().stableEndpoints.begin(),roots.back().stableEndpoints.end());
+
+	std::sort(roots.begin(),roots.end());
+
+	const uint64_t id = ZT_WORLD_ID_EARTH;
+	const uint64_t ts = OSUtils::now();
+
+	// END WORLD SETUP ---------------------------------------------------------
+
+	fprintf(stderr,"INFO: generating and signing id==%llu ts==%llu"ZT_EOL_S,(unsigned long long)id,(unsigned long long)ts);
+
+	World nw = WorldMaker::make(id,ts,currentKP.pub,roots,previousKP);
+
+	Buffer<ZT_WORLD_MAX_SERIALIZED_LENGTH> outtmp;
+	nw.serialize(outtmp,false);
+	fwrite(outtmp.data(),outtmp.size(),1,stdout);
+	fflush(stdout);
+
+	fprintf(stderr,"INFO: wrote %u bytes to stdout"ZT_EOL_S,outtmp.size());
+
+	return 0;
+}
diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp
index 9fcc2e49..39abe720 100644
--- a/node/IncomingPacket.cpp
+++ b/node/IncomingPacket.cpp
@@ -33,7 +33,6 @@
 #include "../include/ZeroTierOne.h"
 
 #include "Constants.hpp"
-#include "Defaults.hpp"
 #include "RuntimeEnvironment.hpp"
 #include "IncomingPacket.hpp"
 #include "Topology.hpp"
diff --git a/node/Node.cpp b/node/Node.cpp
index 1eb21914..7496b045 100644
--- a/node/Node.cpp
+++ b/node/Node.cpp
@@ -46,7 +46,6 @@
 #include "Address.hpp"
 #include "Identity.hpp"
 #include "SelfAwareness.hpp"
-#include "Defaults.hpp"
 
 const struct sockaddr_storage ZT_SOCKADDR_NULL = {0};
 
@@ -64,8 +63,7 @@ Node::Node(
 	ZT_WirePacketSendFunction wirePacketSendFunction,
 	ZT_VirtualNetworkFrameFunction virtualNetworkFrameFunction,
 	ZT_VirtualNetworkConfigFunction virtualNetworkConfigFunction,
-	ZT_EventCallback eventCallback,
-	const char *overrideRootTopology) :
+	ZT_EventCallback eventCallback) :
 	_RR(this),
 	RR(&_RR),
 	_uPtr(uptr),
@@ -125,21 +123,6 @@ Node::Node(
 		throw;
 	}
 
-	Dictionary rt;
-	if (overrideRootTopology) {
-		rt.fromString(std::string(overrideRootTopology));
-	} else {
-		std::string rttmp(dataStoreGet("root-topology"));
-		if (rttmp.length() > 0) {
-			rt.fromString(rttmp);
-			if (!Topology::authenticateRootTopology(rt))
-				rt.clear();
-		}
-		if ((!rt.size())||(!rt.contains("rootservers")))
-			rt.fromString(ZT_DEFAULTS.defaultRootTopology);
-	}
-	RR->topology->setRootServers(Dictionary(rt.get("rootservers","")));
-
 	postEvent(ZT_EVENT_UP);
 }
 
@@ -609,12 +592,11 @@ enum ZT_ResultCode ZT_Node_new(
 	ZT_WirePacketSendFunction wirePacketSendFunction,
 	ZT_VirtualNetworkFrameFunction virtualNetworkFrameFunction,
 	ZT_VirtualNetworkConfigFunction virtualNetworkConfigFunction,
-	ZT_EventCallback eventCallback,
-	const char *overrideRootTopology)
+	ZT_EventCallback eventCallback)
 {
 	*node = (ZT_Node *)0;
 	try {
-		*node = reinterpret_cast<ZT_Node *>(new ZeroTier::Node(now,uptr,dataStoreGetFunction,dataStorePutFunction,wirePacketSendFunction,virtualNetworkFrameFunction,virtualNetworkConfigFunction,eventCallback,overrideRootTopology));
+		*node = reinterpret_cast<ZT_Node *>(new ZeroTier::Node(now,uptr,dataStoreGetFunction,dataStorePutFunction,wirePacketSendFunction,virtualNetworkFrameFunction,virtualNetworkConfigFunction,eventCallback));
 		return ZT_RESULT_OK;
 	} catch (std::bad_alloc &exc) {
 		return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY;
diff --git a/node/Node.hpp b/node/Node.hpp
index 0ae176c0..c7038ed4 100644
--- a/node/Node.hpp
+++ b/node/Node.hpp
@@ -71,8 +71,7 @@ public:
 		ZT_WirePacketSendFunction wirePacketSendFunction,
 		ZT_VirtualNetworkFrameFunction virtualNetworkFrameFunction,
 		ZT_VirtualNetworkConfigFunction virtualNetworkConfigFunction,
-		ZT_EventCallback eventCallback,
-		const char *overrideRootTopology);
+		ZT_EventCallback eventCallback);
 
 	~Node();
 
diff --git a/node/Packet.hpp b/node/Packet.hpp
index 958d0f3e..939d84a5 100644
--- a/node/Packet.hpp
+++ b/node/Packet.hpp
@@ -540,6 +540,8 @@ public:
 		 *   <[...] binary serialized identity (see Identity)>
 		 *   <[1] destination address type>
 		 *   [<[...] destination address>]
+		 *   <[8] 64-bit world ID of current world>
+		 *   <[8] 64-bit timestamp of current world>
 		 *
 		 * This is the only message that ever must be sent in the clear, since it
 		 * is used to push an identity to a new peer.
@@ -564,8 +566,8 @@ public:
 		 *   <[2] software revision (of responder)>
 		 *   <[1] destination address type (for this OK, not copied from HELLO)>
 		 *   [<[...] destination address>]
-		 *   <[8] 64-bit world ID of current world>
-		 *   <[8] 64-bit timestamp of current world>
+		 *   <[8] 64-bit world ID of current world (of responder)>
+		 *   <[8] 64-bit timestamp of current world (of responder)>
 		 *
 		 * ERROR has no payload.
 		 */
diff --git a/node/Topology.cpp b/node/Topology.cpp
index 5aedae86..0cf4cfe8 100644
--- a/node/Topology.cpp
+++ b/node/Topology.cpp
@@ -42,23 +42,6 @@ Topology::Topology(const RuntimeEnvironment *renv) :
 	RR(renv),
 	_amRoot(false)
 {
-	try {
-		std::string dsWorld(RR->node->dataStoreGet("world"));
-		Buffer<ZT_WORLD_MAX_SERIALIZED_LENGTH> dswtmp(dsWorld.data(),dsWorld.length());
-		_world.deserialize(dswtmp,0);
-	} catch ( ... ) {
-		_world = World(); // set to null if cached world is invalid
-	}
-	{
-		World defaultWorld;
-		Buffer<ZT_DEFAULT_WORLD_LENGTH> wtmp(ZT_DEFAULT_WORLD,ZT_DEFAULT_WORLD_LENGTH);
-		defaultWorld.deserialize(wtmp,0); // throws on error, which would indicate a bad static variable up top
-		if (_world.verifyUpdate(defaultWorld)) {
-			_world = defaultWorld;
-			RR->node->dataStorePut("world",ZT_DEFAULT_WORLD,ZT_DEFAULT_WORLD_LENGTH,false);
-		}
-	}
-
 	std::string alls(RR->node->dataStoreGet("peers.save"));
 	const uint8_t *all = reinterpret_cast<const uint8_t *>(alls.data());
 	RR->node->dataStoreDelete("peers.save");
@@ -97,19 +80,24 @@ Topology::Topology(const RuntimeEnvironment *renv) :
 
 	clean(RR->node->now());
 
-	for(std::vector<World::Root>::const_iterator r(_world.roots().begin());r!=_world.roots().end();++r) {
-		if (r->identity == RR->identity)
-			_amRoot = true;
-		_rootAddresses.push_back(r->identity.address());
-		SharedPtr<Peer> *rp = _peers.get(r->identity.address());
-		if (rp) {
-			_rootPeers.push_back(*rp);
-		} else if (r->identity.address() != RR->identity.address()) {
-			SharedPtr<Peer> newrp(new Peer(RR->identity,r->identity));
-			_peers.set(r->identity.address(),newrp);
-			_rootPeers.push_back(newrp);
-		}
+	std::string dsWorld(RR->node->dataStoreGet("world"));
+	World cachedWorld;
+	try {
+		Buffer<ZT_WORLD_MAX_SERIALIZED_LENGTH> dswtmp(dsWorld.data(),dsWorld.length());
+		cachedWorld.deserialize(dswtmp,0);
+	} catch ( ... ) {
+		cachedWorld = World(); // clear if cached world is invalid
 	}
+	World defaultWorld;
+	{
+		Buffer<ZT_DEFAULT_WORLD_LENGTH> wtmp(ZT_DEFAULT_WORLD,ZT_DEFAULT_WORLD_LENGTH);
+		defaultWorld.deserialize(wtmp,0); // throws on error, which would indicate a bad static variable up top
+	}
+	if (cachedWorld.shouldBeReplacedBy(defaultWorld,false)) {
+		_setWorld(defaultWorld);
+		if (dsWorld.length() > 0)
+			RR->node->dataStoreDelete("world");
+	} else _setWorld(cachedWorld);
 }
 
 Topology::~Topology()
@@ -283,6 +271,16 @@ keep_searching_for_roots:
 	return bestRoot;
 }
 
+bool Topology::worldUpdateIfValid(const World &newWorld)
+{
+	Mutex::Lock _l(_lock);
+	if (_world.shouldBeReplacedBy(newWorld,true)) {
+		_setWorld(newWorld);
+		return true;
+	}
+	return false;
+}
+
 void Topology::clean(uint64_t now)
 {
 	Mutex::Lock _l(_lock);
@@ -320,4 +318,26 @@ void Topology::_saveIdentity(const Identity &id)
 	}
 }
 
+void Topology::_setWorld(const World &newWorld)
+{
+	// assumed _lock is locked (or in constructor)
+	_world = newWorld;
+	_amRoot = false;
+	_rootAddresses.clear();
+	_rootPeers.clear();
+	for(std::vector<World::Root>::const_iterator r(_world.roots().begin());r!=_world.roots().end();++r) {
+		if (r->identity == RR->identity)
+			_amRoot = true;
+		_rootAddresses.push_back(r->identity.address());
+		SharedPtr<Peer> *rp = _peers.get(r->identity.address());
+		if (rp) {
+			_rootPeers.push_back(*rp);
+		} else if (r->identity.address() != RR->identity.address()) {
+			SharedPtr<Peer> newrp(new Peer(RR->identity,r->identity));
+			_peers.set(r->identity.address(),newrp);
+			_rootPeers.push_back(newrp);
+		}
+	}
+}
+
 } // namespace ZeroTier
diff --git a/node/Topology.hpp b/node/Topology.hpp
index ed8f3d86..3abc27e4 100644
--- a/node/Topology.hpp
+++ b/node/Topology.hpp
@@ -31,10 +31,10 @@
 #include <stdio.h>
 #include <string.h>
 
-#include <map>
 #include <vector>
 #include <stdexcept>
 #include <algorithm>
+#include <utility>
 
 #include "Constants.hpp"
 
@@ -146,6 +146,23 @@ public:
 		return _world;
 	}
 
+	/**
+	 * @return Pair containing world ID and world timestamp (faster than world().id() etc.)
+	 */
+	inline std::pair<uint64_t,uint64_t> worldIdentification() const
+	{
+		Mutex::Lock _l(_lock);
+		return std::pair<uint64_t,uint64_t>(_world.id(),_world.timestamp());
+	}
+
+	/**
+	 * Validate new world and update if newer and signature is okay
+	 *
+	 * @param newWorld Potential new world definition revision
+	 * @return True if an update actually occurred
+	 */
+	bool worldUpdateIfValid(const World &newWorld);
+
 	/**
 	 * Clean and flush database
 	 */
@@ -176,7 +193,7 @@ public:
 	}
 
 	/**
-	 * @return All currently active peers by address
+	 * @return All currently active peers by address (unsorted)
 	 */
 	inline std::vector< std::pair< Address,SharedPtr<Peer> > > allPeers() const
 	{
@@ -187,6 +204,7 @@ public:
 private:
 	Identity _getIdentity(const Address &zta);
 	void _saveIdentity(const Identity &id);
+	void _setWorld(const World &newWorld);
 
 	const RuntimeEnvironment *RR;
 
diff --git a/node/World.hpp b/node/World.hpp
index 0d26021f..7ccd2c53 100644
--- a/node/World.hpp
+++ b/node/World.hpp
@@ -55,7 +55,7 @@
 /**
  * The (more than) maximum length of a serialized World
  */
-#define ZT_WORLD_MAX_SERIALIZED_LENGTH (((1024 + (32 * ZT_WORLD_MAX_STABLE_ENDPOINTS_PER_ROOT)) * ZT_WORLD_MAX_ROOTS) + ZT_C25519_PUBLIC_KEY_LEN + ZT_C25519_SIGNATURE_LEN + 64)
+#define ZT_WORLD_MAX_SERIALIZED_LENGTH (((1024 + (32 * ZT_WORLD_MAX_STABLE_ENDPOINTS_PER_ROOT)) * ZT_WORLD_MAX_ROOTS) + ZT_C25519_PUBLIC_KEY_LEN + ZT_C25519_SIGNATURE_LEN + 128)
 
 /**
  * World ID indicating null / empty World object
@@ -68,15 +68,11 @@
 #define ZT_WORLD_ID_TESTNET 1
 
 /**
- * World ID for Earth -- its approximate distance from the sun in kilometers
+ * World ID for Earth
  *
  * This is the ID for the ZeroTier World used on planet Earth. It is unrelated
- * to the public network 8056c2e21c000001 of the same name.
- *
- * It's advisable to create a new World for network regions spaced more than
- * 2-3 light seconds, since RTT times in excess of 5s are problematic for some
- * protocols. Earth could therefore include its low and high orbits, the Moon,
- * and nearby Lagrange points.
+ * to the public network 8056c2e21c000001 of the same name. It was chosen
+ * from Earth's approximate distance from the sun in kilometers.
  */
 #define ZT_WORLD_ID_EARTH 149604618
 
@@ -90,9 +86,24 @@ namespace ZeroTier {
 /**
  * A world definition (formerly known as a root topology)
  *
- * A world consists of a set of root servers and a signature scheme enabling
- * it to be updated going forward. It defines a single ZeroTier VL1 network
- * area within which any device can reach any other.
+ * Think of a World as a single data center. Within this data center a set
+ * of distributed fault tolerant root servers provide stable anchor points
+ * for a peer to peer network that provides VLAN service. Updates to a world
+ * definition can be published by signing them with the previous revision's
+ * signing key, and should be very infrequent.
+ *
+ * The maximum data center size is approximately 2.5 cubic light seconds,
+ * since many protocols have issues with >5s RTT latencies.
+ *
+ * ZeroTier operates a World for Earth capable of encompassing the planet, its
+ * orbits, the Moon (about 1.3 light seconds), and nearby Lagrange points. A
+ * world ID for Mars and nearby space is defined but not yet used, and a test
+ * world ID is provided for testing purposes.
+ *
+ * If you absolutely must run your own "unofficial" ZeroTier network, please
+ * define your world IDs above 0xffffffff (4294967295). Code to make a World
+ * is in mkworld.cpp in the parent directory and must be edited to change
+ * settings.
  */
 class World
 {
@@ -130,24 +141,28 @@ public:
 	inline uint64_t timestamp() const throw() { return _ts; }
 
 	/**
-	 * Verify a world update
+	 * Check whether a world update should replace this one
 	 *
 	 * A new world update is valid if it is for the same world ID, is newer,
 	 * and is signed by the current world's signing key. If this world object
 	 * is null, it can always be updated.
 	 *
 	 * @param update Candidate update
+	 * @param fullSignatureCheck Perform full cryptographic signature check (true == yes, false == skip)
 	 * @return True if update is newer than current and is properly signed
 	 */
-	inline bool verifyUpdate(const World &update)
+	inline bool shouldBeReplacedBy(const World &update,bool fullSignatureCheck)
 	{
 		if (_id == ZT_WORLD_ID_NULL)
 			return true;
-		if ((update._id != _id)||(update._ts <= _ts))
-			return false;
-		Buffer<ZT_WORLD_MAX_SERIALIZED_LENGTH> tmp;
-		update.serialize(tmp);
-		return C25519::verify(_updateSigningKey,tmp.data(),tmp.size(),update._signature);
+		if ((_id == update._id)&&(_ts < update._ts)) {
+			if (fullSignatureCheck) {
+				Buffer<ZT_WORLD_MAX_SERIALIZED_LENGTH> tmp;
+				update.serialize(tmp,true);
+				return C25519::verify(_updateSigningKey,tmp.data(),tmp.size(),update._signature);
+			} else return true;
+		}
+		return false;
 	}
 
 	/**
@@ -156,13 +171,16 @@ public:
 	inline operator bool() const throw() { return (_id != ZT_WORLD_ID_NULL); }
 
 	template<unsigned int C>
-	inline void serialize(Buffer<C> &b) const
+	inline void serialize(Buffer<C> &b,bool forSign = false) const
 	{
+		if (forSign)
+			b.append((uint64_t)0x7f7f7f7f7f7f7f7fULL);
 		b.append((uint8_t)0x01); // version -- only one valid value for now
 		b.append((uint64_t)_id);
 		b.append((uint64_t)_ts);
 		b.append(_updateSigningKey.data,ZT_C25519_PUBLIC_KEY_LEN);
-		b.append(_signature.data,ZT_C25519_SIGNATURE_LEN);
+		if (!forSign)
+			b.append(_signature.data,ZT_C25519_SIGNATURE_LEN);
 		b.append((uint8_t)_roots.size());
 		for(std::vector<Root>::const_iterator r(_roots.begin());r!=_roots.end();++r) {
 			r->identity.serialize(b);
@@ -170,6 +188,8 @@ public:
 			for(std::vector<InetAddress>::const_iterator ep(r->stableEndpoints.begin());ep!=r->stableEndpoints.end();++ep)
 				ep->serialize(b);
 		}
+		if (forSign)
+			b.append((uint64_t)0xf7f7f7f7f7f7f7f7ULL);
 	}
 
 	template<unsigned int C>
diff --git a/one.cpp b/one.cpp
index a4d5190c..c8661fda 100644
--- a/one.cpp
+++ b/one.cpp
@@ -911,7 +911,6 @@ static void printHelp(const char *cn,FILE *out)
 	fprintf(out,"  -v                - Show version"ZT_EOL_S);
 	fprintf(out,"  -U                - Run as unprivileged user (skip privilege check)"ZT_EOL_S);
 	fprintf(out,"  -p<port>          - Port for UDP and TCP/HTTP (default: 9993, 0 for random)"ZT_EOL_S);
-	//fprintf(out,"  -T<path>          - Override root topology, do not authenticate or update"ZT_EOL_S);
 
 #ifdef __UNIX_LIKE__
 	fprintf(out,"  -d                - Fork and run as daemon (Unix-ish OSes)"ZT_EOL_S);
@@ -974,7 +973,6 @@ int main(int argc,char **argv)
 	if ((strstr(argv[0],"zerotier-cli"))||(strstr(argv[0],"ZEROTIER-CLI")))
 		return cli(argc,argv);
 
-	std::string overrideRootTopology;
 	std::string homeDir;
 	unsigned int port = ZT_DEFAULT_PORT;
 	bool skipRootCheck = false;
@@ -1001,18 +999,6 @@ int main(int argc,char **argv)
 					skipRootCheck = true;
 					break;
 
-				case 'T': // Override root topology
-					if (argv[i][2]) {
-						if (!OSUtils::readFile(argv[i] + 2,overrideRootTopology)) {
-							fprintf(stderr,"%s: cannot read root topology from %s"ZT_EOL_S,argv[0],argv[i] + 2);
-							return 1;
-						}
-					} else {
-						printHelp(argv[0],stdout);
-						return 1;
-					}
-					break;
-
 				case 'v': // Display version
 					printf("%d.%d.%d"ZT_EOL_S,ZEROTIER_ONE_VERSION_MAJOR,ZEROTIER_ONE_VERSION_MINOR,ZEROTIER_ONE_VERSION_REVISION);
 					return 0;
@@ -1169,7 +1155,7 @@ int main(int argc,char **argv)
 
 	try {
 		for(;;) {
-			zt1Service = OneService::newInstance(homeDir.c_str(),port,(overrideRootTopology.length() > 0) ? overrideRootTopology.c_str() : (const char *)0);
+			zt1Service = OneService::newInstance(homeDir.c_str(),port);
 			switch(zt1Service->run()) {
 				case OneService::ONE_STILL_RUNNING: // shouldn't happen, run() won't return until done
 				case OneService::ONE_NORMAL_TERMINATION:
diff --git a/service/OneService.cpp b/service/OneService.cpp
index 071a2cbc..6b28c41e 100644
--- a/service/OneService.cpp
+++ b/service/OneService.cpp
@@ -418,14 +418,13 @@ struct TcpConnection
 class OneServiceImpl : public OneService
 {
 public:
-	OneServiceImpl(const char *hp,unsigned int port,const char *overrideRootTopology) :
+	OneServiceImpl(const char *hp,unsigned int port) :
 		_homePath((hp) ? hp : "."),
 		_tcpFallbackResolver(ZT_TCP_FALLBACK_RELAY),
 #ifdef ZT_ENABLE_NETWORK_CONTROLLER
 		_controller((SqliteNetworkController *)0),
 #endif
 		_phy(this,false,true),
-		_overrideRootTopology((overrideRootTopology) ? overrideRootTopology : ""),
 		_node((Node *)0),
 		_controlPlane((ControlPlane *)0),
 		_lastDirectReceiveFromGlobal(0),
@@ -550,8 +549,7 @@ public:
 				SnodeWirePacketSendFunction,
 				SnodeVirtualNetworkFrameFunction,
 				SnodeVirtualNetworkConfigFunction,
-				SnodeEventCallback,
-				((_overrideRootTopology.length() > 0) ? _overrideRootTopology.c_str() : (const char *)0));
+				SnodeEventCallback);
 
 #ifdef ZT_ENABLE_NETWORK_CONTROLLER
 			_controller = new SqliteNetworkController(_node,(_homePath + ZT_PATH_SEPARATOR_S + ZT_CONTROLLER_DB_PATH).c_str(),(_homePath + ZT_PATH_SEPARATOR_S + "circuitTestResults.d").c_str());
@@ -1329,7 +1327,6 @@ private:
 	SqliteNetworkController *_controller;
 #endif
 	Phy<OneServiceImpl *> _phy;
-	std::string _overrideRootTopology;
 	Node *_node;
 	InetAddress _v4LocalAddress,_v6LocalAddress;
 	PhySocket *_v4UdpSocket;
@@ -1526,7 +1523,7 @@ std::string OneService::autoUpdateUrl()
 	return std::string();
 }
 
-OneService *OneService::newInstance(const char *hp,unsigned int port,const char *overrideRootTopology) { return new OneServiceImpl(hp,port,overrideRootTopology); }
+OneService *OneService::newInstance(const char *hp,unsigned int port) { return new OneServiceImpl(hp,port); }
 OneService::~OneService() {}
 
 } // namespace ZeroTier
diff --git a/service/OneService.hpp b/service/OneService.hpp
index 70d024bc..2f76ebaa 100644
--- a/service/OneService.hpp
+++ b/service/OneService.hpp
@@ -95,12 +95,10 @@ public:
 	 *
 	 * @param hp Home path
 	 * @param port TCP and UDP port for packets and HTTP control (if 0, pick random port)
-	 * @param overrideRootTopology String-serialized root topology (for testing, default: NULL)
 	 */
 	static OneService *newInstance(
 		const char *hp,
-		unsigned int port,
-		const char *overrideRootTopology = (const char *)0);
+		unsigned int port);
 
 	virtual ~OneService();
 
-- 
cgit v1.2.3


From 5d2f523e81a56a33405d2b98ccef9d384e269f34 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Tue, 13 Oct 2015 12:10:44 -0700
Subject: World stuff...

---
 .gitignore               |   3 +-
 include/ZeroTierOne.h    |   5 --
 make-mac.mk              |   4 --
 mkworld.cpp              | 170 -----------------------------------------------
 node/IncomingPacket.cpp  |  34 ++++++++--
 node/Node.cpp            |  39 ++++++-----
 node/Packet.hpp          |  35 +---------
 node/Peer.cpp            |  71 +++-----------------
 node/Peer.hpp            |  25 ++-----
 node/RemotePath.hpp      |  26 ++------
 node/Topology.cpp        |  11 ++-
 node/Topology.hpp        |  15 +++--
 service/ControlPlane.cpp |   2 -
 world/2015-10-13.bin     | Bin 0 -> 494 bytes
 world/2015-10-13.out     |   6 ++
 world/mkworld.cpp        | 170 +++++++++++++++++++++++++++++++++++++++++++++++
 16 files changed, 271 insertions(+), 345 deletions(-)
 delete mode 100644 mkworld.cpp
 create mode 100644 world/2015-10-13.bin
 create mode 100644 world/2015-10-13.out
 create mode 100644 world/mkworld.cpp

(limited to 'include')

diff --git a/.gitignore b/.gitignore
index 1cb16da5..2dbec1e5 100755
--- a/.gitignore
+++ b/.gitignore
@@ -31,7 +31,8 @@ Thumbs.db
 /ZeroTierOneInstaller-*
 /examples/docker/zerotier-one
 /examples/docker/test-*.env
-/mkworld
+/world/mkworld
+/world/*.c25519
 
 # Miscellaneous file types that we don't want to check in
 *.log
diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h
index f69ab54c..38db3222 100644
--- a/include/ZeroTierOne.h
+++ b/include/ZeroTierOne.h
@@ -582,11 +582,6 @@ typedef struct
 	 */
 	uint64_t lastReceive;
 
-	/**
-	 * Is path fixed? (i.e. not learned, static)
-	 */
-	int fixed;
-
 	/**
 	 * Is path active?
 	 */
diff --git a/make-mac.mk b/make-mac.mk
index 9fb613d8..174216fb 100644
--- a/make-mac.mk
+++ b/make-mac.mk
@@ -79,10 +79,6 @@ selftest: $(OBJS) selftest.o
 	$(CXX) $(CXXFLAGS) -o zerotier-selftest selftest.o $(OBJS) $(LIBS)
 	$(STRIP) zerotier-selftest
 
-mkworld: $(OBJS)
-	rm -f mkworld
-	$(CXX) $(CXXFLAGS) -o mkworld mkworld.cpp $(OBJS) $(LIBS)
-
 # Requires Packages: http://s.sudre.free.fr/Software/Packages/about.html
 mac-dist-pkg: FORCE
 	packagesbuild "ext/installfiles/mac/ZeroTier One.pkgproj"
diff --git a/mkworld.cpp b/mkworld.cpp
deleted file mode 100644
index baf693fc..00000000
--- a/mkworld.cpp
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * ZeroTier One - Network Virtualization Everywhere
- * Copyright (C) 2011-2015  ZeroTier, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * --
- *
- * ZeroTier may be used and distributed under the terms of the GPLv3, which
- * are available at: http://www.gnu.org/licenses/gpl-3.0.html
- *
- * If you would like to embed ZeroTier into a commercial application or
- * redistribute it in a modified binary form, please contact ZeroTier Networks
- * LLC. Start here: http://www.zerotier.com/
- */
-
-/*
- * This utility makes the World from the configuration specified below.
- * It probably won't be much use to anyone outside ZeroTier, Inc. except
- * for testing and experimentation purposes.
- *
- * If you want to make your own World you must edit this file.
- *
- * When run, it expects two files in the current directory:
- *
- * previous.c25519 - key pair to sign this world (key from previous world)
- * current.c25519 - key pair whose public key should be embedded in this world
- *
- * If these files do not exist, they are both created with the same key pair
- * and a self-signed initial World is born.
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <stdint.h>
-
-#include <string>
-#include <vector>
-#include <algorithm>
-
-#include "node/Constants.hpp"
-#include "node/World.hpp"
-#include "node/C25519.hpp"
-#include "node/Identity.hpp"
-#include "node/InetAddress.hpp"
-#include "osdep/OSUtils.hpp"
-
-using namespace ZeroTier;
-
-class WorldMaker : public World
-{
-public:
-	static inline World make(uint64_t id,uint64_t ts,const C25519::Public &sk,const std::vector<World::Root> &roots,const C25519::Pair &signWith)
-	{
-		WorldMaker w;
-		w._id = id;
-		w._ts = ts;
-		w._updateSigningKey = sk;
-		w._roots = roots;
-
-		Buffer<ZT_WORLD_MAX_SERIALIZED_LENGTH> tmp;
-		w.serialize(tmp,true);
-		w._signature = C25519::sign(signWith,tmp.data(),tmp.size());
-
-		return w;
-	}
-};
-
-int main(int argc,char **argv)
-{
-	std::string previous,current;
-	if ((!OSUtils::readFile("previous.c25519",previous))||(!OSUtils::readFile("current.c25519",current))) {
-		C25519::Pair np(C25519::generate());
-		previous = std::string();
-		previous.append((const char *)np.pub.data,ZT_C25519_PUBLIC_KEY_LEN);
-		previous.append((const char *)np.priv.data,ZT_C25519_PRIVATE_KEY_LEN);
-		current = previous;
-		OSUtils::writeFile("previous.c25519",previous);
-		OSUtils::writeFile("current.c25519",current);
-		fprintf(stderr,"INFO: created initial world keys: previous.c25519, current.c25519"ZT_EOL_S);
-	}
-
-	if ((previous.length() != (ZT_C25519_PUBLIC_KEY_LEN + ZT_C25519_PRIVATE_KEY_LEN))||(current.length() != (ZT_C25519_PUBLIC_KEY_LEN + ZT_C25519_PRIVATE_KEY_LEN))) {
-		fprintf(stderr,"FATAL: previous.c25519 or current.c25519 empty or invalid"ZT_EOL_S);
-		return 1;
-	}
-	C25519::Pair previousKP;
-	memcpy(previousKP.pub.data,previous.data(),ZT_C25519_PUBLIC_KEY_LEN);
-	memcpy(previousKP.priv.data,previous.data() + ZT_C25519_PUBLIC_KEY_LEN,ZT_C25519_PRIVATE_KEY_LEN);
-	C25519::Pair currentKP;
-	memcpy(currentKP.pub.data,current.data(),ZT_C25519_PUBLIC_KEY_LEN);
-	memcpy(currentKP.priv.data,current.data() + ZT_C25519_PUBLIC_KEY_LEN,ZT_C25519_PRIVATE_KEY_LEN);
-
-	// EDIT BELOW HERE ---------------------------------------------------------
-
-	std::vector<World::Root> roots;
-
-	// old US-SFO
-	roots.push_back(World::Root());
-	roots.back().identity = Identity("7e19876aba:0:2a6e2b2318930f60eb097f70d0f4b028b2cd6d3d0c63c014b9039ff35390e41181f216fb2e6fa8d95c1ee9667156411905c3dccfea78d8c6dfafba688170b3fa");
-	roots.back().stableEndpoints.push_back(InetAddress("198.199.97.220/9993"));
-	std::sort(roots.back().stableEndpoints.begin(),roots.back().stableEndpoints.end());
-
-	// old EU-PARIS
-	roots.push_back(World::Root());
-	roots.back().identity = Identity("8841408a2e:0:bb1d31f2c323e264e9e64172c1a74f77899555ed10751cd56e86405cde118d02dffe555d462ccf6a85b5631c12350c8d5dc409ba10b9025d0f445cf449d92b1c");
-	roots.back().stableEndpoints.push_back(InetAddress("107.191.46.210/9993"));
-	std::sort(roots.back().stableEndpoints.begin(),roots.back().stableEndpoints.end());
-
-	// old US-NYC
-	roots.push_back(World::Root());
-	roots.back().identity = Identity("8acf059fe3:0:482f6ee5dfe902319b419de5bdc765209c0ecda38c4d6e4fcf0d33658398b4527dcd22f93112fb9befd02fd78bf7261b333fc105d192a623ca9e50fc60b374a5");
-	roots.back().stableEndpoints.push_back(InetAddress("162.243.77.111/9993"));
-	std::sort(roots.back().stableEndpoints.begin(),roots.back().stableEndpoints.end());
-
-	// old AP-SNG
-	roots.push_back(World::Root());
-	roots.back().identity = Identity("9d219039f3:0:01f0922a98e3b34ebcbff333269dc265d7a020aab69d72be4d4acc9c8c9294785771256cd1d942a90d1bd1d2dca3ea84ef7d85afe6611fb43ff0b74126d90a6e");
-	roots.back().stableEndpoints.push_back(InetAddress("128.199.197.217/9993"));
-	std::sort(roots.back().stableEndpoints.begin(),roots.back().stableEndpoints.end());
-
-	std::sort(roots.begin(),roots.end());
-
-	const uint64_t id = ZT_WORLD_ID_EARTH;
-	const uint64_t ts = OSUtils::now();
-
-	// END WORLD SETUP ---------------------------------------------------------
-
-	fprintf(stderr,"INFO: generating and signing id==%llu ts==%llu"ZT_EOL_S,(unsigned long long)id,(unsigned long long)ts);
-
-	World nw = WorldMaker::make(id,ts,currentKP.pub,roots,previousKP);
-
-	Buffer<ZT_WORLD_MAX_SERIALIZED_LENGTH> outtmp;
-	nw.serialize(outtmp,false);
-	World testw;
-	testw.deserialize(outtmp,0);
-	if (testw != nw) {
-		fprintf(stderr,"FATAL: serialization test failed!"ZT_EOL_S);
-		return 1;
-	}
-	fwrite(outtmp.data(),outtmp.size(),1,stdout);
-	fflush(stdout);
-
-	fprintf(stderr,"INFO: wrote %u bytes to stdout"ZT_EOL_S,outtmp.size());
-
-	fprintf(stderr,ZT_EOL_S);
-	fprintf(stderr,"#define ZT_DEFAULT_WORLD_LENGTH %u"ZT_EOL_S,outtmp.size());
-	fprintf(stderr,"static const unsigned char ZT_DEFAULT_WORLD[ZT_DEFAULT_WORLD_LENGTH] = {");
-	for(unsigned int i=0;i<outtmp.size();++i) {
-		const unsigned char *d = (const unsigned char *)outtmp.data();
-		if (i > 0)
-			fprintf(stderr,",");
-		fprintf(stderr,"0x%.2x",(unsigned int)d[i]);
-	}
-	fprintf(stderr,"};"ZT_EOL_S);
-
-	return 0;
-}
diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp
index 39abe720..3c6268ed 100644
--- a/node/IncomingPacket.cpp
+++ b/node/IncomingPacket.cpp
@@ -42,6 +42,7 @@
 #include "SelfAwareness.hpp"
 #include "Salsa20.hpp"
 #include "SHA512.hpp"
+#include "World.hpp"
 
 namespace ZeroTier {
 
@@ -199,10 +200,18 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR)
 		const uint64_t timestamp = at<uint64_t>(ZT_PROTO_VERB_HELLO_IDX_TIMESTAMP);
 
 		Identity id;
-		const unsigned int destAddrPtr = ZT_PROTO_VERB_HELLO_IDX_IDENTITY + id.deserialize(*this,ZT_PROTO_VERB_HELLO_IDX_IDENTITY);
 		InetAddress destAddr;
-		if (destAddrPtr < size()) // ZeroTier One < 1.0.3 did not include this field
-			destAddr.deserialize(*this,destAddrPtr);
+		uint64_t worldId = ZT_WORLD_ID_NULL;
+		uint64_t worldTimestamp = 0;
+		{
+			unsigned int ptr = ZT_PROTO_VERB_HELLO_IDX_IDENTITY + id.deserialize(*this,ZT_PROTO_VERB_HELLO_IDX_IDENTITY);
+			if (ptr < size()) // ZeroTier One < 1.0.3 did not include physical destination address info
+				ptr += destAddr.deserialize(*this,ptr);
+			if ((ptr + 16) <= size()) { // older versions also did not include World IDs or timestamps
+				worldId = at<uint64_t>(ptr); ptr += 8;
+				worldTimestamp = at<uint64_t>(ptr);
+			}
+		}
 
 		if (protoVersion < ZT_PROTO_VERSION_MIN) {
 			TRACE("dropped HELLO from %s(%s): protocol version too old",id.address().toString().c_str(),_remoteAddress.toString().c_str());
@@ -286,8 +295,23 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR)
 		outp.append((unsigned char)ZEROTIER_ONE_VERSION_MINOR);
 		outp.append((uint16_t)ZEROTIER_ONE_VERSION_REVISION);
 		_remoteAddress.serialize(outp);
-		outp.armor(peer->key(),true);
-		RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
+
+		if ((worldId != ZT_WORLD_ID_NULL)&&(worldId == RR->topology->worldId())) {
+			if (RR->topology->worldTimestamp() > worldTimestamp) {
+				World w(RR->topology->world());
+				const unsigned int sizeAt = outp.size();
+				outp.addSize(2); // make room for 16-bit size field
+				w.serialize(outp,false);
+				outp.setAt<uint16_t>(sizeAt,(uint16_t)(outp.size() - sizeAt));
+			} else {
+				outp.append((uint16_t)0); // no world update needed
+			}
+
+			outp.armor(peer->key(),true);
+			RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
+		} else {
+			TRACE("dropped HELLO from %s(%s): world ID mismatch: peer is %llu and we are %llu",source().toString().c_str(),_remoteAddress.toString().c_str(),worldId,RR->topology->worldId());
+		}
 	} catch ( ... ) {
 		TRACE("dropped HELLO from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str());
 	}
diff --git a/node/Node.cpp b/node/Node.cpp
index 7496b045..5468f102 100644
--- a/node/Node.cpp
+++ b/node/Node.cpp
@@ -177,37 +177,47 @@ public:
 		RR(renv),
 		_now(now),
 		_relays(relays),
-		_rootAddresses(RR->topology->rootAddresses())
+		_world(RR->topology->world())
 	{
 	}
 
-	uint64_t lastReceiveFromUpstream;
+	uint64_t lastReceiveFromUpstream; // tracks last time we got a packet from an 'upstream' peer like a root or a relay
 
 	inline void operator()(Topology &t,const SharedPtr<Peer> &p)
 	{
-		bool isRelay = false;
-		for(std::vector< std::pair<Address,InetAddress> >::const_iterator r(_relays.begin());r!=_relays.end();++r) {
-			if (r->first == p->address()) {
-				isRelay = true;
+		bool upstream = false;
+		InetAddress stableEndpoint;
+		for(std::vector<World::Root>::const_iterator r(_world.roots().begin());r!=_world.roots().end();++r) {
+			if (r->identity.address() == p->address()) {
+				if (r->stableEndpoints.size() > 0)
+					stableEndpoint = r->stableEndpoints[(unsigned long)RR->node->prng() % r->stableEndpoints.size()];
+				upstream = true;
 				break;
 			}
 		}
 
-		if ((isRelay)||(std::find(_rootAddresses.begin(),_rootAddresses.end(),p->address()) != _rootAddresses.end())) {
-			p->doPingAndKeepalive(RR,_now);
-			if (p->lastReceive() > lastReceiveFromUpstream)
-				lastReceiveFromUpstream = p->lastReceive();
-		} else {
-			if (p->alive(_now))
-				p->doPingAndKeepalive(RR,_now);
+		if (!upstream) {
+			for(std::vector< std::pair<Address,InetAddress> >::const_iterator r(_relays.begin());r!=_relays.end();++r) {
+				if (r->first == p->address()) {
+					stableEndpoint = r->second;
+					upstream = true;
+					break;
+				}
+			}
 		}
+
+		if ((!p->doPingAndKeepalive(RR,_now))&&(stableEndpoint))
+			p->attemptToContactAt(RR,InetAddress(),stableEndpoint,_now);
+
+		if (upstream)
+			lastReceiveFromUpstream = std::max(p->lastReceive(),lastReceiveFromUpstream);
 	}
 
 private:
 	const RuntimeEnvironment *RR;
 	uint64_t _now;
 	const std::vector< std::pair<Address,InetAddress> > &_relays;
-	std::vector<Address> _rootAddresses;
+	World _world;
 };
 
 ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline)
@@ -376,7 +386,6 @@ ZT_PeerList *Node::peers() const
 			memcpy(&(p->paths[p->pathCount].address),&(path->address()),sizeof(struct sockaddr_storage));
 			p->paths[p->pathCount].lastSend = path->lastSend();
 			p->paths[p->pathCount].lastReceive = path->lastReceived();
-			p->paths[p->pathCount].fixed = path->fixed() ? 1 : 0;
 			p->paths[p->pathCount].active = path->active(_now) ? 1 : 0;
 			p->paths[p->pathCount].preferred = ((bestPath)&&(*path == *bestPath)) ? 1 : 0;
 			++p->pathCount;
diff --git a/node/Packet.hpp b/node/Packet.hpp
index 939d84a5..810f5d67 100644
--- a/node/Packet.hpp
+++ b/node/Packet.hpp
@@ -566,8 +566,8 @@ public:
 		 *   <[2] software revision (of responder)>
 		 *   <[1] destination address type (for this OK, not copied from HELLO)>
 		 *   [<[...] destination address>]
-		 *   <[8] 64-bit world ID of current world (of responder)>
-		 *   <[8] 64-bit timestamp of current world (of responder)>
+		 *   <[2] 16-bit length of world update or 0 if none>
+		 *   [[...] world update]
 		 *
 		 * ERROR has no payload.
 		 */
@@ -1098,36 +1098,7 @@ public:
 		 *
 		 * ERROR has no payload.
 		 */
-		VERB_REQUEST_PROOF_OF_WORK = 19,
-
-		/**
-		 * Generic binary object access:
-		 *   <[8] 64-bit request ID>
-		 *   <[4] 32-bit index in blob to retrieve>
-		 *   <[2] 16-bit max length of block to retrieve>
-		 *   <[2] 16-bit length of blob identifier>
-		 *   <[...] blob identifier>
-		 *
-		 * This is used as a generic remote object retrieval mechanism. It returns
-		 * OK if the object is accessible, INVALID_REQUEST if the index is beyond
-		 * the size of the blob or another element is invalid, and OBJ_NOT_FOUND
-		 * if no blob with the given identifier is available.
-		 *
-		 * Blob identifiers follow a de facto path-like schema, with the following
-		 * names reserved:
-		 *   world - Current world definition (see World.hpp)
-		 *   updates.d/<any> - Software updates (not used yet, but reserved)
-		 *
-		 * OK payload:
-		 *   <[8] 64-bit request ID>
-		 *   <[4] 32-bit total length of blob>
-		 *   <[4] 32-bit index of this data in blob>
-		 *   <[...] data>
-		 *
-		 * ERROR payload:
-		 *   <[8] 64-bit request ID>
-		 */
-		VERB_GET_OBJECT = 20
+		VERB_REQUEST_PROOF_OF_WORK = 19
 	};
 
 	/**
diff --git a/node/Peer.cpp b/node/Peer.cpp
index 111c849e..697ba75d 100644
--- a/node/Peer.cpp
+++ b/node/Peer.cpp
@@ -108,17 +108,16 @@ void Peer::received(
 							// Add new path
 							slot = &(_paths[np++]);
 						} else {
-							// Replace oldest non-fixed path
 							uint64_t slotLRmin = 0xffffffffffffffffULL;
 							for(unsigned int p=0;p<ZT_MAX_PEER_NETWORK_PATHS;++p) {
-								if ((!_paths[p].fixed())&&(_paths[p].lastReceived() <= slotLRmin)) {
+								if (_paths[p].lastReceived() <= slotLRmin) {
 									slotLRmin = _paths[p].lastReceived();
 									slot = &(_paths[p]);
 								}
 							}
 						}
 						if (slot) {
-							*slot = RemotePath(localAddr,remoteAddr,false);
+							*slot = RemotePath(localAddr,remoteAddr);
 							slot->received(now);
 							_numPaths = np;
 							pathIsConfirmed = true;
@@ -172,12 +171,15 @@ void Peer::attemptToContactAt(const RuntimeEnvironment *RR,const InetAddress &lo
 	outp.append(now);
 	RR->identity.serialize(outp,false);
 	atAddress.serialize(outp);
+	outp.append((uint64_t)RR->topology->worldId());
+	outp.append((uint64_t)RR->topology->worldTimestamp());
+
 	outp.armor(_key,false); // HELLO is sent in the clear
 	RR->antiRec->logOutgoingZT(outp.data(),outp.size());
 	RR->node->putPacket(localAddr,atAddress,outp.data(),outp.size());
 }
 
-void Peer::doPingAndKeepalive(const RuntimeEnvironment *RR,uint64_t now)
+RemotePath *Peer::doPingAndKeepalive(const RuntimeEnvironment *RR,uint64_t now)
 {
 	Mutex::Lock _l(_lock);
 	RemotePath *const bestPath = _getBestPath(now);
@@ -193,6 +195,7 @@ void Peer::doPingAndKeepalive(const RuntimeEnvironment *RR,uint64_t now)
 			bestPath->sent(now);
 		}
 	}
+	return bestPath;
 }
 
 void Peer::pushDirectPaths(const RuntimeEnvironment *RR,RemotePath *path,uint64_t now,bool force)
@@ -269,59 +272,6 @@ void Peer::pushDirectPaths(const RuntimeEnvironment *RR,RemotePath *path,uint64_
 	}
 }
 
-void Peer::addPath(const RemotePath &newp,uint64_t now)
-{
-	Mutex::Lock _l(_lock);
-
-	unsigned int np = _numPaths;
-
-	for(unsigned int p=0;p<np;++p) {
-		if (_paths[p].address() == newp.address()) {
-			_paths[p].setFixed(newp.fixed());
-			_sortPaths(now);
-			return;
-		}
-	}
-
-	RemotePath *slot = (RemotePath *)0;
-	if (np < ZT_MAX_PEER_NETWORK_PATHS) {
-		// Add new path
-		slot = &(_paths[np++]);
-	} else {
-		// Replace oldest non-fixed path
-		uint64_t slotLRmin = 0xffffffffffffffffULL;
-		for(unsigned int p=0;p<ZT_MAX_PEER_NETWORK_PATHS;++p) {
-			if ((!_paths[p].fixed())&&(_paths[p].lastReceived() <= slotLRmin)) {
-				slotLRmin = _paths[p].lastReceived();
-				slot = &(_paths[p]);
-			}
-		}
-	}
-	if (slot) {
-		*slot = newp;
-		_numPaths = np;
-	}
-
-	_sortPaths(now);
-}
-
-void Peer::clearPaths(bool fixedToo)
-{
-	if (fixedToo) {
-		_numPaths = 0;
-	} else {
-		unsigned int np = _numPaths;
-		unsigned int x = 0;
-		unsigned int y = 0;
-		while (x < np) {
-			if (_paths[x].fixed())
-				_paths[y++] = _paths[x];
-			++x;
-		}
-		_numPaths = y;
-	}
-}
-
 bool Peer::resetWithinScope(const RuntimeEnvironment *RR,InetAddress::IpScope scope,uint64_t now)
 {
 	Mutex::Lock _l(_lock);
@@ -330,12 +280,9 @@ bool Peer::resetWithinScope(const RuntimeEnvironment *RR,InetAddress::IpScope sc
 	unsigned int y = 0;
 	while (x < np) {
 		if (_paths[x].address().ipScope() == scope) {
-			if (_paths[x].fixed()) {
-				attemptToContactAt(RR,_paths[x].localAddress(),_paths[x].address(),now);
-				_paths[y++] = _paths[x]; // keep fixed paths
-			}
+			attemptToContactAt(RR,_paths[x].localAddress(),_paths[x].address(),now);
 		} else {
-			_paths[y++] = _paths[x]; // keep paths not in this scope
+			_paths[y++] = _paths[x];
 		}
 		++x;
 	}
diff --git a/node/Peer.hpp b/node/Peer.hpp
index 0988561a..cabf0793 100644
--- a/node/Peer.hpp
+++ b/node/Peer.hpp
@@ -133,7 +133,7 @@ public:
 	 * Get the best direct path to this peer
 	 *
 	 * @param now Current time
-	 * @return Best path or NULL if there are no active (or fixed) direct paths
+	 * @return Best path or NULL if there are no active direct paths
 	 */
 	inline RemotePath *getBestPath(uint64_t now)
 	{
@@ -178,8 +178,9 @@ public:
 	 *
 	 * @param RR Runtime environment
 	 * @param now Current time
+	 * @return Current best path or NULL if no active paths
 	 */
-	void doPingAndKeepalive(const RuntimeEnvironment *RR,uint64_t now);
+	RemotePath *doPingAndKeepalive(const RuntimeEnvironment *RR,uint64_t now);
 
 	/**
 	 * Push direct paths if we haven't done so in [rate limit] milliseconds
@@ -290,7 +291,7 @@ public:
 
 	/**
 	 * @param now Current time
-	 * @return True if this peer has at least one active or fixed direct path
+	 * @return True if this peer has at least one active direct path
 	 */
 	inline bool hasActiveDirectPath(uint64_t now) const
 		throw()
@@ -303,27 +304,9 @@ public:
 		return false;
 	}
 
-	/**
-	 * Add a path (if we don't already have it)
-	 *
-	 * @param p New path to add
-	 * @param now Current time
-	 */
-	void addPath(const RemotePath &newp,uint64_t now);
-
-	/**
-	 * Clear paths
-	 *
-	 * @param fixedToo If true, clear fixed paths as well as learned ones
-	 */
-	void clearPaths(bool fixedToo);
-
 	/**
 	 * Reset paths within a given scope
 	 *
-	 * For fixed paths in this scope, a packet is sent. Non-fixed paths in this
-	 * scope are forgotten.
-	 *
 	 * @param RR Runtime environment
 	 * @param scope IP scope of paths to reset
 	 * @param now Current time
diff --git a/node/RemotePath.hpp b/node/RemotePath.hpp
index d2f99997..8b37621a 100644
--- a/node/RemotePath.hpp
+++ b/node/RemotePath.hpp
@@ -39,8 +39,6 @@
 #include "AntiRecursion.hpp"
 #include "RuntimeEnvironment.hpp"
 
-#define ZT_REMOTEPATH_FLAG_FIXED 0x0001
-
 namespace ZeroTier {
 
 /**
@@ -58,34 +56,18 @@ public:
 		_localAddress(),
 		_flags(0) {}
 
-	RemotePath(const InetAddress &localAddress,const InetAddress &addr,bool fixed) :
+	RemotePath(const InetAddress &localAddress,const InetAddress &addr) :
 		Path(addr,0,TRUST_NORMAL),
 		_lastSend(0),
 		_lastReceived(0),
 		_localAddress(localAddress),
-		_flags(fixed ? ZT_REMOTEPATH_FLAG_FIXED : 0) {}
+		_flags(0) {}
 
 	inline const InetAddress &localAddress() const throw() { return _localAddress; }
 
 	inline uint64_t lastSend() const throw() { return _lastSend; }
 	inline uint64_t lastReceived() const throw() { return _lastReceived; }
 
-	/**
-	 * @return Is this a fixed path?
-	 */
-	inline bool fixed() const throw() { return ((_flags & ZT_REMOTEPATH_FLAG_FIXED) != 0); }
-
-	/**
-	 * @param f New value of fixed flag
-	 */
-	inline void setFixed(const bool f)
-		throw()
-	{
-		if (f)
-			_flags |= ZT_REMOTEPATH_FLAG_FIXED;
-		else _flags &= ~ZT_REMOTEPATH_FLAG_FIXED;
-	}
-
 	/**
 	 * Called when a packet is sent to this remote path
 	 *
@@ -112,12 +94,12 @@ public:
 
 	/**
 	 * @param now Current time
-	 * @return True if this path is fixed or has received data in last ACTIVITY_TIMEOUT ms
+	 * @return True if this path appears active
 	 */
 	inline bool active(uint64_t now) const
 		throw()
 	{
-		return ( ((_flags & ZT_REMOTEPATH_FLAG_FIXED) != 0) || ((now - _lastReceived) < ZT_PEER_ACTIVITY_TIMEOUT) );
+		return ((now - _lastReceived) < ZT_PEER_ACTIVITY_TIMEOUT);
 	}
 
 	/**
diff --git a/node/Topology.cpp b/node/Topology.cpp
index 0cf4cfe8..cdecfdae 100644
--- a/node/Topology.cpp
+++ b/node/Topology.cpp
@@ -35,8 +35,8 @@
 namespace ZeroTier {
 
 // Default World
-#define ZT_DEFAULT_WORLD_LENGTH 1
-static const unsigned char ZT_DEFAULT_WORLD[ZT_DEFAULT_WORLD_LENGTH] = { 0 };
+#define ZT_DEFAULT_WORLD_LENGTH 494
+static const unsigned char ZT_DEFAULT_WORLD[ZT_DEFAULT_WORLD_LENGTH] = {0x01,0x00,0x00,0x00,0x00,0x08,0xea,0xc9,0x0a,0x00,0x00,0x01,0x4f,0xdf,0xbf,0xfc,0xbb,0x6c,0x7e,0x15,0x67,0x85,0x1b,0xb4,0x65,0x04,0x01,0xaf,0x56,0xbf,0xe7,0x63,0x9d,0x77,0xef,0xa4,0x1e,0x61,0x53,0x88,0xcb,0x8d,0x78,0xe5,0x47,0x38,0x98,0x5a,0x6c,0x8a,0xdd,0xe6,0x9c,0x65,0xdf,0x1a,0x80,0x63,0xce,0x2e,0x4d,0x48,0x24,0x3d,0x68,0x87,0x96,0x13,0x89,0xba,0x25,0x6f,0xc9,0xb0,0x9f,0x20,0xc5,0x4c,0x51,0x7b,0x30,0xb7,0x5f,0xba,0xca,0xa4,0xc5,0x48,0xa3,0x15,0xab,0x2f,0x1d,0x64,0xe8,0x04,0x42,0xb3,0x1c,0x51,0x8b,0x2a,0x04,0x01,0xf8,0xe1,0x81,0xaf,0x60,0x2f,0x70,0x3e,0xcd,0x0b,0x21,0x38,0x19,0x62,0x02,0xbd,0x0e,0x33,0x1d,0x0a,0x7b,0xf1,0xec,0xad,0xef,0x54,0xb3,0x7b,0x17,0x84,0xaa,0xda,0x0a,0x85,0x5d,0x0b,0x1c,0x05,0x83,0xb9,0x0e,0x3e,0xe3,0xb4,0xd1,0x8b,0x5b,0x64,0xf7,0xcf,0xe1,0xff,0x5d,0xc2,0x2a,0xcf,0x60,0x7b,0x09,0xb4,0xa3,0x86,0x3c,0x5a,0x7e,0x31,0xa0,0xc7,0xb4,0x86,0xe3,0x41,0x33,0x04,0x7e,0x19,0x87,0x6a,0xba,0x00,0x2a,0x6e,0x2b,0x23,0x18,0x93,0x0f,0x60,0xeb,0x09,0x7f,0x70,0xd0,0xf4,0xb0,0x28,0xb2,0xcd,0x6d,0x3d,0x0c,0x63,0xc0,0x14,0xb9,0x03,0x9f,0xf3,0x53,0x90,0xe4,0x11,0x81,0xf2,0x16,0xfb,0x2e,0x6f,0xa8,0xd9,0x5c,0x1e,0xe9,0x66,0x71,0x56,0x41,0x19,0x05,0xc3,0xdc,0xcf,0xea,0x78,0xd8,0xc6,0xdf,0xaf,0xba,0x68,0x81,0x70,0xb3,0xfa,0x00,0x01,0x04,0xc6,0xc7,0x61,0xdc,0x27,0x09,0x88,0x41,0x40,0x8a,0x2e,0x00,0xbb,0x1d,0x31,0xf2,0xc3,0x23,0xe2,0x64,0xe9,0xe6,0x41,0x72,0xc1,0xa7,0x4f,0x77,0x89,0x95,0x55,0xed,0x10,0x75,0x1c,0xd5,0x6e,0x86,0x40,0x5c,0xde,0x11,0x8d,0x02,0xdf,0xfe,0x55,0x5d,0x46,0x2c,0xcf,0x6a,0x85,0xb5,0x63,0x1c,0x12,0x35,0x0c,0x8d,0x5d,0xc4,0x09,0xba,0x10,0xb9,0x02,0x5d,0x0f,0x44,0x5c,0xf4,0x49,0xd9,0x2b,0x1c,0x00,0x01,0x04,0x6b,0xbf,0x2e,0xd2,0x27,0x09,0x8a,0xcf,0x05,0x9f,0xe3,0x00,0x48,0x2f,0x6e,0xe5,0xdf,0xe9,0x02,0x31,0x9b,0x41,0x9d,0xe5,0xbd,0xc7,0x65,0x20,0x9c,0x0e,0xcd,0xa3,0x8c,0x4d,0x6e,0x4f,0xcf,0x0d,0x33,0x65,0x83,0x98,0xb4,0x52,0x7d,0xcd,0x22,0xf9,0x31,0x12,0xfb,0x9b,0xef,0xd0,0x2f,0xd7,0x8b,0xf7,0x26,0x1b,0x33,0x3f,0xc1,0x05,0xd1,0x92,0xa6,0x23,0xca,0x9e,0x50,0xfc,0x60,0xb3,0x74,0xa5,0x00,0x01,0x04,0xa2,0xf3,0x4d,0x6f,0x27,0x09,0x9d,0x21,0x90,0x39,0xf3,0x00,0x01,0xf0,0x92,0x2a,0x98,0xe3,0xb3,0x4e,0xbc,0xbf,0xf3,0x33,0x26,0x9d,0xc2,0x65,0xd7,0xa0,0x20,0xaa,0xb6,0x9d,0x72,0xbe,0x4d,0x4a,0xcc,0x9c,0x8c,0x92,0x94,0x78,0x57,0x71,0x25,0x6c,0xd1,0xd9,0x42,0xa9,0x0d,0x1b,0xd1,0xd2,0xdc,0xa3,0xea,0x84,0xef,0x7d,0x85,0xaf,0xe6,0x61,0x1f,0xb4,0x3f,0xf0,0xb7,0x41,0x26,0xd9,0x0a,0x6e,0x00,0x01,0x04,0x80,0xc7,0xc5,0xd9,0x27,0x09};
 
 Topology::Topology(const RuntimeEnvironment *renv) :
 	RR(renv),
@@ -276,6 +276,13 @@ bool Topology::worldUpdateIfValid(const World &newWorld)
 	Mutex::Lock _l(_lock);
 	if (_world.shouldBeReplacedBy(newWorld,true)) {
 		_setWorld(newWorld);
+		try {
+			Buffer<ZT_WORLD_MAX_SERIALIZED_LENGTH> dswtmp;
+			newWorld.serialize(dswtmp,false);
+			RR->node->dataStorePut("world",dswtmp.data(),dswtmp.size(),false);
+		} catch ( ... ) {
+			RR->node->dataStoreDelete("world");
+		}
 		return true;
 	}
 	return false;
diff --git a/node/Topology.hpp b/node/Topology.hpp
index 3abc27e4..6f0170f0 100644
--- a/node/Topology.hpp
+++ b/node/Topology.hpp
@@ -147,12 +147,19 @@ public:
 	}
 
 	/**
-	 * @return Pair containing world ID and world timestamp (faster than world().id() etc.)
+	 * @return Current world ID
 	 */
-	inline std::pair<uint64_t,uint64_t> worldIdentification() const
+	inline uint64_t worldId() const
 	{
-		Mutex::Lock _l(_lock);
-		return std::pair<uint64_t,uint64_t>(_world.id(),_world.timestamp());
+		return _world.id(); // safe to read without lock, and used from within eachPeer() so don't lock
+	}
+
+	/**
+	 * @return Current world timestamp
+	 */
+	inline uint64_t worldTimestamp() const
+	{
+		return _world.timestamp(); // safe to read without lock, and used from within eachPeer() so don't lock
 	}
 
 	/**
diff --git a/service/ControlPlane.cpp b/service/ControlPlane.cpp
index 6e731bdc..dd755a30 100644
--- a/service/ControlPlane.cpp
+++ b/service/ControlPlane.cpp
@@ -182,14 +182,12 @@ static std::string _jsonEnumerate(unsigned int depth,const ZT_PeerPhysicalPath *
 			"%s\t\"address\": \"%s\",\n"
 			"%s\t\"lastSend\": %llu,\n"
 			"%s\t\"lastReceive\": %llu,\n"
-			"%s\t\"fixed\": %s,\n"
 			"%s\t\"active\": %s,\n"
 			"%s\t\"preferred\": %s\n"
 			"%s}",
 			prefix,_jsonEscape(reinterpret_cast<const InetAddress *>(&(pp[i].address))->toString()).c_str(),
 			prefix,pp[i].lastSend,
 			prefix,pp[i].lastReceive,
-			prefix,(pp[i].fixed == 0) ? "false" : "true",
 			prefix,(pp[i].active == 0) ? "false" : "true",
 			prefix,(pp[i].preferred == 0) ? "false" : "true",
 			prefix);
diff --git a/world/2015-10-13.bin b/world/2015-10-13.bin
new file mode 100644
index 00000000..433a7763
Binary files /dev/null and b/world/2015-10-13.bin differ
diff --git a/world/2015-10-13.out b/world/2015-10-13.out
new file mode 100644
index 00000000..754e0692
--- /dev/null
+++ b/world/2015-10-13.out
@@ -0,0 +1,6 @@
+INFO: created initial world keys: previous.c25519, current.c25519
+INFO: generating and signing id==149604618 ts==1442567945403
+INFO: wrote 494 bytes to stdout
+
+#define ZT_DEFAULT_WORLD_LENGTH 494
+static const unsigned char ZT_DEFAULT_WORLD[ZT_DEFAULT_WORLD_LENGTH] = {0x01,0x00,0x00,0x00,0x00,0x08,0xea,0xc9,0x0a,0x00,0x00,0x01,0x4f,0xdf,0xbf,0xfc,0xbb,0x6c,0x7e,0x15,0x67,0x85,0x1b,0xb4,0x65,0x04,0x01,0xaf,0x56,0xbf,0xe7,0x63,0x9d,0x77,0xef,0xa4,0x1e,0x61,0x53,0x88,0xcb,0x8d,0x78,0xe5,0x47,0x38,0x98,0x5a,0x6c,0x8a,0xdd,0xe6,0x9c,0x65,0xdf,0x1a,0x80,0x63,0xce,0x2e,0x4d,0x48,0x24,0x3d,0x68,0x87,0x96,0x13,0x89,0xba,0x25,0x6f,0xc9,0xb0,0x9f,0x20,0xc5,0x4c,0x51,0x7b,0x30,0xb7,0x5f,0xba,0xca,0xa4,0xc5,0x48,0xa3,0x15,0xab,0x2f,0x1d,0x64,0xe8,0x04,0x42,0xb3,0x1c,0x51,0x8b,0x2a,0x04,0x01,0xf8,0xe1,0x81,0xaf,0x60,0x2f,0x70,0x3e,0xcd,0x0b,0x21,0x38,0x19,0x62,0x02,0xbd,0x0e,0x33,0x1d,0x0a,0x7b,0xf1,0xec,0xad,0xef,0x54,0xb3,0x7b,0x17,0x84,0xaa,0xda,0x0a,0x85,0x5d,0x0b,0x1c,0x05,0x83,0xb9,0x0e,0x3e,0xe3,0xb4,0xd1,0x8b,0x5b,0x64,0xf7,0xcf,0xe1,0xff,0x5d,0xc2,0x2a,0xcf,0x60,0x7b,0x09,0xb4,0xa3,0x86,0x3c,0x5a,0x7e,0x31,0xa0,0xc7,0xb4,0x86,0xe3,0x41,0x33,0x04,0x7e,0x19,0x87,0x6a,0xba,0x00,0x2a,0x6e,0x2b,0x23,0x18,0x93,0x0f,0x60,0xeb,0x09,0x7f,0x70,0xd0,0xf4,0xb0,0x28,0xb2,0xcd,0x6d,0x3d,0x0c,0x63,0xc0,0x14,0xb9,0x03,0x9f,0xf3,0x53,0x90,0xe4,0x11,0x81,0xf2,0x16,0xfb,0x2e,0x6f,0xa8,0xd9,0x5c,0x1e,0xe9,0x66,0x71,0x56,0x41,0x19,0x05,0xc3,0xdc,0xcf,0xea,0x78,0xd8,0xc6,0xdf,0xaf,0xba,0x68,0x81,0x70,0xb3,0xfa,0x00,0x01,0x04,0xc6,0xc7,0x61,0xdc,0x27,0x09,0x88,0x41,0x40,0x8a,0x2e,0x00,0xbb,0x1d,0x31,0xf2,0xc3,0x23,0xe2,0x64,0xe9,0xe6,0x41,0x72,0xc1,0xa7,0x4f,0x77,0x89,0x95,0x55,0xed,0x10,0x75,0x1c,0xd5,0x6e,0x86,0x40,0x5c,0xde,0x11,0x8d,0x02,0xdf,0xfe,0x55,0x5d,0x46,0x2c,0xcf,0x6a,0x85,0xb5,0x63,0x1c,0x12,0x35,0x0c,0x8d,0x5d,0xc4,0x09,0xba,0x10,0xb9,0x02,0x5d,0x0f,0x44,0x5c,0xf4,0x49,0xd9,0x2b,0x1c,0x00,0x01,0x04,0x6b,0xbf,0x2e,0xd2,0x27,0x09,0x8a,0xcf,0x05,0x9f,0xe3,0x00,0x48,0x2f,0x6e,0xe5,0xdf,0xe9,0x02,0x31,0x9b,0x41,0x9d,0xe5,0xbd,0xc7,0x65,0x20,0x9c,0x0e,0xcd,0xa3,0x8c,0x4d,0x6e,0x4f,0xcf,0x0d,0x33,0x65,0x83,0x98,0xb4,0x52,0x7d,0xcd,0x22,0xf9,0x31,0x12,0xfb,0x9b,0xef,0xd0,0x2f,0xd7,0x8b,0xf7,0x26,0x1b,0x33,0x3f,0xc1,0x05,0xd1,0x92,0xa6,0x23,0xca,0x9e,0x50,0xfc,0x60,0xb3,0x74,0xa5,0x00,0x01,0x04,0xa2,0xf3,0x4d,0x6f,0x27,0x09,0x9d,0x21,0x90,0x39,0xf3,0x00,0x01,0xf0,0x92,0x2a,0x98,0xe3,0xb3,0x4e,0xbc,0xbf,0xf3,0x33,0x26,0x9d,0xc2,0x65,0xd7,0xa0,0x20,0xaa,0xb6,0x9d,0x72,0xbe,0x4d,0x4a,0xcc,0x9c,0x8c,0x92,0x94,0x78,0x57,0x71,0x25,0x6c,0xd1,0xd9,0x42,0xa9,0x0d,0x1b,0xd1,0xd2,0xdc,0xa3,0xea,0x84,0xef,0x7d,0x85,0xaf,0xe6,0x61,0x1f,0xb4,0x3f,0xf0,0xb7,0x41,0x26,0xd9,0x0a,0x6e,0x00,0x01,0x04,0x80,0xc7,0xc5,0xd9,0x27,0x09};
diff --git a/world/mkworld.cpp b/world/mkworld.cpp
new file mode 100644
index 00000000..baf693fc
--- /dev/null
+++ b/world/mkworld.cpp
@@ -0,0 +1,170 @@
+/*
+ * ZeroTier One - Network Virtualization Everywhere
+ * Copyright (C) 2011-2015  ZeroTier, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * ZeroTier may be used and distributed under the terms of the GPLv3, which
+ * are available at: http://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * If you would like to embed ZeroTier into a commercial application or
+ * redistribute it in a modified binary form, please contact ZeroTier Networks
+ * LLC. Start here: http://www.zerotier.com/
+ */
+
+/*
+ * This utility makes the World from the configuration specified below.
+ * It probably won't be much use to anyone outside ZeroTier, Inc. except
+ * for testing and experimentation purposes.
+ *
+ * If you want to make your own World you must edit this file.
+ *
+ * When run, it expects two files in the current directory:
+ *
+ * previous.c25519 - key pair to sign this world (key from previous world)
+ * current.c25519 - key pair whose public key should be embedded in this world
+ *
+ * If these files do not exist, they are both created with the same key pair
+ * and a self-signed initial World is born.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+
+#include <string>
+#include <vector>
+#include <algorithm>
+
+#include "node/Constants.hpp"
+#include "node/World.hpp"
+#include "node/C25519.hpp"
+#include "node/Identity.hpp"
+#include "node/InetAddress.hpp"
+#include "osdep/OSUtils.hpp"
+
+using namespace ZeroTier;
+
+class WorldMaker : public World
+{
+public:
+	static inline World make(uint64_t id,uint64_t ts,const C25519::Public &sk,const std::vector<World::Root> &roots,const C25519::Pair &signWith)
+	{
+		WorldMaker w;
+		w._id = id;
+		w._ts = ts;
+		w._updateSigningKey = sk;
+		w._roots = roots;
+
+		Buffer<ZT_WORLD_MAX_SERIALIZED_LENGTH> tmp;
+		w.serialize(tmp,true);
+		w._signature = C25519::sign(signWith,tmp.data(),tmp.size());
+
+		return w;
+	}
+};
+
+int main(int argc,char **argv)
+{
+	std::string previous,current;
+	if ((!OSUtils::readFile("previous.c25519",previous))||(!OSUtils::readFile("current.c25519",current))) {
+		C25519::Pair np(C25519::generate());
+		previous = std::string();
+		previous.append((const char *)np.pub.data,ZT_C25519_PUBLIC_KEY_LEN);
+		previous.append((const char *)np.priv.data,ZT_C25519_PRIVATE_KEY_LEN);
+		current = previous;
+		OSUtils::writeFile("previous.c25519",previous);
+		OSUtils::writeFile("current.c25519",current);
+		fprintf(stderr,"INFO: created initial world keys: previous.c25519, current.c25519"ZT_EOL_S);
+	}
+
+	if ((previous.length() != (ZT_C25519_PUBLIC_KEY_LEN + ZT_C25519_PRIVATE_KEY_LEN))||(current.length() != (ZT_C25519_PUBLIC_KEY_LEN + ZT_C25519_PRIVATE_KEY_LEN))) {
+		fprintf(stderr,"FATAL: previous.c25519 or current.c25519 empty or invalid"ZT_EOL_S);
+		return 1;
+	}
+	C25519::Pair previousKP;
+	memcpy(previousKP.pub.data,previous.data(),ZT_C25519_PUBLIC_KEY_LEN);
+	memcpy(previousKP.priv.data,previous.data() + ZT_C25519_PUBLIC_KEY_LEN,ZT_C25519_PRIVATE_KEY_LEN);
+	C25519::Pair currentKP;
+	memcpy(currentKP.pub.data,current.data(),ZT_C25519_PUBLIC_KEY_LEN);
+	memcpy(currentKP.priv.data,current.data() + ZT_C25519_PUBLIC_KEY_LEN,ZT_C25519_PRIVATE_KEY_LEN);
+
+	// EDIT BELOW HERE ---------------------------------------------------------
+
+	std::vector<World::Root> roots;
+
+	// old US-SFO
+	roots.push_back(World::Root());
+	roots.back().identity = Identity("7e19876aba:0:2a6e2b2318930f60eb097f70d0f4b028b2cd6d3d0c63c014b9039ff35390e41181f216fb2e6fa8d95c1ee9667156411905c3dccfea78d8c6dfafba688170b3fa");
+	roots.back().stableEndpoints.push_back(InetAddress("198.199.97.220/9993"));
+	std::sort(roots.back().stableEndpoints.begin(),roots.back().stableEndpoints.end());
+
+	// old EU-PARIS
+	roots.push_back(World::Root());
+	roots.back().identity = Identity("8841408a2e:0:bb1d31f2c323e264e9e64172c1a74f77899555ed10751cd56e86405cde118d02dffe555d462ccf6a85b5631c12350c8d5dc409ba10b9025d0f445cf449d92b1c");
+	roots.back().stableEndpoints.push_back(InetAddress("107.191.46.210/9993"));
+	std::sort(roots.back().stableEndpoints.begin(),roots.back().stableEndpoints.end());
+
+	// old US-NYC
+	roots.push_back(World::Root());
+	roots.back().identity = Identity("8acf059fe3:0:482f6ee5dfe902319b419de5bdc765209c0ecda38c4d6e4fcf0d33658398b4527dcd22f93112fb9befd02fd78bf7261b333fc105d192a623ca9e50fc60b374a5");
+	roots.back().stableEndpoints.push_back(InetAddress("162.243.77.111/9993"));
+	std::sort(roots.back().stableEndpoints.begin(),roots.back().stableEndpoints.end());
+
+	// old AP-SNG
+	roots.push_back(World::Root());
+	roots.back().identity = Identity("9d219039f3:0:01f0922a98e3b34ebcbff333269dc265d7a020aab69d72be4d4acc9c8c9294785771256cd1d942a90d1bd1d2dca3ea84ef7d85afe6611fb43ff0b74126d90a6e");
+	roots.back().stableEndpoints.push_back(InetAddress("128.199.197.217/9993"));
+	std::sort(roots.back().stableEndpoints.begin(),roots.back().stableEndpoints.end());
+
+	std::sort(roots.begin(),roots.end());
+
+	const uint64_t id = ZT_WORLD_ID_EARTH;
+	const uint64_t ts = OSUtils::now();
+
+	// END WORLD SETUP ---------------------------------------------------------
+
+	fprintf(stderr,"INFO: generating and signing id==%llu ts==%llu"ZT_EOL_S,(unsigned long long)id,(unsigned long long)ts);
+
+	World nw = WorldMaker::make(id,ts,currentKP.pub,roots,previousKP);
+
+	Buffer<ZT_WORLD_MAX_SERIALIZED_LENGTH> outtmp;
+	nw.serialize(outtmp,false);
+	World testw;
+	testw.deserialize(outtmp,0);
+	if (testw != nw) {
+		fprintf(stderr,"FATAL: serialization test failed!"ZT_EOL_S);
+		return 1;
+	}
+	fwrite(outtmp.data(),outtmp.size(),1,stdout);
+	fflush(stdout);
+
+	fprintf(stderr,"INFO: wrote %u bytes to stdout"ZT_EOL_S,outtmp.size());
+
+	fprintf(stderr,ZT_EOL_S);
+	fprintf(stderr,"#define ZT_DEFAULT_WORLD_LENGTH %u"ZT_EOL_S,outtmp.size());
+	fprintf(stderr,"static const unsigned char ZT_DEFAULT_WORLD[ZT_DEFAULT_WORLD_LENGTH] = {");
+	for(unsigned int i=0;i<outtmp.size();++i) {
+		const unsigned char *d = (const unsigned char *)outtmp.data();
+		if (i > 0)
+			fprintf(stderr,",");
+		fprintf(stderr,"0x%.2x",(unsigned int)d[i]);
+	}
+	fprintf(stderr,"};"ZT_EOL_S);
+
+	return 0;
+}
-- 
cgit v1.2.3


From 70d8e3ad94076c51b05b4165cbd68a91806dbde7 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Tue, 13 Oct 2015 12:31:38 -0700
Subject: Expose world ID and world timestamp in ZT_NodeStatus

---
 include/ZeroTierOne.h | 10 ++++++++++
 node/Node.cpp         |  2 ++
 2 files changed, 12 insertions(+)

(limited to 'include')

diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h
index 38db3222..42c904eb 100644
--- a/include/ZeroTierOne.h
+++ b/include/ZeroTierOne.h
@@ -275,6 +275,16 @@ typedef struct
 	 */
 	uint64_t address;
 
+	/**
+	 * Current world ID
+	 */
+	uint64_t worldId;
+
+	/**
+	 * Current world revision/timestamp
+	 */
+	uint64_t worldTimestamp;
+
 	/**
 	 * Public identity in string-serialized form (safe to send to others)
 	 *
diff --git a/node/Node.cpp b/node/Node.cpp
index 5468f102..be37b7c7 100644
--- a/node/Node.cpp
+++ b/node/Node.cpp
@@ -345,6 +345,8 @@ uint64_t Node::address() const
 void Node::status(ZT_NodeStatus *status) const
 {
 	status->address = RR->identity.address().toInt();
+	status->worldId = RR->topology->worldId();
+	status->worldTimestamp = RR->topology->worldTimestamp();
 	status->publicIdentity = RR->publicIdentityStr.c_str();
 	status->secretIdentity = RR->secretIdentityStr.c_str();
 	status->online = _online ? 1 : 0;
-- 
cgit v1.2.3


From 57e29857cf79019af03f6a3dfe0bf6fd36e2fab2 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Tue, 20 Oct 2015 15:27:53 -0700
Subject: Cluster work -- integrating with the rest of the code.

---
 include/ZeroTierOne.h       | 121 ++++++++++++++++++++++++++-
 make-mac.mk                 |   2 +-
 node/Cluster.cpp            | 193 +++++++++++++++++++++++++++++++++++++++-----
 node/Cluster.hpp            | 118 ++++++++++++---------------
 node/IncomingPacket.cpp     |  14 +++-
 node/Node.cpp               | 155 +++++++++++++++++++++++++++++++++--
 node/Node.hpp               |  14 ++++
 node/Peer.cpp               |   6 ++
 node/RuntimeEnvironment.hpp |  24 ++++--
 9 files changed, 539 insertions(+), 108 deletions(-)

(limited to 'include')

diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h
index 42c904eb..135c8e11 100644
--- a/include/ZeroTierOne.h
+++ b/include/ZeroTierOne.h
@@ -128,6 +128,16 @@ extern "C" {
  */
 #define ZT_CIRCUIT_TEST_MAX_HOP_BREADTH 256
 
+/**
+ * Maximum number of cluster members (and max member ID plus one)
+ */
+#define ZT_CLUSTER_MAX_MEMBERS 256
+
+/**
+ * Maximum allowed cluster message length in bytes
+ */
+#define ZT_CLUSTER_MAX_MESSAGE_LENGTH 65535
+
 /**
  * A null/empty sockaddr (all zero) to signify an unspecified socket address
  */
@@ -174,7 +184,17 @@ enum ZT_ResultCode
 	/**
 	 * Network ID not valid
 	 */
-	ZT_RESULT_ERROR_NETWORK_NOT_FOUND = 1000
+	ZT_RESULT_ERROR_NETWORK_NOT_FOUND = 1000,
+
+	/**
+	 * The requested operation is not supported on this version or build
+	 */
+	ZT_RESULT_ERROR_UNSUPPORTED_OPERATION = 1001,
+
+	/**
+	 * The requestion operation was given a bad parameter or was called in an invalid state
+	 */
+	ZT_RESULT_ERROR_BAD_PARAMETER = 1002
 };
 
 /**
@@ -1320,6 +1340,105 @@ enum ZT_ResultCode ZT_Node_circuitTestBegin(ZT_Node *node,ZT_CircuitTest *test,v
  */
 void ZT_Node_circuitTestEnd(ZT_Node *node,ZT_CircuitTest *test);
 
+/**
+ * Initialize cluster operation
+ *
+ * This initializes the internal structures and state for cluster operation.
+ * It takes two function pointers. The first is to a function that can be
+ * used to send data to cluster peers (mechanism is not defined by Node),
+ * and the second is to a function that can be used to get the location of
+ * a physical address in X,Y,Z coordinate space (e.g. as cartesian coordinates
+ * projected from the center of the Earth).
+ *
+ * Send function takes an arbitrary pointer followed by the cluster member ID
+ * to send data to, a pointer to the data, and the length of the data. The
+ * maximum message length is ZT_CLUSTER_MAX_MESSAGE_LENGTH (65535). Messages
+ * must be delivered whole and may be dropped or transposed, though high
+ * failure rates are undesirable and can cause problems. Validity checking or
+ * CRC is also not required since the Node validates the authenticity of
+ * cluster messages using cryptogrphic methods and will silently drop invalid
+ * messages.
+ *
+ * Address to location function is optional and if NULL geo-handoff is not
+ * enabled (in this case x, y, and z in clusterInit are also unused). It
+ * takes an arbitrary pointer followed by a physical address and three result
+ * parameters for x, y, and z. It returns zero on failure or nonzero if these
+ * three coordinates have been set. Coordinate space is arbitrary and can be
+ * e.g. coordinates on Earth relative to Earth's center. These can be obtained
+ * from latitutde and longitude with versions of the Haversine formula.
+ *
+ * See: http://stackoverflow.com/questions/1185408/converting-from-longitude-latitude-to-cartesian-coordinates
+ *
+ * Neither the send nor the address to location function should block. If the
+ * address to location function does not have a location for an address, it
+ * should return zero and then look up the address for future use since it
+ * will be called again in (typically) 1-3 minutes.
+ *
+ * Note that both functions can be called from any thread from which the
+ * various Node functions are called, and so must be thread safe if multiple
+ * threads are being used.
+ *
+ * @param node Node instance
+ * @param myId My cluster member ID (less than or equal to ZT_CLUSTER_MAX_MEMBERS)
+ * @param zeroTierPhysicalEndpoints Preferred physical address(es) for ZeroTier clients to contact this cluster member (for peer redirect)
+ * @param numZeroTierPhysicalEndpoints Number of physical endpoints in zeroTierPhysicalEndpoints[] (max allowed: 255)
+ * @param x My cluster member's X location
+ * @param y My cluster member's Y location
+ * @param z My cluster member's Z location
+ * @param sendFunction Function to be called to send data to other cluster members
+ * @param sendFunctionArg First argument to sendFunction()
+ * @param addressToLocationFunction Function to be called to get the location of a physical address or NULL to disable geo-handoff
+ * @param addressToLocationFunctionArg First argument to addressToLocationFunction()
+ * @return OK or UNSUPPORTED_OPERATION if this Node was not built with cluster support
+ */
+enum ZT_ResultCode ZT_Node_clusterInit(
+	ZT_Node *node,
+	unsigned int myId,
+	const struct sockaddr_storage *zeroTierPhysicalEndpoints,
+	unsigned int numZeroTierPhysicalEndpoints,
+	int x,
+	int y,
+	int z,
+	void (*sendFunction)(void *,unsigned int,const void *,unsigned int),
+	void *sendFunctionArg,
+	int (*addressToLocationFunction)(void *,const struct sockaddr_storage *,int *,int *,int *),
+	void *addressToLocationFunctionArg);
+
+/**
+ * Add a member to this cluster
+ *
+ * Calling this without having called clusterInit() will do nothing.
+ *
+ * @param node Node instance
+ * @param memberId Member ID (must be less than or equal to ZT_CLUSTER_MAX_MEMBERS)
+ * @return OK or error if clustering is disabled, ID invalid, etc.
+ */
+enum ZT_ResultCode ZT_Node_clusterAddMember(ZT_Node *node,unsigned int memberId);
+
+/**
+ * Remove a member from this cluster
+ *
+ * Calling this without having called clusterInit() will do nothing.
+ *
+ * @param node Node instance
+ * @param memberId Member ID to remove (nothing happens if not present)
+ */
+void ZT_Node_clusterRemoveMember(ZT_Node *node,unsigned int memberId);
+
+/**
+ * Handle an incoming cluster state message
+ *
+ * The message itself contains cluster member IDs, and invalid or badly
+ * addressed messages will be silently discarded.
+ *
+ * Calling this without having called clusterInit() will do nothing.
+ *
+ * @param node Node instance
+ * @param msg Cluster message
+ * @param len Length of cluster message
+ */
+void ZT_Node_clusterHandleIncomingMessage(ZT_Node *node,const void *msg,unsigned int len);
+
 /**
  * Get ZeroTier One version
  *
diff --git a/make-mac.mk b/make-mac.mk
index 174216fb..e53212c0 100644
--- a/make-mac.mk
+++ b/make-mac.mk
@@ -6,7 +6,7 @@ ifeq ($(origin CXX),default)
 endif
 
 INCLUDES=
-DEFS=
+DEFS=-DZT_ENABLE_CLUSTER
 LIBS=
 ARCH_FLAGS=-arch x86_64
 
diff --git a/node/Cluster.cpp b/node/Cluster.cpp
index bfa39d22..5b76d1f0 100644
--- a/node/Cluster.cpp
+++ b/node/Cluster.cpp
@@ -31,10 +31,13 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <math.h>
 
 #include <algorithm>
 #include <utility>
 
+#include "../version.h"
+
 #include "Cluster.hpp"
 #include "RuntimeEnvironment.hpp"
 #include "MulticastGroup.hpp"
@@ -42,22 +45,44 @@
 #include "Salsa20.hpp"
 #include "Poly1305.hpp"
 #include "Packet.hpp"
+#include "Identity.hpp"
 #include "Peer.hpp"
 #include "Switch.hpp"
 #include "Node.hpp"
 
 namespace ZeroTier {
 
-Cluster::Cluster(const RuntimeEnvironment *renv,uint16_t id,DistanceAlgorithm da,int32_t x,int32_t y,int32_t z,void (*sendFunction)(void *,uint16_t,const void *,unsigned int),void *arg) :
+static inline double _dist3d(int x1,int y1,int z1,int x2,int y2,int z2)
+	throw()
+{
+	double dx = ((double)x2 - (double)x1);
+	double dy = ((double)y2 - (double)y1);
+	double dz = ((double)z2 - (double)z1);
+	return sqrt((dx * dx) + (dy * dy) + (dz * dz));
+}
+
+Cluster::Cluster(
+	const RuntimeEnvironment *renv,
+	uint16_t id,
+	const std::vector<InetAddress> &zeroTierPhysicalEndpoints,
+	int32_t x,
+	int32_t y,
+	int32_t z,
+	void (*sendFunction)(void *,unsigned int,const void *,unsigned int),
+	void *sendFunctionArg,
+	int (*addressToLocationFunction)(void *,const struct sockaddr_storage *,int *,int *,int *),
+	void *addressToLocationFunctionArg) :
 	RR(renv),
 	_sendFunction(sendFunction),
-	_arg(arg),
+	_sendFunctionArg(sendFunctionArg),
+	_addressToLocationFunction(addressToLocationFunction),
+	_addressToLocationFunctionArg(addressToLocationFunctionArg),
 	_x(x),
 	_y(y),
 	_z(z),
-	_da(da),
 	_id(id),
-	_members(new _Member[65536])
+	_zeroTierPhysicalEndpoints(zeroTierPhysicalEndpoints),
+	_members(new _Member[ZT_CLUSTER_MAX_MEMBERS])
 {
 	uint16_t stmp[ZT_SHA512_DIGEST_LEN / sizeof(uint16_t)];
 
@@ -114,16 +139,20 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len)
 		s20.decrypt12(reinterpret_cast<const char *>(msg) + 24,const_cast<void *>(dmsg.data()),dmsg.size());
 	}
 
-	if (dmsg.size() < 2)
+	if (dmsg.size() < 4)
 		return;
 	const uint16_t fromMemberId = dmsg.at<uint16_t>(0);
 	unsigned int ptr = 2;
+	if (fromMemberId == _id)
+		return;
+	const uint16_t toMemberId = dmsg.at<uint16_t>(ptr);
+	ptr += 2;
+	if (toMemberId != _id)
+		return;
 
 	_Member &m = _members[fromMemberId];
 	Mutex::Lock mlck(m.lock);
 
-	m.lastReceivedFrom = RR->node->now();
-
 	try {
 		while (ptr < dmsg.size()) {
 			const unsigned int mlen = dmsg.at<uint16_t>(ptr); ptr += 2;
@@ -143,11 +172,13 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len)
 						ptr += 8; // skip local clock, not used
 						m.load = dmsg.at<uint64_t>(ptr); ptr += 8;
 						ptr += 8; // skip flags, unused
-						m.physicalAddressCount = dmsg[ptr++];
-						if (m.physicalAddressCount > ZT_CLUSTER_MEMBER_MAX_PHYSICAL_ADDRS)
-							m.physicalAddressCount = ZT_CLUSTER_MEMBER_MAX_PHYSICAL_ADDRS;
-						for(unsigned int i=0;i<m.physicalAddressCount;++i)
-							ptr += m.physicalAddresses[i].deserialize(dmsg,ptr);
+						unsigned int physicalAddressCount = dmsg[ptr++];
+						for(unsigned int i=0;i<physicalAddressCount;++i) {
+							m.zeroTierPhysicalEndpoints.push_back(InetAddress());
+							ptr += m.zeroTierPhysicalEndpoints.back().deserialize(dmsg,ptr);
+							if (!(m.zeroTierPhysicalEndpoints.back()))
+								m.zeroTierPhysicalEndpoints.pop_back();
+						}
 						m.lastReceivedAliveAnnouncement = RR->node->now();
 					}	break;
 
@@ -298,7 +329,7 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len)
 	}
 }
 
-void Cluster::replicateHavePeer(const Address &peerAddress)
+void Cluster::replicateHavePeer(const Identity &peerId)
 {
 }
 
@@ -312,23 +343,59 @@ void Cluster::replicateCertificateOfNetworkMembership(const CertificateOfMembers
 
 void Cluster::doPeriodicTasks()
 {
-	// Go ahead and flush whenever possible right now
+	const uint64_t now = RR->node->now();
+
 	{
 		Mutex::Lock _l(_memberIds_m);
 		for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
 			Mutex::Lock _l2(_members[*mid].lock);
-			_flush(*mid);
+
+			if ((now - _members[*mid].lastAnnouncedAliveTo) >= ((ZT_CLUSTER_TIMEOUT / 2) - 1000)) {
+				Buffer<2048> alive;
+				alive.append((uint16_t)ZEROTIER_ONE_VERSION_MAJOR);
+				alive.append((uint16_t)ZEROTIER_ONE_VERSION_MINOR);
+				alive.append((uint16_t)ZEROTIER_ONE_VERSION_REVISION);
+				alive.append((uint8_t)ZT_PROTO_VERSION);
+				if (_addressToLocationFunction) {
+					alive.append((int32_t)_x);
+					alive.append((int32_t)_y);
+					alive.append((int32_t)_z);
+				} else {
+					alive.append((int32_t)0);
+					alive.append((int32_t)0);
+					alive.append((int32_t)0);
+				}
+				alive.append((uint64_t)now);
+				alive.append((uint64_t)0); // TODO: compute and send load average
+				alive.append((uint64_t)0); // unused/reserved flags
+				alive.append((uint8_t)_zeroTierPhysicalEndpoints.size());
+				for(std::vector<InetAddress>::const_iterator pe(_zeroTierPhysicalEndpoints.begin());pe!=_zeroTierPhysicalEndpoints.end();++pe)
+					pe->serialize(alive);
+				_send(*mid,alive.data(),alive.size());
+				_members[*mid].lastAnnouncedAliveTo = now;
+			}
+
+			_flush(*mid); // does nothing if nothing to flush
 		}
 	}
 }
 
 void Cluster::addMember(uint16_t memberId)
 {
+	if (memberId >= ZT_CLUSTER_MAX_MEMBERS)
+		return;
+
 	Mutex::Lock _l2(_members[memberId].lock);
 
-	Mutex::Lock _l(_memberIds_m);
-	_memberIds.push_back(memberId);
-	std::sort(_memberIds.begin(),_memberIds.end());
+	{
+		Mutex::Lock _l(_memberIds_m);
+		if (std::find(_memberIds.begin(),_memberIds.end(),memberId) != _memberIds.end())
+			return;
+		_memberIds.push_back(memberId);
+		std::sort(_memberIds.begin(),_memberIds.end());
+	}
+
+	_members[memberId].clear();
 
 	// Generate this member's message key from the master and its ID
 	uint16_t stmp[ZT_SHA512_DIGEST_LEN / sizeof(uint16_t)];
@@ -346,6 +413,89 @@ void Cluster::addMember(uint16_t memberId)
 	_members[memberId].q.append(iv,16);
 	_members[memberId].q.addSize(8); // room for MAC
 	_members[memberId].q.append((uint16_t)_id);
+	_members[memberId].q.append((uint16_t)memberId);
+}
+
+void Cluster::removeMember(uint16_t memberId)
+{
+	Mutex::Lock _l(_memberIds_m);
+	std::vector<uint16_t> newMemberIds;
+	for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
+		if (*mid != memberId)
+			newMemberIds.push_back(*mid);
+	}
+	_memberIds = newMemberIds;
+}
+
+bool Cluster::redirectPeer(const SharedPtr<Peer> &peer,const InetAddress &peerPhysicalAddress,bool offload)
+{
+	if (!peerPhysicalAddress) // sanity check
+		return false;
+	if (_addressToLocationFunction) {
+		// Pick based on location if it can be determined
+		int px = 0,py = 0,pz = 0;
+		if (_addressToLocationFunction(_addressToLocationFunctionArg,reinterpret_cast<const struct sockaddr_storage *>(&peerPhysicalAddress),&px,&py,&pz) == 0) {
+			// No geo-info so no change
+			return false;
+		}
+
+		// Find member closest to this peer
+		const uint64_t now = RR->node->now();
+		std::vector<InetAddress> best; // initial "best" is for peer to stay put
+		const double currentDistance = _dist3d(_x,_y,_z,px,py,pz);
+		double bestDistance = (offload ? 2147483648.0 : currentDistance);
+		unsigned int bestMember = _id;
+		{
+			Mutex::Lock _l(_memberIds_m);
+			for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
+				_Member &m = _members[*mid];
+				Mutex::Lock _ml(m.lock);
+
+				// Consider member if it's alive and has sent us a location and one or more physical endpoints to send peers to
+				if ( ((now - m.lastReceivedAliveAnnouncement) < ZT_CLUSTER_TIMEOUT) && ((m.x != 0)||(m.y != 0)||(m.z != 0)) && (m.zeroTierPhysicalEndpoints.size() > 0) ) {
+					double mdist = _dist3d(m.x,m.y,m.z,px,py,pz);
+					if (mdist < bestDistance) {
+						bestMember = *mid;
+						best = m.zeroTierPhysicalEndpoints;
+					}
+				}
+			}
+		}
+
+		if (best.size() > 0) {
+			TRACE("peer %s is at [%d,%d,%d], distance to us is %f, sending to %u instead for better distance %f",peer->address().toString().c_str(),px,py,pz,currentDistance,bestMember,bestDistance);
+
+			/* if (peer->remoteVersionProtocol() >= 5) {
+				// If it's a newer peer send VERB_PUSH_DIRECT_PATHS which is more idiomatic
+			} else { */
+				// Otherwise send VERB_RENDEZVOUS for ourselves, which will trick peers into trying other endpoints for us even if they're too old for PUSH_DIRECT_PATHS
+				for(std::vector<InetAddress>::const_iterator a(best.begin());a!=best.end();++a) {
+					if ((a->ss_family == AF_INET)||(a->ss_family == AF_INET6)) {
+						Packet outp(peer->address(),RR->identity.address(),Packet::VERB_RENDEZVOUS);
+						outp.append((uint8_t)0); // no flags
+						RR->identity.address().appendTo(outp); // HACK: rendezvous with ourselves! with really old peers this will only work if I'm a root server!
+						outp.append((uint16_t)a->port());
+						if (a->ss_family == AF_INET) {
+							outp.append((uint8_t)4);
+							outp.append(a->rawIpData(),4);
+						} else {
+							outp.append((uint8_t)16);
+							outp.append(a->rawIpData(),16);
+						}
+						RR->sw->send(outp,true,0);
+					}
+				}
+			//}
+
+			return true;
+		} else {
+			TRACE("peer %s is at [%d,%d,%d], distance to us is %f and this seems to be the best",peer->address().toString().c_str(),px,py,pz,currentDistance);
+			return false;
+		}
+	} else {
+		// TODO: pick based on load if no location info?
+		return false;
+	}
 }
 
 void Cluster::_send(uint16_t memberId,const void *msg,unsigned int len)
@@ -366,7 +516,7 @@ void Cluster::_flush(uint16_t memberId)
 {
 	_Member &m = _members[memberId];
 	// assumes m.lock is locked!
-	if (m.q.size() > 26) { // 16-byte IV + 8-byte MAC + 2-byte cluster member ID (latter two bytes are inside crypto envelope)
+	if (m.q.size() > (24 + 2 + 2)) { // 16-byte IV + 8-byte MAC + 2 byte from-member-ID + 2 byte to-member-ID
 		// Create key from member's key and IV
 		char keytmp[32];
 		memcpy(keytmp,m.key,32);
@@ -389,7 +539,7 @@ void Cluster::_flush(uint16_t memberId)
 		memcpy(m.q.field(16,8),mac,8);
 
 		// Send!
-		_sendFunction(_arg,memberId,m.q.data(),m.q.size());
+		_sendFunction(_sendFunctionArg,memberId,m.q.data(),m.q.size());
 
 		// Prepare for more
 		m.q.clear();
@@ -397,7 +547,8 @@ void Cluster::_flush(uint16_t memberId)
 		Utils::getSecureRandom(iv,16);
 		m.q.append(iv,16);
 		m.q.addSize(8); // room for MAC
-		m.q.append((uint16_t)_id);
+		m.q.append((uint16_t)_id); // from member ID
+		m.q.append((uint16_t)memberId); // to member ID
 	}
 }
 
diff --git a/node/Cluster.hpp b/node/Cluster.hpp
index 016730e3..f253e6f6 100644
--- a/node/Cluster.hpp
+++ b/node/Cluster.hpp
@@ -34,43 +34,32 @@
 #include <algorithm>
 
 #include "Constants.hpp"
+#include "../include/ZeroTierOne.h"
 #include "Address.hpp"
 #include "InetAddress.hpp"
 #include "SHA512.hpp"
 #include "Utils.hpp"
 #include "Buffer.hpp"
 #include "Mutex.hpp"
+#include "SharedPtr.hpp"
 
 /**
  * Timeout for cluster members being considered "alive"
  */
-#define ZT_CLUSTER_TIMEOUT ZT_PEER_ACTIVITY_TIMEOUT
+#define ZT_CLUSTER_TIMEOUT 30000
 
 /**
- * Maximum cluster message length in bytes
- *
- * Cluster nodes speak via TCP, with data encapsulated into individually
- * encrypted and authenticated messages. The maximum message size is
- * 65535 (0xffff) since the TCP stream uses 16-bit message size headers
- * (and this is a reasonable chunk size anyway).
- */
-#define ZT_CLUSTER_MAX_MESSAGE_LENGTH 65535
-
-/**
- * Maximum number of physical addresses we will cache for a cluster member
- */
-#define ZT_CLUSTER_MEMBER_MAX_PHYSICAL_ADDRS 8
-
-/**
- * How frequently should doPeriodicTasks() be ideally called? (ms)
+ * Desired period between doPeriodicTasks() in milliseconds
  */
-#define ZT_CLUSTER_PERIODIC_TASK_DEADLINE 10
+#define ZT_CLUSTER_PERIODIC_TASK_PERIOD 50
 
 namespace ZeroTier {
 
 class RuntimeEnvironment;
 class CertificateOfMembership;
 class MulticastGroup;
+class Peer;
+class Identity;
 
 /**
  * Multi-homing cluster state replication and packet relaying
@@ -95,22 +84,6 @@ class MulticastGroup;
 class Cluster
 {
 public:
-	/**
-	 * Which distance algorithm is this cluster using?
-	 */
-	enum DistanceAlgorithm
-	{
-		/**
-		 * Simple linear distance in three dimensions
-		 */
-		DISTANCE_SIMPLE = 0,
-
-		/**
-		 * Haversine formula using X,Y as lat,long and ignoring Z
-		 */
-		DISTANCE_HAVERSINE = 1
-	};
-
 	/**
 	 * State message types
 	 */
@@ -184,25 +157,18 @@ public:
 
 	/**
 	 * Construct a new cluster
-	 *
-	 * @param renv Runtime environment
-	 * @param id This member's ID in the cluster
-	 * @param da Distance algorithm this cluster uses to compute distance and hand off peers
-	 * @param x My X
-	 * @param y My Y
-	 * @param z My Z
-	 * @param sendFunction Function to call to send messages to other cluster members
-	 * @param arg First argument to sendFunction
 	 */
 	Cluster(
 		const RuntimeEnvironment *renv,
 		uint16_t id,
-		DistanceAlgorithm da,
+		const std::vector<InetAddress> &zeroTierPhysicalEndpoints,
 		int32_t x,
 		int32_t y,
 		int32_t z,
-		void (*sendFunction)(void *,uint16_t,const void *,unsigned int),
-		void *arg);
+		void (*sendFunction)(void *,unsigned int,const void *,unsigned int),
+		void *sendFunctionArg,
+		int (*addressToLocationFunction)(void *,const struct sockaddr_storage *,int *,int *,int *),
+		void *addressToLocationFunctionArg);
 
 	~Cluster();
 
@@ -222,9 +188,9 @@ public:
 	/**
 	 * Advertise to the cluster that we have this peer
 	 *
-	 * @param peerAddress Peer address that we have
+	 * @param peerId Identity of peer that we have
 	 */
-	void replicateHavePeer(const Address &peerAddress);
+	void replicateHavePeer(const Identity &peerId);
 
 	/**
 	 * Advertise a multicast LIKE to the cluster
@@ -243,7 +209,7 @@ public:
 	void replicateCertificateOfNetworkMembership(const CertificateOfMembership &com);
 
 	/**
-	 * Call every ~ZT_CLUSTER_PERIODIC_TASK_DEADLINE milliseconds.
+	 * Call every ~ZT_CLUSTER_PERIODIC_TASK_PERIOD milliseconds.
 	 */
 	void doPeriodicTasks();
 
@@ -254,6 +220,23 @@ public:
 	 */
 	void addMember(uint16_t memberId);
 
+	/**
+	 * Remove a member ID from this cluster
+	 *
+	 * @param memberId Member ID to remove
+	 */
+	void removeMember(uint16_t memberId);
+
+	/**
+	 * Redirect this peer to a better cluster member if needed
+	 *
+	 * @param peer Peer to (possibly) redirect
+	 * @param peerPhysicalAddress Physical address of peer's current best path (where packet was most recently received or getBestPath()->address())
+	 * @param offload Always redirect if possible -- can be used to offload peers during shutdown
+	 * @return True if peer was redirected
+	 */
+	bool redirectPeer(const SharedPtr<Peer> &peer,const InetAddress &peerPhysicalAddress,bool offload);
+
 private:
 	void _send(uint16_t memberId,const void *msg,unsigned int len);
 	void _flush(uint16_t memberId);
@@ -262,44 +245,45 @@ private:
 	uint16_t _masterSecret[ZT_SHA512_DIGEST_LEN / sizeof(uint16_t)];
 	unsigned char _key[ZT_PEER_SECRET_KEY_LENGTH];
 	const RuntimeEnvironment *RR;
-	void (*_sendFunction)(void *,uint16_t,const void *,unsigned int);
-	void *_arg;
+	void (*_sendFunction)(void *,unsigned int,const void *,unsigned int);
+	void *_sendFunctionArg;
+	int (*_addressToLocationFunction)(void *,const struct sockaddr_storage *,int *,int *,int *);
+	void *_addressToLocationFunctionArg;
 	const int32_t _x;
 	const int32_t _y;
 	const int32_t _z;
-	const DistanceAlgorithm _da;
 	const uint16_t _id;
+	const std::vector<InetAddress> _zeroTierPhysicalEndpoints;
 
 	struct _Member
 	{
 		unsigned char key[ZT_PEER_SECRET_KEY_LENGTH];
 
-		uint64_t lastReceivedFrom;
 		uint64_t lastReceivedAliveAnnouncement;
-		uint64_t lastSentTo;
 		uint64_t lastAnnouncedAliveTo;
 
 		uint64_t load;
 		int32_t x,y,z;
 
-		InetAddress physicalAddresses[ZT_CLUSTER_MEMBER_MAX_PHYSICAL_ADDRS];
-		unsigned int physicalAddressCount;
+		std::vector<InetAddress> zeroTierPhysicalEndpoints;
 
 		Buffer<ZT_CLUSTER_MAX_MESSAGE_LENGTH> q;
 
 		Mutex lock;
 
-		_Member() :
-			lastReceivedFrom(0),
-			lastReceivedAliveAnnouncement(0),
-			lastSentTo(0),
-			lastAnnouncedAliveTo(0),
-			load(0),
-			x(0),
-			y(0),
-			z(0),
-			physicalAddressCount(0) {}
-
+		inline void clear()
+		{
+			lastReceivedAliveAnnouncement = 0;
+			lastAnnouncedAliveTo = 0;
+			load = 0;
+			x = 0;
+			y = 0;
+			z = 0;
+			zeroTierPhysicalEndpoints.clear();
+			q.clear();
+		}
+
+		_Member() { this->clear(); }
 		~_Member() { Utils::burn(key,sizeof(key)); }
 	};
 
diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp
index 19747bbd..a4d45068 100644
--- a/node/IncomingPacket.cpp
+++ b/node/IncomingPacket.cpp
@@ -871,6 +871,8 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha
 		}
 		peer->setLastDirectPathPushReceived(now);
 
+		const RemotePath *currentBest = peer->getBestPath();
+
 		unsigned int count = at<uint16_t>(ZT_PACKET_IDX_PAYLOAD);
 		unsigned int ptr = ZT_PACKET_IDX_PAYLOAD + 2;
 		unsigned int v4Count = 0,v6Count = 0;
@@ -889,16 +891,20 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha
 					InetAddress a(field(ptr,4),4,at<uint16_t>(ptr + 4));
 					if ( ((flags & 0x01) == 0) && (Path::isAddressValidForPath(a)) ) {
 						TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str());
-						if (v4Count++ < ZT_PUSH_DIRECT_PATHS_MAX_ENDPOINTS_PER_TYPE)
-							peer->attemptToContactAt(RR,_localAddress,a,RR->node->now());
+						if (v4Count++ < ZT_PUSH_DIRECT_PATHS_MAX_ENDPOINTS_PER_TYPE) {
+							if ((!currentBest)||(currentBest->address() != a))
+								peer->attemptToContactAt(RR,_localAddress,a,RR->node->now());
+						}
 					}
 				}	break;
 				case 6: {
 					InetAddress a(field(ptr,16),16,at<uint16_t>(ptr + 16));
 					if ( ((flags & 0x01) == 0) && (Path::isAddressValidForPath(a)) ) {
 						TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str());
-						if (v6Count++ < ZT_PUSH_DIRECT_PATHS_MAX_ENDPOINTS_PER_TYPE)
-							peer->attemptToContactAt(RR,_localAddress,a,RR->node->now());
+						if (v6Count++ < ZT_PUSH_DIRECT_PATHS_MAX_ENDPOINTS_PER_TYPE) {
+							if ((!currentBest)||(currentBest->address() != a))
+								peer->attemptToContactAt(RR,_localAddress,a,RR->node->now());
+						}
 					}
 				}	break;
 			}
diff --git a/node/Node.cpp b/node/Node.cpp
index 26d5513e..6eea3d3d 100644
--- a/node/Node.cpp
+++ b/node/Node.cpp
@@ -46,6 +46,7 @@
 #include "Address.hpp"
 #include "Identity.hpp"
 #include "SelfAwareness.hpp"
+#include "Cluster.hpp"
 
 const struct sockaddr_storage ZT_SOCKADDR_NULL = {0};
 
@@ -135,6 +136,9 @@ Node::~Node()
 	delete RR->antiRec;
 	delete RR->mc;
 	delete RR->sw;
+#ifdef ZT_ENABLE_CLUSTER
+	delete RR->cluster;
+#endif
 }
 
 ZT_ResultCode Node::processWirePacket(
@@ -329,7 +333,18 @@ ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextB
 	}
 
 	try {
-		*nextBackgroundTaskDeadline = now + (uint64_t)std::max(std::min(timeUntilNextPingCheck,RR->sw->doTimerTasks(now)),(unsigned long)ZT_CORE_TIMER_TASK_GRANULARITY);
+#ifdef ZT_ENABLE_CLUSTER
+		// If clustering is enabled we have to call cluster->doPeriodicTasks() very often, so we override normal timer deadline behavior
+		if (RR->cluster) {
+			RR->sw->doTimerTasks(now);
+			RR->cluster->doPeriodicTasks();
+			*nextBackgroundTaskDeadline = now + ZT_CLUSTER_PERIODIC_TASK_PERIOD; // this is really short so just tick at this rate
+		} else {
+#endif
+			*nextBackgroundTaskDeadline = now + (uint64_t)std::max(std::min(timeUntilNextPingCheck,RR->sw->doTimerTasks(now)),(unsigned long)ZT_CORE_TIMER_TASK_GRANULARITY);
+#ifdef ZT_ENABLE_CLUSTER
+		}
+#endif
 	} catch ( ... ) {
 		return ZT_RESULT_FATAL_ERROR_INTERNAL;
 	}
@@ -554,6 +569,62 @@ void Node::circuitTestEnd(ZT_CircuitTest *test)
 	}
 }
 
+ZT_ResultCode Node::clusterInit(
+	unsigned int myId,
+	const struct sockaddr_storage *zeroTierPhysicalEndpoints,
+	unsigned int numZeroTierPhysicalEndpoints,
+	int x,
+	int y,
+	int z,
+	void (*sendFunction)(void *,unsigned int,const void *,unsigned int),
+	void *sendFunctionArg,
+	int (*addressToLocationFunction)(void *,const struct sockaddr_storage *,int *,int *,int *),
+	void *addressToLocationFunctionArg)
+{
+#ifdef ZT_ENABLE_CLUSTER
+	if (RR->cluster)
+		return ZT_RESULT_ERROR_BAD_PARAMETER;
+
+	std::vector<InetAddress> eps;
+	for(unsigned int i=0;i<numZeroTierPhysicalEndpoints;++i)
+		eps.push_back(InetAddress(zeroTierPhysicalEndpoints[i]));
+	std::sort(eps.begin(),eps.end());
+	RR->cluster = new Cluster(RR,myId,eps,x,y,z,sendFunction,sendFunctionArg,addressToLocationFunction,addressToLocationFunctionArg);
+
+	return ZT_RESULT_OK;
+#else
+	return ZT_RESULT_ERROR_UNSUPPORTED_OPERATION;
+#endif
+}
+
+ZT_ResultCode Node::clusterAddMember(unsigned int memberId)
+{
+#ifdef ZT_ENABLE_CLUSTER
+	if (!RR->cluster)
+		return ZT_RESULT_ERROR_BAD_PARAMETER;
+	RR->cluster->addMember((uint16_t)memberId);
+	return ZT_RESULT_OK;
+#else
+	return ZT_RESULT_ERROR_UNSUPPORTED_OPERATION;
+#endif
+}
+
+void Node::clusterRemoveMember(unsigned int memberId)
+{
+#ifdef ZT_ENABLE_CLUSTER
+	if (RR->cluster)
+		RR->cluster->removeMember((uint16_t)memberId);
+#endif
+}
+
+void Node::clusterHandleIncomingMessage(const void *msg,unsigned int len)
+{
+#ifdef ZT_ENABLE_CLUSTER
+	if (RR->cluster)
+		RR->cluster->handleIncomingStateMessage(msg,len);
+#endif
+}
+
 /****************************************************************************/
 /* Node methods used only within node/                                      */
 /****************************************************************************/
@@ -806,6 +877,22 @@ void ZT_Node_freeQueryResult(ZT_Node *node,void *qr)
 	} catch ( ... ) {}
 }
 
+int ZT_Node_addLocalInterfaceAddress(ZT_Node *node,const struct sockaddr_storage *addr,int metric, enum ZT_LocalInterfaceAddressTrust trust)
+{
+	try {
+		return reinterpret_cast<ZeroTier::Node *>(node)->addLocalInterfaceAddress(addr,metric,trust);
+	} catch ( ... ) {
+		return 0;
+	}
+}
+
+void ZT_Node_clearLocalInterfaceAddresses(ZT_Node *node)
+{
+	try {
+		reinterpret_cast<ZeroTier::Node *>(node)->clearLocalInterfaceAddresses();
+	} catch ( ... ) {}
+}
+
 void ZT_Node_setNetconfMaster(ZT_Node *node,void *networkControllerInstance)
 {
 	try {
@@ -829,19 +916,75 @@ void ZT_Node_circuitTestEnd(ZT_Node *node,ZT_CircuitTest *test)
 	} catch ( ... ) {}
 }
 
-int ZT_Node_addLocalInterfaceAddress(ZT_Node *node,const struct sockaddr_storage *addr,int metric, enum ZT_LocalInterfaceAddressTrust trust)
+enum ZT_ResultCode ZT_Node_clusterInit(
+	ZT_Node *node,
+	unsigned int myId,
+	const struct sockaddr_storage *zeroTierPhysicalEndpoints,
+	unsigned int numZeroTierPhysicalEndpoints,
+	int x,
+	int y,
+	int z,
+	void (*sendFunction)(void *,unsigned int,const void *,unsigned int),
+	void *sendFunctionArg,
+	int (*addressToLocationFunction)(void *,const struct sockaddr_storage *,int *,int *,int *),
+	void *addressToLocationFunctionArg)
 {
 	try {
-		return reinterpret_cast<ZeroTier::Node *>(node)->addLocalInterfaceAddress(addr,metric,trust);
+		return reinterpret_cast<ZeroTier::Node *>(node)->clusterInit(myId,zeroTierPhysicalEndpoints,numZeroTierPhysicalEndpoints,x,y,z,sendFunction,sendFunctionArg,addressToLocationFunction,addressToLocationFunctionArg);
 	} catch ( ... ) {
-		return 0;
+		return ZT_RESULT_FATAL_ERROR_INTERNAL;
 	}
 }
 
-void ZT_Node_clearLocalInterfaceAddresses(ZT_Node *node)
+/**
+ * Add a member to this cluster
+ *
+ * Calling this without having called clusterInit() will do nothing.
+ *
+ * @param node Node instance
+ * @param memberId Member ID (must be less than or equal to ZT_CLUSTER_MAX_MEMBERS)
+ * @return OK or error if clustering is disabled, ID invalid, etc.
+ */
+enum ZT_ResultCode ZT_Node_clusterAddMember(ZT_Node *node,unsigned int memberId)
 {
 	try {
-		reinterpret_cast<ZeroTier::Node *>(node)->clearLocalInterfaceAddresses();
+		return reinterpret_cast<ZeroTier::Node *>(node)->clusterAddMember(memberId);
+	} catch ( ... ) {
+		return ZT_RESULT_FATAL_ERROR_INTERNAL;
+	}
+}
+
+/**
+ * Remove a member from this cluster
+ *
+ * Calling this without having called clusterInit() will do nothing.
+ *
+ * @param node Node instance
+ * @param memberId Member ID to remove (nothing happens if not present)
+ */
+void ZT_Node_clusterRemoveMember(ZT_Node *node,unsigned int memberId)
+{
+	try {
+		reinterpret_cast<ZeroTier::Node *>(node)->clusterRemoveMember(memberId);
+	} catch ( ... ) {}
+}
+
+/**
+ * Handle an incoming cluster state message
+ *
+ * The message itself contains cluster member IDs, and invalid or badly
+ * addressed messages will be silently discarded.
+ *
+ * Calling this without having called clusterInit() will do nothing.
+ *
+ * @param node Node instance
+ * @param msg Cluster message
+ * @param len Length of cluster message
+ */
+void ZT_Node_clusterHandleIncomingMessage(ZT_Node *node,const void *msg,unsigned int len)
+{
+	try {
+		reinterpret_cast<ZeroTier::Node *>(node)->clusterHandleIncomingMessage(msg,len);
 	} catch ( ... ) {}
 }
 
diff --git a/node/Node.hpp b/node/Node.hpp
index c7038ed4..b8bd4dc5 100644
--- a/node/Node.hpp
+++ b/node/Node.hpp
@@ -110,6 +110,20 @@ public:
 	void setNetconfMaster(void *networkControllerInstance);
 	ZT_ResultCode circuitTestBegin(ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *));
 	void circuitTestEnd(ZT_CircuitTest *test);
+	ZT_ResultCode clusterInit(
+		unsigned int myId,
+		const struct sockaddr_storage *zeroTierPhysicalEndpoints,
+		unsigned int numZeroTierPhysicalEndpoints,
+		int x,
+		int y,
+		int z,
+		void (*sendFunction)(void *,unsigned int,const void *,unsigned int),
+		void *sendFunctionArg,
+		int (*addressToLocationFunction)(void *,const struct sockaddr_storage *,int *,int *,int *),
+		void *addressToLocationFunctionArg);
+	ZT_ResultCode clusterAddMember(unsigned int memberId);
+	void clusterRemoveMember(unsigned int memberId);
+	void clusterHandleIncomingMessage(const void *msg,unsigned int len);
 
 	// Internal functions ------------------------------------------------------
 
diff --git a/node/Peer.cpp b/node/Peer.cpp
index 6f566be4..0ba379c6 100644
--- a/node/Peer.cpp
+++ b/node/Peer.cpp
@@ -213,6 +213,12 @@ bool Peer::doPingAndKeepalive(const RuntimeEnvironment *RR,uint64_t now,int inet
 
 void Peer::pushDirectPaths(const RuntimeEnvironment *RR,RemotePath *path,uint64_t now,bool force)
 {
+#ifdef ZT_ENABLE_CLUSTER
+	// Cluster mode disables normal PUSH_DIRECT_PATHS in favor of cluster-based peer redirection
+	if (RR->cluster)
+		return;
+#endif
+
 	Mutex::Lock _l(_lock);
 
 	if (((now - _lastDirectPathPushSent) >= ZT_DIRECT_PATH_PUSH_INTERVAL)||(force)) {
diff --git a/node/RuntimeEnvironment.hpp b/node/RuntimeEnvironment.hpp
index e5d1f446..2ec88f72 100644
--- a/node/RuntimeEnvironment.hpp
+++ b/node/RuntimeEnvironment.hpp
@@ -43,6 +43,7 @@ class Multicaster;
 class AntiRecursion;
 class NetworkController;
 class SelfAwareness;
+class Cluster;
 
 /**
  * Holds global state for an instance of ZeroTier::Node
@@ -51,14 +52,17 @@ class RuntimeEnvironment
 {
 public:
 	RuntimeEnvironment(Node *n) :
-		node(n),
-		identity(),
-		localNetworkController((NetworkController *)0),
-		sw((Switch *)0),
-		mc((Multicaster *)0),
-		antiRec((AntiRecursion *)0),
-		topology((Topology *)0),
-		sa((SelfAwareness *)0)
+		node(n)
+		,identity()
+		,localNetworkController((NetworkController *)0)
+		,sw((Switch *)0)
+		,mc((Multicaster *)0)
+		,antiRec((AntiRecursion *)0)
+		,topology((Topology *)0)
+		,sa((SelfAwareness *)0)
+#ifdef ZT_ENABLE_CLUSTER
+		,cluster((Cluster *)0)
+#endif
 	{
 	}
 
@@ -86,6 +90,10 @@ public:
 	AntiRecursion *antiRec;
 	Topology *topology;
 	SelfAwareness *sa;
+
+#ifdef ZT_ENABLE_CLUSTER
+	Cluster *cluster;
+#endif
 };
 
 } // namespace ZeroTier
-- 
cgit v1.2.3


From 7711eba297955d4cf3852f36fe7c6d118da38171 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Thu, 22 Oct 2015 16:02:01 -0700
Subject: More cluster wiring...

---
 include/ZeroTierOne.h           |   4 +-
 node/Cluster.cpp                |   4 +-
 node/Cluster.hpp                |   4 +-
 osdep/Phy.hpp                   |   9 +++
 service/ClusterDefinition.hpp   | 125 ++++++++++++++++++++++++++++++++++
 service/ClusterGeoIpService.cpp |   1 -
 service/OneService.cpp          | 145 +++++++++++++++++++++++++++++++++++-----
 service/OneService.hpp          |   3 +
 8 files changed, 272 insertions(+), 23 deletions(-)
 create mode 100644 service/ClusterDefinition.hpp

(limited to 'include')

diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h
index 135c8e11..4de4a765 100644
--- a/include/ZeroTierOne.h
+++ b/include/ZeroTierOne.h
@@ -131,12 +131,12 @@ extern "C" {
 /**
  * Maximum number of cluster members (and max member ID plus one)
  */
-#define ZT_CLUSTER_MAX_MEMBERS 256
+#define ZT_CLUSTER_MAX_MEMBERS 128
 
 /**
  * Maximum allowed cluster message length in bytes
  */
-#define ZT_CLUSTER_MAX_MESSAGE_LENGTH 65535
+#define ZT_CLUSTER_MAX_MESSAGE_LENGTH (1444 * 6)
 
 /**
  * A null/empty sockaddr (all zero) to signify an unspecified socket address
diff --git a/node/Cluster.cpp b/node/Cluster.cpp
index d9514db5..e7aa5a41 100644
--- a/node/Cluster.cpp
+++ b/node/Cluster.cpp
@@ -504,7 +504,7 @@ void Cluster::doPeriodicTasks()
 
 void Cluster::addMember(uint16_t memberId)
 {
-	if (memberId >= ZT_CLUSTER_MAX_MEMBERS)
+	if ((memberId >= ZT_CLUSTER_MAX_MEMBERS)||(memberId == _id))
 		return;
 
 	Mutex::Lock _l2(_members[memberId].lock);
@@ -622,6 +622,8 @@ bool Cluster::redirectPeer(const Address &peerAddress,const InetAddress &peerPhy
 
 void Cluster::_send(uint16_t memberId,StateMessageType type,const void *msg,unsigned int len)
 {
+	if ((len + 3) > (ZT_CLUSTER_MAX_MESSAGE_LENGTH - (24 + 2 + 2))) // sanity check
+		return;
 	_Member &m = _members[memberId];
 	// assumes m.lock is locked!
 	if ((m.q.size() + len + 3) > ZT_CLUSTER_MAX_MESSAGE_LENGTH)
diff --git a/node/Cluster.hpp b/node/Cluster.hpp
index 2e60fd6b..6c9a2917 100644
--- a/node/Cluster.hpp
+++ b/node/Cluster.hpp
@@ -47,7 +47,7 @@
 /**
  * Timeout for cluster members being considered "alive"
  */
-#define ZT_CLUSTER_TIMEOUT 30000
+#define ZT_CLUSTER_TIMEOUT 10000
 
 /**
  * How often should we announce that we have a peer?
@@ -57,7 +57,7 @@
 /**
  * Desired period between doPeriodicTasks() in milliseconds
  */
-#define ZT_CLUSTER_PERIODIC_TASK_PERIOD 50
+#define ZT_CLUSTER_PERIODIC_TASK_PERIOD 100
 
 namespace ZeroTier {
 
diff --git a/osdep/Phy.hpp b/osdep/Phy.hpp
index 7f790e5d..6737034e 100644
--- a/osdep/Phy.hpp
+++ b/osdep/Phy.hpp
@@ -64,6 +64,12 @@
 #include <netinet/in.h>
 #include <netinet/tcp.h>
 
+#if defined(__linux__) || defined(linux) || defined(__LINUX__) || defined(__linux)
+#ifndef IPV6_DONTFRAG
+#define IPV6_DONTFRAG 62
+#endif
+#endif
+
 #define ZT_PHY_SOCKFD_TYPE int
 #define ZT_PHY_SOCKFD_NULL (-1)
 #define ZT_PHY_SOCKFD_VALID(s) ((s) > -1)
@@ -374,6 +380,9 @@ public:
 				f = 1; setsockopt(s,IPPROTO_IPV6,IPV6_V6ONLY,(void *)&f,sizeof(f));
 #ifdef IPV6_MTU_DISCOVER
 				f = 0; setsockopt(s,IPPROTO_IPV6,IPV6_MTU_DISCOVER,&f,sizeof(f));
+#endif
+#ifdef IPV6_DONTFRAG
+				f = 0; setsockopt(s,IPPROTO_IPV6,IPV6_DONTFRAG,&f,sizeof(f));
 #endif
 			}
 			f = 0; setsockopt(s,SOL_SOCKET,SO_REUSEADDR,(void *)&f,sizeof(f));
diff --git a/service/ClusterDefinition.hpp b/service/ClusterDefinition.hpp
new file mode 100644
index 00000000..d02894e4
--- /dev/null
+++ b/service/ClusterDefinition.hpp
@@ -0,0 +1,125 @@
+/*
+ * ZeroTier One - Network Virtualization Everywhere
+ * Copyright (C) 2011-2015  ZeroTier, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * ZeroTier may be used and distributed under the terms of the GPLv3, which
+ * are available at: http://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * If you would like to embed ZeroTier into a commercial application or
+ * redistribute it in a modified binary form, please contact ZeroTier Networks
+ * LLC. Start here: http://www.zerotier.com/
+ */
+
+#ifndef ZT_CLUSTERDEFINITION_HPP
+#define ZT_CLUSTERDEFINITION_HPP
+
+#ifdef ZT_ENABLE_CLUSTER
+
+#include <vector>
+#include <algorithm>
+
+#include "../node/Constants.hpp"
+#include "../node/Utils.hpp"
+#include "../osdep/OSUtils.hpp"
+
+namespace ZeroTier {
+
+/**
+ * Parser for cluster definition file
+ */
+class ClusterDefinition
+{
+public:
+	struct MemberDefinition
+	{
+		MemberDefinition() : id(0),x(0),y(0),z(0) { name[0] = (char)0; }
+
+		unsigned int id;
+		int x,y,z;
+		char name[256];
+		InetAddress clusterEndpoint;
+		std::vector<InetAddress> zeroTierEndpoints;
+	};
+
+	ClusterDefinition(uint64_t myAddress,const char *pathToClusterFile)
+	{
+		std::string cf;
+		if (!OSUtils::readFile(pathToClusterFile,cf))
+			return;
+
+		char myAddressStr[64];
+		Utils::snprintf(myAddressStr,sizeof(myAddressStr),"%.10llx",myAddress);
+
+		std::vector<std::string> lines(Utils::split(cf.c_str(),"\r\n","",""));
+		for(std::vector<std::string>::iterator l(lines.begin());l!=lines.end();++l) {
+			std::vector<std::string> fields(Utils::split(l->c_str()," \t","",""));
+			if ((fields.size() < 5)||(fields[0][0] == '#')||(fields[0] != myAddressStr))
+				continue;
+
+			int id = Utils::strToUInt(fields[1].c_str());
+			if ((id < 0)||(id > ZT_CLUSTER_MAX_MEMBERS))
+				continue;
+			MemberDefinition &md = _md[id];
+
+			md.id = (unsigned int)id;
+			if (fields.size() >= 6) {
+				std::vector<std::string> xyz(Utils::split(fields[5].c_str(),",","",""));
+				md.x = (xyz.size() > 0) ? Utils::strToInt(xyz[0].c_str()) : 0;
+				md.y = (xyz.size() > 1) ? Utils::strToInt(xyz[1].c_str()) : 0;
+				md.z = (xyz.size() > 2) ? Utils::strToInt(xyz[2].c_str()) : 0;
+			}
+			Utils::scopy(md.name,sizeof(md.name),fields[2].c_str());
+			md.clusterEndpoint.fromString(fields[3]);
+			if (!md.clusterEndpoint)
+				continue;
+			std::vector<std::string> zips(Utils::split(fields[4].c_str(),",","",""));
+			for(std::vector<std::string>::iterator zip(zips.begin());zip!=zips.end();++zip) {
+				InetAddress i;
+				i.fromString(*zip);
+				if (i)
+					md.zeroTierEndpoints.push_back(i);
+			}
+
+			_ids.push_back((unsigned int)id);
+		}
+
+		std::sort(_ids.begin(),_ids.end());
+	}
+
+	inline const MemberDefinition &operator[](unsigned int id) const throw() { return _md[id]; }
+	inline unsigned int size() const throw() { return (unsigned int)_ids.size(); }
+	inline const std::vector<unsigned int> &ids() const throw() { return _ids; }
+
+	inline std::vector<MemberDefinition> members() const
+	{
+		std::vector<MemberDefinition> m;
+		for(std::vector<unsigned int>::const_iterator i(_ids.begin());i!=_ids.end();++i)
+			m.push_back(_md[*i]);
+		return m;
+	}
+
+private:
+	MemberDefinition _md[ZT_CLUSTER_MAX_MEMBERS];
+	std::vector<unsigned int> _ids;
+};
+
+} // namespace ZeroTier
+
+#endif // ZT_ENABLE_CLUSTER
+
+#endif
diff --git a/service/ClusterGeoIpService.cpp b/service/ClusterGeoIpService.cpp
index 83afe770..0c815272 100644
--- a/service/ClusterGeoIpService.cpp
+++ b/service/ClusterGeoIpService.cpp
@@ -165,7 +165,6 @@ void ClusterGeoIpService::threadMain()
 									{
 										Mutex::Lock _l2(_cache_m);
 										_cache[rip] = ce;
-										std::cout << ">> " << linebuf << std::endl;
 									}
 								}
 							}
diff --git a/service/OneService.cpp b/service/OneService.cpp
index 6b28c41e..a64d680b 100644
--- a/service/OneService.cpp
+++ b/service/OneService.cpp
@@ -58,6 +58,8 @@
 
 #include "OneService.hpp"
 #include "ControlPlane.hpp"
+#include "ClusterGeoIpService.hpp"
+#include "ClusterDefinition.hpp"
 
 /**
  * Uncomment to enable UDP breakage switch
@@ -366,6 +368,11 @@ static int SnodeDataStorePutFunction(ZT_Node *node,void *uptr,const char *name,c
 static int SnodeWirePacketSendFunction(ZT_Node *node,void *uptr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *addr,const void *data,unsigned int len);
 static void SnodeVirtualNetworkFrameFunction(ZT_Node *node,void *uptr,uint64_t nwid,uint64_t sourceMac,uint64_t destMac,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len);
 
+#ifdef ZT_ENABLE_CLUSTER
+static void SclusterSendFunction(void *uptr,unsigned int toMemberId,const void *data,unsigned int len);
+static int SclusterGeoIpFunction(void *uptr,const struct sockaddr_storage *addr,int *x,int *y,int *z);
+#endif
+
 static void StapFrameHandler(void *uptr,uint64_t nwid,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len);
 
 static int ShttpOnMessageBegin(http_parser *parser);
@@ -419,26 +426,32 @@ class OneServiceImpl : public OneService
 {
 public:
 	OneServiceImpl(const char *hp,unsigned int port) :
-		_homePath((hp) ? hp : "."),
-		_tcpFallbackResolver(ZT_TCP_FALLBACK_RELAY),
+		_homePath((hp) ? hp : ".")
+		,_tcpFallbackResolver(ZT_TCP_FALLBACK_RELAY)
 #ifdef ZT_ENABLE_NETWORK_CONTROLLER
-		_controller((SqliteNetworkController *)0),
+		,_controller((SqliteNetworkController *)0)
 #endif
-		_phy(this,false,true),
-		_node((Node *)0),
-		_controlPlane((ControlPlane *)0),
-		_lastDirectReceiveFromGlobal(0),
-		_lastSendToGlobal(0),
-		_lastRestart(0),
-		_nextBackgroundTaskDeadline(0),
-		_tcpFallbackTunnel((TcpConnection *)0),
-		_termReason(ONE_STILL_RUNNING),
-		_port(0),
+		,_phy(this,false,true)
+		,_node((Node *)0)
+		,_controlPlane((ControlPlane *)0)
+		,_lastDirectReceiveFromGlobal(0)
+		,_lastSendToGlobal(0)
+		,_lastRestart(0)
+		,_nextBackgroundTaskDeadline(0)
+		,_tcpFallbackTunnel((TcpConnection *)0)
+		,_termReason(ONE_STILL_RUNNING)
+		,_port(0)
 #ifdef ZT_USE_MINIUPNPC
-		_v4UpnpUdpSocket((PhySocket *)0),
-		_upnpClient((UPNPClient *)0),
+		,_v4UpnpUdpSocket((PhySocket *)0)
+		,_upnpClient((UPNPClient *)0)
 #endif
-		_run(true)
+#ifdef ZT_ENABLE_CLUSTER
+		,_clusterMessageSocket((PhySocket *)0)
+		,_clusterGeoIpService((ClusterGeoIpService *)0)
+		,_clusterDefinition((ClusterDefinition *)0)
+		,_clusterMemberId(0)
+#endif
+		,_run(true)
 	{
 		const int portTrials = (port == 0) ? 256 : 1; // if port is 0, pick random
 		for(int k=0;k<portTrials;++k) {
@@ -510,12 +523,19 @@ public:
 		_phy.close(_v6UdpSocket);
 		_phy.close(_v4TcpListenSocket);
 		_phy.close(_v6TcpListenSocket);
+#ifdef ZT_ENABLE_CLUSTER
+		_phy.close(_clusterMessageSocket);
+#endif
 #ifdef ZT_USE_MINIUPNPC
 		_phy.close(_v4UpnpUdpSocket);
 		delete _upnpClient;
 #endif
 #ifdef ZT_ENABLE_NETWORK_CONTROLLER
 		delete _controller;
+#endif
+#ifdef ZT_ENABLE_CLUSTER
+		delete _clusterGeoIpService;
+		delete _clusterDefinition;
 #endif
 	}
 
@@ -556,6 +576,70 @@ public:
 			_node->setNetconfMaster((void *)_controller);
 #endif
 
+#ifdef ZT_ENABLE_CLUSTER
+		if (OSUtils::fileExists((_homePath + ZT_PATH_SEPARATOR_S + "cluster").c_str())) {
+			_clusterDefinition = new ClusterDefinition(_node->address(),(_homePath + ZT_PATH_SEPARATOR_S + "cluster").c_str());
+			if (_clusterDefinition->size() > 0) {
+				std::vector<ClusterDefinition::MemberDefinition> members(_clusterDefinition->members());
+				for(std::vector<ClusterDefinition::MemberDefinition>::iterator m(members.begin());m!=members.end();++m) {
+					PhySocket *cs = _phy.udpBind(reinterpret_cast<const struct sockaddr *>(&(m->clusterEndpoint)));
+					if (cs) {
+						if (_clusterMessageSocket) {
+							_phy.close(_clusterMessageSocket,false);
+							_phy.close(cs,false);
+
+							Mutex::Lock _l(_termReason_m);
+							_termReason = ONE_UNRECOVERABLE_ERROR;
+							_fatalErrorMessage = "Cluster: can't determine my cluster member ID: able to bind more than one cluster message socket IP/port!";
+							return _termReason;
+						}
+						_clusterMessageSocket = cs;
+						_clusterMemberId = m->id;
+					}
+				}
+
+				if (!_clusterMessageSocket) {
+					Mutex::Lock _l(_termReason_m);
+					_termReason = ONE_UNRECOVERABLE_ERROR;
+					_fatalErrorMessage = "Cluster: can't determine my cluster member ID: unable to bind to any cluster message socket IP/port.";
+					return _termReason;
+				}
+
+				if (OSUtils::fileExists((_homePath + ZT_PATH_SEPARATOR_S + "cluster-geo.exe").c_str()))
+					_clusterGeoIpService = new ClusterGeoIpService((_homePath + ZT_PATH_SEPARATOR_S + "cluster-geo.exe").c_str());
+
+				const ClusterDefinition::MemberDefinition &me = (*_clusterDefinition)[_clusterMemberId];
+				InetAddress endpoints[255];
+				unsigned int numEndpoints = 0;
+				for(std::vector<InetAddress>::const_iterator i(me.zeroTierEndpoints.begin());i!=me.zeroTierEndpoints.end();++i)
+					endpoints[numEndpoints++] = *i;
+
+				if (_node->clusterInit(
+					_clusterMemberId,
+					reinterpret_cast<const struct sockaddr_storage *>(endpoints),
+					numEndpoints,
+					me.x,
+					me.y,
+					me.z,
+					&SclusterSendFunction,
+					this,
+					(_clusterGeoIpService) ? &SclusterGeoIpFunction : 0,
+					this) == ZT_RESULT_OK) {
+
+					std::vector<ClusterDefinition::MemberDefinition> members(_clusterDefinition->members());
+					for(std::vector<ClusterDefinition::MemberDefinition>::iterator m(members.begin());m!=members.end();++m) {
+						if (m->id != _clusterMemberId)
+							_node->clusterAddMember(m->id);
+					}
+
+				}
+			} else {
+				delete _clusterDefinition;
+				_clusterDefinition = (ClusterDefinition *)0;
+			}
+		}
+#endif
+
 			_controlPlane = new ControlPlane(this,_node,(_homePath + ZT_PATH_SEPARATOR_S + "ui").c_str());
 			_controlPlane->addAuthToken(authToken.c_str());
 
@@ -781,10 +865,18 @@ public:
 
 	inline void phyOnDatagram(PhySocket *sock,void **uptr,const struct sockaddr *from,void *data,unsigned long len)
 	{
+#ifdef ZT_ENABLE_CLUSTER
+		if (sock == _clusterMessageSocket) {
+			_node->clusterHandleIncomingMessage(data,len);
+			return;
+		}
+#endif
+
 #ifdef ZT_BREAK_UDP
 		if (OSUtils::fileExists("/tmp/ZT_BREAK_UDP"))
 			return;
 #endif
+
 		if ((len >= 16)&&(reinterpret_cast<const InetAddress *>(from)->ipScope() == InetAddress::IP_SCOPE_GLOBAL))
 			_lastDirectReceiveFromGlobal = OSUtils::now();
 		ZT_ResultCode rc = _node->processWirePacket(
@@ -1303,7 +1395,6 @@ public:
 			_phy.close(tc->sock); // will call close handler, which deletes from _tcpConnections
 	}
 
-private:
 	std::string _dataStorePrepPath(const char *name) const
 	{
 		std::string p(_homePath);
@@ -1358,6 +1449,13 @@ private:
 	UPNPClient *_upnpClient;
 #endif
 
+#ifdef ZT_ENABLE_CLUSTER
+	PhySocket *_clusterMessageSocket;
+	ClusterGeoIpService *_clusterGeoIpService;
+	ClusterDefinition *_clusterDefinition;
+	unsigned int _clusterMemberId;
+#endif
+
 	bool _run;
 	Mutex _run_m;
 };
@@ -1375,6 +1473,19 @@ static int SnodeWirePacketSendFunction(ZT_Node *node,void *uptr,const struct soc
 static void SnodeVirtualNetworkFrameFunction(ZT_Node *node,void *uptr,uint64_t nwid,uint64_t sourceMac,uint64_t destMac,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len)
 { reinterpret_cast<OneServiceImpl *>(uptr)->nodeVirtualNetworkFrameFunction(nwid,sourceMac,destMac,etherType,vlanId,data,len); }
 
+static void SclusterSendFunction(void *uptr,unsigned int toMemberId,const void *data,unsigned int len)
+{
+	OneServiceImpl *const impl = reinterpret_cast<OneServiceImpl *>(uptr);
+	const ClusterDefinition::MemberDefinition &md = (*(impl->_clusterDefinition))[toMemberId];
+	if (md.clusterEndpoint)
+		impl->_phy.udpSend(impl->_clusterMessageSocket,reinterpret_cast<const struct sockaddr *>(&(md.clusterEndpoint)),data,len);
+}
+static int SclusterGeoIpFunction(void *uptr,const struct sockaddr_storage *addr,int *x,int *y,int *z)
+{
+	OneServiceImpl *const impl = reinterpret_cast<OneServiceImpl *>(uptr);
+	return (int)(impl->_clusterGeoIpService->locate(*(reinterpret_cast<const InetAddress *>(addr)),*x,*y,*z));
+}
+
 static void StapFrameHandler(void *uptr,uint64_t nwid,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len)
 { reinterpret_cast<OneServiceImpl *>(uptr)->tapFrameHandler(nwid,from,to,etherType,vlanId,data,len); }
 
diff --git a/service/OneService.hpp b/service/OneService.hpp
index 2f76ebaa..4f7b988b 100644
--- a/service/OneService.hpp
+++ b/service/OneService.hpp
@@ -43,6 +43,9 @@ namespace ZeroTier {
  * periodically checked and updates are automatically downloaded, verified
  * against a built-in list of update signing keys, and installed. This is
  * only supported for certain platforms.
+ *
+ * If built with ZT_ENABLE_CLUSTER, a 'cluster' file is checked and if
+ * present is read to determine the identity of other cluster members.
  */
 class OneService
 {
-- 
cgit v1.2.3


From 865acfa40f65626f41bbd718b645f57c7d6db9b2 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Mon, 26 Oct 2015 12:41:08 -0700
Subject: Cluster status plumbing.

---
 include/ZeroTierOne.h | 77 +++++++++++++++++++++++++++++++++++++++++++++++++++
 node/Cluster.cpp      | 56 +++++++++++++++++++++++++++++++++++++
 node/Cluster.hpp      |  7 +++++
 node/Topology.cpp     | 15 ++++++++++
 node/Topology.hpp     |  5 ++++
 5 files changed, 160 insertions(+)

(limited to 'include')

diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h
index 4de4a765..86bffcdf 100644
--- a/include/ZeroTierOne.h
+++ b/include/ZeroTierOne.h
@@ -133,6 +133,11 @@ extern "C" {
  */
 #define ZT_CLUSTER_MAX_MEMBERS 128
 
+/**
+ * Maximum number of physical ZeroTier addresses a cluster member can report
+ */
+#define ZT_CLUSTER_MAX_ZT_PHYSICAL_ADDRESSES 16
+
 /**
  * Maximum allowed cluster message length in bytes
  */
@@ -879,6 +884,78 @@ typedef struct {
 	unsigned int nextHopCount;
 } ZT_CircuitTestReport;
 
+/**
+ * A cluster member's status
+ */
+typedef struct {
+	/**
+	 * This cluster member's ID (from 0 to 1-ZT_CLUSTER_MAX_MEMBERS)
+	 */
+	unsigned int id;
+
+	/**
+	 * Number of milliseconds since last 'alive' heartbeat message received via cluster backplane address
+	 */
+	unsigned int msSinceLastHeartbeat;
+
+	/**
+	 * Non-zero if cluster member is alive
+	 */
+	int alive;
+
+	/**
+	 * X, Y, and Z coordinates of this member (if specified, otherwise zero)
+	 *
+	 * What these mean depends on the location scheme being used for
+	 * location-aware clustering. At present this is GeoIP and these
+	 * will be the X, Y, and Z coordinates of the location on a spherical
+	 * approximation of Earth where Earth's core is the origin (in km).
+	 * They don't have to be perfect and need only be comparable with others
+	 * to find shortest path via the standard vector distance formula.
+	 */
+	int x,y,z;
+
+	/**
+	 * Cluster member's last reported load
+	 */
+	uint64_t load;
+
+	/**
+	 * Number of peers this cluster member "has"
+	 */
+	uint64_t peers;
+
+	/**
+	 * Physical ZeroTier endpoints for this member (where peers are sent when directed here)
+	 */
+	struct sockaddr_storage zeroTierPhysicalEndpoints[ZT_CLUSTER_MAX_ZT_PHYSICAL_ADDRESSES];
+
+	/**
+	 * Number of physical ZeroTier endpoints this member is announcing
+	 */
+	unsigned int numZeroTierPhysicalEndpoints;
+} ZT_ClusterMemberStatus;
+
+/**
+ * ZeroTier cluster status
+ */
+typedef struct {
+	/**
+	 * My cluster member ID (a record for 'self' is included in member[])
+	 */
+	unsigned int myId;
+
+	/**
+	 * Number of cluster members
+	 */
+	unsigned int clusterSize;
+
+	/**
+	 * Cluster member statuses
+	 */
+	ZT_ClusterMemberStatus member[ZT_CLUSTER_MAX_MEMBERS];
+} ZT_ClusterStatus;
+
 /**
  * An instance of a ZeroTier One node (opaque)
  */
diff --git a/node/Cluster.cpp b/node/Cluster.cpp
index 900804b7..47794214 100644
--- a/node/Cluster.cpp
+++ b/node/Cluster.cpp
@@ -638,6 +638,62 @@ bool Cluster::redirectPeer(const Address &peerAddress,const InetAddress &peerPhy
 	}
 }
 
+void Cluster::status(ZT_ClusterStatus &status) const
+{
+	const uint64_t now = RR->node->now();
+	memset(&status,0,sizeof(ZT_ClusterStatus));
+	ZT_ClusterMemberStatus *ms[ZT_CLUSTER_MAX_MEMBERS];
+	memset(ms,0,sizeof(ms));
+
+	status.myId = _id;
+
+	ms[_id] = &(status.member[status.clusterSize++]);
+	ms[_id]->id = _id;
+	ms[_id]->alive = 1;
+	ms[_id]->x = _x;
+	ms[_id]->y = _y;
+	ms[_id]->z = _z;
+	ms[_id]->peers = RR->topology->countAlive();
+	for(std::vector<InetAddress>::const_iterator ep(_zeroTierPhysicalEndpoints.begin());ep!=_zeroTierPhysicalEndpoints.end();++ep) {
+		if (ms[_id]->numZeroTierPhysicalEndpoints >= ZT_CLUSTER_MAX_ZT_PHYSICAL_ADDRESSES) // sanity check
+			break;
+		memcpy(&(ms[_id]->zeroTierPhysicalEndpoints[ms[_id]->numZeroTierPhysicalEndpoints++]),&(*ep),sizeof(struct sockaddr_storage));
+	}
+
+	{
+		Mutex::Lock _l1(_memberIds_m);
+		for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
+			if (status.clusterSize >= ZT_CLUSTER_MAX_MEMBERS) // sanity check
+				break;
+			ZT_ClusterMemberStatus *s = ms[*mid] = &(status.member[status.clusterSize++]);
+			_Member &m = _members[*mid];
+			Mutex::Lock ml(m.lock);
+
+			s->id = *mid;
+			s->msSinceLastHeartbeat = (unsigned int)std::min((uint64_t)(~((unsigned int)0)),(now - m.lastReceivedAliveAnnouncement));
+			s->alive = (s->msSinceLastHeartbeat < ZT_CLUSTER_TIMEOUT) ? 1 : 0;
+			s->x = m.x;
+			s->y = m.y;
+			s->z = m.z;
+			s->load = m.load;
+			for(std::vector<InetAddress>::const_iterator ep(m.zeroTierPhysicalEndpoints.begin());ep!=m.zeroTierPhysicalEndpoints.end();++ep) {
+				if (s->numZeroTierPhysicalEndpoints >= ZT_CLUSTER_MAX_ZT_PHYSICAL_ADDRESSES) // sanity check
+					break;
+				memcpy(&(s->zeroTierPhysicalEndpoints[s->numZeroTierPhysicalEndpoints++]),&(*ep),sizeof(struct sockaddr_storage));
+			}
+		}
+	}
+
+	{
+		Mutex::Lock _l2(_peerAffinities_m);
+		for(std::vector<_PeerAffinity>::const_iterator pi(_peerAffinities.begin());pi!=_peerAffinities.end();++pi) {
+			unsigned int mid = pi->clusterMemberId();
+			if ((ms[mid])&&(mid != _id)&&((now - pi->timestamp) < ZT_PEER_ACTIVITY_TIMEOUT))
+				++ms[mid]->peers;
+		}
+	}
+}
+
 void Cluster::_send(uint16_t memberId,StateMessageType type,const void *msg,unsigned int len)
 {
 	if ((len + 3) > (ZT_CLUSTER_MAX_MESSAGE_LENGTH - (24 + 2 + 2))) // sanity check
diff --git a/node/Cluster.hpp b/node/Cluster.hpp
index daa81185..be346659 100644
--- a/node/Cluster.hpp
+++ b/node/Cluster.hpp
@@ -254,6 +254,13 @@ public:
 	 */
 	bool redirectPeer(const Address &peerAddress,const InetAddress &peerPhysicalAddress,bool offload);
 
+	/**
+	 * Fill out ZT_ClusterStatus structure (from core API)
+	 *
+	 * @param status Reference to structure to hold result (anything there is replaced)
+	 */
+	void status(ZT_ClusterStatus &status) const;
+
 private:
 	void _send(uint16_t memberId,StateMessageType type,const void *msg,unsigned int len);
 	void _flush(uint16_t memberId);
diff --git a/node/Topology.cpp b/node/Topology.cpp
index 624c4987..e56d1f47 100644
--- a/node/Topology.cpp
+++ b/node/Topology.cpp
@@ -332,6 +332,21 @@ void Topology::clean(uint64_t now)
 	}
 }
 
+unsigned long Topology::countAlive() const
+{
+	const uint64_t now = RR->node->now();
+	unsigned long cnt = 0;
+	Mutex::Lock _l(_lock);
+	Hashtable< Address,SharedPtr<Peer> >::Iterator i(const_cast<Topology *>(this)->_peers);
+	Address *a = (Address *)0;
+	SharedPtr<Peer> *p = (SharedPtr<Peer> *)0;
+	while (i.next(a,p)) {
+		if ((*p)->alive(now))
+			++cnt;
+	}
+	return cnt;
+}
+
 Identity Topology::_getIdentity(const Address &zta)
 {
 	char p[128];
diff --git a/node/Topology.hpp b/node/Topology.hpp
index 324ec000..ee9827b9 100644
--- a/node/Topology.hpp
+++ b/node/Topology.hpp
@@ -192,6 +192,11 @@ public:
 	 */
 	void clean(uint64_t now);
 
+	/**
+	 * @return Number of 'alive' peers
+	 */
+	unsigned long countAlive() const;
+
 	/**
 	 * Apply a function or function object to all peers
 	 *
-- 
cgit v1.2.3


From 5ff7733f84302bbd0ce0a578b3040122d1140f0f Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Mon, 26 Oct 2015 12:49:17 -0700
Subject: More plumbing of cluster status.

---
 include/ZeroTierOne.h | 11 +++++++++++
 node/Node.cpp         | 48 +++++++++++++++++++-----------------------------
 node/Node.hpp         |  1 +
 3 files changed, 31 insertions(+), 29 deletions(-)

(limited to 'include')

diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h
index 86bffcdf..68c1e4d7 100644
--- a/include/ZeroTierOne.h
+++ b/include/ZeroTierOne.h
@@ -1516,6 +1516,17 @@ void ZT_Node_clusterRemoveMember(ZT_Node *node,unsigned int memberId);
  */
 void ZT_Node_clusterHandleIncomingMessage(ZT_Node *node,const void *msg,unsigned int len);
 
+/**
+ * Get the current status of the cluster from this node's point of view
+ *
+ * Calling this without clusterInit() or without cluster support will just
+ * zero out the structure and show a cluster size of zero.
+ *
+ * @param node Node instance
+ * @param cs Cluster status structure to fill with data
+ */
+void ZT_Node_clusterStatus(ZT_Node *node,ZT_ClusterStatus *cs);
+
 /**
  * Get ZeroTier One version
  *
diff --git a/node/Node.cpp b/node/Node.cpp
index c54d783f..2b298903 100644
--- a/node/Node.cpp
+++ b/node/Node.cpp
@@ -636,6 +636,18 @@ void Node::clusterHandleIncomingMessage(const void *msg,unsigned int len)
 #endif
 }
 
+void Node::clusterStatus(ZT_ClusterStatus *cs)
+{
+	if (!cs)
+		return;
+#ifdef ZT_ENABLE_CLUSTER
+	if (RR->cluster)
+		RR->cluster->status(*cs);
+	else
+#endif
+	memset(cs,0,sizeof(ZT_ClusterStatus));
+}
+
 /****************************************************************************/
 /* Node methods used only within node/                                      */
 /****************************************************************************/
@@ -947,15 +959,6 @@ enum ZT_ResultCode ZT_Node_clusterInit(
 	}
 }
 
-/**
- * Add a member to this cluster
- *
- * Calling this without having called clusterInit() will do nothing.
- *
- * @param node Node instance
- * @param memberId Member ID (must be less than or equal to ZT_CLUSTER_MAX_MEMBERS)
- * @return OK or error if clustering is disabled, ID invalid, etc.
- */
 enum ZT_ResultCode ZT_Node_clusterAddMember(ZT_Node *node,unsigned int memberId)
 {
 	try {
@@ -965,14 +968,6 @@ enum ZT_ResultCode ZT_Node_clusterAddMember(ZT_Node *node,unsigned int memberId)
 	}
 }
 
-/**
- * Remove a member from this cluster
- *
- * Calling this without having called clusterInit() will do nothing.
- *
- * @param node Node instance
- * @param memberId Member ID to remove (nothing happens if not present)
- */
 void ZT_Node_clusterRemoveMember(ZT_Node *node,unsigned int memberId)
 {
 	try {
@@ -980,18 +975,6 @@ void ZT_Node_clusterRemoveMember(ZT_Node *node,unsigned int memberId)
 	} catch ( ... ) {}
 }
 
-/**
- * Handle an incoming cluster state message
- *
- * The message itself contains cluster member IDs, and invalid or badly
- * addressed messages will be silently discarded.
- *
- * Calling this without having called clusterInit() will do nothing.
- *
- * @param node Node instance
- * @param msg Cluster message
- * @param len Length of cluster message
- */
 void ZT_Node_clusterHandleIncomingMessage(ZT_Node *node,const void *msg,unsigned int len)
 {
 	try {
@@ -999,6 +982,13 @@ void ZT_Node_clusterHandleIncomingMessage(ZT_Node *node,const void *msg,unsigned
 	} catch ( ... ) {}
 }
 
+void ZT_Node_clusterStatus(ZT_Node *node,ZT_ClusterStatus *cs)
+{
+	try {
+		reinterpret_cast<ZeroTier::Node *>(node)->clusterStatus(cs);
+	} catch ( ... ) {}
+}
+
 void ZT_version(int *major,int *minor,int *revision,unsigned long *featureFlags)
 {
 	if (major) *major = ZEROTIER_ONE_VERSION_MAJOR;
diff --git a/node/Node.hpp b/node/Node.hpp
index b8bd4dc5..4094a79e 100644
--- a/node/Node.hpp
+++ b/node/Node.hpp
@@ -124,6 +124,7 @@ public:
 	ZT_ResultCode clusterAddMember(unsigned int memberId);
 	void clusterRemoveMember(unsigned int memberId);
 	void clusterHandleIncomingMessage(const void *msg,unsigned int len);
+	void clusterStatus(ZT_ClusterStatus *cs);
 
 	// Internal functions ------------------------------------------------------
 
-- 
cgit v1.2.3


From debed1ac2dce5822df234ce92bf4692ff1a081db Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Mon, 26 Oct 2015 13:06:10 -0700
Subject: Expose cluster status in /status JSON response.

---
 include/ZeroTierOne.h    |  2 +-
 node/Cluster.cpp         |  4 ++--
 service/ControlPlane.cpp | 34 ++++++++++++++++++++++++++++++++--
 3 files changed, 35 insertions(+), 5 deletions(-)

(limited to 'include')

diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h
index 68c1e4d7..7af4f760 100644
--- a/include/ZeroTierOne.h
+++ b/include/ZeroTierOne.h
@@ -953,7 +953,7 @@ typedef struct {
 	/**
 	 * Cluster member statuses
 	 */
-	ZT_ClusterMemberStatus member[ZT_CLUSTER_MAX_MEMBERS];
+	ZT_ClusterMemberStatus members[ZT_CLUSTER_MAX_MEMBERS];
 } ZT_ClusterStatus;
 
 /**
diff --git a/node/Cluster.cpp b/node/Cluster.cpp
index 47794214..9d25593a 100644
--- a/node/Cluster.cpp
+++ b/node/Cluster.cpp
@@ -647,7 +647,7 @@ void Cluster::status(ZT_ClusterStatus &status) const
 
 	status.myId = _id;
 
-	ms[_id] = &(status.member[status.clusterSize++]);
+	ms[_id] = &(status.members[status.clusterSize++]);
 	ms[_id]->id = _id;
 	ms[_id]->alive = 1;
 	ms[_id]->x = _x;
@@ -665,7 +665,7 @@ void Cluster::status(ZT_ClusterStatus &status) const
 		for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
 			if (status.clusterSize >= ZT_CLUSTER_MAX_MEMBERS) // sanity check
 				break;
-			ZT_ClusterMemberStatus *s = ms[*mid] = &(status.member[status.clusterSize++]);
+			ZT_ClusterMemberStatus *s = ms[*mid] = &(status.members[status.clusterSize++]);
 			_Member &m = _members[*mid];
 			Mutex::Lock ml(m.lock);
 
diff --git a/service/ControlPlane.cpp b/service/ControlPlane.cpp
index 7affb08c..9770db90 100644
--- a/service/ControlPlane.cpp
+++ b/service/ControlPlane.cpp
@@ -354,8 +354,36 @@ unsigned int ControlPlane::handleRequest(
 
 			if (ps[0] == "status") {
 				responseContentType = "application/json";
+
 				ZT_NodeStatus status;
 				_node->status(&status);
+
+				std::string clusterJson;
+#ifdef ZT_ENABLE_CLUSTER
+				{
+					ZT_ClusterStatus cs;
+					_node->clusterStatus(&cs);
+
+					char t[4096];
+					Utils::snprintf(t,sizeof(t),"{\n\t\t\"myId\": %u,\n\t\t\"clusterSize\": %u,\n\t\t\"members: [\n",cs.myId,cs.clusterSize);
+					clusterJson.append(t);
+					for(unsigned int i=0;i<cs.clusterSize;++i) {
+						Utils::snprintf(t,sizeof(t),"\t\t\t{\n\t\t\t\t\"id\": %u,\n\t\t\t\t\"msSinceLastHeartbeat\": %u,\n\t\t\t\t\"alive\": %s,\n\t\t\t\t\"x\": %d,\n\t\t\t\t\"y\": %d,\n\t\t\t\t\"z\": %d,\n\t\t\t\t\"load\": %llu\n\t\t\t\t\"peers\": %llu\n\t\t\t}%s",
+							cs.members[i].id,
+							cs.members[i].msSinceLastHeartbeat,
+							(cs.members[i].alive != 0) ? "true" : "false",
+							cs.members[i].x,
+							cs.members[i].y,
+							cs.members[i].z,
+							cs.members[i].load,
+							cs.members[i].peers,
+							(i == (cs.clusterSize - 1)) ? "," : "");
+						clusterJson.append(t);
+					}
+					clusterJson.append(" ]\n\t\t}");
+				}
+#endif
+
 				Utils::snprintf(json,sizeof(json),
 					"{\n"
 					"\t\"address\": \"%.10llx\",\n"
@@ -368,7 +396,8 @@ unsigned int ControlPlane::handleRequest(
 					"\t\"versionMinor\": %d,\n"
 					"\t\"versionRev\": %d,\n"
 					"\t\"version\": \"%d.%d.%d\",\n"
-					"\t\"clock\": %llu\n"
+					"\t\"clock\": %llu,\n"
+					"\t\"cluster\": %s\n"
 					"}\n",
 					status.address,
 					status.publicIdentity,
@@ -380,7 +409,8 @@ unsigned int ControlPlane::handleRequest(
 					ZEROTIER_ONE_VERSION_MINOR,
 					ZEROTIER_ONE_VERSION_REVISION,
 					ZEROTIER_ONE_VERSION_MAJOR,ZEROTIER_ONE_VERSION_MINOR,ZEROTIER_ONE_VERSION_REVISION,
-					(unsigned long long)OSUtils::now());
+					(unsigned long long)OSUtils::now(),
+					((clusterJson.length() > 0) ? clusterJson.c_str() : "null"));
 				responseBody = json;
 				scode = 200;
 			} else if (ps[0] == "config") {
-- 
cgit v1.2.3


From cfe166ef359c0d92b1521e6c127e2b92238c0731 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Tue, 27 Oct 2015 12:29:01 -0700
Subject: Tweak some size limits.

---
 include/ZeroTierOne.h | 2 +-
 node/Cluster.hpp      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'include')

diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h
index 7af4f760..7371b9f0 100644
--- a/include/ZeroTierOne.h
+++ b/include/ZeroTierOne.h
@@ -141,7 +141,7 @@ extern "C" {
 /**
  * Maximum allowed cluster message length in bytes
  */
-#define ZT_CLUSTER_MAX_MESSAGE_LENGTH (1444 * 6)
+#define ZT_CLUSTER_MAX_MESSAGE_LENGTH (1444 * 4)
 
 /**
  * A null/empty sockaddr (all zero) to signify an unspecified socket address
diff --git a/node/Cluster.hpp b/node/Cluster.hpp
index 7d0c6a08..bb7d3b39 100644
--- a/node/Cluster.hpp
+++ b/node/Cluster.hpp
@@ -57,7 +57,7 @@
 /**
  * Desired period between doPeriodicTasks() in milliseconds
  */
-#define ZT_CLUSTER_PERIODIC_TASK_PERIOD 100
+#define ZT_CLUSTER_PERIODIC_TASK_PERIOD 50
 
 namespace ZeroTier {
 
-- 
cgit v1.2.3


From f692cec763d67caae54a4f47446657c390563319 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Tue, 27 Oct 2015 14:04:12 -0700
Subject: Change how cluster relays packets -- just PROXY_UNITE and then send
 packet via normal ZeroTier front plane -- more efficient and eliminates
 fragmentation issues.

---
 include/ZeroTierOne.h |   2 +-
 node/Cluster.cpp      | 195 +++++++++++++++++++++++++-------------------------
 node/Cluster.hpp      |  22 +++---
 node/Switch.cpp       |  37 +++++-----
 node/Switch.hpp       |   8 +--
 5 files changed, 133 insertions(+), 131 deletions(-)

(limited to 'include')

diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h
index 7371b9f0..3457634b 100644
--- a/include/ZeroTierOne.h
+++ b/include/ZeroTierOne.h
@@ -141,7 +141,7 @@ extern "C" {
 /**
  * Maximum allowed cluster message length in bytes
  */
-#define ZT_CLUSTER_MAX_MESSAGE_LENGTH (1444 * 4)
+#define ZT_CLUSTER_MAX_MESSAGE_LENGTH (1500 - 48)
 
 /**
  * A null/empty sockaddr (all zero) to signify an unspecified socket address
diff --git a/node/Cluster.cpp b/node/Cluster.cpp
index eef02bc7..c18663bc 100644
--- a/node/Cluster.cpp
+++ b/node/Cluster.cpp
@@ -250,102 +250,87 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len)
 							}
 						}	break;
 
-						case STATE_MESSAGE_RELAY: {
+						case STATE_MESSAGE_PROXY_UNITE: {
+							const Address localPeerAddress(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); ptr += ZT_ADDRESS_LENGTH;
+							const Address remotePeerAddress(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); ptr += ZT_ADDRESS_LENGTH;
 							const unsigned int numRemotePeerPaths = dmsg[ptr++];
 							InetAddress remotePeerPaths[256]; // size is 8-bit, so 256 is max
 							for(unsigned int i=0;i<numRemotePeerPaths;++i)
 								ptr += remotePeerPaths[i].deserialize(dmsg,ptr);
-							const unsigned int packetLen = dmsg.at<uint16_t>(ptr); ptr += 2;
-							const void *packet = (const void *)dmsg.field(ptr,packetLen); ptr += packetLen;
-
-							if (packetLen >= ZT_PROTO_MIN_FRAGMENT_LENGTH) { // ignore anything too short to contain a dest address
-								const Address destinationAddress(reinterpret_cast<const char *>(packet) + 8,ZT_ADDRESS_LENGTH);
-								TRACE("[%u] relay %u bytes to %s (%u remote paths included)",(unsigned int)fromMemberId,packetLen,destinationAddress.toString().c_str(),numRemotePeerPaths);
-
-								SharedPtr<Peer> destinationPeer(RR->topology->getPeer(destinationAddress));
-								if (destinationPeer) {
-									if (
-									    (destinationPeer->send(RR,packet,packetLen,RR->node->now()))&&
-									    (numRemotePeerPaths > 0)&&
-									    (packetLen >= 18)&&
-									    (reinterpret_cast<const unsigned char *>(packet)[ZT_PACKET_FRAGMENT_IDX_FRAGMENT_INDICATOR] == ZT_PACKET_FRAGMENT_INDICATOR)
-									   ) {
-										// If remote peer paths were sent with this relayed packet, we do
-										// RENDEZVOUS. It's handled here for cluster-relayed packets since
-										// we don't have both Peer records so this is a different path.
-
-										const Address remotePeerAddress(reinterpret_cast<const char *>(packet) + 13,ZT_ADDRESS_LENGTH);
-
-										InetAddress bestDestV4,bestDestV6;
-										destinationPeer->getBestActiveAddresses(RR->node->now(),bestDestV4,bestDestV6);
-										InetAddress bestRemoteV4,bestRemoteV6;
-										for(unsigned int i=0;i<numRemotePeerPaths;++i) {
-											if ((bestRemoteV4)&&(bestRemoteV6))
-												break;
-											switch(remotePeerPaths[i].ss_family) {
-												case AF_INET:
-													if (!bestRemoteV4)
-														bestRemoteV4 = remotePeerPaths[i];
-													break;
-												case AF_INET6:
-													if (!bestRemoteV6)
-														bestRemoteV6 = remotePeerPaths[i];
-													break;
-											}
-										}
-
-										Packet rendezvousForDest(destinationAddress,RR->identity.address(),Packet::VERB_RENDEZVOUS);
-										rendezvousForDest.append((uint8_t)0);
-										remotePeerAddress.appendTo(rendezvousForDest);
-
-										Buffer<2048> rendezvousForOtherEnd;
-										remotePeerAddress.appendTo(rendezvousForOtherEnd);
-										rendezvousForOtherEnd.append((uint8_t)Packet::VERB_RENDEZVOUS);
-										const unsigned int rendezvousForOtherEndPayloadSizePtr = rendezvousForOtherEnd.size();
-										rendezvousForOtherEnd.addSize(2); // space for actual packet payload length
-										rendezvousForOtherEnd.append((uint8_t)0); // flags == 0
-										destinationAddress.appendTo(rendezvousForOtherEnd);
-
-										bool haveMatch = false;
-										if ((bestDestV6)&&(bestRemoteV6)) {
-											haveMatch = true;
-
-											rendezvousForDest.append((uint16_t)bestRemoteV6.port());
-											rendezvousForDest.append((uint8_t)16);
-											rendezvousForDest.append(bestRemoteV6.rawIpData(),16);
-
-											rendezvousForOtherEnd.append((uint16_t)bestDestV6.port());
-											rendezvousForOtherEnd.append((uint8_t)16);
-											rendezvousForOtherEnd.append(bestDestV6.rawIpData(),16);
-											rendezvousForOtherEnd.setAt<uint16_t>(rendezvousForOtherEndPayloadSizePtr,(uint16_t)(9 + 16));
-										} else if ((bestDestV4)&&(bestRemoteV4)) {
-											haveMatch = true;
-
-											rendezvousForDest.append((uint16_t)bestRemoteV4.port());
-											rendezvousForDest.append((uint8_t)4);
-											rendezvousForDest.append(bestRemoteV4.rawIpData(),4);
-
-											rendezvousForOtherEnd.append((uint16_t)bestDestV4.port());
-											rendezvousForOtherEnd.append((uint8_t)4);
-											rendezvousForOtherEnd.append(bestDestV4.rawIpData(),4);
-											rendezvousForOtherEnd.setAt<uint16_t>(rendezvousForOtherEndPayloadSizePtr,(uint16_t)(9 + 4));
-										}
 
-										if (haveMatch) {
-											_send(fromMemberId,STATE_MESSAGE_PROXY_SEND,rendezvousForOtherEnd.data(),rendezvousForOtherEnd.size());
-											RR->sw->send(rendezvousForDest,true,0);
-										}
+							TRACE("[%u] requested proxy unite between local peer %s and remote peer %s",(unsigned int)fromMemberId,localPeerAddress.toString().c_str(),remotePeerAddress.toString().c_str());
+
+							SharedPtr<Peer> localPeer(RR->topology->getPeer(localPeerAddress));
+							if ((localPeer)&&(numRemotePeerPaths > 0)) {
+								InetAddress bestLocalV4,bestLocalV6;
+								localPeer->getBestActiveAddresses(RR->node->now(),bestLocalV4,bestLocalV6);
+
+								InetAddress bestRemoteV4,bestRemoteV6;
+								for(unsigned int i=0;i<numRemotePeerPaths;++i) {
+									if ((bestRemoteV4)&&(bestRemoteV6))
+										break;
+									switch(remotePeerPaths[i].ss_family) {
+										case AF_INET:
+											if (!bestRemoteV4)
+												bestRemoteV4 = remotePeerPaths[i];
+											break;
+										case AF_INET6:
+											if (!bestRemoteV6)
+												bestRemoteV6 = remotePeerPaths[i];
+											break;
 									}
 								}
+
+								Packet rendezvousForLocal(localPeerAddress,RR->identity.address(),Packet::VERB_RENDEZVOUS);
+								rendezvousForLocal.append((uint8_t)0);
+								remotePeerAddress.appendTo(rendezvousForLocal);
+
+								Buffer<2048> rendezvousForRemote;
+								remotePeerAddress.appendTo(rendezvousForRemote);
+								rendezvousForRemote.append((uint8_t)Packet::VERB_RENDEZVOUS);
+								const unsigned int rendezvousForOtherEndPayloadSizePtr = rendezvousForRemote.size();
+								rendezvousForRemote.addSize(2); // space for actual packet payload length
+								rendezvousForRemote.append((uint8_t)0); // flags == 0
+								localPeerAddress.appendTo(rendezvousForRemote);
+
+								bool haveMatch = false;
+								if ((bestLocalV6)&&(bestRemoteV6)) {
+									haveMatch = true;
+
+									rendezvousForLocal.append((uint16_t)bestRemoteV6.port());
+									rendezvousForLocal.append((uint8_t)16);
+									rendezvousForLocal.append(bestRemoteV6.rawIpData(),16);
+
+									rendezvousForRemote.append((uint16_t)bestLocalV6.port());
+									rendezvousForRemote.append((uint8_t)16);
+									rendezvousForRemote.append(bestLocalV6.rawIpData(),16);
+									rendezvousForRemote.setAt<uint16_t>(rendezvousForOtherEndPayloadSizePtr,(uint16_t)(9 + 16));
+								} else if ((bestLocalV4)&&(bestRemoteV4)) {
+									haveMatch = true;
+
+									rendezvousForLocal.append((uint16_t)bestRemoteV4.port());
+									rendezvousForLocal.append((uint8_t)4);
+									rendezvousForLocal.append(bestRemoteV4.rawIpData(),4);
+
+									rendezvousForRemote.append((uint16_t)bestLocalV4.port());
+									rendezvousForRemote.append((uint8_t)4);
+									rendezvousForRemote.append(bestLocalV4.rawIpData(),4);
+									rendezvousForRemote.setAt<uint16_t>(rendezvousForOtherEndPayloadSizePtr,(uint16_t)(9 + 4));
+								}
+
+								if (haveMatch) {
+									_send(fromMemberId,STATE_MESSAGE_PROXY_SEND,rendezvousForRemote.data(),rendezvousForRemote.size());
+									RR->sw->send(rendezvousForLocal,true,0);
+								}
 							}
 						}	break;
 
 						case STATE_MESSAGE_PROXY_SEND: {
-							const Address rcpt(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH);
+							const Address rcpt(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); ptr += ZT_ADDRESS_LENGTH;
 							const Packet::Verb verb = (Packet::Verb)dmsg[ptr++];
 							const unsigned int len = dmsg.at<uint16_t>(ptr); ptr += 2;
 							Packet outp(rcpt,RR->identity.address(),verb);
-							outp.append(dmsg.field(ptr,len),len);
+							outp.append(dmsg.field(ptr,len),len); ptr += len;
 							RR->sw->send(outp,true,0);
 							TRACE("[%u] proxy send %s to %s length %u",(unsigned int)fromMemberId,Packet::verbString(verb),rcpt.toString().c_str(),len);
 						}	break;
@@ -364,13 +349,13 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len)
 	}
 }
 
-bool Cluster::sendViaCluster(const Address &fromPeerAddress,const Address &toPeerAddress,const void *data,unsigned int len)
+bool Cluster::sendViaCluster(const Address &fromPeerAddress,const Address &toPeerAddress,const void *data,unsigned int len,bool unite)
 {
 	if (len > 16384) // sanity check
 		return false;
 
 	uint64_t mostRecentTimestamp = 0;
-	uint16_t canHasPeer = 0;
+	unsigned int canHasPeer = 0;
 
 	{	// Anyone got this peer?
 		Mutex::Lock _l2(_peerAffinities_m);
@@ -387,25 +372,37 @@ bool Cluster::sendViaCluster(const Address &fromPeerAddress,const Address &toPee
 
 	const uint64_t now = RR->node->now();
 	if ((now - mostRecentTimestamp) < ZT_PEER_ACTIVITY_TIMEOUT) {
-		Buffer<16384> buf;
-
-		InetAddress v4,v6;
-		if (fromPeerAddress) {
-			SharedPtr<Peer> fromPeer(RR->topology->getPeer(fromPeerAddress));
-			if (fromPeer)
-				fromPeer->getBestActiveAddresses(now,v4,v6);
+		Buffer<2048> buf;
+
+		if (unite) {
+			InetAddress v4,v6;
+			if (fromPeerAddress) {
+				SharedPtr<Peer> fromPeer(RR->topology->getPeer(fromPeerAddress));
+				if (fromPeer)
+					fromPeer->getBestActiveAddresses(now,v4,v6);
+			}
+			uint8_t addrCount = 0;
+			if (v4)
+				++addrCount;
+			if (v6)
+				++addrCount;
+			if (addrCount) {
+				toPeerAddress.appendTo(buf);
+				fromPeerAddress.appendTo(buf);
+				buf.append(addrCount);
+				if (v4)
+					v4.serialize(buf);
+				if (v6)
+					v6.serialize(buf);
+			}
 		}
-		buf.append((uint8_t)( (v4) ? ((v6) ? 2 : 1) : ((v6) ? 1 : 0) ));
-		if (v4)
-			v4.serialize(buf);
-		if (v6)
-			v6.serialize(buf);
-		buf.append((uint16_t)len);
-		buf.append(data,len);
 
 		{
 			Mutex::Lock _l2(_members[canHasPeer].lock);
-			_send(canHasPeer,STATE_MESSAGE_RELAY,buf.data(),buf.size());
+			if (buf.size() > 0)
+				_send(canHasPeer,STATE_MESSAGE_PROXY_UNITE,buf.data(),buf.size());
+			if (_members[canHasPeer].zeroTierPhysicalEndpoints.size() > 0)
+				RR->node->putPacket(InetAddress(),_members[canHasPeer].zeroTierPhysicalEndpoints.front(),data,len);
 		}
 
 		TRACE("sendViaCluster(): relaying %u bytes from %s to %s by way of %u",len,fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str(),(unsigned int)canHasPeer);
diff --git a/node/Cluster.hpp b/node/Cluster.hpp
index bb7d3b39..282d8120 100644
--- a/node/Cluster.hpp
+++ b/node/Cluster.hpp
@@ -57,7 +57,7 @@
 /**
  * Desired period between doPeriodicTasks() in milliseconds
  */
-#define ZT_CLUSTER_PERIODIC_TASK_PERIOD 50
+#define ZT_CLUSTER_PERIODIC_TASK_PERIOD 100
 
 namespace ZeroTier {
 
@@ -136,13 +136,18 @@ public:
 		STATE_MESSAGE_COM = 4,
 
 		/**
-		 * Relay a packet to a peer:
-		 *   <[1] 8-bit number of sending peer active path addresses>
-		 *   <[...] series of serialized InetAddresses of sending peer's paths>
-		 *   <[2] 16-bit packet length>
-		 *   <[...] packet or packet fragment>
+		 * Request that VERB_RENDEZVOUS be sent to a peer that we have:
+		 *   <[5] ZeroTier address of peer on recipient's side>
+		 *   <[5] ZeroTier address of peer on sender's side>
+		 *   <[1] 8-bit number of sender's peer's active path addresses>
+		 *   <[...] series of serialized InetAddresses of sender's peer's paths>
+		 *
+		 * This requests that we perform NAT-t introduction between a peer that
+		 * we have and one on the sender's side. The sender furnishes contact
+		 * info for its peer, and we send VERB_RENDEZVOUS to both sides: to ours
+		 * directly and with PROXY_SEND to theirs.
 		 */
-		STATE_MESSAGE_RELAY = 5,
+		STATE_MESSAGE_PROXY_UNITE = 5,
 
 		/**
 		 * Request that a cluster member send a packet to a locally-known peer:
@@ -211,9 +216,10 @@ public:
 	 * @param toPeerAddress Destination peer address
 	 * @param data Packet or packet fragment data
 	 * @param len Length of packet or fragment
+	 * @param unite If true, also request proxy unite across cluster
 	 * @return True if this data was sent via another cluster member, false if none have this peer
 	 */
-	bool sendViaCluster(const Address &fromPeerAddress,const Address &toPeerAddress,const void *data,unsigned int len);
+	bool sendViaCluster(const Address &fromPeerAddress,const Address &toPeerAddress,const void *data,unsigned int len,bool unite);
 
 	/**
 	 * Advertise to the cluster that we have this peer
diff --git a/node/Switch.cpp b/node/Switch.cpp
index 709ed802..772eaf02 100644
--- a/node/Switch.cpp
+++ b/node/Switch.cpp
@@ -303,11 +303,10 @@ void Switch::send(const Packet &packet,bool encrypt,uint64_t nwid)
 	}
 }
 
-bool Switch::unite(const Address &p1,const Address &p2,bool force)
+bool Switch::unite(const Address &p1,const Address &p2)
 {
 	if ((p1 == RR->identity.address())||(p2 == RR->identity.address()))
 		return false;
-
 	SharedPtr<Peer> p1p = RR->topology->getPeer(p1);
 	if (!p1p)
 		return false;
@@ -317,14 +316,6 @@ bool Switch::unite(const Address &p1,const Address &p2,bool force)
 
 	const uint64_t now = RR->node->now();
 
-	{
-		Mutex::Lock _l(_lastUniteAttempt_m);
-		uint64_t &luts = _lastUniteAttempt[_LastUniteKey(p1,p2)];
-		if (((now - luts) < ZT_MIN_UNITE_INTERVAL)&&(!force))
-			return false;
-		luts = now;
-	}
-
 	std::pair<InetAddress,InetAddress> cg(Peer::findCommonGround(*p1p,*p2p,now));
 	if ((!(cg.first))||(cg.first.ipScope() != cg.second.ipScope()))
 		return false;
@@ -571,7 +562,7 @@ void Switch::_handleRemotePacketFragment(const InetAddress &localAddr,const Inet
 			SharedPtr<Peer> relayTo = RR->topology->getPeer(destination);
 			if ((!relayTo)||(!relayTo->send(RR,fragment.data(),fragment.size(),RR->node->now()))) {
 #ifdef ZT_ENABLE_CLUSTER
-				if ((RR->cluster)&&(RR->cluster->sendViaCluster(Address(),destination,fragment.data(),fragment.size())))
+				if ((RR->cluster)&&(RR->cluster->sendViaCluster(Address(),destination,fragment.data(),fragment.size(),false)))
 					return; // sent by way of another member of this cluster
 #endif
 
@@ -634,7 +625,8 @@ void Switch::_handleRemotePacketFragment(const InetAddress &localAddr,const Inet
 
 void Switch::_handleRemotePacketHead(const InetAddress &localAddr,const InetAddress &fromAddr,const void *data,unsigned int len)
 {
-	SharedPtr<IncomingPacket> packet(new IncomingPacket(data,len,localAddr,fromAddr,RR->node->now()));
+	const uint64_t now = RR->node->now();
+	SharedPtr<IncomingPacket> packet(new IncomingPacket(data,len,localAddr,fromAddr,now));
 
 	Address source(packet->source());
 	Address destination(packet->destination());
@@ -652,17 +644,18 @@ void Switch::_handleRemotePacketHead(const InetAddress &localAddr,const InetAddr
 			packet->incrementHops();
 
 			SharedPtr<Peer> relayTo = RR->topology->getPeer(destination);
-			if ((relayTo)&&((relayTo->send(RR,packet->data(),packet->size(),RR->node->now())))) {
-				unite(source,destination,false);
+			if ((relayTo)&&((relayTo->send(RR,packet->data(),packet->size(),now)))) {
+				if (_shouldTryUnite(now,source,destination))
+					unite(source,destination);
 			} else {
 #ifdef ZT_ENABLE_CLUSTER
-				if ((RR->cluster)&&(RR->cluster->sendViaCluster(source,destination,packet->data(),packet->size())))
+				if ((RR->cluster)&&(RR->cluster->sendViaCluster(source,destination,packet->data(),packet->size(),_shouldTryUnite(now,source,destination))))
 					return; // sent by way of another member of this cluster
 #endif
 
 				relayTo = RR->topology->getBestRoot(&source,1,true);
 				if (relayTo)
-					relayTo->send(RR,packet->data(),packet->size(),RR->node->now());
+					relayTo->send(RR,packet->data(),packet->size(),now);
 			}
 		} else {
 			TRACE("dropped relay %s(%s) -> %s, max hops exceeded",packet->source().toString().c_str(),fromAddr.toString().c_str(),destination.toString().c_str());
@@ -677,7 +670,7 @@ void Switch::_handleRemotePacketHead(const InetAddress &localAddr,const InetAddr
 		if (!dq.creationTime) {
 			// If we have no other fragments yet, create an entry and save the head
 
-			dq.creationTime = RR->node->now();
+			dq.creationTime = now;
 			dq.frag0 = packet;
 			dq.totalFragments = 0; // 0 == unknown, waiting for Packet::Fragment
 			dq.haveFragments = 1; // head is first bit (left to right)
@@ -805,4 +798,14 @@ bool Switch::_trySend(const Packet &packet,bool encrypt,uint64_t nwid)
 	return false;
 }
 
+bool Switch::_shouldTryUnite(const uint64_t now,const Address &p1,const Address &p2)
+{
+	Mutex::Lock _l(_lastUniteAttempt_m);
+	uint64_t &luts = _lastUniteAttempt[_LastUniteKey(p1,p2)];
+	if ((now - luts) < ZT_MIN_UNITE_INTERVAL)
+		return false;
+	luts = now;
+	return true;
+}
+
 } // namespace ZeroTier
diff --git a/node/Switch.hpp b/node/Switch.hpp
index 3bdc0c47..42e87ca5 100644
--- a/node/Switch.hpp
+++ b/node/Switch.hpp
@@ -127,15 +127,10 @@ public:
 	 * This only works if both peers are known, with known working direct
 	 * links to this peer. The best link for each peer is sent to the other.
 	 *
-	 * A rate limiter is in effect via the _lastUniteAttempt map. If force
-	 * is true, a unite attempt is made even if one has been made less than
-	 * ZT_MIN_UNITE_INTERVAL milliseconds ago.
-	 *
 	 * @param p1 One of two peers (order doesn't matter)
 	 * @param p2 Second of pair
-	 * @param force If true, send now regardless of interval
 	 */
-	bool unite(const Address &p1,const Address &p2,bool force);
+	bool unite(const Address &p1,const Address &p2);
 
 	/**
 	 * Attempt NAT traversal to peer at a given physical address
@@ -185,6 +180,7 @@ private:
 	void _handleRemotePacketHead(const InetAddress &localAddr,const InetAddress &fromAddr,const void *data,unsigned int len);
 	Address _sendWhoisRequest(const Address &addr,const Address *peersAlreadyConsulted,unsigned int numPeersAlreadyConsulted);
 	bool _trySend(const Packet &packet,bool encrypt,uint64_t nwid);
+	bool _shouldTryUnite(const uint64_t now,const Address &p1,const Address &p2);
 
 	const RuntimeEnvironment *const RR;
 	uint64_t _lastBeaconResponse;
-- 
cgit v1.2.3


From 16bc3e03982286232e1df2da17d8b2fc3c5a5c55 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Tue, 27 Oct 2015 15:00:16 -0700
Subject: Factor out RemotePath subclass of Path -- no longer needed, just
 cruft.

---
 include/ZeroTierOne.h   |   3 +-
 node/Cluster.cpp        |   1 +
 node/IncomingPacket.cpp |   5 +-
 node/Multicaster.cpp    |   1 +
 node/Network.cpp        |   1 +
 node/Node.cpp           |  14 ++---
 node/Node.hpp           |   6 +-
 node/Path.cpp           |  45 ++++++++++++++
 node/Path.hpp           | 117 +++++++++++++++++++++++++++++++++--
 node/Peer.cpp           |  32 +++++-----
 node/Peer.hpp           |  24 ++++----
 node/RemotePath.hpp     | 161 ------------------------------------------------
 node/SelfAwareness.cpp  |   2 +-
 node/Switch.cpp         |   2 +-
 objects.mk              |   1 +
 service/OneService.cpp  |   4 +-
 16 files changed, 208 insertions(+), 211 deletions(-)
 create mode 100644 node/Path.cpp
 delete mode 100644 node/RemotePath.hpp

(limited to 'include')

diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h
index 3457634b..01c8bcde 100644
--- a/include/ZeroTierOne.h
+++ b/include/ZeroTierOne.h
@@ -1356,11 +1356,10 @@ void ZT_Node_freeQueryResult(ZT_Node *node,void *qr);
  * reject bad, empty, and unusable addresses.
  *
  * @param addr Local interface address
- * @param metric Local interface metric
  * @param trust How much do you trust the local network under this interface?
  * @return Boolean: non-zero if address was accepted and added
  */
-int ZT_Node_addLocalInterfaceAddress(ZT_Node *node,const struct sockaddr_storage *addr,int metric, enum ZT_LocalInterfaceAddressTrust trust);
+int ZT_Node_addLocalInterfaceAddress(ZT_Node *node,const struct sockaddr_storage *addr,enum ZT_LocalInterfaceAddressTrust trust);
 
 /**
  * Clear local interface addresses
diff --git a/node/Cluster.cpp b/node/Cluster.cpp
index 73ff5846..07ca0ba1 100644
--- a/node/Cluster.cpp
+++ b/node/Cluster.cpp
@@ -48,6 +48,7 @@
 #include "Topology.hpp"
 #include "Packet.hpp"
 #include "Switch.hpp"
+#include "Node.hpp"
 
 namespace ZeroTier {
 
diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp
index 2514cd64..7015535a 100644
--- a/node/IncomingPacket.cpp
+++ b/node/IncomingPacket.cpp
@@ -44,6 +44,7 @@
 #include "SHA512.hpp"
 #include "World.hpp"
 #include "Cluster.hpp"
+#include "Node.hpp"
 
 namespace ZeroTier {
 
@@ -888,7 +889,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha
 {
 	try {
 		const uint64_t now = RR->node->now();
-		const RemotePath *currentBest = peer->getBestPath(now);
+		const Path *currentBest = peer->getBestPath(now);
 
 		unsigned int count = at<uint16_t>(ZT_PACKET_IDX_PAYLOAD);
 		unsigned int ptr = ZT_PACKET_IDX_PAYLOAD + 2;
@@ -1036,7 +1037,7 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt
 				remainingHopsPtr += ZT_ADDRESS_LENGTH;
 				SharedPtr<Peer> nhp(RR->topology->getPeer(nextHop[h]));
 				if (nhp) {
-					RemotePath *const rp = nhp->getBestPath(now);
+					Path *const rp = nhp->getBestPath(now);
 					if (rp)
 						nextHopBestPathAddress[h] = rp->address();
 				}
diff --git a/node/Multicaster.cpp b/node/Multicaster.cpp
index 6e6cd628..e43d7d88 100644
--- a/node/Multicaster.cpp
+++ b/node/Multicaster.cpp
@@ -37,6 +37,7 @@
 #include "Peer.hpp"
 #include "C25519.hpp"
 #include "CertificateOfMembership.hpp"
+#include "Node.hpp"
 
 namespace ZeroTier {
 
diff --git a/node/Network.cpp b/node/Network.cpp
index cd30e386..afbe1074 100644
--- a/node/Network.cpp
+++ b/node/Network.cpp
@@ -37,6 +37,7 @@
 #include "Packet.hpp"
 #include "Buffer.hpp"
 #include "NetworkController.hpp"
+#include "Node.hpp"
 
 #include "../version.h"
 
diff --git a/node/Node.cpp b/node/Node.cpp
index 87871e20..82cda66d 100644
--- a/node/Node.cpp
+++ b/node/Node.cpp
@@ -448,10 +448,10 @@ ZT_PeerList *Node::peers() const
 		p->latency = pi->second->latency();
 		p->role = RR->topology->isRoot(pi->second->identity()) ? ZT_PEER_ROLE_ROOT : ZT_PEER_ROLE_LEAF;
 
-		std::vector<RemotePath> paths(pi->second->paths());
-		RemotePath *bestPath = pi->second->getBestPath(_now);
+		std::vector<Path> paths(pi->second->paths());
+		Path *bestPath = pi->second->getBestPath(_now);
 		p->pathCount = 0;
-		for(std::vector<RemotePath>::iterator path(paths.begin());path!=paths.end();++path) {
+		for(std::vector<Path>::iterator path(paths.begin());path!=paths.end();++path) {
 			memcpy(&(p->paths[p->pathCount].address),&(path->address()),sizeof(struct sockaddr_storage));
 			p->paths[p->pathCount].lastSend = path->lastSend();
 			p->paths[p->pathCount].lastReceive = path->lastReceived();
@@ -499,11 +499,11 @@ void Node::freeQueryResult(void *qr)
 		::free(qr);
 }
 
-int Node::addLocalInterfaceAddress(const struct sockaddr_storage *addr,int metric,ZT_LocalInterfaceAddressTrust trust)
+int Node::addLocalInterfaceAddress(const struct sockaddr_storage *addr,ZT_LocalInterfaceAddressTrust trust)
 {
 	if (Path::isAddressValidForPath(*(reinterpret_cast<const InetAddress *>(addr)))) {
 		Mutex::Lock _l(_directPaths_m);
-		_directPaths.push_back(Path(*(reinterpret_cast<const InetAddress *>(addr)),metric,(Path::Trust)trust));
+		_directPaths.push_back(*(reinterpret_cast<const InetAddress *>(addr)));
 		std::sort(_directPaths.begin(),_directPaths.end());
 		_directPaths.erase(std::unique(_directPaths.begin(),_directPaths.end()),_directPaths.end());
 		return 1;
@@ -900,10 +900,10 @@ void ZT_Node_freeQueryResult(ZT_Node *node,void *qr)
 	} catch ( ... ) {}
 }
 
-int ZT_Node_addLocalInterfaceAddress(ZT_Node *node,const struct sockaddr_storage *addr,int metric, enum ZT_LocalInterfaceAddressTrust trust)
+int ZT_Node_addLocalInterfaceAddress(ZT_Node *node,const struct sockaddr_storage *addr,enum ZT_LocalInterfaceAddressTrust trust)
 {
 	try {
-		return reinterpret_cast<ZeroTier::Node *>(node)->addLocalInterfaceAddress(addr,metric,trust);
+		return reinterpret_cast<ZeroTier::Node *>(node)->addLocalInterfaceAddress(addr,trust);
 	} catch ( ... ) {
 		return 0;
 	}
diff --git a/node/Node.hpp b/node/Node.hpp
index 4094a79e..48c5ead8 100644
--- a/node/Node.hpp
+++ b/node/Node.hpp
@@ -105,7 +105,7 @@ public:
 	ZT_VirtualNetworkConfig *networkConfig(uint64_t nwid) const;
 	ZT_VirtualNetworkList *networks() const;
 	void freeQueryResult(void *qr);
-	int addLocalInterfaceAddress(const struct sockaddr_storage *addr,int metric,ZT_LocalInterfaceAddressTrust trust);
+	int addLocalInterfaceAddress(const struct sockaddr_storage *addr,ZT_LocalInterfaceAddressTrust trust);
 	void clearLocalInterfaceAddresses();
 	void setNetconfMaster(void *networkControllerInstance);
 	ZT_ResultCode circuitTestBegin(ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *));
@@ -207,7 +207,7 @@ public:
 	/**
 	 * @return Potential direct paths to me a.k.a. local interface addresses
 	 */
-	inline std::vector<Path> directPaths() const
+	inline std::vector<InetAddress> directPaths() const
 	{
 		Mutex::Lock _l(_directPaths_m);
 		return _directPaths;
@@ -285,7 +285,7 @@ private:
 	std::vector< ZT_CircuitTest * > _circuitTests;
 	Mutex _circuitTests_m;
 
-	std::vector<Path> _directPaths;
+	std::vector<InetAddress> _directPaths;
 	Mutex _directPaths_m;
 
 	Mutex _backgroundTasksLock;
diff --git a/node/Path.cpp b/node/Path.cpp
new file mode 100644
index 00000000..e2475751
--- /dev/null
+++ b/node/Path.cpp
@@ -0,0 +1,45 @@
+/*
+ * ZeroTier One - Network Virtualization Everywhere
+ * Copyright (C) 2011-2015  ZeroTier, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * ZeroTier may be used and distributed under the terms of the GPLv3, which
+ * are available at: http://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * If you would like to embed ZeroTier into a commercial application or
+ * redistribute it in a modified binary form, please contact ZeroTier Networks
+ * LLC. Start here: http://www.zerotier.com/
+ */
+
+#include "Path.hpp"
+#include "AntiRecursion.hpp"
+#include "RuntimeEnvironment.hpp"
+#include "Node.hpp"
+
+namespace ZeroTier {
+
+bool Path::send(const RuntimeEnvironment *RR,const void *data,unsigned int len,uint64_t now)
+{
+	if (RR->node->putPacket(_localAddress,address(),data,len)) {
+		sent(now);
+		RR->antiRec->logOutgoingZT(data,len);
+		return true;
+	}
+	return false;
+}
+
+} // namespace ZeroTier
diff --git a/node/Path.hpp b/node/Path.hpp
index 6fa3c52e..99f6590b 100644
--- a/node/Path.hpp
+++ b/node/Path.hpp
@@ -28,12 +28,19 @@
 #ifndef ZT_PATH_HPP
 #define ZT_PATH_HPP
 
+#include <stdint.h>
+#include <string.h>
+
+#include <stdexcept>
+#include <algorithm>
+
 #include "Constants.hpp"
 #include "InetAddress.hpp"
-#include "Utils.hpp"
 
 namespace ZeroTier {
 
+class RuntimeEnvironment;
+
 /**
  * Base class for paths
  *
@@ -67,19 +74,87 @@ public:
 	};
 
 	Path() :
+		_lastSend(0),
+		_lastReceived(0),
 		_addr(),
+		_localAddress(),
 		_ipScope(InetAddress::IP_SCOPE_NONE),
-		_trust(TRUST_NORMAL)
+		_trust(TRUST_NORMAL),
+		_flags(0)
 	{
 	}
 
-	Path(const InetAddress &addr,int metric,Trust trust) :
+	Path(const InetAddress &localAddress,const InetAddress &addr,Trust trust) :
+		_lastSend(0),
+		_lastReceived(0),
 		_addr(addr),
+		_localAddress(localAddress),
 		_ipScope(addr.ipScope()),
-		_trust(trust)
+		_trust(trust),
+		_flags(0)
+	{
+	}
+
+	/**
+	 * Called when a packet is sent to this remote path
+	 *
+	 * This is called automatically by Path::send().
+	 *
+	 * @param t Time of send
+	 */
+	inline void sent(uint64_t t)
+		throw()
 	{
+		_lastSend = t;
 	}
 
+	/**
+	 * Called when a packet is received from this remote path
+	 *
+	 * @param t Time of receive
+	 */
+	inline void received(uint64_t t)
+		throw()
+	{
+		_lastReceived = t;
+	}
+
+	/**
+	 * @param now Current time
+	 * @return True if this path appears active
+	 */
+	inline bool active(uint64_t now) const
+		throw()
+	{
+		return ((now - _lastReceived) < ZT_PEER_ACTIVITY_TIMEOUT);
+	}
+
+	/**
+	 * Send a packet via this path
+	 *
+	 * @param RR Runtime environment
+	 * @param data Packet data
+	 * @param len Packet length
+	 * @param now Current time
+	 * @return True if transport reported success
+	 */
+	bool send(const RuntimeEnvironment *RR,const void *data,unsigned int len,uint64_t now);
+
+	/**
+	 * @return Address of local side of this path or NULL if unspecified
+	 */
+	inline const InetAddress &localAddress() const throw() { return _localAddress; }
+
+	/**
+	 * @return Time of last send to this path
+	 */
+	inline uint64_t lastSend() const throw() { return _lastSend; }
+
+	/**
+	 * @return Time of last receive from this path
+	 */
+	inline uint64_t lastReceived() const throw() { return _lastReceived; }
+
 	/**
 	 * @return Physical address
 	 */
@@ -157,10 +232,42 @@ public:
 		return false;
 	}
 
-protected:
+	template<unsigned int C>
+	inline void serialize(Buffer<C> &b) const
+	{
+		b.append((uint8_t)0); // version
+		b.append((uint64_t)_lastSend);
+		b.append((uint64_t)_lastReceived);
+		_addr.serialize(b);
+		_localAddress.serialize(b);
+		b.append((uint8_t)_trust);
+		b.append((uint16_t)_flags);
+	}
+
+	template<unsigned int C>
+	inline unsigned int deserialize(const Buffer<C> &b,unsigned int startAt = 0)
+	{
+		unsigned int p = startAt;
+		if (b[p++] != 0)
+			throw std::invalid_argument("invalid serialized Path");
+		_lastSend = b.template at<uint64_t>(p); p += 8;
+		_lastReceived = b.template at<uint64_t>(p); p += 8;
+		p += _addr.deserialize(b,p);
+		p += _localAddress.deserialize(b,p);
+		_ipScope = _addr.ipScope();
+		_trust = (Path::Trust)b[p++];
+		_flags = b.template at<uint16_t>(p); p += 2;
+		return (p - startAt);
+	}
+
+private:
+	uint64_t _lastSend;
+	uint64_t _lastReceived;
 	InetAddress _addr;
+	InetAddress _localAddress;
 	InetAddress::IpScope _ipScope; // memoize this since it's a computed value checked often
 	Trust _trust;
+	uint16_t _flags;
 };
 
 } // namespace ZeroTier
diff --git a/node/Peer.cpp b/node/Peer.cpp
index 009e2be5..ebdb6026 100644
--- a/node/Peer.cpp
+++ b/node/Peer.cpp
@@ -169,7 +169,7 @@ void Peer::received(
 			if (!pathIsConfirmed) {
 				if (verb == Packet::VERB_OK) {
 
-					RemotePath *slot = (RemotePath *)0;
+					Path *slot = (Path *)0;
 					if (np < ZT_MAX_PEER_NETWORK_PATHS) {
 						slot = &(_paths[np++]);
 					} else {
@@ -182,7 +182,7 @@ void Peer::received(
 						}
 					}
 					if (slot) {
-						*slot = RemotePath(localAddr,remoteAddr);
+						*slot = Path(localAddr,remoteAddr,Path::TRUST_NORMAL);
 						slot->received(now);
 						_numPaths = np;
 						pathIsConfirmed = true;
@@ -240,7 +240,7 @@ void Peer::attemptToContactAt(const RuntimeEnvironment *RR,const InetAddress &lo
 
 bool Peer::doPingAndKeepalive(const RuntimeEnvironment *RR,uint64_t now,int inetAddressFamily)
 {
-	RemotePath *p = (RemotePath *)0;
+	Path *p = (Path *)0;
 
 	Mutex::Lock _l(_lock);
 	if (inetAddressFamily != 0) {
@@ -268,7 +268,7 @@ bool Peer::doPingAndKeepalive(const RuntimeEnvironment *RR,uint64_t now,int inet
 	return false;
 }
 
-void Peer::pushDirectPaths(const RuntimeEnvironment *RR,RemotePath *path,uint64_t now,bool force)
+void Peer::pushDirectPaths(const RuntimeEnvironment *RR,Path *path,uint64_t now,bool force)
 {
 #ifdef ZT_ENABLE_CLUSTER
 	// Cluster mode disables normal PUSH_DIRECT_PATHS in favor of cluster-based peer redirection
@@ -281,7 +281,7 @@ void Peer::pushDirectPaths(const RuntimeEnvironment *RR,RemotePath *path,uint64_
 	if (((now - _lastDirectPathPushSent) >= ZT_DIRECT_PATH_PUSH_INTERVAL)||(force)) {
 		_lastDirectPathPushSent = now;
 
-		std::vector<Path> dps(RR->node->directPaths());
+		std::vector<InetAddress> dps(RR->node->directPaths());
 		if (dps.empty())
 			return;
 
@@ -291,13 +291,13 @@ void Peer::pushDirectPaths(const RuntimeEnvironment *RR,RemotePath *path,uint64_
 			for(std::vector<Path>::const_iterator p(dps.begin());p!=dps.end();++p) {
 				if (ps.length() > 0)
 					ps.push_back(',');
-				ps.append(p->address().toString());
+				ps.append(p->toString());
 			}
 			TRACE("pushing %u direct paths to %s: %s",(unsigned int)dps.size(),_id.address().toString().c_str(),ps.c_str());
 		}
 #endif
 
-		std::vector<Path>::const_iterator p(dps.begin());
+		std::vector<InetAddress>::const_iterator p(dps.begin());
 		while (p != dps.end()) {
 			Packet outp(_id.address(),RR->identity.address(),Packet::VERB_PUSH_DIRECT_PATHS);
 			outp.addSize(2); // leave room for count
@@ -305,7 +305,7 @@ void Peer::pushDirectPaths(const RuntimeEnvironment *RR,RemotePath *path,uint64_
 			unsigned int count = 0;
 			while ((p != dps.end())&&((outp.size() + 24) < ZT_PROTO_MAX_PACKET_LENGTH)) {
 				uint8_t addressType = 4;
-				switch(p->address().ss_family) {
+				switch(p->ss_family) {
 					case AF_INET:
 						break;
 					case AF_INET6:
@@ -317,6 +317,7 @@ void Peer::pushDirectPaths(const RuntimeEnvironment *RR,RemotePath *path,uint64_
 				}
 
 				uint8_t flags = 0;
+				/* TODO: path trust is not implemented yet
 				switch(p->trust()) {
 					default:
 						break;
@@ -327,13 +328,14 @@ void Peer::pushDirectPaths(const RuntimeEnvironment *RR,RemotePath *path,uint64_
 						flags |= (0x04 | 0x08); // no encryption, no authentication (redundant but go ahead and set both)
 						break;
 				}
+				*/
 
 				outp.append(flags);
 				outp.append((uint16_t)0); // no extensions
 				outp.append(addressType);
 				outp.append((uint8_t)((addressType == 4) ? 6 : 18));
-				outp.append(p->address().rawIpData(),((addressType == 4) ? 4 : 16));
-				outp.append((uint16_t)p->address().port());
+				outp.append(p->rawIpData(),((addressType == 4) ? 4 : 16));
+				outp.append((uint16_t)p->port());
 
 				++count;
 				++p;
@@ -506,7 +508,7 @@ struct _SortPathsByQuality
 {
 	uint64_t _now;
 	_SortPathsByQuality(const uint64_t now) : _now(now) {}
-	inline bool operator()(const RemotePath &a,const RemotePath &b) const
+	inline bool operator()(const Path &a,const Path &b) const
 	{
 		const uint64_t qa = (
 			((uint64_t)a.active(_now) << 63) |
@@ -526,7 +528,7 @@ void Peer::_sortPaths(const uint64_t now)
 	std::sort(&(_paths[0]),&(_paths[_numPaths]),_SortPathsByQuality(now));
 }
 
-RemotePath *Peer::_getBestPath(const uint64_t now)
+Path *Peer::_getBestPath(const uint64_t now)
 {
 	// assumes _lock is locked
 	if ((now - _lastPathSort) >= ZT_PEER_PATH_SORT_INTERVAL)
@@ -538,10 +540,10 @@ RemotePath *Peer::_getBestPath(const uint64_t now)
 		if (_paths[0].active(now))
 			return &(_paths[0]);
 	}
-	return (RemotePath *)0;
+	return (Path *)0;
 }
 
-RemotePath *Peer::_getBestPath(const uint64_t now,int inetAddressFamily)
+Path *Peer::_getBestPath(const uint64_t now,int inetAddressFamily)
 {
 	// assumes _lock is locked
 	if ((now - _lastPathSort) >= ZT_PEER_PATH_SORT_INTERVAL)
@@ -553,7 +555,7 @@ RemotePath *Peer::_getBestPath(const uint64_t now,int inetAddressFamily)
 		}
 		_sortPaths(now);
 	}
-	return (RemotePath *)0;
+	return (Path *)0;
 }
 
 } // namespace ZeroTier
diff --git a/node/Peer.hpp b/node/Peer.hpp
index 04b541af..aa75b3f4 100644
--- a/node/Peer.hpp
+++ b/node/Peer.hpp
@@ -41,7 +41,7 @@
 
 #include "RuntimeEnvironment.hpp"
 #include "CertificateOfMembership.hpp"
-#include "RemotePath.hpp"
+#include "Path.hpp"
 #include "Address.hpp"
 #include "Utils.hpp"
 #include "Identity.hpp"
@@ -135,7 +135,7 @@ public:
 	 * @param now Current time
 	 * @return Best path or NULL if there are no active direct paths
 	 */
-	inline RemotePath *getBestPath(uint64_t now)
+	inline Path *getBestPath(uint64_t now)
 	{
 		Mutex::Lock _l(_lock);
 		return _getBestPath(now);
@@ -150,14 +150,14 @@ public:
 	 * @param now Current time
 	 * @return Path used on success or NULL on failure
 	 */
-	inline RemotePath *send(const RuntimeEnvironment *RR,const void *data,unsigned int len,uint64_t now)
+	inline Path *send(const RuntimeEnvironment *RR,const void *data,unsigned int len,uint64_t now)
 	{
-		RemotePath *bestPath = getBestPath(now);
+		Path *bestPath = getBestPath(now);
 		if (bestPath) {
 			if (bestPath->send(RR,data,len,now))
 				return bestPath;
 		}
-		return (RemotePath *)0;
+		return (Path *)0;
 	}
 
 	/**
@@ -191,14 +191,14 @@ public:
 	 * @param now Current time
 	 * @param force If true, push regardless of rate limit
 	 */
-	void pushDirectPaths(const RuntimeEnvironment *RR,RemotePath *path,uint64_t now,bool force);
+	void pushDirectPaths(const RuntimeEnvironment *RR,Path *path,uint64_t now,bool force);
 
 	/**
 	 * @return All known direct paths to this peer
 	 */
-	inline std::vector<RemotePath> paths() const
+	inline std::vector<Path> paths() const
 	{
-		std::vector<RemotePath> pp;
+		std::vector<Path> pp;
 		Mutex::Lock _l(_lock);
 		for(unsigned int p=0,np=_numPaths;p<np;++p)
 			pp.push_back(_paths[p]);
@@ -533,7 +533,7 @@ public:
 				p += np->_paths[np->_numPaths++].deserialize(b,p);
 			} else {
 				// Skip any paths beyond max, but still read stream
-				RemotePath foo;
+				Path foo;
 				p += foo.deserialize(b,p);
 			}
 		}
@@ -557,8 +557,8 @@ public:
 
 private:
 	void _sortPaths(const uint64_t now);
-	RemotePath *_getBestPath(const uint64_t now);
-	RemotePath *_getBestPath(const uint64_t now,int inetAddressFamily);
+	Path *_getBestPath(const uint64_t now);
+	Path *_getBestPath(const uint64_t now,int inetAddressFamily);
 
 	unsigned char _key[ZT_PEER_SECRET_KEY_LENGTH]; // computed with key agreement, not serialized
 
@@ -575,7 +575,7 @@ private:
 	uint16_t _vMinor;
 	uint16_t _vRevision;
 	Identity _id;
-	RemotePath _paths[ZT_MAX_PEER_NETWORK_PATHS];
+	Path _paths[ZT_MAX_PEER_NETWORK_PATHS];
 	unsigned int _numPaths;
 	unsigned int _latency;
 
diff --git a/node/RemotePath.hpp b/node/RemotePath.hpp
deleted file mode 100644
index 8b37621a..00000000
--- a/node/RemotePath.hpp
+++ /dev/null
@@ -1,161 +0,0 @@
-/*
- * ZeroTier One - Network Virtualization Everywhere
- * Copyright (C) 2011-2015  ZeroTier, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * --
- *
- * ZeroTier may be used and distributed under the terms of the GPLv3, which
- * are available at: http://www.gnu.org/licenses/gpl-3.0.html
- *
- * If you would like to embed ZeroTier into a commercial application or
- * redistribute it in a modified binary form, please contact ZeroTier Networks
- * LLC. Start here: http://www.zerotier.com/
- */
-
-#ifndef ZT_REMOTEPATH_HPP
-#define ZT_REMOTEPATH_HPP
-
-#include <stdint.h>
-#include <string.h>
-
-#include <stdexcept>
-#include <algorithm>
-
-#include "Path.hpp"
-#include "Node.hpp"
-#include "AntiRecursion.hpp"
-#include "RuntimeEnvironment.hpp"
-
-namespace ZeroTier {
-
-/**
- * Path to a remote peer
- *
- * This extends Path to include status information about path activity.
- */
-class RemotePath : public Path
-{
-public:
-	RemotePath() :
-		Path(),
-		_lastSend(0),
-		_lastReceived(0),
-		_localAddress(),
-		_flags(0) {}
-
-	RemotePath(const InetAddress &localAddress,const InetAddress &addr) :
-		Path(addr,0,TRUST_NORMAL),
-		_lastSend(0),
-		_lastReceived(0),
-		_localAddress(localAddress),
-		_flags(0) {}
-
-	inline const InetAddress &localAddress() const throw() { return _localAddress; }
-
-	inline uint64_t lastSend() const throw() { return _lastSend; }
-	inline uint64_t lastReceived() const throw() { return _lastReceived; }
-
-	/**
-	 * Called when a packet is sent to this remote path
-	 *
-	 * This is called automatically by RemotePath::send().
-	 *
-	 * @param t Time of send
-	 */
-	inline void sent(uint64_t t)
-		throw()
-	{
-		_lastSend = t;
-	}
-
-	/**
-	 * Called when a packet is received from this remote path
-	 *
-	 * @param t Time of receive
-	 */
-	inline void received(uint64_t t)
-		throw()
-	{
-		_lastReceived = t;
-	}
-
-	/**
-	 * @param now Current time
-	 * @return True if this path appears active
-	 */
-	inline bool active(uint64_t now) const
-		throw()
-	{
-		return ((now - _lastReceived) < ZT_PEER_ACTIVITY_TIMEOUT);
-	}
-
-	/**
-	 * Send a packet via this path
-	 *
-	 * @param RR Runtime environment
-	 * @param data Packet data
-	 * @param len Packet length
-	 * @param now Current time
-	 * @return True if transport reported success
-	 */
-	inline bool send(const RuntimeEnvironment *RR,const void *data,unsigned int len,uint64_t now)
-	{
-		if (RR->node->putPacket(_localAddress,address(),data,len)) {
-			sent(now);
-			RR->antiRec->logOutgoingZT(data,len);
-			return true;
-		}
-		return false;
-	}
-
-	template<unsigned int C>
-	inline void serialize(Buffer<C> &b) const
-	{
-		b.append((uint8_t)1); // version
-		_addr.serialize(b);
-		b.append((uint8_t)_trust);
-		b.append((uint64_t)_lastSend);
-		b.append((uint64_t)_lastReceived);
-		_localAddress.serialize(b);
-		b.append((uint16_t)_flags);
-	}
-
-	template<unsigned int C>
-	inline unsigned int deserialize(const Buffer<C> &b,unsigned int startAt = 0)
-	{
-		unsigned int p = startAt;
-		if (b[p++] != 1)
-			throw std::invalid_argument("invalid serialized RemotePath");
-		p += _addr.deserialize(b,p);
-		_ipScope = _addr.ipScope();
-		_trust = (Path::Trust)b[p++];
-		_lastSend = b.template at<uint64_t>(p); p += 8;
-		_lastReceived = b.template at<uint64_t>(p); p += 8;
-		p += _localAddress.deserialize(b,p);
-		_flags = b.template at<uint16_t>(p); p += 2;
-		return (p - startAt);
-	}
-
-protected:
-	uint64_t _lastSend;
-	uint64_t _lastReceived;
-	InetAddress _localAddress;
-	uint16_t _flags;
-};
-
-} // namespace ZeroTier
-
-#endif
diff --git a/node/SelfAwareness.cpp b/node/SelfAwareness.cpp
index 81d19369..b4841544 100644
--- a/node/SelfAwareness.cpp
+++ b/node/SelfAwareness.cpp
@@ -125,7 +125,7 @@ void SelfAwareness::iam(const Address &reporter,const InetAddress &reporterPhysi
 		// they are still considered alive so that we will re-establish direct links.
 		SharedPtr<Peer> r(RR->topology->getBestRoot());
 		if (r) {
-			RemotePath *rp = r->getBestPath(now);
+			Path *rp = r->getBestPath(now);
 			if (rp) {
 				for(std::vector< SharedPtr<Peer> >::const_iterator p(rset.peersReset.begin());p!=rset.peersReset.end();++p) {
 					if ((*p)->alive(now)) {
diff --git a/node/Switch.cpp b/node/Switch.cpp
index 772eaf02..2f72f57a 100644
--- a/node/Switch.cpp
+++ b/node/Switch.cpp
@@ -736,7 +736,7 @@ bool Switch::_trySend(const Packet &packet,bool encrypt,uint64_t nwid)
 				return false; // sanity check: unconfigured network? why are we trying to talk to it?
 		}
 
-		RemotePath *viaPath = peer->getBestPath(now);
+		Path *viaPath = peer->getBestPath(now);
 		SharedPtr<Peer> relay;
 		if (!viaPath) {
 			// See if this network has a preferred relay (if packet has an associated network)
diff --git a/objects.mk b/objects.mk
index 6dd5ea30..540072d5 100644
--- a/objects.mk
+++ b/objects.mk
@@ -15,6 +15,7 @@ OBJS=\
 	node/Node.o \
 	node/OutboundMulticast.o \
 	node/Packet.o \
+	node/Path.o \
 	node/Peer.o \
 	node/Poly1305.o \
 	node/Salsa20.o \
diff --git a/service/OneService.cpp b/service/OneService.cpp
index 1765b5c4..4e3f24c7 100644
--- a/service/OneService.cpp
+++ b/service/OneService.cpp
@@ -731,7 +731,7 @@ public:
 #ifdef ZT_USE_MINIUPNPC
 					std::vector<InetAddress> upnpAddresses(_upnpClient->get());
 					for(std::vector<InetAddress>::const_iterator ext(upnpAddresses.begin());ext!=upnpAddresses.end();++ext)
-						_node->addLocalInterfaceAddress(reinterpret_cast<const struct sockaddr_storage *>(&(*ext)),0,ZT_LOCAL_INTERFACE_ADDRESS_TRUST_NORMAL);
+						_node->addLocalInterfaceAddress(reinterpret_cast<const struct sockaddr_storage *>(&(*ext)),ZT_LOCAL_INTERFACE_ADDRESS_TRUST_NORMAL);
 #endif
 
 					struct ifaddrs *ifatbl = (struct ifaddrs *)0;
@@ -749,7 +749,7 @@ public:
 								if (!isZT) {
 									InetAddress ip(ifa->ifa_addr);
 									ip.setPort(_port);
-									_node->addLocalInterfaceAddress(reinterpret_cast<const struct sockaddr_storage *>(&ip),0,ZT_LOCAL_INTERFACE_ADDRESS_TRUST_NORMAL);
+									_node->addLocalInterfaceAddress(reinterpret_cast<const struct sockaddr_storage *>(&ip),ZT_LOCAL_INTERFACE_ADDRESS_TRUST_NORMAL);
 								}
 							}
 							ifa = ifa->ifa_next;
-- 
cgit v1.2.3


From a994573a436ec2835781d13ecd2307e18c67855b Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Thu, 29 Oct 2015 09:42:15 -0700
Subject: Eliminate some more dead code. We may do path trust, but not like
 that.

---
 include/ZeroTierOne.h  | 17 +----------------
 node/Node.cpp          |  6 +++---
 node/Node.hpp          |  2 +-
 node/Path.hpp          | 16 +++-------------
 node/Peer.cpp          |  2 +-
 service/OneService.cpp |  4 ++--
 6 files changed, 11 insertions(+), 36 deletions(-)

(limited to 'include')

diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h
index 01c8bcde..e9b38c52 100644
--- a/include/ZeroTierOne.h
+++ b/include/ZeroTierOne.h
@@ -424,15 +424,6 @@ enum ZT_VirtualNetworkConfigOperation
 	ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY = 4
 };
 
-/**
- * Local interface trust levels
- */
-enum ZT_LocalInterfaceAddressTrust {
-	ZT_LOCAL_INTERFACE_ADDRESS_TRUST_NORMAL = 0,
-	ZT_LOCAL_INTERFACE_ADDRESS_TRUST_PRIVACY = 10,
-	ZT_LOCAL_INTERFACE_ADDRESS_TRUST_ULTIMATE = 20
-};
-
 /**
  * What trust hierarchy role does this peer have?
  */
@@ -1337,11 +1328,6 @@ void ZT_Node_freeQueryResult(ZT_Node *node,void *qr);
 /**
  * Add a local interface address
  *
- * Local interface addresses may be added if you want remote peers
- * with whom you have a trust relatinship (e.g. common network membership)
- * to receive information about these endpoints as potential endpoints for
- * direct communication.
- *
  * Take care that these are never ZeroTier interface addresses, otherwise
  * strange things might happen or they simply won't work.
  *
@@ -1356,10 +1342,9 @@ void ZT_Node_freeQueryResult(ZT_Node *node,void *qr);
  * reject bad, empty, and unusable addresses.
  *
  * @param addr Local interface address
- * @param trust How much do you trust the local network under this interface?
  * @return Boolean: non-zero if address was accepted and added
  */
-int ZT_Node_addLocalInterfaceAddress(ZT_Node *node,const struct sockaddr_storage *addr,enum ZT_LocalInterfaceAddressTrust trust);
+int ZT_Node_addLocalInterfaceAddress(ZT_Node *node,const struct sockaddr_storage *addr);
 
 /**
  * Clear local interface addresses
diff --git a/node/Node.cpp b/node/Node.cpp
index 82cda66d..42180e99 100644
--- a/node/Node.cpp
+++ b/node/Node.cpp
@@ -499,7 +499,7 @@ void Node::freeQueryResult(void *qr)
 		::free(qr);
 }
 
-int Node::addLocalInterfaceAddress(const struct sockaddr_storage *addr,ZT_LocalInterfaceAddressTrust trust)
+int Node::addLocalInterfaceAddress(const struct sockaddr_storage *addr)
 {
 	if (Path::isAddressValidForPath(*(reinterpret_cast<const InetAddress *>(addr)))) {
 		Mutex::Lock _l(_directPaths_m);
@@ -900,10 +900,10 @@ void ZT_Node_freeQueryResult(ZT_Node *node,void *qr)
 	} catch ( ... ) {}
 }
 
-int ZT_Node_addLocalInterfaceAddress(ZT_Node *node,const struct sockaddr_storage *addr,enum ZT_LocalInterfaceAddressTrust trust)
+int ZT_Node_addLocalInterfaceAddress(ZT_Node *node,const struct sockaddr_storage *addr)
 {
 	try {
-		return reinterpret_cast<ZeroTier::Node *>(node)->addLocalInterfaceAddress(addr,trust);
+		return reinterpret_cast<ZeroTier::Node *>(node)->addLocalInterfaceAddress(addr);
 	} catch ( ... ) {
 		return 0;
 	}
diff --git a/node/Node.hpp b/node/Node.hpp
index 48c5ead8..9b85b832 100644
--- a/node/Node.hpp
+++ b/node/Node.hpp
@@ -105,7 +105,7 @@ public:
 	ZT_VirtualNetworkConfig *networkConfig(uint64_t nwid) const;
 	ZT_VirtualNetworkList *networks() const;
 	void freeQueryResult(void *qr);
-	int addLocalInterfaceAddress(const struct sockaddr_storage *addr,ZT_LocalInterfaceAddressTrust trust);
+	int addLocalInterfaceAddress(const struct sockaddr_storage *addr);
 	void clearLocalInterfaceAddresses();
 	void setNetconfMaster(void *networkControllerInstance);
 	ZT_ResultCode circuitTestBegin(ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *));
diff --git a/node/Path.hpp b/node/Path.hpp
index 2b05b812..c0182990 100644
--- a/node/Path.hpp
+++ b/node/Path.hpp
@@ -79,18 +79,16 @@ public:
 		_addr(),
 		_localAddress(),
 		_ipScope(InetAddress::IP_SCOPE_NONE),
-		_trust(TRUST_NORMAL),
 		_flags(0)
 	{
 	}
 
-	Path(const InetAddress &localAddress,const InetAddress &addr,Trust trust) :
+	Path(const InetAddress &localAddress,const InetAddress &addr) :
 		_lastSend(0),
 		_lastReceived(0),
 		_addr(addr),
 		_localAddress(localAddress),
 		_ipScope(addr.ipScope()),
-		_trust(trust),
 		_flags(0)
 	{
 	}
@@ -187,11 +185,6 @@ public:
 		return ( ((int)_ipScope * 2) + ((_addr.ss_family == AF_INET6) ? 1 : 0) );
 	}
 
-	/**
-	 * @return Path trust level
-	 */
-	inline Trust trust() const throw() { return _trust; }
-
 	/**
 	 * @return True if path is considered reliable (no NAT keepalives etc. are needed)
 	 */
@@ -243,12 +236,11 @@ public:
 	template<unsigned int C>
 	inline void serialize(Buffer<C> &b) const
 	{
-		b.append((uint8_t)0); // version
+		b.append((uint8_t)1); // version
 		b.append((uint64_t)_lastSend);
 		b.append((uint64_t)_lastReceived);
 		_addr.serialize(b);
 		_localAddress.serialize(b);
-		b.append((uint8_t)_trust);
 		b.append((uint16_t)_flags);
 	}
 
@@ -256,14 +248,13 @@ public:
 	inline unsigned int deserialize(const Buffer<C> &b,unsigned int startAt = 0)
 	{
 		unsigned int p = startAt;
-		if (b[p++] != 0)
+		if (b[p++] != 1)
 			throw std::invalid_argument("invalid serialized Path");
 		_lastSend = b.template at<uint64_t>(p); p += 8;
 		_lastReceived = b.template at<uint64_t>(p); p += 8;
 		p += _addr.deserialize(b,p);
 		p += _localAddress.deserialize(b,p);
 		_ipScope = _addr.ipScope();
-		_trust = (Path::Trust)b[p++];
 		_flags = b.template at<uint16_t>(p); p += 2;
 		return (p - startAt);
 	}
@@ -274,7 +265,6 @@ private:
 	InetAddress _addr;
 	InetAddress _localAddress;
 	InetAddress::IpScope _ipScope; // memoize this since it's a computed value checked often
-	Trust _trust;
 	uint16_t _flags;
 };
 
diff --git a/node/Peer.cpp b/node/Peer.cpp
index 976c7c44..9d0d78e5 100644
--- a/node/Peer.cpp
+++ b/node/Peer.cpp
@@ -179,7 +179,7 @@ void Peer::received(
 						}
 					}
 					if (slot) {
-						*slot = Path(localAddr,remoteAddr,Path::TRUST_NORMAL);
+						*slot = Path(localAddr,remoteAddr);
 						slot->received(now);
 						_numPaths = np;
 						pathIsConfirmed = true;
diff --git a/service/OneService.cpp b/service/OneService.cpp
index 4e3f24c7..8c8ff1ed 100644
--- a/service/OneService.cpp
+++ b/service/OneService.cpp
@@ -731,7 +731,7 @@ public:
 #ifdef ZT_USE_MINIUPNPC
 					std::vector<InetAddress> upnpAddresses(_upnpClient->get());
 					for(std::vector<InetAddress>::const_iterator ext(upnpAddresses.begin());ext!=upnpAddresses.end();++ext)
-						_node->addLocalInterfaceAddress(reinterpret_cast<const struct sockaddr_storage *>(&(*ext)),ZT_LOCAL_INTERFACE_ADDRESS_TRUST_NORMAL);
+						_node->addLocalInterfaceAddress(reinterpret_cast<const struct sockaddr_storage *>(&(*ext)));
 #endif
 
 					struct ifaddrs *ifatbl = (struct ifaddrs *)0;
@@ -749,7 +749,7 @@ public:
 								if (!isZT) {
 									InetAddress ip(ifa->ifa_addr);
 									ip.setPort(_port);
-									_node->addLocalInterfaceAddress(reinterpret_cast<const struct sockaddr_storage *>(&ip),ZT_LOCAL_INTERFACE_ADDRESS_TRUST_NORMAL);
+									_node->addLocalInterfaceAddress(reinterpret_cast<const struct sockaddr_storage *>(&ip));
 								}
 							}
 							ifa = ifa->ifa_next;
-- 
cgit v1.2.3