Tons of code cleanup, refactor Network to use EthernetTapFactory, probably also fix GitHub issue #90

13 files changed, 160 insertions, 469 deletions

diff --git a/node/Constants.hpp b/node/Constants.hpp
index 524a8428..7650de8b 100644
--- a/node/Constants.hpp
+++ b/node/Constants.hpp
@@ -156,6 +156,11 @@
 #define ZT_IF_MTU 2800
 
 /**
+ * Default interface metric for ZeroTier taps -- should be higher than physical ports
+ */
+#define ZT_DEFAULT_IF_METRIC 32768
+
+/**
  * Maximum number of packet fragments we'll support
  * 
  * The actual spec allows 16, but this is the most we'll support right
diff --git a/node/Defaults.cpp b/node/Defaults.cpp
index 4387c1ae..9176a414 100644
--- a/node/Defaults.cpp
+++ b/node/Defaults.cpp
@@ -52,57 +52,6 @@ static inline std::map< Identity,std::vector< std::pair<InetAddress,bool> > > _m
 	// Nothing special about a supernode... except that they are
 	// designated as such and trusted to provide WHOIS lookup.
 
-#ifdef ZT_USE_TESTNET
-
-	// If ZT_USE_TESTNET is defined we talk to test rather than live supernode
-	// instances. The testnet may not always be running, so this is probably not
-	// of any interest to users. Testnet servers run on port 7773 (UDP) and
-	// 773 (TCP).
-
-	// cthulhu.zerotier.com - New York, New York, USA
-	addrs.clear();
-	if (!id.fromString("0bfa76f104:0:aff4d4604f2a2538d414a1d69fc722a28bea049d52192aded117c28b0f6c1052db9d36c488c5fe5e2071f2def8f86b6db64db09e819f90fdaedbfcb9f3bcdef9"))
-		throw std::runtime_error("invalid identity in Defaults");
-	addrs.push_back(std::pair<InetAddress,bool>(InetAddress("162.243.77.111",7773),false));
-	addrs.push_back(std::pair<InetAddress,bool>(InetAddress("162.243.77.111",773),true));
-	sn[id] = addrs;
-
-	// nyarlathotep.zerotier.com - San Francisco, California, USA
-	addrs.clear();
-	if (!id.fromString("9f2b042cdb:0:8993f9348bb9642afa9a60995a35ef19817894fd0b6859201c0e56e399288867c8f0d01ae2858f9dc6f95eee6d42e2f6d08c44551404906b25679aa6db1faee7"))
-		throw std::runtime_error("invalid identity in Defaults");
-	addrs.push_back(std::pair<InetAddress,bool>(InetAddress("198.199.97.220",7773),false));
-	addrs.push_back(std::pair<InetAddress,bool>(InetAddress("198.199.97.220",773),true));
-	sn[id] = addrs;
-
-	// shub-niggurath.zerotier.com - Amsterdam, Netherlands
-	addrs.clear();
-	if (!id.fromString("916a4ca17d:0:b679a8d6761096ba4958fea0036dc4dbb76cb8cbf1ce9bc352cc594c3c24987bb3b30b5448d1f494f5e90a6cdaac9d28317cb4088780278ef20bc7c366cb214a"))
-		throw std::runtime_error("invalid identity in Defaults");
-	addrs.push_back(std::pair<InetAddress,bool>(InetAddress("198.211.127.172",7773),false));
-	addrs.push_back(std::pair<InetAddress,bool>(InetAddress("198.211.127.172",773),true));
-	sn[id] = addrs;
-
-	// yig.zerotier.com - Sydney, Australia
-	addrs.clear();
-	if (!id.fromString("3b62c7a69a:0:d967595a3b96d780151764e6ffb47af2fa8865f8e344fba4a684c10dd2e70014e26312f5b8a1590c13bfeb909a1fd35b96a84a8a43e0704cd8d01d9c2b791359"))
-		throw std::runtime_error("invalid identity in Defaults");
-	addrs.push_back(std::pair<InetAddress,bool>(InetAddress("108.61.212.61",7773),false));
-	addrs.push_back(std::pair<InetAddress,bool>(InetAddress("108.61.212.61",773),true));
-	sn[id] = addrs;
-
-	// shoggoth.zerotier.com - Tokyo, Japan
-	addrs.clear();
-	if (!id.fromString("345ad16512:0:9e796aec6e083726f45fbfdc10bcf18c0dc7a7914c9ce29f5eb5abcf41bfcb6b3698b68131d347235ae488804317df9c6102e2753841b973037d1e4685dce9fc"))
-		throw std::runtime_error("invalid identity in Defaults");
-	addrs.push_back(std::pair<InetAddress,bool>(InetAddress("108.61.200.101",7773),false));
-	addrs.push_back(std::pair<InetAddress,bool>(InetAddress("108.61.200.101",773),true));
-	sn[id] = addrs;
-
-#else
-
-	// Normally we use the live supernodes.
-
 	// cthulhu.zerotier.com - New York, New York, USA
 	addrs.clear();
 	if (!id.fromString("8acf059fe3:0:482f6ee5dfe902319b419de5bdc765209c0ecda38c4d6e4fcf0d33658398b4527dcd22f93112fb9befd02fd78bf7261b333fc105d192a623ca9e50fc60b374a5"))
@@ -143,8 +92,6 @@ static inline std::map< Identity,std::vector< std::pair<InetAddress,bool> > > _m
 	addrs.push_back(std::pair<InetAddress,bool>(InetAddress("108.61.200.101",443),true));
 	sn[id] = addrs;
 
-#endif
-
 	return sn;
 }
 
diff --git a/node/EthernetTap.hpp b/node/EthernetTap.hpp
index ba534a7e..87ab607d 100644
--- a/node/EthernetTap.hpp
+++ b/node/EthernetTap.hpp
@@ -171,14 +171,16 @@ public:
 	virtual void put(const MAC &from,const MAC &to,unsigned int etherType,const void *data,unsigned int len) = 0;
 
 	/**
-	 * @return OS-specific device or connection name
+	 * @return OS-specific device or connection name (e.g. zt0, tap0, etc.)
 	 */
 	virtual std::string deviceName() const = 0;
 
 	/**
-	 * @return OS-internal persistent device ID or empty string if not applicable to this platform or not persistent
+	 * Change this device's user-visible name (if supported)
+	 *
+	 * @param friendlyName New name
 	 */
-	virtual std::string persistentId() const = 0;
+	virtual void setFriendlyName(const char *friendlyName) = 0;
 
 	/**
 	 * Fill or modify a set to contain multicast groups for this device
diff --git a/node/EthernetTapFactory.hpp b/node/EthernetTapFactory.hpp
index fde458f9..17d697f5 100644
--- a/node/EthernetTapFactory.hpp
+++ b/node/EthernetTapFactory.hpp
@@ -89,11 +89,12 @@ public:
 		void *arg) = 0;
 
 	/**
-	 * Close an ethernet tap device
+	 * Close an ethernet tap device and delete/free the tap object
 	 *
 	 * @param tap Tap instance
+	 * @param destroyPersistentDevices If true, destroy persistent device (on platforms where applicable)
 	 */
-	virtual void close(EthernetTap *tap) = 0;
+	virtual void close(EthernetTap *tap,bool destroyPersistentDevices) = 0;
 
 	/**
 	 * @return All currently open tap device names
diff --git a/node/Network.cpp b/node/Network.cpp
index f45d6c2f..43565e9f 100644
--- a/node/Network.cpp
+++ b/node/Network.cpp
@@ -37,12 +37,8 @@
 #include "Switch.hpp"
 #include "Packet.hpp"
 #include "Buffer.hpp"
-
-#ifdef __WINDOWS__
-#include "WindowsEthernetTap.hpp"
-#else
-#include "UnixEthernetTap.hpp"
-#endif
+#include "EthernetTap.hpp"
+#include "EthernetTapFactory.hpp"
 
 #define ZT_NETWORK_CERT_WRITE_BUF_SIZE 131072
 
@@ -69,26 +65,13 @@ Network::~Network()
 {
 	Thread::join(_setupThread);
 
-#ifdef __WINDOWS__
-	std::string devPersistentId;
-	if (_tap) {
-		devPersistentId = _tap->persistentId();
-		delete _tap;
-	}
-#else
 	if (_tap)
-		delete _tap;
-#endif
+		_r->tapFactory->close(_tap,_destroyOnDelete);
 
 	if (_destroyOnDelete) {
 		Utils::rm(std::string(_r->homePath + ZT_PATH_SEPARATOR_S + "networks.d" + ZT_PATH_SEPARATOR_S + idString() + ".conf"));
 		Utils::rm(std::string(_r->homePath + ZT_PATH_SEPARATOR_S + "networks.d" + ZT_PATH_SEPARATOR_S + idString() + ".mcerts"));
-#ifdef __WINDOWS__
-		if (devPersistentId.length())
-			WindowsEthernetTap::deletePersistentTapDevice(_r,devPersistentId.c_str());
-#endif
 	} else {
-		// Causes flush of membership certs to disk
 		clean();
 		_dumpMulticastCerts();
 	}
@@ -113,10 +96,16 @@ SharedPtr<Network> Network::newInstance(const RuntimeEnvironment *renv,NodeConfi
 	nw->_destroyOnDelete = false;
 	nw->_netconfFailure = NETCONF_FAILURE_NONE;
 
-	if (nw->controller() == renv->identity.address()) // netconf masters can't really join networks
+	if (nw->controller() == renv->identity.address()) // TODO: fix Switch to allow packets to self
 		throw std::runtime_error("cannot join a network for which I am the netconf master");
 
-	nw->_setupThread = Thread::start<Network>(nw.ptr());
+	try {
+		nw->_restoreState();
+		nw->requestConfiguration();
+	} catch ( ... ) {
+		TRACE("exception in network setup thread in _restoreState() or requestConfiguration()!");
+		nw->_lastConfigUpdate = 0; // call requestConfiguration() again
+	}
 
 	return nw;
 }
@@ -127,7 +116,7 @@ bool Network::updateMulticastGroups()
 	EthernetTap *t = _tap;
 	if (t) {
 		// Grab current groups from the local tap
-		bool updated = _tap->updateMulticastGroups(_multicastGroups);
+		bool updated = t->updateMulticastGroups(_multicastGroups);
 
 		// Merge in learned groups from any hosts bridged in behind us
 		for(std::map<MulticastGroup,uint64_t>::const_iterator mg(_bridgedMulticastGroups.begin());mg!=_bridgedMulticastGroups.end();++mg)
@@ -154,21 +143,11 @@ bool Network::setConfiguration(const Dictionary &conf,bool saveToDisk)
 {
 	Mutex::Lock _l(_lock);
 
-	EthernetTap *t = _tap;
-	if (!t) {
-		TRACE("BUG: setConfiguration() called while tap is null!");
-		return false; // can't accept config in initialization state
-	}
-
 	try {
-		SharedPtr<NetworkConfig> newConfig(new NetworkConfig(conf));
+		SharedPtr<NetworkConfig> newConfig(new NetworkConfig(conf)); // throws if invalid
 		if ((newConfig->networkId() == _id)&&(newConfig->issuedTo() == _r->identity.address())) {
 			_config = newConfig;
 
-			if (newConfig->staticIps().size())
-				t->setIps(newConfig->staticIps());
-			t->setDisplayName((std::string("ZeroTier One [") + newConfig->name() + "]").c_str());
-
 			_lastConfigUpdate = Utils::now();
 			_netconfFailure = NETCONF_FAILURE_NONE;
 
@@ -181,6 +160,17 @@ bool Network::setConfiguration(const Dictionary &conf,bool saveToDisk)
 				}
 			}
 
+			EthernetTap *t = _tap;
+			if (t) {
+				char fname[1024];
+				_mkNetworkFriendlyName(fname,sizeof(fname));
+				t->setIps(newConfig->staticIps());
+				t->setFriendlyName(fname);
+			} else {
+				if (!_setupThread)
+					_setupThread = Thread::start<Network>(this);
+			}
+
 			return true;
 		} else {
 			LOG("ignored invalid configuration for network %.16llx (configuration contains mismatched network ID or issued-to address)",(unsigned long long)_id);
@@ -196,9 +186,6 @@ bool Network::setConfiguration(const Dictionary &conf,bool saveToDisk)
 
 void Network::requestConfiguration()
 {
-	if (!_tap)
-		return; // don't bother requesting until we are initialized
-
 	if (controller() == _r->identity.address()) {
 		// netconf master cannot be a member of its own nets
 		LOG("unable to request network configuration for network %.16llx: I am the network master, cannot query self",(unsigned long long)_id);
@@ -346,51 +333,46 @@ void Network::_pushMembershipCertificate(const Address &peer,bool force,uint64_t
 	}
 }
 
+// Ethernet tap creation thread -- required on some platforms where tap
+// creation may be time consuming (e.g. Windows).
 void Network::threadMain()
 	throw()
 {
-	// Setup thread -- this exits when tap is constructed. It's here
-	// because opening the tap can take some time on some platforms.
+	char fname[1024],lcentry[128];
+	Utils::snprintf(lcentry,sizeof(lcentry),"_dev_for_%.16llx",(unsigned long long)_id);
 
+	EthernetTap *t;
 	try {
-#ifdef __WINDOWS__
-		// Windows tags interfaces by their network IDs, which are shoved into the
-		// registry to mark persistent instance of the tap device.
-		char tag[24];
-		Utils::snprintf(tag,sizeof(tag),"%.16llx",(unsigned long long)_id);
-		_tap = new WindowsEthernetTap(_r,tag,_mac,ZT_IF_MTU,&_CBhandleTapData,this);
-#else
-		// Unix tries to get the same device name next time, if possible.
-		std::string tagstr;
-		char lcentry[128];
-		Utils::snprintf(lcentry,sizeof(lcentry),"_dev_for_%.16llx",(unsigned long long)_id);
-		tagstr = _nc->getLocalConfig(lcentry);
-
-		const char *tag = (tagstr.length() > 0) ? tagstr.c_str() : (const char *)0;
-		_tap = new UnixEthernetTap(_r,tag,_mac,ZT_IF_MTU,&_CBhandleTapData,this);
-
-		std::string dn(_tap->deviceName());
-		if ((!tag)||(dn != tag))
+		std::string desiredDevice(_nc->getLocalConfig(lcentry));
+		_mkNetworkFriendlyName(fname,sizeof(fname));
+
+		t = _r->tapFactory->open(_mac,ZT_IF_MTU,ZT_DEFAULT_IF_METRIC,_id,(desiredDevice.length() > 0) ? desiredDevice.c_str() : (const char *)0,fname,_CBhandleTapData,this);
+
+		std::string dn(t->deviceName());
+		if ((dn.length())&&(dn != desiredDevice))
 			_nc->putLocalConfig(lcentry,dn);
-#endif
 	} catch (std::exception &exc) {
-		delete _tap;
-		_tap = (EthernetTap *)0;
+		delete t;
+		t = (EthernetTap *)0;
 		LOG("network %.16llx failed to initialize: %s",_id,exc.what());
 		_netconfFailure = NETCONF_FAILURE_INIT_FAILED;
 	} catch ( ... ) {
-		delete _tap;
-		_tap = (EthernetTap *)0;
+		delete t;
+		t = (EthernetTap *)0;
 		LOG("network %.16llx failed to initialize: unknown error",_id);
 		_netconfFailure = NETCONF_FAILURE_INIT_FAILED;
 	}
 
-	try {
-		_restoreState();
-		requestConfiguration();
-	} catch ( ... ) {
-		TRACE("BUG: exception in network setup thread in _restoreState() or requestConfiguration()!");
-		_lastConfigUpdate = 0; // call requestConfiguration() again
+	{
+		Mutex::Lock _l(_lock);
+		if (_tap) // the tap creation thread can technically be re-launched, though this isn't done right now
+			_r->tapFactory->close(_tap,_destroyOnDelete);
+		_tap = t;
+		if (t) {
+			if (_config)
+				t->setIps(_config->staticIps());
+			t->setEnabled(_enabled);
+		}
 	}
 }
 
@@ -423,14 +405,12 @@ void Network::setEnabled(bool enabled)
 {
 	Mutex::Lock _l(_lock);
 	_enabled = enabled;
-	// TODO: bring OS network device to "down" state if enabled == false
+	if (_tap)
+		_tap->setEnabled(enabled);
 }
 
 void Network::_restoreState()
 {
-	if (!_id)
-		return; // sanity check
-
 	Buffer<ZT_NETWORK_CERT_WRITE_BUF_SIZE> buf;
 
 	std::string idstr(idString());
@@ -448,7 +428,7 @@ void Network::_restoreState()
 		} else {
 			// If the conf file isn't present, "touch" it so we'll remember
 			// the existence of this network.
-			FILE *tmp = fopen(confPath.c_str(),"wb");
+			FILE *tmp = fopen(confPath.c_str(),"w");
 			if (tmp)
 				fclose(tmp);
 		}
diff --git a/node/Network.hpp b/node/Network.hpp
index 03b011f3..1d62bdf1 100644
--- a/node/Network.hpp
+++ b/node/Network.hpp
@@ -426,6 +426,14 @@ private:
 	void _restoreState();
 	void _dumpMulticastCerts();
 
+	inline void _mkNetworkFriendlyName(char *buf,unsigned int len)
+	{
+		// assumes _lock is locked
+		if (_config)
+			Utils::snprintf(buf,len,"ZeroTier One [%s]",_config->name().c_str());
+		else Utils::snprintf(buf,len,"ZeroTier One [%.16llx]",(unsigned long long)_id);
+	}
+
 	uint64_t _id;
 	NodeConfig *_nc; // parent NodeConfig object
 	MAC _mac; // local MAC address
@@ -439,8 +447,8 @@ private:
 	std::map<Address,CertificateOfMembership> _membershipCertificates;
 	std::map<Address,uint64_t> _lastPushedMembershipCertificate;
 
-	std::map<MAC,Address> _bridgeRoutes;
-	std::map<MulticastGroup,uint64_t> _bridgedMulticastGroups;
+	std::map<MAC,Address> _bridgeRoutes; // remote addresses where given MACs are reachable
+	std::map<MulticastGroup,uint64_t> _bridgedMulticastGroups; // multicast groups of interest on our side of the bridge
 
 	SharedPtr<NetworkConfig> _config;
 	volatile uint64_t _lastConfigUpdate;
diff --git a/node/Node.cpp b/node/Node.cpp
index 92b49d76..f2e6b3e4 100644
--- a/node/Node.cpp
+++ b/node/Node.cpp
@@ -67,7 +67,6 @@
 #include "EthernetTap.hpp"
 #include "CMWC4096.hpp"
 #include "NodeConfig.hpp"
-#include "SysEnv.hpp"
 #include "Network.hpp"
 #include "MulticastGroup.hpp"
 #include "Mutex.hpp"
@@ -77,6 +76,7 @@
 #include "Buffer.hpp"
 #include "IpcConnection.hpp"
 #include "AntiRecursion.hpp"
+#include "RoutingTable.hpp"
 
 namespace ZeroTier {
 
@@ -218,6 +218,7 @@ const char *Node::NodeControlClient::authTokenDefaultSystemPath()
 struct _NodeImpl
 {
 	RuntimeEnvironment renv;
+
 	unsigned int udpPort,tcpPort;
 	std::string reasonForTerminationStr;
 	volatile Node::ReasonForTermination reasonForTermination;
@@ -225,6 +226,7 @@ struct _NodeImpl
 	volatile bool running;
 	volatile bool resynchronize;
 
+	// This function performs final node tear-down
 	inline Node::ReasonForTermination terminate()
 	{
 		RuntimeEnvironment *_r = &renv;
@@ -238,16 +240,15 @@ struct _NodeImpl
 #ifndef __WINDOWS__
 		delete renv.netconfService;
 #endif
-		delete renv.updater;
-		delete renv.nc;
-		delete renv.sysEnv;
-		delete renv.topology;
-		delete renv.sm;
-		delete renv.sw;
-		delete renv.mc;
-		delete renv.antiRec;
-		delete renv.prng;
-		delete renv.log;
+		delete renv.updater;  renv.updater = (SoftwareUpdater *)0;
+		delete renv.nc;       renv.nc = (NodeConfig *)0;            // shut down all networks, close taps, etc.
+		delete renv.topology; renv.topology = (Topology *)0;        // now we no longer need routing info
+		delete renv.sm;       renv.sm = (SocketManager *)0;         // close all sockets
+		delete renv.sw;       renv.sw = (Switch *)0;                // order matters less from here down
+		delete renv.mc;       renv.mc = (Multicaster *)0;
+		delete renv.antiRec;  renv.antiRec = (AntiRecursion *)0;
+		delete renv.prng;     renv.prng = (CMWC4096 *)0;
+		delete renv.log;      renv.log = (Logger *)0;               // but stop logging last of all
 
 		return reasonForTermination;
 	}
@@ -260,7 +261,7 @@ struct _NodeImpl
 	}
 };
 
-#ifndef __WINDOWS__
+#ifndef __WINDOWS__ // "services" are not supported on Windows
 static void _netconfServiceMessageHandler(void *renv,Service &svc,const Dictionary &msg)
 {
 	if (!renv)
@@ -347,7 +348,13 @@ static void _netconfServiceMessageHandler(void *renv,Service &svc,const Dictiona
 }
 #endif // !__WINDOWS__
 
-Node::Node(const char *hp,unsigned int udpPort,unsigned int tcpPort,bool resetIdentity)
+Node::Node(
+	const char *hp,
+	EthernetTapFactory *tf,
+	RoutingTable *rt,
+	unsigned int udpPort,
+	unsigned int tcpPort,
+	bool resetIdentity)
 	throw() :
 	_impl(new _NodeImpl)
 {
@@ -357,6 +364,9 @@ Node::Node(const char *hp,unsigned int udpPort,unsigned int tcpPort,bool resetId
 		impl->renv.homePath = hp;
 	else impl->renv.homePath = ZT_DEFAULTS.defaultHomePath;
 
+	impl->renv.tapFactory = tf;
+	impl->renv.routingTable = rt;
+
 	if (resetIdentity) {
 		// Forget identity and peer database, peer keys, etc.
 		Utils::rm((impl->renv.homePath + ZT_PATH_SEPARATOR_S + "identity.public").c_str());
@@ -364,13 +374,14 @@ Node::Node(const char *hp,unsigned int udpPort,unsigned int tcpPort,bool resetId
 		Utils::rm((impl->renv.homePath + ZT_PATH_SEPARATOR_S + "peers.persist").c_str());
 
 		// Truncate network config information in networks.d but leave the files since we
-		// still want to remember any networks we have joined. This will force re-config.
+		// still want to remember any networks we have joined. This will force those networks
+		// to be reconfigured with our newly regenerated identity after startup.
 		std::string networksDotD(impl->renv.homePath + ZT_PATH_SEPARATOR_S + "networks.d");
 		std::map< std::string,bool > nwfiles(Utils::listDirectory(networksDotD.c_str()));
 		for(std::map<std::string,bool>::iterator nwf(nwfiles.begin());nwf!=nwfiles.end();++nwf) {
-			FILE *foo = fopen((networksDotD + ZT_PATH_SEPARATOR_S + nwf->first).c_str(),"w");
-			if (foo)
-				fclose(foo);
+			FILE *trun = fopen((networksDotD + ZT_PATH_SEPARATOR_S + nwf->first).c_str(),"w");
+			if (trun)
+				fclose(trun);
 		}
 	}
 
@@ -470,13 +481,11 @@ Node::ReasonForTermination Node::run()
 		}
 		Utils::lockDownFile(configAuthTokenPath.c_str(),false);
 
-		// Create the objects that make up runtime state.
 		_r->antiRec = new AntiRecursion();
 		_r->mc = new Multicaster();
 		_r->sw = new Switch(_r);
 		_r->sm = new SocketManager(impl->udpPort,impl->tcpPort,&_CBztTraffic,_r);
 		_r->topology = new Topology(_r,Utils::fileExists((_r->homePath + ZT_PATH_SEPARATOR_S + "iddb.d").c_str()));
-		_r->sysEnv = new SysEnv();
 		try {
 			_r->nc = new NodeConfig(_r,configAuthToken.c_str());
 		} catch (std::exception &exc) {
@@ -568,7 +577,7 @@ Node::ReasonForTermination Node::run()
 			// If our network environment looks like it changed, resynchronize.
 			if ((resynchronize)||((now - lastNetworkFingerprintCheck) >= ZT_NETWORK_FINGERPRINT_CHECK_DELAY)) {
 				lastNetworkFingerprintCheck = now;
-				uint64_t fp = _r->sysEnv->getNetworkConfigurationFingerprint(_r->nc->networkTapDeviceNames());
+				uint64_t fp = _r->routingTable->networkEnvironmentFingerprint(_r->nc->networkTapDeviceNames());
 				if (fp != networkConfigurationFingerprint) {
 					LOG("netconf fingerprint change: %.16llx != %.16llx, resyncing with network",networkConfigurationFingerprint,fp);
 					networkConfigurationFingerprint = fp;
@@ -588,7 +597,7 @@ Node::ReasonForTermination Node::run()
 			}
 
 			if (resynchronize) {
-				_r->tcpTunnelingEnabled = false; // turn off TCP tunneling master switch at first
+				_r->tcpTunnelingEnabled = false; // turn off TCP tunneling master switch at first, will be reenabled on persistent UDP failure
 				_r->timeOfLastResynchronize = now;
 			}
 
@@ -643,17 +652,15 @@ Node::ReasonForTermination Node::run()
 
 				/* Periodically ping all our non-stale direct peers unless we're a supernode.
 				 * Supernodes only ping each other (which is done above). */
-				if (!_r->topology->amSupernode()) {
-					if ((now - lastPingCheck) >= ZT_PING_CHECK_DELAY) {
-						lastPingCheck = now;
-						try {
-							_r->topology->eachPeer(Topology::PingPeersThatNeedPing(_r,now));
-							_r->topology->eachPeer(Topology::OpenPeersThatNeedFirewallOpener(_r,now));
-						} catch (std::exception &exc) {
-							LOG("unexpected exception running ping check cycle: %s",exc.what());
-						} catch ( ... ) {
-							LOG("unexpected exception running ping check cycle: (unkonwn)");
-						}
+				if ((!_r->topology->amSupernode())&&((now - lastPingCheck) >= ZT_PING_CHECK_DELAY)) {
+					lastPingCheck = now;
+					try {
+						_r->topology->eachPeer(Topology::PingPeersThatNeedPing(_r,now));
+						_r->topology->eachPeer(Topology::OpenPeersThatNeedFirewallOpener(_r,now));
+					} catch (std::exception &exc) {
+						LOG("unexpected exception running ping check cycle: %s",exc.what());
+					} catch ( ... ) {
+						LOG("unexpected exception running ping check cycle: (unkonwn)");
 					}
 				}
 			}
diff --git a/node/Node.hpp b/node/Node.hpp
index cc2886e7..6cc3ef40 100644
--- a/node/Node.hpp
+++ b/node/Node.hpp
@@ -33,6 +33,9 @@
 
 namespace ZeroTier {
 
+class EthernetTapFactory;
+class RoutingTable;
+
 /**
  * A ZeroTier One node
  *
@@ -150,14 +153,25 @@ public:
 	/**
 	 * Create a new node
 	 *
-	 * The node is not executed until run() is called.
+	 * The node is not executed until run() is called. The supplied tap factory
+	 * and routing table must not be freed until the node is no longer
+	 * executing. Node does not delete these objects, so the caller still owns
+	 * them.
 	 *
-	 * @param hp Home directory path or NULL for system-wide default for this platform (default: NULL)
-	 * @param udpPort UDP port or 0 to disable (default: 9993)
-	 * @param tcpPort TCP port or 0 to disable (default: 0)
-	 * @param resetIdentity If true, delete identity before starting and regenerate (default: false)
+	 * @param hp Home directory path or NULL for system-wide default for this platform
+	 * @param tf Ethernet tap factory for platform network stack
+	 * @param rt Routing table interface for platform network stack
+	 * @param udpPort UDP port or 0 to disable
+	 * @param tcpPort TCP port or 0 to disable
+	 * @param resetIdentity If true, delete identity before starting and regenerate
 	 */
-	Node(const char *hp = (const char *)0,unsigned int udpPort = 9993,unsigned int tcpPort = 0,bool resetIdentity = false)
+	Node(
+		const char *hp,
+		EthernetTapFactory *tf,
+		RoutingTable *rt,
+		unsigned int udpPort,
+		unsigned int tcpPort,
+		bool resetIdentity)
 		throw();
 
 	~Node();
diff --git a/node/NodeConfig.hpp b/node/NodeConfig.hpp
index ada135ac..b9bc8f5f 100644
--- a/node/NodeConfig.hpp
+++ b/node/NodeConfig.hpp
@@ -120,16 +120,16 @@ public:
 	}
 
 	/**
-	 * @return Set of network tap device names from our virtual networks (not other taps on system)
+	 * @return Sorted vector of network tap device names from our virtual networks (not other taps on system)
 	 */
-	inline std::set<std::string> networkTapDeviceNames() const
+	inline std::vector<std::string> networkTapDeviceNames() const
 	{
-		std::set<std::string> tapDevs;
+		std::vector<std::string> tapDevs;
 		Mutex::Lock _l(_networks_m);
 		for(std::map< uint64_t,SharedPtr<Network> >::const_iterator n(_networks.begin());n!=_networks.end();++n) {
 			std::string dn(n->second->tapDeviceName());
 			if (dn.length())
-				tapDevs.insert(dn);
+				tapDevs.push_back(dn);
 		}
 		return tapDevs;
 	}
diff --git a/node/RuntimeEnvironment.hpp b/node/RuntimeEnvironment.hpp
index 8887b081..78d950ce 100644
--- a/node/RuntimeEnvironment.hpp
+++ b/node/RuntimeEnvironment.hpp
@@ -39,7 +39,6 @@ class NodeConfig;
 class Logger;
 class Switch;
 class Topology;
-class SysEnv;
 class CMWC4096;
 class Service;
 class Node;
@@ -47,6 +46,8 @@ class Multicaster;
 class SoftwareUpdater;
 class SocketManager;
 class AntiRecursion;
+class EthernetTapFactory;
+class RoutingTable;
 
 /**
  * Holds global state for an instance of ZeroTier::Node
@@ -68,6 +69,8 @@ public:
 		tcpTunnelingEnabled(false),
 		timeOfLastResynchronize(0),
 		timeOfLastPacketReceived(0),
+		tapFactory((EthernetTapFactory *)0),
+		routingTable((RoutingTable *)0),
 		log((Logger *)0),
 		prng((CMWC4096 *)0),
 		antiRec((AntiRecursion *)0),
@@ -75,7 +78,6 @@ public:
 		sw((Switch *)0),
 		sm((SocketManager *)0),
 		topology((Topology *)0),
-		sysEnv((SysEnv *)0),
 		nc((NodeConfig *)0),
 		updater((SoftwareUpdater *)0)
 #ifndef __WINDOWS__
@@ -103,6 +105,10 @@ public:
 	// via an ugly const_cast<>.
 	volatile uint64_t timeOfLastPacketReceived;
 
+	// These are passed in from outside and are not created or deleted by the ZeroTier node core
+	EthernetTapFactory *tapFactory;
+	RoutingTable *routingTable;
+
 	/*
 	 * Order matters a bit here. These are constructed in this order
 	 * and then deleted in the opposite order on Node exit. The order ensures
@@ -118,7 +124,6 @@ public:
 	Switch *sw;
 	SocketManager *sm;
 	Topology *topology;
-	SysEnv *sysEnv;
 	NodeConfig *nc;
 	Node *node;
 	SoftwareUpdater *updater; // null if software updates are not enabled
diff --git a/node/SysEnv.cpp b/node/SysEnv.cpp
deleted file mode 100644
index fdf52f11..00000000
--- a/node/SysEnv.cpp
+++ /dev/null
@@ -1,224 +0,0 @@
-/*
- * ZeroTier One - Global Peer to Peer Ethernet
- * Copyright (C) 2011-2014  ZeroTier Networks LLC
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * --
- *
- * ZeroTier may be used and distributed under the terms of the GPLv3, which
- * are available at: http://www.gnu.org/licenses/gpl-3.0.html
- *
- * If you would like to embed ZeroTier into a commercial application or
- * redistribute it in a modified binary form, please contact ZeroTier Networks
- * LLC. Start here: http://www.zerotier.com/
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <fcntl.h>
-#include <errno.h>
-#include <sys/types.h>
-
-#include <set>
-#include <string>
-
-#include "Constants.hpp"
-#include "SysEnv.hpp"
-#include "Utils.hpp"
-#include "RuntimeEnvironment.hpp"
-#include "NodeConfig.hpp"
-
-#ifdef __UNIX_LIKE__
-#include <arpa/inet.h>
-#include <sys/socket.h>
-#include <unistd.h>
-#include <signal.h>
-#endif
-
-#ifdef __APPLE__
-#include <sys/sysctl.h>
-#include <sys/uio.h>
-#include <sys/param.h>
-#include <net/route.h>
-#endif
-
-#ifdef __WINDOWS__
-#include <Windows.h>
-#include <WinSock2.h>
-#endif
-
-namespace ZeroTier {
-
-SysEnv::SysEnv()
-{
-}
-
-SysEnv::~SysEnv()
-{
-}
-
-#ifdef __APPLE__
-
-uint64_t SysEnv::getNetworkConfigurationFingerprint(const std::set<std::string> &ignoreDevices)
-{
-	int mib[6];
-	size_t needed;
-	uint64_t fingerprint = 5381; // djb2 hash algorithm is used below
-
-	// Right now this just scans for changes in default routes. This is not
-	// totally robust -- it will miss cases where we switch from one 10.0.0.0/24
-	// network with gateway .1 to another -- but most of the time it'll pick
-	// up shifts in connectivity. Combined with sleep/wake detection this seems
-	// pretty solid so far on Mac for detecting when you change locations.
-
-	mib[0] = CTL_NET;
-	mib[1] = PF_ROUTE;
-	mib[2] = 0;
-	mib[3] = AF_UNSPEC;
-	mib[4] = NET_RT_DUMP;
-	mib[5] = 0;
-	if (!sysctl(mib,6,NULL,&needed,NULL,0)) {
-		char *buf = (char *)malloc(needed);
-		if (buf) {
-			if (!sysctl(mib,6,buf,&needed,NULL,0)) {
-		    struct rt_msghdr *rtm;
-				for(char *next=buf,*end=buf+needed;next<end;) {
-					rtm = (struct rt_msghdr *)next;
-					char *saptr = (char *)(rtm + 1);
-					char *saend = next + rtm->rtm_msglen;
-					if (((rtm->rtm_addrs & RTA_DST))&&((rtm->rtm_addrs & RTA_GATEWAY))) {
-						int sano = 0;
-						struct sockaddr *dst = (struct sockaddr *)0;
-						struct sockaddr *gateway = (struct sockaddr *)0;
-						while (saptr < saend) {
-							struct sockaddr *sa = (struct sockaddr *)saptr;
-							if (!sa->sa_len)
-								break;
-							if (sano == 0)
-								dst = sa;
-							else if (sano == 1)
-								gateway = sa;
-							else if (sano > 1)
-								break;
-							++sano;
-							saptr += sa->sa_len;
-						}
-						if ((dst)&&(gateway)) {
-							if ((dst->sa_family == AF_INET)&&(gateway->sa_family == AF_INET)&&(!((struct sockaddr_in *)dst)->sin_addr.s_addr)) {
-								fingerprint = ((fingerprint << 5) + fingerprint) + (uint64_t)((struct sockaddr_in *)gateway)->sin_addr.s_addr;
-							} else if ((dst->sa_family == AF_INET6)&&(gateway->sa_family == AF_INET6)&&(Utils::isZero(((struct sockaddr_in6 *)dst)->sin6_addr.s6_addr,16))) {
-								for(unsigned int i=0;i<16;++i)
-									fingerprint = ((fingerprint << 5) + fingerprint) + (uint64_t)((struct sockaddr_in6 *)gateway)->sin6_addr.s6_addr[i];
-							}
-						}
-					}
-					next = saend;
-				}
-			}
-			free(buf);
-		}
-	}
-
-	return fingerprint;
-}
-
-#endif // __APPLE__
-
-#if defined(__linux__) || defined(linux) || defined(__LINUX__) || defined(__linux)
-
-uint64_t SysEnv::getNetworkConfigurationFingerprint(const std::set<std::string> &ignoreDevices)
-{
-	char buf[16384];
-	uint64_t fingerprint = 5381; // djb2 hash algorithm is used below
-	char *t1,*t2;
-
-	try {
-		// Include default IPv4 route if available
-		int fd = open("/proc/net/route",O_RDONLY);
-		if (fd > 0) {
-			long n = read(fd,buf,sizeof(buf) - 1);
-			::close(fd);
-			if (n > 0) {
-				buf[n] = 0;
-				for(char *line=strtok_r(buf,"\r\n",&t1);(line);line=strtok_r((char *)0,"\r\n",&t1)) {
-					int fno = 0;
-					for(char *field=strtok_r(line," \t",&t2);(field);field=strtok_r((char *)0," \t",&t2)) {
-						if (fno == 0) { // device name
-							if ((ignoreDevices.count(std::string(field)))||(!strcmp(field,"lo")))
-								break;
-						} else if ((fno == 1)||(fno == 2)) { // destination, gateway
-							if (strlen(field) == 8) { // ignore header junk, use only hex route info
-								while (*field)
-									fingerprint = ((fingerprint << 5) + fingerprint) + (uint64_t)*(field++);
-							}
-						} else if (fno > 2)
-							break;
-						++fno;
-					}
-				}
-			}
-		}
-
-		// Include IPs of IPv6 enabled interfaces if available
-		fd = open("/proc/net/if_inet6",O_RDONLY);
-		if (fd > 0) {
-			long n = read(fd,buf,sizeof(buf) - 1);
-			::close(fd);
-			if (n > 0) {
-				buf[n] = 0;
-				for(char *line=strtok_r(buf,"\r\n",&t1);(line);line=strtok_r((char *)0,"\r\n",&t1)) {
-					int fno = 0;
-					const char *v6ip = (const char *)0;
-					const char *devname = (const char *)0;
-					for(char *field=strtok_r(line," \t",&t2);(field);field=strtok_r((char *)0," \t",&t2)) {
-						switch(fno) {
-							case 0:
-								v6ip = field;
-								break;
-							case 5:
-								devname = field;
-								break;
-						}
-						++fno;
-					}
-
-					if ((v6ip)&&(devname)) {
-						if ((!(ignoreDevices.count(std::string(devname))))&&(strcmp(devname,"lo"))) {
-							while (*v6ip)
-								fingerprint = ((fingerprint << 5) + fingerprint) + (uint64_t)*(v6ip++);
-						}
-					}
-				}
-			}
-		}
-	} catch ( ... ) {}
-
-	return fingerprint;
-}
-
-#endif // __linux__
-
-#ifdef __WINDOWS__
-
-uint64_t SysEnv::getNetworkConfigurationFingerprint(const std::set<std::string> &ignoreDevices)
-{
-	// TODO: windows version
-	return 1;
-}
-
-#endif // __WINDOWS__
-
-} // namespace ZeroTier
diff --git a/node/SysEnv.hpp b/node/SysEnv.hpp
deleted file mode 100644
index dc73101b..00000000
--- a/node/SysEnv.hpp
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * ZeroTier One - Global Peer to Peer Ethernet
- * Copyright (C) 2011-2014  ZeroTier Networks LLC
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * --
- *
- * ZeroTier may be used and distributed under the terms of the GPLv3, which
- * are available at: http://www.gnu.org/licenses/gpl-3.0.html
- *
- * If you would like to embed ZeroTier into a commercial application or
- * redistribute it in a modified binary form, please contact ZeroTier Networks
- * LLC. Start here: http://www.zerotier.com/
- */
-
-#ifndef ZT_SYSENV_HPP
-#define ZT_SYSENV_HPP
-
-#include <stdint.h>
-
-#include <set>
-
-#include "NonCopyable.hpp"
-
-namespace ZeroTier {
-
-class RuntimeEnvironment;
-
-/**
- * Local system environment monitoring utilities
- */
-class SysEnv : NonCopyable
-{
-public:
-	SysEnv();
-	~SysEnv();
-
-	/**
-	 * This computes a CRC-type code from gathered information about your network settings
-	 *
-	 * @param ignoreDevices Ignore these local network devices by OS-specific name (e.g. our taps)
-	 * @return Fingerprint of currently running network environment
-	 */
-	uint64_t getNetworkConfigurationFingerprint(const std::set<std::string> &ignoreDevices);
-};
-
-} // namespace ZeroTier
-
-#endif
diff --git a/node/Thread.hpp b/node/Thread.hpp
index 6426d5bb..a796af33 100644
--- a/node/Thread.hpp
+++ b/node/Thread.hpp
@@ -87,6 +87,8 @@ public:
 			CancelSynchronousIo(t._th);

 	}

 

+	inline operator bool() const throw() { return (_th != NULL); }

+

 private:

 	HANDLE _th;

 	DWORD _tid;

@@ -123,18 +125,21 @@ public:
 		throw()

 	{

 		memset(&_tid,0,sizeof(_tid));

+		_started = false;

 	}

 

 	Thread(const Thread &t)

 		throw()

 	{

 		memcpy(&_tid,&(t._tid),sizeof(_tid));

+		_started = t._started;

 	}

 

 	inline Thread &operator=(const Thread &t)

 		throw()

 	{

 		memcpy(&_tid,&(t._tid),sizeof(_tid));

+		_started = t._started;

 		return *this;

 	}

 

@@ -151,19 +156,21 @@ public:
 		throw(std::runtime_error)

 	{

 		Thread t;

+		t._started = true;

 		if (pthread_create(&t._tid,(const pthread_attr_t *)0,&___zt_threadMain<C>,instance))

 			throw std::runtime_error("pthread_create() failed, unable to create thread");

 		return t;

 	}

 

 	/**

-	 * Join to a thread, waiting for it to terminate

+	 * Join to a thread, waiting for it to terminate (does nothing on null Thread values)

 	 *

 	 * @param t Thread to join

 	 */

 	static inline void join(const Thread &t)

 	{

-		pthread_join(t._tid,(void **)0);

+		if (t._started)

+			pthread_join(t._tid,(void **)0);

 	}

 

 	/**

@@ -171,13 +178,13 @@ public:
 	 *

 	 * @param ms Number of milliseconds to sleep

 	 */

-	static inline void sleep(unsigned long ms)

-	{

-		usleep(ms * 1000);

-	}

+	static inline void sleep(unsigned long ms) { usleep(ms * 1000); }

+

+	inline operator bool() const throw() { return (_started); }

 

 private:

 	pthread_t _tid;

+	volatile bool _started;

 };

 

 } // namespace ZeroTier