Refactor controller to use split-out DB for better performance and less ugly.

4 files changed, 211 insertions, 162 deletions

diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp
index 8f5cefd1..21cd78e4 100644
--- a/controller/EmbeddedNetworkController.cpp
+++ b/controller/EmbeddedNetworkController.cpp
@@ -64,6 +64,7 @@ using json = nlohmann::json;
 namespace ZeroTier {
 
 // JSON blob I/O
+/*
 static json _readJson(const std::string &path)
 {
 	std::string buf;
@@ -78,6 +79,7 @@ static bool _writeJson(const std::string &path,const json &obj)
 {
 	return OSUtils::writeFile(path.c_str(),obj.dump(2));
 }
+*/
 
 // Get JSON values as unsigned integers, strings, or booleans, doing type conversion if possible
 static uint64_t _jI(const json &jv,const uint64_t dfl)
@@ -475,55 +477,17 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule)
 }
 
 EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *dbPath) :
-	_node(node),
-	_path(dbPath),
-	_daemonRun(true)
+	_db(dbPath),
+	_node(node)
 {
 	OSUtils::mkdir(dbPath);
 	OSUtils::lockDownFile(dbPath,true); // networks might contain auth tokens, etc., so restrict directory permissions
-	_daemon = Thread::start(this);
 }
 
 EmbeddedNetworkController::~EmbeddedNetworkController()
 {
 }
 
-void EmbeddedNetworkController::threadMain()
-	throw()
-{
-	uint64_t lastUpdatedNetworkMemberCache = 0;
-	while (_daemonRun) {
-		// Every 60 seconds we rescan the filesystem for network members and rebuild our cache
-		if ((OSUtils::now() - lastUpdatedNetworkMemberCache) >= 60000) {
-			const std::vector<std::string> networks(OSUtils::listSubdirectories((_path + ZT_PATH_SEPARATOR_S + "network").c_str()));
-			for(auto n=networks.begin();n!=networks.end();++n) {
-				if (n->length() == 16) {
-					const std::vector<std::string> members(OSUtils::listSubdirectories((*n + ZT_PATH_SEPARATOR_S + "member").c_str()));
-					std::map<Address,nlohmann::json> newCache;
-					for(auto m=members.begin();m!=members.end();++m) {
-						if (m->length() == ZT_ADDRESS_LENGTH_HEX) {
-							const Address maddr(*m);
-							try {
-								const json mj(_readJson((_path + ZT_PATH_SEPARATOR_S + "network" + ZT_PATH_SEPARATOR_S + *n + ZT_PATH_SEPARATOR_S + "member" + ZT_PATH_SEPARATOR_S + *m + ZT_PATH_SEPARATOR_S + "config.json")));
-								if ((mj.is_object())&&(mj.size() > 0)) {
-									newCache[maddr] = mj;
-								}
-							} catch ( ... ) {}
-						}
-					}
-					{
-						Mutex::Lock _l(_networkMemberCache_m);
-						_networkMemberCache[Utils::hexStrToU64(n->c_str())] = newCache;
-					}
-				}
-			}
-			lastUpdatedNetworkMemberCache = OSUtils::now();
-		}
-
-		Thread::sleep(25);
-	}
-}
-
 NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest(const InetAddress &fromAddr,const Identity &signingId,const Identity &identity,uint64_t nwid,const Dictionary<ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY> &metaData,NetworkConfig &nc)
 {
 	if (((!signingId)||(!signingId.hasPrivate()))||(signingId.address().toInt() != (nwid >> 24))) {
@@ -541,12 +505,17 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest(
 		lrt = now;
 	}
 
-	json network(_readJson(_networkJP(nwid,false)));
+	char nwids[24];
+	Utils::snprintf(nwids,sizeof(nwids),"%.16llx",nwid);
+	json network;
+	json member;
+	{
+		Mutex::Lock _l(_db_m);
+		network = _db.get("network",nwids,0);
+		member = _db.get("network",nwids,"member",identity.address().toString(),0);
+	}
 	if (!network.size())
 		return NetworkController::NETCONF_QUERY_OBJECT_NOT_FOUND;
-
-	const std::string memberJP(_memberJP(nwid,identity.address(),true));
-	json member(_readJson(memberJP));
 	_initMember(member);
 
 	{
@@ -673,7 +642,8 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest(
 
 	// If they are not authorized, STOP!
 	if (!authorizedBy) {
-		_writeJson(memberJP,member);
+		Mutex::Lock _l(_db_m);
+		_db.put("network",nwids,"member",identity.address().toString(),member);
 		return NetworkController::NETCONF_QUERY_ACCESS_DENIED;
 	}
 
@@ -966,7 +936,10 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest(
 		return NETCONF_QUERY_INTERNAL_SERVER_ERROR;
 	}
 
-	_writeJson(memberJP,member);
+	{
+		Mutex::Lock _l(_db_m);
+		_db.put("network",nwids,"member",identity.address().toString(),member);
+	}
 	return NetworkController::NETCONF_QUERY_OK;
 }
 
@@ -985,7 +958,11 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET(
 			char nwids[24];
 			Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid);
 
-			json network(_readJson(_networkJP(nwid,false)));
+			json network;
+			{
+				Mutex::Lock _l(_db_m);
+				network = _db.get("network",nwids,0);
+			}
 			if (!network.size())
 				return 404;
 
@@ -996,7 +973,11 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET(
 					if (path.size() >= 4) {
 						const uint64_t address = Utils::hexStrToU64(path[3].c_str());
 
-						json member(_readJson(_memberJP(nwid,Address(address),false)));
+						json member;
+						{
+							Mutex::Lock _l(_db_m);
+							member = _db.get("network",nwids,"member",Address(address).toString(),0);
+						}
 						if (!member.size())
 							return 404;
 
@@ -1007,19 +988,19 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET(
 						return 200;
 					} else {
 
+						Mutex::Lock _l(_db_m);
+
 						responseBody = "{";
-						std::vector<std::string> members(OSUtils::listSubdirectories((_networkBP(nwid,false) + ZT_PATH_SEPARATOR_S + "member").c_str()));
-						for(std::vector<std::string>::iterator i(members.begin());i!=members.end();++i) {
-							if (i->length() == ZT_ADDRESS_LENGTH_HEX) {
-								json member(_readJson(_memberJP(nwid,Address(Utils::hexStrToU64(i->c_str())),false)));
-								if (member.size()) {
-									responseBody.append((responseBody.length() == 1) ? "\"" : ",\"");
-									responseBody.append(*i);
-									responseBody.append("\":");
-									responseBody.append(_jS(member["revision"],"0"));
-								}
+						std::string pfx(std::string("network/") + nwids + "member/");
+						_db.filter(pfx,120000,[&responseBody](const std::string &n,const json &member) {
+							if (member.size() > 0) {
+								responseBody.append((responseBody.length() == 1) ? "\"" : ",\"");
+								responseBody.append(_jS(member["id"],""));
+								responseBody.append("\":");
+								responseBody.append(_jS(member["revision"],"0"));
 							}
-						}
+							return true; // never delete
+						});
 						responseBody.push_back('}');
 						responseContentType = "application/json";
 
@@ -1056,7 +1037,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET(
 			}
 		} else if (path.size() == 1) {
 
-			responseBody = "[";
+			/*
 			std::vector<std::string> networks(OSUtils::listSubdirectories((_path + ZT_PATH_SEPARATOR_S + "network").c_str()));
 			for(auto i(networks.begin());i!=networks.end();++i) {
 				if (i->length() == 16) {
@@ -1065,6 +1046,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET(
 					responseBody.append("\"");
 				}
 			}
+			*/
 			responseBody.push_back(']');
 			responseContentType = "application/json";
 			return 200;
@@ -1122,7 +1104,11 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
 			Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid);
 
 			if (path.size() >= 3) {
-				json network(_readJson(_networkJP(nwid,false)));
+				json network;
+				{
+					Mutex::Lock _l(_db_m);
+					network = _db.get("network",nwids,0);
+				}
 				if (!network.size())
 					return 404;
 
@@ -1131,7 +1117,13 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
 					char addrs[24];
 					Utils::snprintf(addrs,sizeof(addrs),"%.10llx",(unsigned long long)address);
 
-					json member(_readJson(_memberJP(nwid,Address(address),true)));
+					json member;
+					{
+						Mutex::Lock _l(_db_m);
+						member = _db.get("network",nwids,"member",Address(address).toString(),0);
+					}
+					if (!member.size())
+						return 404;
 					_initMember(member);
 
 					try {
@@ -1154,7 +1146,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
 						}
 
 						if (b.count("ipAssignments")) {
-							auto ipa = b["ipAssignments"];
+							json &ipa = b["ipAssignments"];
 							if (ipa.is_array()) {
 								json mipa(json::array());
 								for(unsigned long i=0;i<ipa.size();++i) {
@@ -1169,7 +1161,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
 						}
 
 						if (b.count("tags")) {
-							auto tags = b["tags"];
+							json &tags = b["tags"];
 							if (tags.is_array()) {
 								std::map<uint64_t,uint64_t> mtags;
 								for(unsigned long i=0;i<tags.size();++i) {
@@ -1189,7 +1181,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
 						}
 
 						if (b.count("capabilities")) {
-							auto capabilities = b["capabilities"];
+							json &capabilities = b["capabilities"];
 							if (capabilities.is_array()) {
 								json mcaps = json::array();
 								for(unsigned long i=0;i<capabilities.size();++i) {
@@ -1210,14 +1202,12 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
 					member["address"] = addrs; // legacy
 					member["nwid"] = nwids;
 					member["lastModified"] = now;
-					auto revj = member["revision"];
+					json &revj = member["revision"];
 					member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL);
 
-					_writeJson(_memberJP(nwid,Address(address),true).c_str(),member);
-
 					{
-						Mutex::Lock _l(_networkMemberCache_m);
-						_networkMemberCache[nwid][Address(address)] = member;
+						Mutex::Lock _l(_db_m);
+						_db.put("network",nwids,"member",Address(address).toString(),member);
 					}
 
 					// Add non-persisted fields
@@ -1281,26 +1271,33 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
 			} else {
 				// POST to network ID
 
-				// Magic ID ending with ______ picks a random unused network ID
-				if (path[1].substr(10) == "______") {
-					nwid = 0;
-					uint64_t nwidPrefix = (Utils::hexStrToU64(path[1].substr(0,10).c_str()) << 24) & 0xffffffffff000000ULL;
-					uint64_t nwidPostfix = 0;
-					for(unsigned long k=0;k<100000;++k) { // sanity limit on trials
-						Utils::getSecureRandom(&nwidPostfix,sizeof(nwidPostfix));
-						uint64_t tryNwid = nwidPrefix | (nwidPostfix & 0xffffffULL);
-						if ((tryNwid & 0xffffffULL) == 0ULL) tryNwid |= 1ULL;
-						Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)tryNwid);
-						if (!OSUtils::fileExists(_networkJP(tryNwid,false).c_str())) {
-							nwid = tryNwid;
-							break;
+				json network;
+				{
+					Mutex::Lock _l(_db_m);
+
+					// Magic ID ending with ______ picks a random unused network ID
+					if (path[1].substr(10) == "______") {
+						nwid = 0;
+						uint64_t nwidPrefix = (Utils::hexStrToU64(path[1].substr(0,10).c_str()) << 24) & 0xffffffffff000000ULL;
+						uint64_t nwidPostfix = 0;
+						for(unsigned long k=0;k<100000;++k) { // sanity limit on trials
+							Utils::getSecureRandom(&nwidPostfix,sizeof(nwidPostfix));
+							uint64_t tryNwid = nwidPrefix | (nwidPostfix & 0xffffffULL);
+							if ((tryNwid & 0xffffffULL) == 0ULL) tryNwid |= 1ULL;
+							Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)tryNwid);
+							if (_db.get("network",nwids,120000).size() <= 0) {
+								nwid = tryNwid;
+								break;
+							}
 						}
+						if (!nwid)
+							return 503;
 					}
-					if (!nwid)
-						return 503;
-				}
 
-				json network(_readJson(_networkJP(nwid,true)));
+					network = _db.get("network",nwids,0);
+					if (!network.size())
+						return 404;
+				}
 				_initNetwork(network);
 
 				try {
@@ -1481,11 +1478,14 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
 
 				network["id"] = nwids;
 				network["nwid"] = nwids; // legacy
-				auto rev = network["revision"];
+				json &rev = network["revision"];
 				network["revision"] = (rev.is_number() ? ((uint64_t)rev + 1ULL) : 1ULL);
 				network["lastModified"] = now;
 
-				_writeJson(_networkJP(nwid,true),network);
+				{
+					Mutex::Lock _l(_db_m);
+					_db.put("network",nwids,network);
+				}
 
 				_NetworkMemberInfo nmi;
 				_getNetworkMemberInfo(now,nwid,nmi);
@@ -1518,7 +1518,13 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE(
 		if ((path.size() >= 2)&&(path[1].length() == 16)) {
 			const uint64_t nwid = Utils::hexStrToU64(path[1].c_str());
 
-			json network(_readJson(_networkJP(nwid,false)));
+			char nwids[24];
+			Utils::snprintf(nwids,sizeof(nwids),"%.16llx",nwid);
+			json network;
+			{
+				Mutex::Lock _l(_db_m);
+				network = _db.get("network",nwids,0);
+			}
 			if (!network.size())
 				return 404;
 
@@ -1526,22 +1532,23 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE(
 				if ((path.size() == 4)&&(path[2] == "member")&&(path[3].length() == 10)) {
 					const uint64_t address = Utils::hexStrToU64(path[3].c_str());
 
-					json member(_readJson(_memberJP(nwid,Address(address),false)));
+					Mutex::Lock _l(_db_m);
+
+					json member = _db.get("network",nwids,"member",Address(address).toString(),0);
 					if (!member.size())
 						return 404;
-
-					OSUtils::rmDashRf(_memberBP(nwid,Address(address),false).c_str());
+					_db.erase("network",nwids,"member",Address(address).toString());
 
 					responseBody = member.dump(2);
 					responseContentType = "application/json";
 					return 200;
 				}
 			} else {
-				OSUtils::rmDashRf(_networkBP(nwid,false).c_str());
-				{
-					Mutex::Lock _l(_networkMemberCache_m);
-					_networkMemberCache.erase(nwid);
-				}
+				Mutex::Lock _l(_db_m);
+				std::string pfx("network/"); pfx.append(nwids);
+				_db.filter(pfx,120000,[](const std::string &n,const json &obj) {
+					return false; // delete
+				});
 				responseBody = network.dump(2);
 				responseContentType = "application/json";
 				return 200;
@@ -1618,42 +1625,46 @@ void EmbeddedNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTes
 
 void EmbeddedNetworkController::_getNetworkMemberInfo(uint64_t now,uint64_t nwid,_NetworkMemberInfo &nmi)
 {
-	Mutex::Lock _mcl(_networkMemberCache_m);
-	std::map< Address,nlohmann::json > &memberCacheEntry = _networkMemberCache[nwid];
-	nmi.totalMemberCount = memberCacheEntry.size();
-	for(std::map< Address,nlohmann::json >::iterator nm(memberCacheEntry.begin());nm!=memberCacheEntry.end();++nm) {
-		if (_jB(nm->second["authorized"],false)) {
-			++nmi.authorizedMemberCount;
-
-			if (nm->second.count("recentLog")) {
-				json &mlog = nm->second["recentLog"];
-				if ((mlog.is_array())&&(mlog.size() > 0)) {
-					json &mlog1 = mlog[0];
-					if (mlog1.is_object()) {
-						if ((now - _jI(mlog1["ts"],0ULL)) < ZT_NETCONF_NODE_ACTIVE_THRESHOLD)
-							++nmi.activeMemberCount;
+	char pfx[256];
+	Utils::snprintf(pfx,sizeof(pfx),"network/%.16llx/member",nwid);
+
+	Mutex::Lock _l(_db_m);
+	_db.filter(pfx,120000,[&nmi,&now](const std::string &n,const json &member) {
+		try {
+			if (_jB(member["authorized"],false)) {
+				++nmi.authorizedMemberCount;
+
+				if (member.count("recentLog")) {
+					const json &mlog = member["recentLog"];
+					if ((mlog.is_array())&&(mlog.size() > 0)) {
+						const json &mlog1 = mlog[0];
+						if (mlog1.is_object()) {
+							if ((now - _jI(mlog1["ts"],0ULL)) < ZT_NETCONF_NODE_ACTIVE_THRESHOLD)
+								++nmi.activeMemberCount;
+						}
 					}
 				}
-			}
 
-			if (_jB(nm->second["activeBridge"],false)) {
-				nmi.activeBridges.insert(nm->first);
-			}
+				if (_jB(member["activeBridge"],false)) {
+					nmi.activeBridges.insert(_jS(member["id"],"0000000000"));
+				}
 
-			if (nm->second.count("ipAssignments")) {
-				json &mips = nm->second["ipAssignments"];
-				if (mips.is_array()) {
-					for(unsigned long i=0;i<mips.size();++i) {
-						InetAddress mip(_jS(mips[i],""));
-						if ((mip.ss_family == AF_INET)||(mip.ss_family == AF_INET6))
-							nmi.allocatedIps.insert(mip);
+				if (member.count("ipAssignments")) {
+					const json &mips = member["ipAssignments"];
+					if (mips.is_array()) {
+						for(unsigned long i=0;i<mips.size();++i) {
+							InetAddress mip(_jS(mips[i],""));
+							if ((mip.ss_family == AF_INET)||(mip.ss_family == AF_INET6))
+								nmi.allocatedIps.insert(mip);
+						}
 					}
 				}
+			} else {
+				nmi.mostRecentDeauthTime = std::max(nmi.mostRecentDeauthTime,_jI(member["lastDeauthorizedTime"],0ULL));
 			}
-		} else {
-			nmi.mostRecentDeauthTime = std::max(nmi.mostRecentDeauthTime,_jI(nm->second["lastDeauthorizedTime"],0ULL));
-		}
-	}
+		} catch ( ... ) {}
+		return true;
+	});
 }
 
 } // namespace ZeroTier
diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp
index 99e1836d..f364c078 100644
--- a/controller/EmbeddedNetworkController.hpp
+++ b/controller/EmbeddedNetworkController.hpp
@@ -40,6 +40,8 @@
 
 #include "../ext/json/json.hpp"
 
+#include "JSONDB.hpp"
+
 namespace ZeroTier {
 
 class Node;
@@ -50,10 +52,6 @@ public:
 	EmbeddedNetworkController(Node *node,const char *dbPath);
 	virtual ~EmbeddedNetworkController();
 
-	// Thread main method -- do not call directly
-	void threadMain()
-		throw();
-
 	virtual NetworkController::ResultCode doNetworkConfigRequest(
 		const InetAddress &fromAddr,
 		const Identity &signingId,
@@ -88,6 +86,7 @@ private:
 	static void _circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report);
 
 	// Network base path and network JSON path
+	/*
 	inline std::string _networkBP(const uint64_t nwid,bool create)
 	{
 		char tmp[64];
@@ -124,6 +123,10 @@ private:
 	// In-memory cache of network members
 	std::map< uint64_t,std::map< Address,nlohmann::json > > _networkMemberCache;
 	Mutex _networkMemberCache_m;
+	*/
+
+	JSONDB _db;
+	Mutex _db_m;
 
 	// Gathers a bunch of statistics about members of a network, IP assignments, etc. that we need in various places
 	// This does lock _networkMemberCache_m
@@ -203,9 +206,6 @@ private:
 	// Last request time by address, for rate limitation
 	std::map< std::pair<uint64_t,uint64_t>,uint64_t > _lastRequestTime;
 	Mutex _lastRequestTime_m;
-
-	Thread _daemon;
-	volatile bool _daemonRun;
 };
 
 } // namespace ZeroTier
diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp
index 40bc21ac..bb0b98ed 100644
--- a/controller/JSONDB.cpp
+++ b/controller/JSONDB.cpp
@@ -27,7 +27,7 @@ bool JSONDB::put(const std::string &n,const nlohmann::json &obj)
 	if (!_isValidObjectName(n))
 		return false;
 
-	std::string path(_genPath(n,false));
+	std::string path(_genPath(n,true));
 	if (!path.length())
 		return false;
 
@@ -36,10 +36,9 @@ bool JSONDB::put(const std::string &n,const nlohmann::json &obj)
 		return false;
 
 	_E &e = _db[n];
-
+	e.obj = obj;
 	e.lastModifiedOnDisk = OSUtils::getLastModified(path.c_str());
 	e.lastCheck = OSUtils::now();
-	e.obj = obj;
 
 	return true;
 }
@@ -56,7 +55,6 @@ const nlohmann::json &JSONDB::get(const std::string &n,unsigned long maxSinceChe
 	if (e != _db.end()) {
 		if ((now - e->second.lastCheck) <= (uint64_t)maxSinceCheck)
 			return e->second.obj;
-
 		std::string path(_genPath(n,false));
 		if (!path.length()) // sanity check
 			return _EMPTY_JSON;
@@ -64,15 +62,13 @@ const nlohmann::json &JSONDB::get(const std::string &n,unsigned long maxSinceChe
 		// We are somewhat tolerant to momentary disk failures here. This may
 		// occur over e.g. EC2's elastic filesystem (NFS).
 		const uint64_t lm = OSUtils::getLastModified(path.c_str());
-		if ((lm)&&(e->second.lastModifiedOnDisk != lm)) {
+		if (e->second.lastModifiedOnDisk != lm) {
 			if (OSUtils::readFile(path.c_str(),buf)) {
 				try {
-					e->second.lastModifiedOnDisk = lm;
-					e->second.lastCheck = now;
 					e->second.obj = nlohmann::json::parse(buf);
-				} catch ( ... ) {
-					e->second.obj = _EMPTY_JSON;
-				}
+					e->second.lastModifiedOnDisk = lm; // don't update these if there is a parse error -- try again and again ASAP
+					e->second.lastCheck = now;
+				} catch ( ... ) {} // parse errors result in "holding pattern" behavior
 			}
 		}
 
@@ -86,22 +82,50 @@ const nlohmann::json &JSONDB::get(const std::string &n,unsigned long maxSinceChe
 			return _EMPTY_JSON;
 
 		const uint64_t lm = OSUtils::getLastModified(path.c_str());
-		if (!lm)
-			return _EMPTY_JSON;
-
 		_E &e2 = _db[n];
-		e2.lastModifiedOnDisk = lm;
-		e2.lastCheck = now;
 		try {
 			e2.obj = nlohmann::json::parse(buf);
 		} catch ( ... ) {
 			e2.obj = _EMPTY_JSON;
 		}
+		e2.lastModifiedOnDisk = lm;
+		e2.lastCheck = now;
 
 		return e2.obj;
 	}
 }
 
+void JSONDB::erase(const std::string &n)
+{
+	if (!_isValidObjectName(n))
+		return;
+
+	std::string path(_genPath(n,true));
+	if (!path.length())
+		return;
+
+	OSUtils::rm(path.c_str());
+	_db.erase(n);
+}
+
+void JSONDB::_reload(const std::string &p)
+{
+	std::map<std::string,char> l(OSUtils::listDirectoryFull(p.c_str()));
+	for(std::map<std::string,char>::iterator li(l.begin());li!=l.end();++li) {
+		if (li->second == 'f') {
+			// assume p starts with _basePath, which it always does -- will throw otherwise
+			std::string n(p.substr(_basePath.length()));
+			while ((n.length() > 0)&&(n[0] == ZT_PATH_SEPARATOR)) n = n.substr(1);
+			if (ZT_PATH_SEPARATOR != '/') std::replace(n.begin(),n.end(),ZT_PATH_SEPARATOR,'/');
+			if ((n.length() > 0)&&(n[n.length() - 1] != '/')) n.push_back('/');
+			n.append(li->first);
+			this->get(n,0); // causes load and cache or update
+		} else if (li->second == 'd') {
+			this->_reload(p + ZT_PATH_SEPARATOR + li->first);
+		}
+	}
+}
+
 bool JSONDB::_isValidObjectName(const std::string &n)
 {
 	if (n.length() == 0)
@@ -139,8 +163,4 @@ std::string JSONDB::_genPath(const std::string &n,bool create)
 	return p;
 }
 
-void JSONDB::_reloadAll(const std::string &path)
-{
-}
-
 } // namespace ZeroTier
diff --git a/controller/JSONDB.hpp b/controller/JSONDB.hpp
index b189c530..e33cd768 100644
--- a/controller/JSONDB.hpp
+++ b/controller/JSONDB.hpp
@@ -41,13 +41,21 @@ public:
 	JSONDB(const std::string &basePath) :
 		_basePath(basePath)
 	{
-		this->_reloadAll(_basePath);
+		_reload(_basePath);
+	}
+
+	inline void reload()
+	{
+		_db.clear();
+		_reload(_basePath);
 	}
 
 	bool put(const std::string &n,const nlohmann::json &obj);
 
 	inline bool put(const std::string &n1,const std::string &n2,const nlohmann::json &obj) { return this->put((n1 + "/" + n2),obj); }
 	inline bool put(const std::string &n1,const std::string &n2,const std::string &n3,const nlohmann::json &obj) { return this->put((n1 + "/" + n2 + "/" + n3),obj); }
+	inline bool put(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,const nlohmann::json &obj) { return this->put((n1 + "/" + n2 + "/" + n3 + "/" + n4),obj); }
+	inline bool put(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,const std::string &n5,const nlohmann::json &obj) { return this->put((n1 + "/" + n2 + "/" + n3 + "/" + n4 + "/" + n5),obj); }
 
 	const nlohmann::json &get(const std::string &n,unsigned long maxSinceCheck = 0);
 
@@ -56,27 +64,37 @@ public:
 	inline const nlohmann::json &get(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,unsigned long maxSinceCheck = 0) { return this->get((n1 + "/" + n2 + "/" + n3 + "/" + n4),maxSinceCheck); }
 	inline const nlohmann::json &get(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,const std::string &n5,unsigned long maxSinceCheck = 0) { return this->get((n1 + "/" + n2 + "/" + n3 + "/" + n4 + "/" + n5),maxSinceCheck); }
 
+	void erase(const std::string &n);
+
+	inline void erase(const std::string &n1,const std::string &n2) { this->erase(n1 + "/" + n2); }
+	inline void erase(const std::string &n1,const std::string &n2,const std::string &n3) { this->erase(n1 + "/" + n2 + "/" + n3); }
+	inline void erase(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4) { this->erase(n1 + "/" + n2 + "/" + n3 + "/" + n4); }
+	inline void erase(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,const std::string &n5) { this->erase(n1 + "/" + n2 + "/" + n3 + "/" + n4 + "/" + n5); }
+
 	template<typename F>
-	inline void each(F func,unsigned long maxSinceCheck = 0)
+	inline void filter(const std::string &prefix,unsigned long maxSinceCheck,F func)
 	{
-		const uint64_t now = OSUtils::now();
-		for(std::map<std::string,_E>::const_iterator i(_db.begin());i!=_db.end();++i) {
-			if ((now - i->second.lastCheck) > (uint64_t)maxSinceCheck)
-				this->get(i->first);
-			func(i->first,i->second.obj);
+		for(std::map<std::string,_E>::iterator i(_db.lower_bound(prefix));i!=_db.end();) {
+			if (i->first.substr(0,prefix.length()) == prefix) {
+				if (!func(i->first,get(i->second.obj,maxSinceCheck))) {
+					std::map<std::string,_E>::iterator i2(i); ++i2;
+					this->erase(i->first);
+					i = i2;
+				} else ++i;
+			} else break;
 		}
 	}
 
 private:
+	void _reload(const std::string &p);
 	bool _isValidObjectName(const std::string &n);
 	std::string _genPath(const std::string &n,bool create);
-	void _reloadAll(const std::string &path);
 
 	struct _E
 	{
+		nlohmann::json obj;
 		uint64_t lastModifiedOnDisk;
 		uint64_t lastCheck;
-		nlohmann::json obj;
 	};
 
 	std::string _basePath;