From 1613f42d0082cf6438ad0c62d89405ab82625f98 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 7 Nov 2017 14:44:46 -0800 Subject: Re-integrate in-filesystem DB into new controller DB structure. --- controller/FileDB.hpp | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 controller/FileDB.hpp (limited to 'controller/FileDB.hpp') diff --git a/controller/FileDB.hpp b/controller/FileDB.hpp new file mode 100644 index 00000000..fe9869b9 --- /dev/null +++ b/controller/FileDB.hpp @@ -0,0 +1,47 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2015 ZeroTier, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef ZT_CONTROLLER_FILEDB_HPP +#define ZT_CONTROLLER_FILEDB_HPP + +#include "DB.hpp" + +namespace ZeroTier +{ + +class FileDB : public DB +{ +public: + FileDB(EmbeddedNetworkController *const nc,const Address &myAddress,const char *path); + virtual ~FileDB(); + + virtual bool waitForReady(); + + virtual void save(const nlohmann::json &record); + + virtual void eraseNetwork(const uint64_t networkId); + + virtual void eraseMember(const uint64_t networkId,const uint64_t memberId); + +protected: + std::string _networksPath; +}; + +} // namespace ZeroTier + +#endif -- cgit v1.2.3 From 4166d8ca35ded34180d60b56105a853dd6b02ff4 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 8 Nov 2017 11:06:14 -0800 Subject: Fix a deadlock and some more work on RethinkDB (for central) integration. --- controller/DB.hpp | 4 +- controller/EmbeddedNetworkController.cpp | 31 +++---------- controller/FileDB.cpp | 7 ++- controller/FileDB.hpp | 4 +- controller/RethinkDB.cpp | 74 ++++++++++++++++++++++++++++++-- controller/RethinkDB.hpp | 10 ++++- node/Peer.cpp | 59 ++++--------------------- 7 files changed, 106 insertions(+), 83 deletions(-) (limited to 'controller/FileDB.hpp') diff --git a/controller/DB.hpp b/controller/DB.hpp index dfc8ac95..fe06c24d 100644 --- a/controller/DB.hpp +++ b/controller/DB.hpp @@ -78,12 +78,14 @@ public: void networks(std::vector &networks); - virtual void save(const nlohmann::json &record) = 0; + virtual void save(nlohmann::json *orig,nlohmann::json &record) = 0; virtual void eraseNetwork(const uint64_t networkId) = 0; virtual void eraseMember(const uint64_t networkId,const uint64_t memberId) = 0; + virtual void nodeIsOnline(const uint64_t memberId) = 0; + protected: struct _Network { diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 5707e6e0..a2795d96 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -734,12 +734,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( member["nwid"] = nwids; _removeMemberNonPersistedFields(member); - if (member != origMember) { - json &revj = member["revision"]; - member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); - _db->save(member); - } - + _db->save(&origMember,member); _addMemberNonPersistedFields(nwid,address,member,now); responseBody = OSUtils::jsonDump(member); responseContentType = "application/json"; @@ -986,12 +981,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( network["nwid"] = nwids; // legacy _removeNetworkNonPersistedFields(network); - if (network != origNetwork) { - json &revj = network["revision"]; - network["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); - _db->save(network); - } - + _db->save(&origNetwork,network); ControllerDB::NetworkSummaryInfo ns; _db->summary(nwid,ns); _addNetworkNonPersistedFields(nwid,network,now,ns); @@ -1116,7 +1106,7 @@ void EmbeddedNetworkController::handleRemoteTrace(const ZT_RemoteTrace &rt) d["objtype"] = "trace"; d["ts"] = now; d["nodeId"] = Utils::hex10(rt.origin,tmp); - _db->save(d); + _db->save((nlohmann::json *)0,d); } catch ( ... ) { // drop invalid trace messages if an error occurs } @@ -1185,6 +1175,8 @@ void EmbeddedNetworkController::_request( ms.lastRequestTime = now; } + _db->nodeIsOnline(identity.address().toInt()); + Utils::hex(nwid,nwids); _db->get(nwid,network,identity.address().toInt(),member,ns); if ((!network.is_object())||(network.size() == 0)) { @@ -1299,11 +1291,7 @@ void EmbeddedNetworkController::_request( } else { // If they are not authorized, STOP! _removeMemberNonPersistedFields(member); - if (origMember != member) { - json &revj = member["revision"]; - member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); - _db->save(member); - } + _db->save(&origMember,member); _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED); return; } @@ -1666,12 +1654,7 @@ void EmbeddedNetworkController::_request( } _removeMemberNonPersistedFields(member); - if (member != origMember) { - json &revj = member["revision"]; - member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); - _db->save(member); - } - + _db->save(&origMember,member); _sender->ncSendConfig(nwid,requestPacketId,identity.address(),*(nc.get()),metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,0) < 6); } diff --git a/controller/FileDB.cpp b/controller/FileDB.cpp index b48d5e87..646fa2fe 100644 --- a/controller/FileDB.cpp +++ b/controller/FileDB.cpp @@ -69,7 +69,7 @@ bool FileDB::waitForReady() return true; } -void FileDB::save(const nlohmann::json &record) +void FileDB::save(nlohmann::json *orig,nlohmann::json &record) { char p1[16384],p2[16384]; try { @@ -126,4 +126,9 @@ void FileDB::eraseMember(const uint64_t networkId,const uint64_t memberId) { } +void FileDB::nodeIsOnline(const uint64_t memberId) +{ + // Nothing to do here right now in the filesystem store mode since we can just get this from the peer list +} + } // namespace ZeroTier diff --git a/controller/FileDB.hpp b/controller/FileDB.hpp index fe9869b9..76a47936 100644 --- a/controller/FileDB.hpp +++ b/controller/FileDB.hpp @@ -32,12 +32,14 @@ public: virtual bool waitForReady(); - virtual void save(const nlohmann::json &record); + virtual void save(nlohmann::json *orig,nlohmann::json &record); virtual void eraseNetwork(const uint64_t networkId); virtual void eraseMember(const uint64_t networkId,const uint64_t memberId); + virtual void nodeIsOnline(const uint64_t memberId); + protected: std::string _networksPath; }; diff --git a/controller/RethinkDB.cpp b/controller/RethinkDB.cpp index d1012167..031bd516 100644 --- a/controller/RethinkDB.cpp +++ b/controller/RethinkDB.cpp @@ -215,6 +215,47 @@ RethinkDB::RethinkDB(EmbeddedNetworkController *const nc,const Address &myAddres }); } + _onlineNotificationThread = std::thread([this]() { + try { + std::unique_ptr rdb; + while (_run == 1) { + try { + if (!rdb) + rdb = R::connect(this->_host,this->_port,this->_auth); + if (rdb) { + std::lock_guard l(_lastOnline_l); + R::Array batch; + R::Object tmpobj; + for(auto i=_lastOnline.begin();i!=_lastOnline.end();++i) { + char nodeId[16]; + Utils::hex10(i->first,nodeId); + tmpobj["id"] = nodeId; + tmpobj["ts"] = i->second; + batch.emplace_back(tmpobj); + if (batch.size() >= 256) { + R::db(this->_db).table("NodeLastOnline").insert(R::args(batch),R::optargs("conflict","update")).run(*rdb); + batch.clear(); + } + } + if (batch.size() > 0) + R::db(this->_db).table("NodeLastOnline").insert(R::args(batch),R::optargs("conflict","update")).run(*rdb); + _lastOnline.clear(); + } + } catch (std::exception &e) { + fprintf(stderr,"ERROR: controller RethinkDB (node status update): %s" ZT_EOL_S,e.what()); + rdb.reset(); + } catch (R::Error &e) { + fprintf(stderr,"ERROR: controller RethinkDB (node status update): %s" ZT_EOL_S,e.message.c_str()); + rdb.reset(); + } catch ( ... ) { + fprintf(stderr,"ERROR: controller RethinkDB (node status update): unknown exception" ZT_EOL_S); + rdb.reset(); + } + std::this_thread::sleep_for(std::chrono::milliseconds(250)); + } + } catch ( ... ) {} + }); + _heartbeatThread = std::thread([this]() { try { char tmp[1024]; @@ -251,9 +292,10 @@ RethinkDB::~RethinkDB() _membersDbWatcher.join(); _networksDbWatcher.join(); _heartbeatThread.join(); + _onlineNotificationThread.join(); } -void RethinkDB::waitForReady() +bool RethinkDB::waitForReady() { while (_ready > 0) { if (!_waitNoticePrinted) { @@ -263,12 +305,32 @@ void RethinkDB::waitForReady() _readyLock.lock(); _readyLock.unlock(); } + return true; } -void RethinkDB::save(const nlohmann::json &record) +void RethinkDB::save(nlohmann::json *orig,nlohmann::json &record) { + if (!record.is_object()) // sanity check + return; waitForReady(); - _commitQueue.post(new nlohmann::json(record)); + if (orig) { + if (*orig != record) { + nlohmann::json *q = new nlohmann::json(); + try { + record["revision"] = OSUtils::jsonInt(record["revision"],0ULL) + 1; + for(auto kv=record.begin();kv!=record.end();++kv) { + if ((kv.key() == "id")||(kv.key() == "nwid")||(kv.key() == "objtype")||((*q)[kv.key()] != kv.value())) + (*q)[kv.key()] = kv.value(); + } + } catch ( ... ) { + delete q; + throw; + } + } + } else { + record["revision"] = 1; + _commitQueue.post(new nlohmann::json(record)); + } } void RethinkDB::eraseNetwork(const uint64_t networkId) @@ -295,6 +357,12 @@ void RethinkDB::eraseMember(const uint64_t networkId,const uint64_t memberId) _commitQueue.post(tmp); } +void RethinkDB::nodeIsOnline(const uint64_t memberId) +{ + std::lock_guard l(_lastOnline_l); + _lastOnline[memberId] = OSUtils::now(); +} + } // namespace ZeroTier #endif // ZT_CONTROLLER_USE_RETHINKDB diff --git a/controller/RethinkDB.hpp b/controller/RethinkDB.hpp index 2309a25c..8c8b16a6 100644 --- a/controller/RethinkDB.hpp +++ b/controller/RethinkDB.hpp @@ -34,14 +34,16 @@ public: RethinkDB(EmbeddedNetworkController *const nc,const Address &myAddress,const char *path); virtual ~RethinkDB(); - virtual void waitForReady(); + virtual bool waitForReady(); - virtual void save(const nlohmann::json &record); + virtual void save(nlohmann::json *orig,nlohmann::json &record); virtual void eraseNetwork(const uint64_t networkId); virtual void eraseMember(const uint64_t networkId,const uint64_t memberId); + virtual void nodeIsOnline(const uint64_t memberId); + protected: std::string _host; std::string _db; @@ -56,6 +58,10 @@ protected: BlockingQueue< nlohmann::json * > _commitQueue; std::thread _commitThread[ZT_CONTROLLER_RETHINKDB_COMMIT_THREADS]; + std::unordered_map< uint64_t,int64_t > _lastOnline; + mutable std::mutex _lastOnline_l; + std::thread _onlineNotificationThread; + std::thread _heartbeatThread; mutable std::mutex _readyLock; // locked until ready diff --git a/node/Peer.cpp b/node/Peer.cpp index a3682a97..2d562f12 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -78,54 +78,6 @@ void Peer::received( { const int64_t now = RR->node->now(); -/* -#ifdef ZT_ENABLE_CLUSTER - bool isClusterSuboptimalPath = false; - if ((RR->cluster)&&(hops == 0)) { - // Note: findBetterEndpoint() is first since we still want to check - // for a better endpoint even if we don't actually send a redirect. - InetAddress redirectTo; - if ( (verb != Packet::VERB_OK) && (verb != Packet::VERB_ERROR) && (verb != Packet::VERB_RENDEZVOUS) && (verb != Packet::VERB_PUSH_DIRECT_PATHS) && (RR->cluster->findBetterEndpoint(redirectTo,_id.address(),path->address(),false)) ) { - if (_vProto >= 5) { - // For newer peers we can send a more idiomatic verb: PUSH_DIRECT_PATHS. - Packet outp(_id.address(),RR->identity.address(),Packet::VERB_PUSH_DIRECT_PATHS); - outp.append((uint16_t)1); // count == 1 - outp.append((uint8_t)ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT); // flags: cluster redirect - outp.append((uint16_t)0); // no extensions - if (redirectTo.ss_family == AF_INET) { - outp.append((uint8_t)4); - outp.append((uint8_t)6); - outp.append(redirectTo.rawIpData(),4); - } else { - outp.append((uint8_t)6); - outp.append((uint8_t)18); - outp.append(redirectTo.rawIpData(),16); - } - outp.append((uint16_t)redirectTo.port()); - outp.armor(_key,true,path->nextOutgoingCounter()); - path->send(RR,tPtr,outp.data(),outp.size(),now); - } else { - // For older peers we use RENDEZVOUS to coax them into contacting us elsewhere. - Packet outp(_id.address(),RR->identity.address(),Packet::VERB_RENDEZVOUS); - outp.append((uint8_t)0); // no flags - RR->identity.address().appendTo(outp); - outp.append((uint16_t)redirectTo.port()); - if (redirectTo.ss_family == AF_INET) { - outp.append((uint8_t)4); - outp.append(redirectTo.rawIpData(),4); - } else { - outp.append((uint8_t)16); - outp.append(redirectTo.rawIpData(),16); - } - outp.armor(_key,true,path->nextOutgoingCounter()); - path->send(RR,tPtr,outp.data(),outp.size(),now); - } - isClusterSuboptimalPath = true; - } - } -#endif -*/ - _lastReceive = now; switch (verb) { case Packet::VERB_FRAME: @@ -163,6 +115,7 @@ void Peer::received( } } + bool attemptToContact = false; if ((!havePath)&&(RR->node->shouldUsePathForZeroTierTraffic(tPtr,_id.address(),path->localSocket(),path->address()))) { Mutex::Lock _l(_paths_m); @@ -201,13 +154,17 @@ void Peer::received( _paths[replacePath].p = path; _paths[replacePath].priority = 1; } else { - attemptToContactAt(tPtr,path->localSocket(),path->address(),now,true,path->nextOutgoingCounter()); - path->sent(now); - RR->t->peerConfirmingUnknownPath(tPtr,networkId,*this,path,packetId,verb); + attemptToContact = true; } } } } + + if (attemptToContact) { + attemptToContactAt(tPtr,path->localSocket(),path->address(),now,true,path->nextOutgoingCounter()); + path->sent(now); + RR->t->peerConfirmingUnknownPath(tPtr,networkId,*this,path,packetId,verb); + } } // If we have a trust relationship periodically push a message enumerating -- cgit v1.2.3 From c12b68a6b2f8597e374a4c1386c2b95f4291932e Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 8 Nov 2017 11:32:01 -0800 Subject: More Central work. --- controller/DB.hpp | 2 +- controller/EmbeddedNetworkController.cpp | 2 +- controller/FileDB.cpp | 2 +- controller/FileDB.hpp | 2 +- controller/RethinkDB.cpp | 14 +++++++------- controller/RethinkDB.hpp | 4 ++-- 6 files changed, 13 insertions(+), 13 deletions(-) (limited to 'controller/FileDB.hpp') diff --git a/controller/DB.hpp b/controller/DB.hpp index fe06c24d..fca41d70 100644 --- a/controller/DB.hpp +++ b/controller/DB.hpp @@ -84,7 +84,7 @@ public: virtual void eraseMember(const uint64_t networkId,const uint64_t memberId) = 0; - virtual void nodeIsOnline(const uint64_t memberId) = 0; + virtual void nodeIsOnline(const uint64_t networkId,const uint64_t memberId) = 0; protected: struct _Network diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index a2795d96..999319af 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1175,7 +1175,7 @@ void EmbeddedNetworkController::_request( ms.lastRequestTime = now; } - _db->nodeIsOnline(identity.address().toInt()); + _db->nodeIsOnline(nwid,identity.address().toInt()); Utils::hex(nwid,nwids); _db->get(nwid,network,identity.address().toInt(),member,ns); diff --git a/controller/FileDB.cpp b/controller/FileDB.cpp index 646fa2fe..b9f36031 100644 --- a/controller/FileDB.cpp +++ b/controller/FileDB.cpp @@ -126,7 +126,7 @@ void FileDB::eraseMember(const uint64_t networkId,const uint64_t memberId) { } -void FileDB::nodeIsOnline(const uint64_t memberId) +void FileDB::nodeIsOnline(const uint64_t networkId,const uint64_t memberId) { // Nothing to do here right now in the filesystem store mode since we can just get this from the peer list } diff --git a/controller/FileDB.hpp b/controller/FileDB.hpp index 76a47936..b438e80e 100644 --- a/controller/FileDB.hpp +++ b/controller/FileDB.hpp @@ -38,7 +38,7 @@ public: virtual void eraseMember(const uint64_t networkId,const uint64_t memberId); - virtual void nodeIsOnline(const uint64_t memberId); + virtual void nodeIsOnline(const uint64_t networkId,const uint64_t memberId); protected: std::string _networksPath; diff --git a/controller/RethinkDB.cpp b/controller/RethinkDB.cpp index 031bd516..2da55177 100644 --- a/controller/RethinkDB.cpp +++ b/controller/RethinkDB.cpp @@ -227,18 +227,18 @@ RethinkDB::RethinkDB(EmbeddedNetworkController *const nc,const Address &myAddres R::Array batch; R::Object tmpobj; for(auto i=_lastOnline.begin();i!=_lastOnline.end();++i) { - char nodeId[16]; - Utils::hex10(i->first,nodeId); - tmpobj["id"] = nodeId; + char tmp[64]; + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%.16llx-%.10llx",i->first.first,i->first.second); + tmpobj["id"] = tmp; tmpobj["ts"] = i->second; batch.emplace_back(tmpobj); if (batch.size() >= 256) { - R::db(this->_db).table("NodeLastOnline").insert(R::args(batch),R::optargs("conflict","update")).run(*rdb); + R::db(this->_db).table("MemberLastRequest",R::optargs("read_mode","outdated")).insert(R::args(batch),R::optargs("conflict","update")).run(*rdb); batch.clear(); } } if (batch.size() > 0) - R::db(this->_db).table("NodeLastOnline").insert(R::args(batch),R::optargs("conflict","update")).run(*rdb); + R::db(this->_db).table("MemberLastRequest",R::optargs("read_mode","outdated")).insert(R::args(batch),R::optargs("conflict","update")).run(*rdb); _lastOnline.clear(); } } catch (std::exception &e) { @@ -357,10 +357,10 @@ void RethinkDB::eraseMember(const uint64_t networkId,const uint64_t memberId) _commitQueue.post(tmp); } -void RethinkDB::nodeIsOnline(const uint64_t memberId) +void RethinkDB::nodeIsOnline(const uint64_t networkId,const uint64_t memberId) { std::lock_guard l(_lastOnline_l); - _lastOnline[memberId] = OSUtils::now(); + _lastOnline[std::pair(networkId,memberId)] = OSUtils::now(); } } // namespace ZeroTier diff --git a/controller/RethinkDB.hpp b/controller/RethinkDB.hpp index 8c8b16a6..6efa5624 100644 --- a/controller/RethinkDB.hpp +++ b/controller/RethinkDB.hpp @@ -42,7 +42,7 @@ public: virtual void eraseMember(const uint64_t networkId,const uint64_t memberId); - virtual void nodeIsOnline(const uint64_t memberId); + virtual void nodeIsOnline(const uint64_t networkId,const uint64_t memberId); protected: std::string _host; @@ -58,7 +58,7 @@ protected: BlockingQueue< nlohmann::json * > _commitQueue; std::thread _commitThread[ZT_CONTROLLER_RETHINKDB_COMMIT_THREADS]; - std::unordered_map< uint64_t,int64_t > _lastOnline; + std::unordered_map< std::pair,int64_t > _lastOnline; mutable std::mutex _lastOnline_l; std::thread _onlineNotificationThread; -- cgit v1.2.3 From f7f658605dfc255c5bad88ed190f7b00b0dd81a2 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 8 Nov 2017 20:19:46 -0500 Subject: Move more ephemeral stuff to a tiny MemberLastRequest table instead of the main Member table. --- controller/DB.hpp | 2 +- controller/EmbeddedNetworkController.cpp | 2 +- controller/FileDB.cpp | 2 +- controller/FileDB.hpp | 2 +- controller/RethinkDB.cpp | 12 ++++++++---- controller/RethinkDB.hpp | 4 ++-- 6 files changed, 14 insertions(+), 10 deletions(-) (limited to 'controller/FileDB.hpp') diff --git a/controller/DB.hpp b/controller/DB.hpp index fca41d70..8731cb5c 100644 --- a/controller/DB.hpp +++ b/controller/DB.hpp @@ -84,7 +84,7 @@ public: virtual void eraseMember(const uint64_t networkId,const uint64_t memberId) = 0; - virtual void nodeIsOnline(const uint64_t networkId,const uint64_t memberId) = 0; + virtual void nodeIsOnline(const uint64_t networkId,const uint64_t memberId,const InetAddress &physicalAddress) = 0; protected: struct _Network diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 999319af..0f82ff63 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1175,7 +1175,7 @@ void EmbeddedNetworkController::_request( ms.lastRequestTime = now; } - _db->nodeIsOnline(nwid,identity.address().toInt()); + _db->nodeIsOnline(nwid,identity.address().toInt(),fromAddr); Utils::hex(nwid,nwids); _db->get(nwid,network,identity.address().toInt(),member,ns); diff --git a/controller/FileDB.cpp b/controller/FileDB.cpp index b9f36031..728cec6b 100644 --- a/controller/FileDB.cpp +++ b/controller/FileDB.cpp @@ -126,7 +126,7 @@ void FileDB::eraseMember(const uint64_t networkId,const uint64_t memberId) { } -void FileDB::nodeIsOnline(const uint64_t networkId,const uint64_t memberId) +void FileDB::nodeIsOnline(const uint64_t networkId,const uint64_t memberId,const InetAddress &physicalAddress) { // Nothing to do here right now in the filesystem store mode since we can just get this from the peer list } diff --git a/controller/FileDB.hpp b/controller/FileDB.hpp index b438e80e..e31b18e6 100644 --- a/controller/FileDB.hpp +++ b/controller/FileDB.hpp @@ -38,7 +38,7 @@ public: virtual void eraseMember(const uint64_t networkId,const uint64_t memberId); - virtual void nodeIsOnline(const uint64_t networkId,const uint64_t memberId); + virtual void nodeIsOnline(const uint64_t networkId,const uint64_t memberId,const InetAddress &physicalAddress); protected: std::string _networksPath; diff --git a/controller/RethinkDB.cpp b/controller/RethinkDB.cpp index 6583f23c..cd968e26 100644 --- a/controller/RethinkDB.cpp +++ b/controller/RethinkDB.cpp @@ -227,10 +227,11 @@ RethinkDB::RethinkDB(EmbeddedNetworkController *const nc,const Address &myAddres R::Array batch; R::Object tmpobj; for(auto i=_lastOnline.begin();i!=_lastOnline.end();++i) { - char tmp[64]; + char tmp[64],tmp2[64]; OSUtils::ztsnprintf(tmp,sizeof(tmp),"%.16llx-%.10llx",i->first.first,i->first.second); tmpobj["id"] = tmp; - tmpobj["ts"] = i->second; + tmpobj["ts"] = i->second.first; + tmpobj["phy"] = i->second.second.toIpString(tmp2); batch.emplace_back(tmpobj); if (batch.size() >= 256) { R::db(this->_db).table("MemberLastRequest",R::optargs("read_mode","outdated")).insert(batch,R::optargs("conflict","update")).run(*rdb); @@ -357,10 +358,13 @@ void RethinkDB::eraseMember(const uint64_t networkId,const uint64_t memberId) _commitQueue.post(tmp); } -void RethinkDB::nodeIsOnline(const uint64_t networkId,const uint64_t memberId) +void RethinkDB::nodeIsOnline(const uint64_t networkId,const uint64_t memberId,const InetAddress &physicalAddress) { std::lock_guard l(_lastOnline_l); - _lastOnline[std::pair(networkId,memberId)] = OSUtils::now(); + std::pair &i = _lastOnline[std::pair(networkId,memberId)]; + i.first = OSUtils::now(); + if (physicalAddress) + i.second = physicalAddress; } } // namespace ZeroTier diff --git a/controller/RethinkDB.hpp b/controller/RethinkDB.hpp index 26987019..a69f462f 100644 --- a/controller/RethinkDB.hpp +++ b/controller/RethinkDB.hpp @@ -48,7 +48,7 @@ public: virtual void eraseMember(const uint64_t networkId,const uint64_t memberId); - virtual void nodeIsOnline(const uint64_t networkId,const uint64_t memberId); + virtual void nodeIsOnline(const uint64_t networkId,const uint64_t memberId,const InetAddress &physicalAddress); protected: struct _PairHasher @@ -69,7 +69,7 @@ protected: BlockingQueue< nlohmann::json * > _commitQueue; std::thread _commitThread[ZT_CONTROLLER_RETHINKDB_COMMIT_THREADS]; - std::unordered_map< std::pair,int64_t,_PairHasher > _lastOnline; + std::unordered_map< std::pair,std::pair,_PairHasher > _lastOnline; mutable std::mutex _lastOnline_l; std::thread _onlineNotificationThread; -- cgit v1.2.3 From 8d9464c4140e5882b0fc9276388401514f29e62a Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 7 Dec 2017 13:39:25 -0800 Subject: docs, and make RethinkDB controller DB driver upsert into the Controller DB and also update the hostname field. --- RELEASE-NOTES.md | 13 +++++++++++ controller/DB.cpp | 5 ++-- controller/DB.hpp | 5 ++-- controller/EmbeddedNetworkController.cpp | 4 ++-- controller/FileDB.cpp | 4 ++-- controller/FileDB.hpp | 2 +- controller/RethinkDB.cpp | 40 ++++++++++++++++++++++++++++---- controller/RethinkDB.hpp | 2 +- 8 files changed, 60 insertions(+), 15 deletions(-) (limited to 'controller/FileDB.hpp') diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 195e8888..86522cea 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,6 +1,19 @@ ZeroTier Release Notes ====== +# 2017-12-XX -- Version 1.2.6 + + * Network Hypervisor + * We've made some improvements to dead path detection and path selection. These also include changes under the hood to prepare for multi-path trunking and faster fail-over support. + * Platform-Specific Changes + * MacOS + * Installer now loads the kernel extension right away so that High Sierra users will see the prompt to authorize it. This is done in the "Security & Privacy" preference pane and must be done driectly on the console (not via remote desktop). + * Windows + * The Windows installer should now install the driver without requiring a special prompt in most cases. This should make it easier for our packages to be accepted into and updated in the Chocolatey repository and should make it easier to perform remote installs. + * The Windows official packages are now signed with an EV certificate (with hardware key) from DigiCert. + * The Windows UI now contains a preview of features to more deeply integrate it with ZeroTier Central. You can enter a ZeroTier Central API key and join networks, etc. from the UI itself. We'll be expanding this in the future and possibly changing it, so this is just a test to see how users respond. + * The `zerotier-idtool` command should now work on Windows. + # 2017-04-20 -- Version 1.2.4 * Managed routes are now only bifurcated for the default route. This is a change in behavior, though few people will probably notice. Bifurcating all managed routes was causing more trouble than it was worth for most users. diff --git a/controller/DB.cpp b/controller/DB.cpp index 2f9a4a89..2f09205b 100644 --- a/controller/DB.cpp +++ b/controller/DB.cpp @@ -27,9 +27,10 @@ using json = nlohmann::json; namespace ZeroTier { -DB::DB(EmbeddedNetworkController *const nc,const Address &myAddress,const char *path) : +DB::DB(EmbeddedNetworkController *const nc,const Identity &myId,const char *path) : _controller(nc), - _myAddress(myAddress), + _myId(myId), + _myAddress(myId.address()), _path((path) ? path : "") { char tmp[32]; diff --git a/controller/DB.hpp b/controller/DB.hpp index 8731cb5c..4c7a16b2 100644 --- a/controller/DB.hpp +++ b/controller/DB.hpp @@ -20,7 +20,7 @@ #define ZT_CONTROLLER_DB_HPP #include "../node/Constants.hpp" -#include "../node/Address.hpp" +#include "../node/Identity.hpp" #include "../node/InetAddress.hpp" #include "../osdep/OSUtils.hpp" #include "../osdep/BlockingQueue.hpp" @@ -58,7 +58,7 @@ public: int64_t mostRecentDeauthTime; }; - DB(EmbeddedNetworkController *const nc,const Address &myAddress,const char *path); + DB(EmbeddedNetworkController *const nc,const Identity &myId,const char *path); virtual ~DB(); virtual bool waitForReady() = 0; @@ -104,6 +104,7 @@ protected: void _fillSummaryInfo(const std::shared_ptr<_Network> &nw,NetworkSummaryInfo &info); EmbeddedNetworkController *const _controller; + const Identity _myId; const Address _myAddress; const std::string _path; std::string _myAddressStr; diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index a3ce9208..d97a1ce2 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -477,10 +477,10 @@ void EmbeddedNetworkController::init(const Identity &signingId,Sender *sender) _signingIdAddressString = signingId.address().toString(tmp); #ifdef ZT_CONTROLLER_USE_RETHINKDB if ((_path.length() > 10)&&(_path.substr(0,10) == "rethinkdb:")) - _db.reset(new RethinkDB(this,_signingId.address(),_path.c_str())); + _db.reset(new RethinkDB(this,_signingId,_path.c_str())); else // else use FileDB after endif #endif - _db.reset(new FileDB(this,_signingId.address(),_path.c_str())); + _db.reset(new FileDB(this,_signingId,_path.c_str())); _db->waitForReady(); } diff --git a/controller/FileDB.cpp b/controller/FileDB.cpp index 3f8564fa..6b02f836 100644 --- a/controller/FileDB.cpp +++ b/controller/FileDB.cpp @@ -21,8 +21,8 @@ namespace ZeroTier { -FileDB::FileDB(EmbeddedNetworkController *const nc,const Address &myAddress,const char *path) : - DB(nc,myAddress,path), +FileDB::FileDB(EmbeddedNetworkController *const nc,const Identity &myId,const char *path) : + DB(nc,myId,path), _networksPath(_path + ZT_PATH_SEPARATOR_S + "network") { OSUtils::mkdir(_path.c_str()); diff --git a/controller/FileDB.hpp b/controller/FileDB.hpp index e31b18e6..eeb1c541 100644 --- a/controller/FileDB.hpp +++ b/controller/FileDB.hpp @@ -27,7 +27,7 @@ namespace ZeroTier class FileDB : public DB { public: - FileDB(EmbeddedNetworkController *const nc,const Address &myAddress,const char *path); + FileDB(EmbeddedNetworkController *const nc,const Identity &myId,const char *path); virtual ~FileDB(); virtual bool waitForReady(); diff --git a/controller/RethinkDB.cpp b/controller/RethinkDB.cpp index e6b58efd..b4f07f53 100644 --- a/controller/RethinkDB.cpp +++ b/controller/RethinkDB.cpp @@ -18,6 +18,8 @@ #ifdef ZT_CONTROLLER_USE_RETHINKDB +#include + #include "RethinkDB.hpp" #include "EmbeddedNetworkController.hpp" @@ -34,8 +36,8 @@ using json = nlohmann::json; namespace ZeroTier { -RethinkDB::RethinkDB(EmbeddedNetworkController *const nc,const Address &myAddress,const char *path) : - DB(nc,myAddress,path), +RethinkDB::RethinkDB(EmbeddedNetworkController *const nc,const Identity &myId,const char *path) : + DB(nc,myId,path), _ready(2), // two tables need to be synchronized before we're ready, so this is ready when it reaches 0 _run(1), _waitNoticePrinted(false) @@ -317,16 +319,44 @@ RethinkDB::RethinkDB(EmbeddedNetworkController *const nc,const Address &myAddres _heartbeatThread = std::thread([this]() { try { - char tmp[1024]; + R::Object controllerRecord; std::unique_ptr rdb; + + { + char publicId[1024]; + char secretId[1024]; + char hostname[1024]; + this->_myId.toString(publicId,false); + this->_myId.toString(secretId,true); + if (gethostname(hostname,sizeof(hostname)) != 0) { + hostname[0] = (char)0; + } else { + for(int i=0;i_myAddressStr.c_str(); + controllerRecord["publicIdentity"] = publicId; + controllerRecord["secretIdentity"] = secretId; + if (hostname[0]) + controllerRecord["clusterHost"] = hostname; + controllerRecord["vMajor"] = ZEROTIER_ONE_VERSION_MAJOR; + controllerRecord["vMinor"] = ZEROTIER_ONE_VERSION_MINOR; + controllerRecord["vRev"] = ZEROTIER_ONE_VERSION_REVISION; + controllerRecord["vBuild"] = ZEROTIER_ONE_VERSION_BUILD; + } + while (_run == 1) { try { if (!rdb) rdb = R::connect(this->_host,this->_port,this->_auth); if (rdb) { - OSUtils::ztsnprintf(tmp,sizeof(tmp),"{\"id\":\"%s\",\"lastAlive\":%lld,\"version\":\"%d.%d.%d\"}",this->_myAddressStr.c_str(),(long long)OSUtils::now(),ZEROTIER_ONE_VERSION_MAJOR,ZEROTIER_ONE_VERSION_MINOR,ZEROTIER_ONE_VERSION_REVISION); + controllerRecord["lastAlive"] = OSUtils::now(); //printf("HEARTBEAT: %s" ZT_EOL_S,tmp); - R::db(this->_db).table("Controller").update(R::Datum::from_json(tmp)).run(*rdb); + R::db(this->_db).table("Controller",R::optargs("read_mode","outdated")).insert(controllerRecord,R::optargs("conflict","update")).run(*rdb); } } catch ( ... ) { rdb.reset(); diff --git a/controller/RethinkDB.hpp b/controller/RethinkDB.hpp index a69f462f..07f0abfb 100644 --- a/controller/RethinkDB.hpp +++ b/controller/RethinkDB.hpp @@ -37,7 +37,7 @@ namespace ZeroTier class RethinkDB : public DB { public: - RethinkDB(EmbeddedNetworkController *const nc,const Address &myAddress,const char *path); + RethinkDB(EmbeddedNetworkController *const nc,const Identity &myId,const char *path); virtual ~RethinkDB(); virtual bool waitForReady(); -- cgit v1.2.3 From f17cc1c6d80657fe8cd98891cf0bb1bbf3ac62bf Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 10 Jan 2018 15:03:39 -0800 Subject: cleanup --- controller/FileDB.cpp | 9 +++++---- controller/FileDB.hpp | 5 +---- controller/RethinkDB.hpp | 4 ---- 3 files changed, 6 insertions(+), 12 deletions(-) (limited to 'controller/FileDB.hpp') diff --git a/controller/FileDB.cpp b/controller/FileDB.cpp index 6942a16e..40b9f914 100644 --- a/controller/FileDB.cpp +++ b/controller/FileDB.cpp @@ -23,12 +23,13 @@ namespace ZeroTier FileDB::FileDB(EmbeddedNetworkController *const nc,const Identity &myId,const char *path) : DB(nc,myId,path), - _networksPath(_path + ZT_PATH_SEPARATOR_S + "network") + _networksPath(_path + ZT_PATH_SEPARATOR_S + "network"), + _tracePath(_path + ZT_PATH_SEPARATOR_S + "trace") { OSUtils::mkdir(_path.c_str()); OSUtils::lockDownFile(_path.c_str(),true); - OSUtils::mkdir((_path + ZT_PATH_SEPARATOR + "network").c_str()); - OSUtils::mkdir((_path + ZT_PATH_SEPARATOR + "trace").c_str()); + OSUtils::mkdir(_networksPath.c_str()); + OSUtils::mkdir(_tracePath.c_str()); std::vector networks(OSUtils::listDirectory(_networksPath.c_str(),false)); std::string buf; @@ -120,7 +121,7 @@ void FileDB::save(nlohmann::json *orig,nlohmann::json &record) } else if (objtype == "trace") { const std::string id = record["id"]; if (id.length() > 0) { - OSUtils::ztsnprintf(p1,sizeof(p1),"%s" ZT_PATH_SEPARATOR_S "trace" ZT_PATH_SEPARATOR_S "%s.json",_path.c_str(),id.c_str()); + OSUtils::ztsnprintf(p1,sizeof(p1),"%s" ZT_PATH_SEPARATOR_S "%s.json",_tracePath.c_str(),id.c_str()); OSUtils::writeFile(p1,OSUtils::jsonDump(record,-1)); } } diff --git a/controller/FileDB.hpp b/controller/FileDB.hpp index eeb1c541..b02da8cb 100644 --- a/controller/FileDB.hpp +++ b/controller/FileDB.hpp @@ -31,17 +31,14 @@ public: virtual ~FileDB(); virtual bool waitForReady(); - virtual void save(nlohmann::json *orig,nlohmann::json &record); - virtual void eraseNetwork(const uint64_t networkId); - virtual void eraseMember(const uint64_t networkId,const uint64_t memberId); - virtual void nodeIsOnline(const uint64_t networkId,const uint64_t memberId,const InetAddress &physicalAddress); protected: std::string _networksPath; + std::string _tracePath; }; } // namespace ZeroTier diff --git a/controller/RethinkDB.hpp b/controller/RethinkDB.hpp index 07f0abfb..bce8bdef 100644 --- a/controller/RethinkDB.hpp +++ b/controller/RethinkDB.hpp @@ -41,13 +41,9 @@ public: virtual ~RethinkDB(); virtual bool waitForReady(); - virtual void save(nlohmann::json *orig,nlohmann::json &record); - virtual void eraseNetwork(const uint64_t networkId); - virtual void eraseMember(const uint64_t networkId,const uint64_t memberId); - virtual void nodeIsOnline(const uint64_t networkId,const uint64_t memberId,const InetAddress &physicalAddress); protected: -- cgit v1.2.3 From 574b24c0826cea575b77f36aa0238d7a26aeac7b Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 8 Mar 2018 22:41:42 -0800 Subject: docs --- controller/DB.cpp | 2 +- controller/DB.hpp | 2 +- controller/EmbeddedNetworkController.cpp | 2 +- controller/EmbeddedNetworkController.hpp | 2 +- controller/FileDB.hpp | 2 +- controller/RethinkDB.cpp | 2 +- controller/RethinkDB.hpp | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) (limited to 'controller/FileDB.hpp') diff --git a/controller/DB.cpp b/controller/DB.cpp index 680b4120..64311be7 100644 --- a/controller/DB.cpp +++ b/controller/DB.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2015 ZeroTier, Inc. + * Copyright (C) 2011-2018 ZeroTier, Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/controller/DB.hpp b/controller/DB.hpp index 86626009..4757bb40 100644 --- a/controller/DB.hpp +++ b/controller/DB.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2015 ZeroTier, Inc. + * Copyright (C) 2011-2018 ZeroTier, Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index f88f8cff..9a07b285 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2015 ZeroTier, Inc-> + * Copyright (C) 2011-2018 ZeroTier, Inc * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 1dda9f45..417005a4 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2015 ZeroTier, Inc. + * Copyright (C) 2011-2018 ZeroTier, Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/controller/FileDB.hpp b/controller/FileDB.hpp index b02da8cb..1e275a36 100644 --- a/controller/FileDB.hpp +++ b/controller/FileDB.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2015 ZeroTier, Inc. + * Copyright (C) 2011-2018 ZeroTier, Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/controller/RethinkDB.cpp b/controller/RethinkDB.cpp index 279d6ec2..f6c8a59c 100644 --- a/controller/RethinkDB.cpp +++ b/controller/RethinkDB.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2015 ZeroTier, Inc. + * Copyright (C) 2011-2018 ZeroTier, Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/controller/RethinkDB.hpp b/controller/RethinkDB.hpp index 01b46a47..b1049ac3 100644 --- a/controller/RethinkDB.hpp +++ b/controller/RethinkDB.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2015 ZeroTier, Inc. + * Copyright (C) 2011-2018 ZeroTier, Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by -- cgit v1.2.3