From 640ad577d1f52140adbe42e87b2da931bf15f430 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 6 Jul 2017 11:56:46 -0700 Subject: . --- attic/ClusterGeoIpService.cpp | 243 ++++++++++++++++++++++++++++++++++++++++ attic/ClusterGeoIpService.hpp | 151 +++++++++++++++++++++++++ make-bsd.mk | 5 - make-linux.mk | 4 - make-mac.mk | 4 - node/Constants.hpp | 5 - node/Path.hpp | 12 -- node/Peer.cpp | 157 -------------------------- node/Peer.hpp | 31 ----- node/Topology.cpp | 5 +- objects.mk | 1 - service/ClusterGeoIpService.cpp | 243 ---------------------------------------- service/ClusterGeoIpService.hpp | 151 ------------------------- 13 files changed, 395 insertions(+), 617 deletions(-) create mode 100644 attic/ClusterGeoIpService.cpp create mode 100644 attic/ClusterGeoIpService.hpp delete mode 100644 service/ClusterGeoIpService.cpp delete mode 100644 service/ClusterGeoIpService.hpp diff --git a/attic/ClusterGeoIpService.cpp b/attic/ClusterGeoIpService.cpp new file mode 100644 index 00000000..2dcc9179 --- /dev/null +++ b/attic/ClusterGeoIpService.cpp @@ -0,0 +1,243 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. + */ + +#ifdef ZT_ENABLE_CLUSTER + +#include + +#include + +#include "ClusterGeoIpService.hpp" + +#include "../node/Utils.hpp" +#include "../osdep/OSUtils.hpp" + +#define ZT_CLUSTERGEOIPSERVICE_FILE_MODIFICATION_CHECK_EVERY 10000 + +namespace ZeroTier { + +ClusterGeoIpService::ClusterGeoIpService() : + _pathToCsv(), + _ipStartColumn(-1), + _ipEndColumn(-1), + _latitudeColumn(-1), + _longitudeColumn(-1), + _lastFileCheckTime(0), + _csvModificationTime(0), + _csvFileSize(0) +{ +} + +ClusterGeoIpService::~ClusterGeoIpService() +{ +} + +bool ClusterGeoIpService::locate(const InetAddress &ip,int &x,int &y,int &z) +{ + Mutex::Lock _l(_lock); + + if ((_pathToCsv.length() > 0)&&((OSUtils::now() - _lastFileCheckTime) > ZT_CLUSTERGEOIPSERVICE_FILE_MODIFICATION_CHECK_EVERY)) { + _lastFileCheckTime = OSUtils::now(); + if ((_csvFileSize != OSUtils::getFileSize(_pathToCsv.c_str()))||(_csvModificationTime != OSUtils::getLastModified(_pathToCsv.c_str()))) + _load(_pathToCsv.c_str(),_ipStartColumn,_ipEndColumn,_latitudeColumn,_longitudeColumn); + } + + /* We search by looking up the upper bound of the sorted vXdb vectors + * and then iterating down for a matching IP range. We stop when we hit + * the beginning or an entry whose start and end are before the IP we + * are searching. */ + + if ((ip.ss_family == AF_INET)&&(_v4db.size() > 0)) { + _V4E key; + key.start = Utils::ntoh((uint32_t)(reinterpret_cast(&ip)->sin_addr.s_addr)); + std::vector<_V4E>::const_iterator i(std::upper_bound(_v4db.begin(),_v4db.end(),key)); + while (i != _v4db.begin()) { + --i; + if ((key.start >= i->start)&&(key.start <= i->end)) { + x = i->x; + y = i->y; + z = i->z; + //printf("%s : %f,%f %d,%d,%d\n",ip.toIpString().c_str(),i->lat,i->lon,x,y,z); + return true; + } else if ((key.start > i->start)&&(key.start > i->end)) + break; + } + } else if ((ip.ss_family == AF_INET6)&&(_v6db.size() > 0)) { + _V6E key; + memcpy(key.start,reinterpret_cast(&ip)->sin6_addr.s6_addr,16); + std::vector<_V6E>::const_iterator i(std::upper_bound(_v6db.begin(),_v6db.end(),key)); + while (i != _v6db.begin()) { + --i; + const int s_vs_s = memcmp(key.start,i->start,16); + const int s_vs_e = memcmp(key.start,i->end,16); + if ((s_vs_s >= 0)&&(s_vs_e <= 0)) { + x = i->x; + y = i->y; + z = i->z; + //printf("%s : %f,%f %d,%d,%d\n",ip.toIpString().c_str(),i->lat,i->lon,x,y,z); + return true; + } else if ((s_vs_s > 0)&&(s_vs_e > 0)) + break; + } + } + + return false; +} + +void ClusterGeoIpService::_parseLine(const char *line,std::vector<_V4E> &v4db,std::vector<_V6E> &v6db,int ipStartColumn,int ipEndColumn,int latitudeColumn,int longitudeColumn) +{ + std::vector ls(OSUtils::split(line,",\t","\\","\"'")); + if ( ((ipStartColumn >= 0)&&(ipStartColumn < (int)ls.size()))&& + ((ipEndColumn >= 0)&&(ipEndColumn < (int)ls.size()))&& + ((latitudeColumn >= 0)&&(latitudeColumn < (int)ls.size()))&& + ((longitudeColumn >= 0)&&(longitudeColumn < (int)ls.size())) ) { + InetAddress ipStart(ls[ipStartColumn].c_str(),0); + InetAddress ipEnd(ls[ipEndColumn].c_str(),0); + const double lat = strtod(ls[latitudeColumn].c_str(),(char **)0); + const double lon = strtod(ls[longitudeColumn].c_str(),(char **)0); + + if ((ipStart.ss_family == ipEnd.ss_family)&&(ipStart)&&(ipEnd)&&(std::isfinite(lat))&&(std::isfinite(lon))) { + const double latRadians = lat * 0.01745329251994; // PI / 180 + const double lonRadians = lon * 0.01745329251994; // PI / 180 + const double cosLat = cos(latRadians); + const int x = (int)round((-6371.0) * cosLat * cos(lonRadians)); // 6371 == Earth's approximate radius in kilometers + const int y = (int)round(6371.0 * sin(latRadians)); + const int z = (int)round(6371.0 * cosLat * sin(lonRadians)); + + if (ipStart.ss_family == AF_INET) { + v4db.push_back(_V4E()); + v4db.back().start = Utils::ntoh((uint32_t)(reinterpret_cast(&ipStart)->sin_addr.s_addr)); + v4db.back().end = Utils::ntoh((uint32_t)(reinterpret_cast(&ipEnd)->sin_addr.s_addr)); + v4db.back().lat = (float)lat; + v4db.back().lon = (float)lon; + v4db.back().x = x; + v4db.back().y = y; + v4db.back().z = z; + //printf("%s - %s : %d,%d,%d\n",ipStart.toIpString().c_str(),ipEnd.toIpString().c_str(),x,y,z); + } else if (ipStart.ss_family == AF_INET6) { + v6db.push_back(_V6E()); + memcpy(v6db.back().start,reinterpret_cast(&ipStart)->sin6_addr.s6_addr,16); + memcpy(v6db.back().end,reinterpret_cast(&ipEnd)->sin6_addr.s6_addr,16); + v6db.back().lat = (float)lat; + v6db.back().lon = (float)lon; + v6db.back().x = x; + v6db.back().y = y; + v6db.back().z = z; + //printf("%s - %s : %d,%d,%d\n",ipStart.toIpString().c_str(),ipEnd.toIpString().c_str(),x,y,z); + } + } + } +} + +long ClusterGeoIpService::_load(const char *pathToCsv,int ipStartColumn,int ipEndColumn,int latitudeColumn,int longitudeColumn) +{ + // assumes _lock is locked + + FILE *f = fopen(pathToCsv,"rb"); + if (!f) + return -1; + + std::vector<_V4E> v4db; + std::vector<_V6E> v6db; + v4db.reserve(16777216); + v6db.reserve(16777216); + + char buf[4096]; + char linebuf[1024]; + unsigned int lineptr = 0; + for(;;) { + int n = (int)fread(buf,1,sizeof(buf),f); + if (n <= 0) + break; + for(int i=0;i 0)||(v6db.size() > 0)) { + std::sort(v4db.begin(),v4db.end()); + std::sort(v6db.begin(),v6db.end()); + + _pathToCsv = pathToCsv; + _ipStartColumn = ipStartColumn; + _ipEndColumn = ipEndColumn; + _latitudeColumn = latitudeColumn; + _longitudeColumn = longitudeColumn; + + _lastFileCheckTime = OSUtils::now(); + _csvModificationTime = OSUtils::getLastModified(pathToCsv); + _csvFileSize = OSUtils::getFileSize(pathToCsv); + + _v4db.swap(v4db); + _v6db.swap(v6db); + + return (long)(_v4db.size() + _v6db.size()); + } else { + return 0; + } +} + +} // namespace ZeroTier + +#endif // ZT_ENABLE_CLUSTER + +/* +int main(int argc,char **argv) +{ + char buf[1024]; + + ZeroTier::ClusterGeoIpService gip; + printf("loading...\n"); + gip.load("/Users/api/Code/ZeroTier/Infrastructure/root-servers/zerotier-one/cluster-geoip.csv",0,1,5,6); + printf("... done!\n"); fflush(stdout); + + while (gets(buf)) { // unsafe, testing only + ZeroTier::InetAddress addr(buf,0); + printf("looking up: %s\n",addr.toString().c_str()); fflush(stdout); + int x = 0,y = 0,z = 0; + if (gip.locate(addr,x,y,z)) { + //printf("%s: %d,%d,%d\n",addr.toString().c_str(),x,y,z); fflush(stdout); + } else { + printf("%s: not found!\n",addr.toString().c_str()); fflush(stdout); + } + } + + return 0; +} +*/ diff --git a/attic/ClusterGeoIpService.hpp b/attic/ClusterGeoIpService.hpp new file mode 100644 index 00000000..380f944f --- /dev/null +++ b/attic/ClusterGeoIpService.hpp @@ -0,0 +1,151 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. + */ + +#ifndef ZT_CLUSTERGEOIPSERVICE_HPP +#define ZT_CLUSTERGEOIPSERVICE_HPP + +#ifdef ZT_ENABLE_CLUSTER + +#include +#include +#include +#include + +#include +#include +#include + +#include "../node/Constants.hpp" +#include "../node/Mutex.hpp" +#include "../node/NonCopyable.hpp" +#include "../node/InetAddress.hpp" + +namespace ZeroTier { + +/** + * Loads a GeoIP CSV into memory for fast lookup, reloading as needed + * + * This was designed around the CSV from https://db-ip.com but can be used + * with any similar GeoIP CSV database that is presented in the form of an + * IP range and lat/long coordinates. + * + * It loads the whole database into memory, which can be kind of large. If + * the CSV file changes, the changes are loaded automatically. + */ +class ClusterGeoIpService : NonCopyable +{ +public: + ClusterGeoIpService(); + ~ClusterGeoIpService(); + + /** + * Load or reload CSV file + * + * CSV column indexes start at zero. CSVs can be quoted with single or + * double quotes. Whitespace before or after commas is ignored. Backslash + * may be used for escaping whitespace as well. + * + * @param pathToCsv Path to (uncompressed) CSV file + * @param ipStartColumn Column with IP range start + * @param ipEndColumn Column with IP range end (inclusive) + * @param latitudeColumn Column with latitude + * @param longitudeColumn Column with longitude + * @return Number of valid records loaded or -1 on error (invalid file, not found, etc.) + */ + inline long load(const char *pathToCsv,int ipStartColumn,int ipEndColumn,int latitudeColumn,int longitudeColumn) + { + Mutex::Lock _l(_lock); + return _load(pathToCsv,ipStartColumn,ipEndColumn,latitudeColumn,longitudeColumn); + } + + /** + * Attempt to locate an IP + * + * This returns true if x, y, and z are set. If the return value is false + * the values of x, y, and z are undefined. + * + * @param ip IPv4 or IPv6 address + * @param x Reference to variable to receive X + * @param y Reference to variable to receive Y + * @param z Reference to variable to receive Z + * @return True if coordinates were set + */ + bool locate(const InetAddress &ip,int &x,int &y,int &z); + + /** + * @return True if IP database/service is available for queries (otherwise locate() will always be false) + */ + inline bool available() const + { + Mutex::Lock _l(_lock); + return ((_v4db.size() + _v6db.size()) > 0); + } + +private: + struct _V4E + { + uint32_t start; + uint32_t end; + float lat,lon; + int16_t x,y,z; + + inline bool operator<(const _V4E &e) const { return (start < e.start); } + }; + + struct _V6E + { + uint8_t start[16]; + uint8_t end[16]; + float lat,lon; + int16_t x,y,z; + + inline bool operator<(const _V6E &e) const { return (memcmp(start,e.start,16) < 0); } + }; + + static void _parseLine(const char *line,std::vector<_V4E> &v4db,std::vector<_V6E> &v6db,int ipStartColumn,int ipEndColumn,int latitudeColumn,int longitudeColumn); + long _load(const char *pathToCsv,int ipStartColumn,int ipEndColumn,int latitudeColumn,int longitudeColumn); + + std::string _pathToCsv; + int _ipStartColumn; + int _ipEndColumn; + int _latitudeColumn; + int _longitudeColumn; + + uint64_t _lastFileCheckTime; + uint64_t _csvModificationTime; + int64_t _csvFileSize; + + std::vector<_V4E> _v4db; + std::vector<_V6E> _v6db; + + Mutex _lock; +}; + +} // namespace ZeroTier + +#endif // ZT_ENABLE_CLUSTER + +#endif diff --git a/make-bsd.mk b/make-bsd.mk index cbab5810..c2fd6062 100644 --- a/make-bsd.mk +++ b/make-bsd.mk @@ -7,11 +7,6 @@ LIBS= include objects.mk ONE_OBJS+=osdep/BSDEthernetTap.o ext/http-parser/http_parser.o -# Build with ZT_ENABLE_CLUSTER=1 to build with cluster support -ifeq ($(ZT_ENABLE_CLUSTER),1) - DEFS+=-DZT_ENABLE_CLUSTER -endif - # "make debug" is a shortcut for this ifeq ($(ZT_DEBUG),1) CFLAGS+=-Wall -Werror -g -pthread $(INCLUDES) $(DEFS) diff --git a/make-linux.mk b/make-linux.mk index c27c600d..7017d31e 100644 --- a/make-linux.mk +++ b/make-linux.mk @@ -38,10 +38,6 @@ endif # Trying to use dynamically linked libhttp-parser causes tons of compatibility problems. ONE_OBJS+=ext/http-parser/http_parser.o -ifeq ($(ZT_ENABLE_CLUSTER),1) - DEFS+=-DZT_ENABLE_CLUSTER -endif - ifeq ($(ZT_SYNOLOGY), 1) DEFS+=-D__SYNOLOGY__ endif diff --git a/make-mac.mk b/make-mac.mk index 5622a41b..196b17cb 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -33,10 +33,6 @@ else DEFS+=-DZT_SOFTWARE_UPDATE_DEFAULT="\"download\"" endif -ifeq ($(ZT_ENABLE_CLUSTER),1) - DEFS+=-DZT_ENABLE_CLUSTER -endif - # Use fast ASM Salsa20/12 for x64 processors DEFS+=-DZT_USE_X64_ASM_SALSA2012 CORE_OBJS+=ext/x64-salsa2012-asm/salsa2012.o diff --git a/node/Constants.hpp b/node/Constants.hpp index 274b9564..12bf8d28 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -218,11 +218,6 @@ */ #define ZT_HOUSEKEEPING_PERIOD 60000 -/** - * How often in ms to write peer state to storage and/or cluster (approximate) - */ -#define ZT_PEER_STATE_WRITE_PERIOD 10000 - /** * How long to remember peer records in RAM if they haven't been used */ diff --git a/node/Path.hpp b/node/Path.hpp index 854b28e2..ac8e4c0e 100644 --- a/node/Path.hpp +++ b/node/Path.hpp @@ -283,18 +283,6 @@ public: */ inline uint64_t lastTrustEstablishedPacketReceived() const { return _lastTrustEstablishedPacketReceived; } - /** - * @param lo Last out send - * @param li Last in send - * @param lt Last trust established packet received - */ - inline void updateFromRemoteState(const uint64_t lo,const uint64_t li,const uint64_t lt) - { - _lastOut = lo; - _lastIn = li; - _lastTrustEstablishedPacketReceived = lt; - } - /** * Return and increment outgoing packet counter (used with Packet::armor()) * diff --git a/node/Peer.cpp b/node/Peer.cpp index 875d651e..fb9a72b1 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -38,8 +38,6 @@ namespace ZeroTier { Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Identity &peerIdentity) : RR(renv), - _lastWroteState(0), - _lastReceivedStateTimestamp(0), _lastReceive(0), _lastNontrivialReceive(0), _lastTriedMemorizedPath(0), @@ -184,7 +182,6 @@ void Peer::received( if (verb == Packet::VERB_OK) { potentialNewPeerPath->lr = now; potentialNewPeerPath->p = path; - _lastWroteState = 0; // force state write now } else { TRACE("got %s via unknown path %s(%s), confirming...",Packet::verbString(verb),_id.address().toString().c_str(),path->address().toString().c_str()); attemptToContactAt(tPtr,path->localSocket(),path->address(),now,true,path->nextOutgoingCounter()); @@ -263,9 +260,6 @@ void Peer::received( } } } - - if ((now - _lastWroteState) > ZT_PEER_STATE_WRITE_PERIOD) - writeState(tPtr,now); } bool Peer::sendDirect(void *tPtr,const void *data,unsigned int len,uint64_t now,bool force) @@ -428,155 +422,4 @@ bool Peer::doPingAndKeepalive(void *tPtr,uint64_t now,int inetAddressFamily) return false; } -void Peer::writeState(void *tPtr,const uint64_t now) -{ - try { - Buffer b; - - b.append((uint8_t)1); // version - b.append(now); - - _id.serialize(b); - - { - Mutex::Lock _l(_paths_m); - unsigned int count = 0; - if (_v4Path.lr) - ++count; - if (_v6Path.lr) - ++count; - b.append((uint8_t)count); - if (_v4Path.lr) { - b.append(_v4Path.lr); - b.append(_v4Path.p->lastOut()); - b.append(_v4Path.p->lastIn()); - b.append(_v4Path.p->lastTrustEstablishedPacketReceived()); - _v4Path.p->address().serialize(b); - } - if (_v6Path.lr) { - b.append(_v6Path.lr); - b.append(_v6Path.p->lastOut()); - b.append(_v6Path.p->lastIn()); - b.append(_v6Path.p->lastTrustEstablishedPacketReceived()); - _v6Path.p->address().serialize(b); - } - } - - // Save space by sending these as time since now at 100ms resolution - b.append((uint16_t)(std::max(now - _lastReceive,(uint64_t)6553500) / 100)); - b.append((uint16_t)(std::max(now - _lastNontrivialReceive,(uint64_t)6553500) / 100)); - b.append((uint16_t)(std::max(now - _lastTriedMemorizedPath,(uint64_t)6553500) / 100)); - b.append((uint16_t)(std::max(now - _lastDirectPathPushSent,(uint64_t)6553500) / 100)); - b.append((uint16_t)(std::max(now - _lastDirectPathPushReceive,(uint64_t)6553500) / 100)); - b.append((uint16_t)(std::max(now - _lastCredentialRequestSent,(uint64_t)6553500) / 100)); - b.append((uint16_t)(std::max(now - _lastWhoisRequestReceived,(uint64_t)6553500) / 100)); - b.append((uint16_t)(std::max(now - _lastEchoRequestReceived,(uint64_t)6553500) / 100)); - b.append((uint16_t)(std::max(now - _lastComRequestReceived,(uint64_t)6553500) / 100)); - b.append((uint16_t)(std::max(now - _lastComRequestSent,(uint64_t)6553500) / 100)); - b.append((uint16_t)(std::max(now - _lastCredentialsReceived,(uint64_t)6553500) / 100)); - b.append((uint16_t)(std::max(now - _lastTrustEstablishedPacketReceived,(uint64_t)6553500) / 100)); - - b.append((uint8_t)_vProto); - b.append((uint8_t)_vMajor); - b.append((uint8_t)_vMinor); - b.append((uint16_t)_vRevision); - - b.append((uint16_t)0); // length of additional fields - - uint64_t tmp[2]; - tmp[0] = _id.address().toInt(); tmp[1] = 0; - //RR->node->stateObjectPut(tPtr,ZT_STATE_OBJECT_PEER_STATE,tmp,b.data(),b.size()); - - _lastWroteState = now; - } catch ( ... ) {} // sanity check, should not be possible -} - -bool Peer::applyStateUpdate(const void *data,unsigned int len) -{ - try { - Buffer b(data,len); - unsigned int ptr = 0; - - if (b[ptr++] != 1) - return false; - const uint64_t ts = b.at(ptr); ptr += 8; - if (ts <= _lastReceivedStateTimestamp) - return false; - - Identity id; - ptr += id.deserialize(b,ptr); - if (id != _id) // sanity check - return false; - - const unsigned int pathCount = (unsigned int)b[ptr++]; - { - Mutex::Lock _l(_paths_m); - for(unsigned int i=0;i(ptr); ptr += 8; - const uint64_t lastOut = b.at(ptr); ptr += 8; - const uint64_t lastIn = b.at(ptr); ptr += 8; - const uint64_t lastTrustEstablishedPacketReceived = b.at(ptr); ptr += 8; - InetAddress addr; - ptr += addr.deserialize(b,ptr); - _PeerPath *p = (_PeerPath *)0; - switch(addr.ss_family) { - case AF_INET: p = &_v4Path; break; - case AF_INET6: p = &_v6Path; break; - } - if (p) { - if ( (!p->p) || (p->p->address() != addr) ) { - p->p = RR->topology->getPath(-1,addr); - } - p->lr = lr; - p->p->updateFromRemoteState(lastOut,lastIn,lastTrustEstablishedPacketReceived); - } - } - } - - _lastReceive = std::max(_lastReceive,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; - _lastNontrivialReceive = std::max(_lastNontrivialReceive,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; - _lastTriedMemorizedPath = std::max(_lastTriedMemorizedPath,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; - _lastDirectPathPushSent = std::max(_lastDirectPathPushSent,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; - _lastDirectPathPushReceive = std::max(_lastDirectPathPushReceive,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; - _lastCredentialRequestSent = std::max(_lastCredentialRequestSent,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; - _lastWhoisRequestReceived = std::max(_lastWhoisRequestReceived,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; - _lastEchoRequestReceived = std::max(_lastEchoRequestReceived,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; - _lastComRequestReceived = std::max(_lastComRequestReceived,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; - _lastComRequestSent = std::max(_lastComRequestSent,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; - _lastCredentialsReceived = std::max(_lastCredentialsReceived,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; - _lastTrustEstablishedPacketReceived = std::max(_lastTrustEstablishedPacketReceived,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; - - _vProto = (uint16_t)b[ptr++]; - _vMajor = (uint16_t)b[ptr++]; - _vMinor = (uint16_t)b[ptr++]; - _vRevision = b.at(ptr); ptr += 2; - - _lastReceivedStateTimestamp = ts; - - return true; - } catch ( ... ) {} // ignore invalid state updates - return false; -} - -SharedPtr Peer::createFromStateUpdate(const RuntimeEnvironment *renv,void *tPtr,const void *data,unsigned int len) -{ - try { - Identity id; - { - Buffer b(data,len); - unsigned int ptr = 0; - if (b[ptr++] != 1) - return SharedPtr(); - ptr += 8; // skip TS, don't care - id.deserialize(b,ptr); - } - if (id) { - const SharedPtr p(new Peer(renv,renv->identity,id)); - if (p->applyStateUpdate(data,len)) - return renv->topology->addPeer(tPtr,p); - } - } catch ( ... ) {} - return SharedPtr(); -} - } // namespace ZeroTier diff --git a/node/Peer.hpp b/node/Peer.hpp index 478c7232..ad2d0ddc 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -195,23 +195,6 @@ public: */ bool doPingAndKeepalive(void *tPtr,uint64_t now,int inetAddressFamily); - /** - * Write object state to external storage and/or cluster network - * - * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call - * @param now Current time - */ - void writeState(void *tPtr,const uint64_t now); - - /** - * Apply a state update received from e.g. a remote cluster member - * - * @param data State update data - * @param len Length of state update - * @return True if state update was applied, false if ignored or invalid - */ - bool applyStateUpdate(const void *data,unsigned int len); - /** * Reset paths within a given IP scope and address family * @@ -440,17 +423,6 @@ public: return false; } - /** - * Create a peer from a remote state update - * - * @param renv Runtime environment - * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call - * @param data State update data - * @param len State update length - * @return Peer or NULL if data was invalid - */ - static SharedPtr createFromStateUpdate(const RuntimeEnvironment *renv,void *tPtr,const void *data,unsigned int len); - private: struct _PeerPath { @@ -463,9 +435,6 @@ private: const RuntimeEnvironment *RR; - uint64_t _lastWroteState; - uint64_t _lastReceivedStateTimestamp; - uint64_t _lastReceive; // direct or indirect uint64_t _lastNontrivialReceive; // frames, things like netconf, etc. uint64_t _lastTriedMemorizedPath; diff --git a/node/Topology.cpp b/node/Topology.cpp index d4632f43..809bc7e7 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -395,11 +395,8 @@ void Topology::doPeriodicTasks(void *tPtr,uint64_t now) Address *a = (Address *)0; SharedPtr *p = (SharedPtr *)0; while (i.next(a,p)) { - if ( (!(*p)->isAlive(now)) && (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),*a) == _upstreamAddresses.end()) ) { + if ( (!(*p)->isAlive(now)) && (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),*a) == _upstreamAddresses.end()) ) _peers.erase(*a); - } else { - (*p)->writeState(tPtr,now); - } } } diff --git a/objects.mk b/objects.mk index c8231f08..3a8bd645 100644 --- a/objects.mk +++ b/objects.mk @@ -31,7 +31,6 @@ ONE_OBJS=\ osdep/ManagedRoute.o \ osdep/Http.o \ osdep/OSUtils.o \ - service/ClusterGeoIpService.o \ service/SoftwareUpdater.o \ service/OneService.o diff --git a/service/ClusterGeoIpService.cpp b/service/ClusterGeoIpService.cpp deleted file mode 100644 index 2dcc9179..00000000 --- a/service/ClusterGeoIpService.cpp +++ /dev/null @@ -1,243 +0,0 @@ -/* - * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - * -- - * - * You can be released from the requirements of the license by purchasing - * a commercial license. Buying such a license is mandatory as soon as you - * develop commercial closed-source software that incorporates or links - * directly against ZeroTier software without disclosing the source code - * of your own application. - */ - -#ifdef ZT_ENABLE_CLUSTER - -#include - -#include - -#include "ClusterGeoIpService.hpp" - -#include "../node/Utils.hpp" -#include "../osdep/OSUtils.hpp" - -#define ZT_CLUSTERGEOIPSERVICE_FILE_MODIFICATION_CHECK_EVERY 10000 - -namespace ZeroTier { - -ClusterGeoIpService::ClusterGeoIpService() : - _pathToCsv(), - _ipStartColumn(-1), - _ipEndColumn(-1), - _latitudeColumn(-1), - _longitudeColumn(-1), - _lastFileCheckTime(0), - _csvModificationTime(0), - _csvFileSize(0) -{ -} - -ClusterGeoIpService::~ClusterGeoIpService() -{ -} - -bool ClusterGeoIpService::locate(const InetAddress &ip,int &x,int &y,int &z) -{ - Mutex::Lock _l(_lock); - - if ((_pathToCsv.length() > 0)&&((OSUtils::now() - _lastFileCheckTime) > ZT_CLUSTERGEOIPSERVICE_FILE_MODIFICATION_CHECK_EVERY)) { - _lastFileCheckTime = OSUtils::now(); - if ((_csvFileSize != OSUtils::getFileSize(_pathToCsv.c_str()))||(_csvModificationTime != OSUtils::getLastModified(_pathToCsv.c_str()))) - _load(_pathToCsv.c_str(),_ipStartColumn,_ipEndColumn,_latitudeColumn,_longitudeColumn); - } - - /* We search by looking up the upper bound of the sorted vXdb vectors - * and then iterating down for a matching IP range. We stop when we hit - * the beginning or an entry whose start and end are before the IP we - * are searching. */ - - if ((ip.ss_family == AF_INET)&&(_v4db.size() > 0)) { - _V4E key; - key.start = Utils::ntoh((uint32_t)(reinterpret_cast(&ip)->sin_addr.s_addr)); - std::vector<_V4E>::const_iterator i(std::upper_bound(_v4db.begin(),_v4db.end(),key)); - while (i != _v4db.begin()) { - --i; - if ((key.start >= i->start)&&(key.start <= i->end)) { - x = i->x; - y = i->y; - z = i->z; - //printf("%s : %f,%f %d,%d,%d\n",ip.toIpString().c_str(),i->lat,i->lon,x,y,z); - return true; - } else if ((key.start > i->start)&&(key.start > i->end)) - break; - } - } else if ((ip.ss_family == AF_INET6)&&(_v6db.size() > 0)) { - _V6E key; - memcpy(key.start,reinterpret_cast(&ip)->sin6_addr.s6_addr,16); - std::vector<_V6E>::const_iterator i(std::upper_bound(_v6db.begin(),_v6db.end(),key)); - while (i != _v6db.begin()) { - --i; - const int s_vs_s = memcmp(key.start,i->start,16); - const int s_vs_e = memcmp(key.start,i->end,16); - if ((s_vs_s >= 0)&&(s_vs_e <= 0)) { - x = i->x; - y = i->y; - z = i->z; - //printf("%s : %f,%f %d,%d,%d\n",ip.toIpString().c_str(),i->lat,i->lon,x,y,z); - return true; - } else if ((s_vs_s > 0)&&(s_vs_e > 0)) - break; - } - } - - return false; -} - -void ClusterGeoIpService::_parseLine(const char *line,std::vector<_V4E> &v4db,std::vector<_V6E> &v6db,int ipStartColumn,int ipEndColumn,int latitudeColumn,int longitudeColumn) -{ - std::vector ls(OSUtils::split(line,",\t","\\","\"'")); - if ( ((ipStartColumn >= 0)&&(ipStartColumn < (int)ls.size()))&& - ((ipEndColumn >= 0)&&(ipEndColumn < (int)ls.size()))&& - ((latitudeColumn >= 0)&&(latitudeColumn < (int)ls.size()))&& - ((longitudeColumn >= 0)&&(longitudeColumn < (int)ls.size())) ) { - InetAddress ipStart(ls[ipStartColumn].c_str(),0); - InetAddress ipEnd(ls[ipEndColumn].c_str(),0); - const double lat = strtod(ls[latitudeColumn].c_str(),(char **)0); - const double lon = strtod(ls[longitudeColumn].c_str(),(char **)0); - - if ((ipStart.ss_family == ipEnd.ss_family)&&(ipStart)&&(ipEnd)&&(std::isfinite(lat))&&(std::isfinite(lon))) { - const double latRadians = lat * 0.01745329251994; // PI / 180 - const double lonRadians = lon * 0.01745329251994; // PI / 180 - const double cosLat = cos(latRadians); - const int x = (int)round((-6371.0) * cosLat * cos(lonRadians)); // 6371 == Earth's approximate radius in kilometers - const int y = (int)round(6371.0 * sin(latRadians)); - const int z = (int)round(6371.0 * cosLat * sin(lonRadians)); - - if (ipStart.ss_family == AF_INET) { - v4db.push_back(_V4E()); - v4db.back().start = Utils::ntoh((uint32_t)(reinterpret_cast(&ipStart)->sin_addr.s_addr)); - v4db.back().end = Utils::ntoh((uint32_t)(reinterpret_cast(&ipEnd)->sin_addr.s_addr)); - v4db.back().lat = (float)lat; - v4db.back().lon = (float)lon; - v4db.back().x = x; - v4db.back().y = y; - v4db.back().z = z; - //printf("%s - %s : %d,%d,%d\n",ipStart.toIpString().c_str(),ipEnd.toIpString().c_str(),x,y,z); - } else if (ipStart.ss_family == AF_INET6) { - v6db.push_back(_V6E()); - memcpy(v6db.back().start,reinterpret_cast(&ipStart)->sin6_addr.s6_addr,16); - memcpy(v6db.back().end,reinterpret_cast(&ipEnd)->sin6_addr.s6_addr,16); - v6db.back().lat = (float)lat; - v6db.back().lon = (float)lon; - v6db.back().x = x; - v6db.back().y = y; - v6db.back().z = z; - //printf("%s - %s : %d,%d,%d\n",ipStart.toIpString().c_str(),ipEnd.toIpString().c_str(),x,y,z); - } - } - } -} - -long ClusterGeoIpService::_load(const char *pathToCsv,int ipStartColumn,int ipEndColumn,int latitudeColumn,int longitudeColumn) -{ - // assumes _lock is locked - - FILE *f = fopen(pathToCsv,"rb"); - if (!f) - return -1; - - std::vector<_V4E> v4db; - std::vector<_V6E> v6db; - v4db.reserve(16777216); - v6db.reserve(16777216); - - char buf[4096]; - char linebuf[1024]; - unsigned int lineptr = 0; - for(;;) { - int n = (int)fread(buf,1,sizeof(buf),f); - if (n <= 0) - break; - for(int i=0;i 0)||(v6db.size() > 0)) { - std::sort(v4db.begin(),v4db.end()); - std::sort(v6db.begin(),v6db.end()); - - _pathToCsv = pathToCsv; - _ipStartColumn = ipStartColumn; - _ipEndColumn = ipEndColumn; - _latitudeColumn = latitudeColumn; - _longitudeColumn = longitudeColumn; - - _lastFileCheckTime = OSUtils::now(); - _csvModificationTime = OSUtils::getLastModified(pathToCsv); - _csvFileSize = OSUtils::getFileSize(pathToCsv); - - _v4db.swap(v4db); - _v6db.swap(v6db); - - return (long)(_v4db.size() + _v6db.size()); - } else { - return 0; - } -} - -} // namespace ZeroTier - -#endif // ZT_ENABLE_CLUSTER - -/* -int main(int argc,char **argv) -{ - char buf[1024]; - - ZeroTier::ClusterGeoIpService gip; - printf("loading...\n"); - gip.load("/Users/api/Code/ZeroTier/Infrastructure/root-servers/zerotier-one/cluster-geoip.csv",0,1,5,6); - printf("... done!\n"); fflush(stdout); - - while (gets(buf)) { // unsafe, testing only - ZeroTier::InetAddress addr(buf,0); - printf("looking up: %s\n",addr.toString().c_str()); fflush(stdout); - int x = 0,y = 0,z = 0; - if (gip.locate(addr,x,y,z)) { - //printf("%s: %d,%d,%d\n",addr.toString().c_str(),x,y,z); fflush(stdout); - } else { - printf("%s: not found!\n",addr.toString().c_str()); fflush(stdout); - } - } - - return 0; -} -*/ diff --git a/service/ClusterGeoIpService.hpp b/service/ClusterGeoIpService.hpp deleted file mode 100644 index 380f944f..00000000 --- a/service/ClusterGeoIpService.hpp +++ /dev/null @@ -1,151 +0,0 @@ -/* - * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - * -- - * - * You can be released from the requirements of the license by purchasing - * a commercial license. Buying such a license is mandatory as soon as you - * develop commercial closed-source software that incorporates or links - * directly against ZeroTier software without disclosing the source code - * of your own application. - */ - -#ifndef ZT_CLUSTERGEOIPSERVICE_HPP -#define ZT_CLUSTERGEOIPSERVICE_HPP - -#ifdef ZT_ENABLE_CLUSTER - -#include -#include -#include -#include - -#include -#include -#include - -#include "../node/Constants.hpp" -#include "../node/Mutex.hpp" -#include "../node/NonCopyable.hpp" -#include "../node/InetAddress.hpp" - -namespace ZeroTier { - -/** - * Loads a GeoIP CSV into memory for fast lookup, reloading as needed - * - * This was designed around the CSV from https://db-ip.com but can be used - * with any similar GeoIP CSV database that is presented in the form of an - * IP range and lat/long coordinates. - * - * It loads the whole database into memory, which can be kind of large. If - * the CSV file changes, the changes are loaded automatically. - */ -class ClusterGeoIpService : NonCopyable -{ -public: - ClusterGeoIpService(); - ~ClusterGeoIpService(); - - /** - * Load or reload CSV file - * - * CSV column indexes start at zero. CSVs can be quoted with single or - * double quotes. Whitespace before or after commas is ignored. Backslash - * may be used for escaping whitespace as well. - * - * @param pathToCsv Path to (uncompressed) CSV file - * @param ipStartColumn Column with IP range start - * @param ipEndColumn Column with IP range end (inclusive) - * @param latitudeColumn Column with latitude - * @param longitudeColumn Column with longitude - * @return Number of valid records loaded or -1 on error (invalid file, not found, etc.) - */ - inline long load(const char *pathToCsv,int ipStartColumn,int ipEndColumn,int latitudeColumn,int longitudeColumn) - { - Mutex::Lock _l(_lock); - return _load(pathToCsv,ipStartColumn,ipEndColumn,latitudeColumn,longitudeColumn); - } - - /** - * Attempt to locate an IP - * - * This returns true if x, y, and z are set. If the return value is false - * the values of x, y, and z are undefined. - * - * @param ip IPv4 or IPv6 address - * @param x Reference to variable to receive X - * @param y Reference to variable to receive Y - * @param z Reference to variable to receive Z - * @return True if coordinates were set - */ - bool locate(const InetAddress &ip,int &x,int &y,int &z); - - /** - * @return True if IP database/service is available for queries (otherwise locate() will always be false) - */ - inline bool available() const - { - Mutex::Lock _l(_lock); - return ((_v4db.size() + _v6db.size()) > 0); - } - -private: - struct _V4E - { - uint32_t start; - uint32_t end; - float lat,lon; - int16_t x,y,z; - - inline bool operator<(const _V4E &e) const { return (start < e.start); } - }; - - struct _V6E - { - uint8_t start[16]; - uint8_t end[16]; - float lat,lon; - int16_t x,y,z; - - inline bool operator<(const _V6E &e) const { return (memcmp(start,e.start,16) < 0); } - }; - - static void _parseLine(const char *line,std::vector<_V4E> &v4db,std::vector<_V6E> &v6db,int ipStartColumn,int ipEndColumn,int latitudeColumn,int longitudeColumn); - long _load(const char *pathToCsv,int ipStartColumn,int ipEndColumn,int latitudeColumn,int longitudeColumn); - - std::string _pathToCsv; - int _ipStartColumn; - int _ipEndColumn; - int _latitudeColumn; - int _longitudeColumn; - - uint64_t _lastFileCheckTime; - uint64_t _csvModificationTime; - int64_t _csvFileSize; - - std::vector<_V4E> _v4db; - std::vector<_V6E> _v6db; - - Mutex _lock; -}; - -} // namespace ZeroTier - -#endif // ZT_ENABLE_CLUSTER - -#endif -- cgit v1.2.3