From a3db7d0728c1bc5181b8a70e8c379632125ee376 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 1 Oct 2015 11:11:52 -0700 Subject: Refactor: move network COMs out of Network and into Peer in prep for tightening up multicast lookup and other things. --- node/Constants.hpp | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'node/Constants.hpp') diff --git a/node/Constants.hpp b/node/Constants.hpp index 4f783550..27c43aa7 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -324,6 +324,13 @@ */ #define ZT_DIRECT_PATH_PUSH_INTERVAL 300000 +/** + * How long (max) to remember network certificates of membership? + * + * This only applies to networks we don't belong to. + */ +#define ZT_PEER_NETWORK_COM_EXPIRATION 3600000 + /** * Sanity limit on maximum bridge routes * -- cgit v1.2.3 From 7d62dbe9f7fa620982c71f44089d319120023e26 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 7 Oct 2015 11:57:59 -0700 Subject: Tune NAT-t keepalives so that timing is better obeyed, clean up a build warning, and fix a potential source of network recursion (though harmless). --- node/Constants.hpp | 2 +- node/Network.cpp | 2 -- node/Peer.cpp | 7 +++++-- 3 files changed, 6 insertions(+), 5 deletions(-) (limited to 'node/Constants.hpp') diff --git a/node/Constants.hpp b/node/Constants.hpp index 27c43aa7..9ab390eb 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -264,7 +264,7 @@ * This is also how often pings will be retried to upstream peers (relays, roots) * constantly until something is heard. */ -#define ZT_PING_CHECK_INVERVAL 6250 +#define ZT_PING_CHECK_INVERVAL 9500 /** * Delay between ordinary case pings of direct links diff --git a/node/Network.cpp b/node/Network.cpp index 9ce58c63..46f93241 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -448,7 +448,6 @@ class _AnnounceMulticastGroupsToAll public: _AnnounceMulticastGroupsToAll(const RuntimeEnvironment *renv,Network *nw) : _now(renv->node->now()), - RR(renv), _network(nw), _rootAddresses(renv->topology->rootAddresses()), _allMulticastGroups(nw->_allMulticastGroups()) @@ -458,7 +457,6 @@ public: private: uint64_t _now; - const RuntimeEnvironment *RR; Network *_network; std::vector
_rootAddresses; std::vector _allMulticastGroups; diff --git a/node/Peer.cpp b/node/Peer.cpp index 15648e0f..111c849e 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -173,6 +173,7 @@ void Peer::attemptToContactAt(const RuntimeEnvironment *RR,const InetAddress &lo RR->identity.serialize(outp,false); atAddress.serialize(outp); outp.armor(_key,false); // HELLO is sent in the clear + RR->antiRec->logOutgoingZT(outp.data(),outp.size()); RR->node->putPacket(localAddr,atAddress,outp.data(),outp.size()); } @@ -182,12 +183,12 @@ void Peer::doPingAndKeepalive(const RuntimeEnvironment *RR,uint64_t now) RemotePath *const bestPath = _getBestPath(now); if (bestPath) { if ((now - bestPath->lastReceived()) >= ZT_PEER_DIRECT_PING_DELAY) { - TRACE("PING %s(%s)",_id.address().toString().c_str(),bestPath->address().toString().c_str()); + TRACE("PING %s(%s) after %llums/%llums send/receive inactivity",_id.address().toString().c_str(),bestPath->address().toString().c_str(),now - bestPath->lastSend(),now - bestPath->lastReceived()); attemptToContactAt(RR,bestPath->localAddress(),bestPath->address(),now); bestPath->sent(now); } else if (((now - bestPath->lastSend()) >= ZT_NAT_KEEPALIVE_DELAY)&&(!bestPath->reliable())) { + TRACE("NAT keepalive %s(%s) after %llums/%llums send/receive inactivity",_id.address().toString().c_str(),bestPath->address().toString().c_str(),now - bestPath->lastSend(),now - bestPath->lastReceived()); _natKeepaliveBuf += (uint32_t)((now * 0x9e3779b1) >> 1); // tumble this around to send constantly varying (meaningless) payloads - TRACE("NAT keepalive %s(%s)",_id.address().toString().c_str(),bestPath->address().toString().c_str()); RR->node->putPacket(bestPath->localAddress(),bestPath->address(),&_natKeepaliveBuf,sizeof(_natKeepaliveBuf)); bestPath->sent(now); } @@ -202,6 +203,8 @@ void Peer::pushDirectPaths(const RuntimeEnvironment *RR,RemotePath *path,uint64_ _lastDirectPathPush = now; std::vector dps(RR->node->directPaths()); + if (dps.empty()) + return; #ifdef ZT_TRACE { -- cgit v1.2.3 From 619e1137480de4682bb46eabaee3ce750c5be3e8 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 14 Oct 2015 14:12:12 -0700 Subject: Work in progress on Cluster for new root infrastructure, multi-homing. --- make-mac.mk | 2 +- node/Cluster.cpp | 398 +++++++++++++++++++++++++++++++++++++++++++++++++++++ node/Cluster.hpp | 331 ++++++++++++++++++++++++++++++++++++++++++++ node/Constants.hpp | 2 +- node/Identity.hpp | 19 ++- node/Packet.hpp | 4 + node/Peer.hpp | 3 +- node/Topology.cpp | 31 +++-- node/Topology.hpp | 19 ++- objects.mk | 1 + 10 files changed, 794 insertions(+), 16 deletions(-) create mode 100644 node/Cluster.cpp create mode 100644 node/Cluster.hpp (limited to 'node/Constants.hpp') diff --git a/make-mac.mk b/make-mac.mk index 174216fb..e53212c0 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -6,7 +6,7 @@ ifeq ($(origin CXX),default) endif INCLUDES= -DEFS= +DEFS=-DZT_ENABLE_CLUSTER LIBS= ARCH_FLAGS=-arch x86_64 diff --git a/node/Cluster.cpp b/node/Cluster.cpp new file mode 100644 index 00000000..98b45265 --- /dev/null +++ b/node/Cluster.cpp @@ -0,0 +1,398 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2015 ZeroTier, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * -- + * + * ZeroTier may be used and distributed under the terms of the GPLv3, which + * are available at: http://www.gnu.org/licenses/gpl-3.0.html + * + * If you would like to embed ZeroTier into a commercial application or + * redistribute it in a modified binary form, please contact ZeroTier Networks + * LLC. Start here: http://www.zerotier.com/ + */ + +#ifdef ZT_ENABLE_CLUSTER + +#include +#include +#include +#include + +#include +#include + +#include "Cluster.hpp" +#include "RuntimeEnvironment.hpp" +#include "MulticastGroup.hpp" +#include "CertificateOfMembership.hpp" +#include "Salsa20.hpp" +#include "Poly1305.hpp" +#include "Packet.hpp" +#include "Peer.hpp" +#include "Switch.hpp" +#include "Node.hpp" + +namespace ZeroTier { + +Cluster::Cluster(const RuntimeEnvironment *renv,uint16_t id,DistanceAlgorithm da,int32_t x,int32_t y,int32_t z,void (*sendFunction)(void *,uint16_t,const void *,unsigned int),void *arg) : + RR(renv), + _sendFunction(sendFunction), + _arg(arg), + _x(x), + _y(y), + _z(z), + _da(da), + _id(id) +{ + uint16_t stmp[ZT_SHA512_DIGEST_LEN / sizeof(uint16_t)]; + + // Generate master secret by hashing the secret from our Identity key pair + RR->identity.sha512PrivateKey(_masterSecret); + + // Generate our inbound message key, which is the master secret XORed with our ID and hashed twice + memcpy(stmp,_masterSecret,sizeof(stmp)); + stmp[0] ^= Utils::hton(id); + SHA512::hash(stmp,stmp,sizeof(stmp)); + SHA512::hash(stmp,stmp,sizeof(stmp)); + memcpy(_key,stmp,sizeof(_key)); + Utils::burn(stmp,sizeof(stmp)); +} + +Cluster::~Cluster() +{ + Utils::burn(_masterSecret,sizeof(_masterSecret)); + Utils::burn(_key,sizeof(_key)); +} + +void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len) +{ + Buffer dmsg; + { + // FORMAT: <[16] iv><[8] MAC><... data> + if ((len < 24)||(len > ZT_CLUSTER_MAX_MESSAGE_LENGTH)) + return; + + // 16-byte IV: first 8 bytes XORed with key, last 8 bytes used as Salsa20 64-bit IV + char keytmp[32]; + memcpy(keytmp,_key,32); + for(int i=0;i<8;++i) + keytmp[i] ^= reinterpret_cast(msg)[i]; + Salsa20 s20(keytmp,256,reinterpret_cast(msg) + 8); + Utils::burn(keytmp,sizeof(keytmp)); + + // One-time-use Poly1305 key from first 32 bytes of Salsa20 keystream (as per DJB/NaCl "standard") + char polykey[ZT_POLY1305_KEY_LEN]; + memset(polykey,0,sizeof(polykey)); + s20.encrypt12(polykey,polykey,sizeof(polykey)); + + // Compute 16-byte MAC + char mac[ZT_POLY1305_MAC_LEN]; + Poly1305::compute(mac,reinterpret_cast(msg) + 24,len - 24,polykey); + + // Check first 8 bytes of MAC against 64-bit MAC in stream + if (!Utils::secureEq(mac,reinterpret_cast(msg) + 16,8)) + return; + + // Decrypt! + dmsg.setSize(len - 16); + s20.decrypt12(reinterpret_cast(msg) + 16,const_cast(dmsg.data()),dmsg.size()); + } + + if (dmsg.size() < 2) + return; + const uint16_t fromMemberId = dmsg.at(0); + unsigned int ptr = 2; + + _Member &m = _members[fromMemberId]; + Mutex::Lock mlck(m.lock); + + m.lastReceivedFrom = RR->node->now(); + + try { + while (ptr < dmsg.size()) { + const unsigned int mlen = dmsg.at(ptr); ptr += 2; + const unsigned int nextPtr = ptr + mlen; + + int mtype = -1; + try { + switch((StateMessageType)(mtype = (int)dmsg[ptr++])) { + default: + break; + + case STATE_MESSAGE_ALIVE: { + ptr += 7; // skip version stuff, not used yet + m.x = dmsg.at(ptr); ptr += 4; + m.y = dmsg.at(ptr); ptr += 4; + m.z = dmsg.at(ptr); ptr += 4; + ptr += 8; // skip local clock, not used + m.load = dmsg.at(ptr); ptr += 8; + ptr += 8; // skip flags, unused + m.physicalAddressCount = dmsg[ptr++]; + if (m.physicalAddressCount > ZT_CLUSTER_MEMBER_MAX_PHYSICAL_ADDRS) + m.physicalAddressCount = ZT_CLUSTER_MEMBER_MAX_PHYSICAL_ADDRS; + for(unsigned int i=0;inode->now(); + } break; + + case STATE_MESSAGE_HAVE_PEER: { + try { + Identity id; + ptr += id.deserialize(dmsg,ptr); + RR->topology->saveIdentity(id); + + { // Add or update peer affinity entry + _PeerAffinity pa(id.address(),fromMemberId,RR->node->now()); + Mutex::Lock _l2(_peerAffinities_m); + std::vector<_PeerAffinity>::iterator i(std::lower_bound(_peerAffinities.begin(),_peerAffinities.end(),pa)); // O(log(n)) + if ((i != _peerAffinities.end())&&(i->key == pa.key)) { + i->timestamp = pa.timestamp; + } else { + _peerAffinities.push_back(pa); + std::sort(_peerAffinities.begin(),_peerAffinities.end()); // probably a more efficient way to insert but okay for now + } + } + } catch ( ... ) { + // ignore invalid identities + } + } break; + + case STATE_MESSAGE_MULTICAST_LIKE: { + const uint64_t nwid = dmsg.at(ptr); ptr += 8; + const Address address(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); ptr += ZT_ADDRESS_LENGTH; + const MAC mac(dmsg.field(ptr,6),6); ptr += 6; + const uint32_t adi = dmsg.at(ptr); ptr += 4; + RR->mc->add(RR->node->now(),nwid,MulticastGroup(mac,adi),address); + } break; + + case STATE_MESSAGE_COM: { + // TODO: not used yet + } break; + + case STATE_MESSAGE_RELAY: { + const unsigned int numRemotePeerPaths = dmsg[ptr++]; + InetAddress remotePeerPaths[256]; // size is 8-bit, so 256 is max + for(unsigned int i=0;i(ptr); ptr += 2; + const void *packet = (const void *)dmsg.field(ptr,packetLen); ptr += packetLen; + + if (packetLen >= ZT_PROTO_MIN_FRAGMENT_LENGTH) { // ignore anything too short to contain a dest address + const Address destinationAddress(reinterpret_cast(packet) + 8,ZT_ADDRESS_LENGTH); + SharedPtr destinationPeer(RR->topology->getPeer(destinationAddress)); + if (destinationPeer) { + RemotePath *destinationPath = destinationPeer->send(RR,packet,packetLen,RR->node->now()); + if ((destinationPath)&&(numRemotePeerPaths > 0)&&(packetLen >= 18)&&(reinterpret_cast(packet)[ZT_PACKET_FRAGMENT_IDX_FRAGMENT_INDICATOR] == ZT_PACKET_FRAGMENT_INDICATOR)) { + // If remote peer paths were sent with this relayed packet, we do + // RENDEZVOUS. It's handled here for cluster-relayed packets since + // we don't have both Peer records so this is a different path. + + const Address remotePeerAddress(reinterpret_cast(packet) + 13,ZT_ADDRESS_LENGTH); + + InetAddress bestDestV4,bestDestV6; + destinationPeer->getBestActiveAddresses(RR->node->now(),bestDestV4,bestDestV6); + InetAddress bestRemoteV4,bestRemoteV6; + for(unsigned int i=0;iidentity.address(),Packet::VERB_RENDEZVOUS); + rendezvousForDest.append((uint8_t)0); + remotePeerAddress.appendTo(rendezvousForDest); + + Buffer<2048> rendezvousForOtherEnd; + rendezvousForOtherEnd.addSize(2); // leave room for payload size + rendezvousForOtherEnd.append((uint8_t)STATE_MESSAGE_PROXY_SEND); + remotePeerAddress.appendTo(rendezvousForOtherEnd); + rendezvousForOtherEnd.append((uint8_t)Packet::VERB_RENDEZVOUS); + const unsigned int rendezvousForOtherEndPayloadSizePtr = rendezvousForOtherEnd.size(); + rendezvousForOtherEnd.addSize(2); // space for actual packet payload length + rendezvousForOtherEnd.append((uint8_t)0); // flags == 0 + destinationAddress.appendTo(rendezvousForOtherEnd); + + bool haveMatch = false; + if ((bestDestV6)&&(bestRemoteV6)) { + haveMatch = true; + + rendezvousForDest.append((uint16_t)bestRemoteV6.port()); + rendezvousForDest.append((uint8_t)16); + rendezvousForDest.append(bestRemoteV6.rawIpData(),16); + + rendezvousForOtherEnd.append((uint16_t)bestDestV6.port()); + rendezvousForOtherEnd.append((uint8_t)16); + rendezvousForOtherEnd.append(bestDestV6.rawIpData(),16); + rendezvousForOtherEnd.setAt(rendezvousForOtherEndPayloadSizePtr,(uint16_t)(9 + 16)); + } else if ((bestDestV4)&&(bestRemoteV4)) { + haveMatch = true; + + rendezvousForDest.append((uint16_t)bestRemoteV4.port()); + rendezvousForDest.append((uint8_t)4); + rendezvousForDest.append(bestRemoteV4.rawIpData(),4); + + rendezvousForOtherEnd.append((uint16_t)bestDestV4.port()); + rendezvousForOtherEnd.append((uint8_t)4); + rendezvousForOtherEnd.append(bestDestV4.rawIpData(),4); + rendezvousForOtherEnd.setAt(rendezvousForOtherEndPayloadSizePtr,(uint16_t)(9 + 4)); + } + + if (haveMatch) { + RR->sw->send(rendezvousForDest,true,0); + rendezvousForOtherEnd.setAt(0,(uint16_t)(rendezvousForOtherEnd.size() - 2)); + _send(fromMemberId,rendezvousForOtherEnd.data(),rendezvousForOtherEnd.size()); + } + } + } + } + } break; + + case STATE_MESSAGE_PROXY_SEND: { + const Address rcpt(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); + const Packet::Verb verb = (Packet::Verb)dmsg[ptr++]; + const unsigned int len = dmsg.at(ptr); ptr += 2; + Packet outp(rcpt,RR->identity.address(),verb); + outp.append(dmsg.field(ptr,len),len); + RR->sw->send(outp,true,0); + } break; + } + } catch ( ... ) { + TRACE("invalid message of size %u type %d (inner decode), discarding",mlen,mtype); + // drop invalids + } + + ptr = nextPtr; + } + } catch ( ... ) { + TRACE("invalid message (outer loop), discarding"); + // drop invalids + } +} + +void Cluster::replicateHavePeer(const Address &peerAddress) +{ +} + +void Cluster::replicateMulticastLike(uint64_t nwid,const Address &peerAddress,const MulticastGroup &group) +{ +} + +void Cluster::replicateCertificateOfNetworkMembership(const CertificateOfMembership &com) +{ +} + +void Cluster::doPeriodicTasks() +{ + // Go ahead and flush whenever possible right now + { + Mutex::Lock _l(_memberIds_m); + for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { + Mutex::Lock _l2(_members[*mid].lock); + _flush(*mid); + } + } +} + +void Cluster::addMember(uint16_t memberId) +{ + Mutex::Lock _l2(_members[memberId].lock); + + Mutex::Lock _l(_memberIds_m); + _memberIds.push_back(memberId); + std::sort(_memberIds.begin(),_memberIds.end()); + + // Generate this member's message key from the master and its ID + uint16_t stmp[ZT_SHA512_DIGEST_LEN / sizeof(uint16_t)]; + memcpy(stmp,_masterSecret,sizeof(stmp)); + stmp[0] ^= Utils::hton(memberId); + SHA512::hash(stmp,stmp,sizeof(stmp)); + SHA512::hash(stmp,stmp,sizeof(stmp)); + memcpy(_members[memberId].key,stmp,sizeof(_members[memberId].key)); + Utils::burn(stmp,sizeof(stmp)); + + // Prepare q + _members[memberId].q.clear(); + char iv[16]; + Utils::getSecureRandom(iv,16); + _members[memberId].q.append(iv,16); + _members[memberId].q.addSize(8); // room for MAC +} + +void Cluster::_send(uint16_t memberId,const void *msg,unsigned int len) +{ + _Member &m = _members[memberId]; + // assumes m.lock is locked! + for(;;) { + if ((m.q.size() + len) > ZT_CLUSTER_MAX_MESSAGE_LENGTH) + _flush(memberId); + else { + m.q.append(msg,len); + break; + } + } +} + +void Cluster::_flush(uint16_t memberId) +{ + _Member &m = _members[memberId]; + // assumes m.lock is locked! + if (m.q.size() > 24) { + // Create key from member's key and IV + char keytmp[32]; + memcpy(keytmp,m.key,32); + for(int i=0;i<8;++i) + keytmp[i] ^= m.q[i]; + Salsa20 s20(keytmp,256,m.q.field(8,8)); + Utils::burn(keytmp,sizeof(keytmp)); + + // One-time-use Poly1305 key from first 32 bytes of Salsa20 keystream (as per DJB/NaCl "standard") + char polykey[ZT_POLY1305_KEY_LEN]; + memset(polykey,0,sizeof(polykey)); + s20.encrypt12(polykey,polykey,sizeof(polykey)); + + // Encrypt m.q in place + s20.encrypt12(reinterpret_cast(m.q.data()) + 24,const_cast(reinterpret_cast(m.q.data())) + 24,m.q.size() - 24); + + // Add MAC for authentication (encrypt-then-MAC) + char mac[ZT_POLY1305_MAC_LEN]; + Poly1305::compute(mac,reinterpret_cast(m.q.data()) + 24,m.q.size() - 24,polykey); + memcpy(m.q.field(16,8),mac,8); + + // Send! + _sendFunction(_arg,memberId,m.q.data(),m.q.size()); + + // Prepare for more + m.q.clear(); + char iv[16]; + Utils::getSecureRandom(iv,16); + m.q.append(iv,16); + m.q.addSize(8); // room for MAC + } +} + +} // namespace ZeroTier + +#endif // ZT_ENABLE_CLUSTER diff --git a/node/Cluster.hpp b/node/Cluster.hpp new file mode 100644 index 00000000..13d9e2f5 --- /dev/null +++ b/node/Cluster.hpp @@ -0,0 +1,331 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2015 ZeroTier, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * -- + * + * ZeroTier may be used and distributed under the terms of the GPLv3, which + * are available at: http://www.gnu.org/licenses/gpl-3.0.html + * + * If you would like to embed ZeroTier into a commercial application or + * redistribute it in a modified binary form, please contact ZeroTier Networks + * LLC. Start here: http://www.zerotier.com/ + */ + +#ifndef ZT_CLUSTER_HPP +#define ZT_CLUSTER_HPP + +#ifdef ZT_ENABLE_CLUSTER + +#include +#include + +#include "Constants.hpp" +#include "Address.hpp" +#include "InetAddress.hpp" +#include "SHA512.hpp" +#include "Utils.hpp" +#include "Buffer.hpp" +#include "Mutex.hpp" + +/** + * Timeout for cluster members being considered "alive" + */ +#define ZT_CLUSTER_TIMEOUT ZT_PEER_ACTIVITY_TIMEOUT + +/** + * Maximum cluster message length in bytes + * + * Cluster nodes speak via TCP, with data encapsulated into individually + * encrypted and authenticated messages. The maximum message size is + * 65535 (0xffff) since the TCP stream uses 16-bit message size headers + * (and this is a reasonable chunk size anyway). + */ +#define ZT_CLUSTER_MAX_MESSAGE_LENGTH 65535 + +/** + * Maximum number of physical addresses we will cache for a cluster member + */ +#define ZT_CLUSTER_MEMBER_MAX_PHYSICAL_ADDRS 8 + +namespace ZeroTier { + +class RuntimeEnvironment; +class CertificateOfMembership; +class MulticastGroup; + +/** + * Multi-homing cluster state replication and packet relaying + * + * Multi-homing means more than one node sharing the same ZeroTier identity. + * There is nothing in the protocol to prevent this, but to make it work well + * requires the devices sharing an identity to cooperate and share some + * information. + * + * There are three use cases we want to fulfill: + * + * (1) Multi-homing of root servers with handoff for efficient routing, + * HA, and load balancing across many commodity nodes. + * (2) Multi-homing of network controllers for the same reason. + * (3) Multi-homing of nodes on virtual networks, such as domain servers + * and other important endpoints. + * + * These use cases are in order of escalating difficulty. The initial + * version of Cluster is aimed at satisfying the first, though you are + * free to try #2 and #3. + */ +class Cluster +{ +public: + /** + * Which distance algorithm is this cluster using? + */ + enum DistanceAlgorithm + { + /** + * Simple linear distance in three dimensions + */ + DISTANCE_SIMPLE = 0, + + /** + * Haversine formula using X,Y as lat,long and ignoring Z + */ + DISTANCE_HAVERSINE = 1 + }; + + /** + * State message types + */ + enum StateMessageType + { + STATE_MESSAGE_NOP = 0, + + /** + * This cluster member is alive: + * <[2] version minor> + * <[2] version major> + * <[2] version revision> + * <[1] protocol version> + * <[4] X location (signed 32-bit)> + * <[4] Y location (signed 32-bit)> + * <[4] Z location (signed 32-bit)> + * <[8] local clock at this member> + * <[8] load average> + * <[8] flags (currently unused, must be zero)> + * <[1] number of preferred ZeroTier endpoints> + * <[...] InetAddress(es) of preferred ZeroTier endpoint(s)> + */ + STATE_MESSAGE_ALIVE = 1, + + /** + * Cluster member has this peer: + * <[...] binary serialized peer identity> + */ + STATE_MESSAGE_HAVE_PEER = 2, + + /** + * Peer subscription to multicast group: + * <[8] network ID> + * <[5] peer ZeroTier address> + * <[6] MAC address of multicast group> + * <[4] 32-bit multicast group ADI> + */ + STATE_MESSAGE_MULTICAST_LIKE = 3, + + /** + * Certificate of network membership for a peer: + * <[...] serialized COM> + */ + STATE_MESSAGE_COM = 4, + + /** + * Relay a packet to a peer: + * <[1] 8-bit number of sending peer active path addresses> + * <[...] series of serialized InetAddresses of sending peer's paths> + * <[2] 16-bit packet length> + * <[...] packet or packet fragment> + */ + STATE_MESSAGE_RELAY = 5, + + /** + * Request to send a packet to a locally-known peer: + * <[5] ZeroTier address of recipient> + * <[1] packet verb> + * <[2] length of packet payload> + * <[...] packet payload> + * + * This differs from RELAY in that it requests the receiving cluster + * member to actually compose a ZeroTier Packet from itself to the + * provided recipient. RELAY simply says "please forward this blob." + * RELAY is used to implement peer-to-peer relaying with RENDEZVOUS, + * while PROXY_SEND is used to implement proxy sending (which right + * now is only used to send RENDEZVOUS). + */ + STATE_MESSAGE_PROXY_SEND = 6 + }; + + /** + * Construct a new cluster + * + * @param renv Runtime environment + * @param id This member's ID in the cluster + * @param da Distance algorithm this cluster uses to compute distance and hand off peers + * @param x My X + * @param y My Y + * @param z My Z + * @param sendFunction Function to call to send messages to other cluster members + * @param arg First argument to sendFunction + */ + Cluster( + const RuntimeEnvironment *renv, + uint16_t id, + DistanceAlgorithm da, + int32_t x, + int32_t y, + int32_t z, + void (*sendFunction)(void *,uint16_t,const void *,unsigned int), + void *arg); + + ~Cluster(); + + /** + * @return This cluster member's ID + */ + inline uint16_t id() const throw() { return _id; } + + /** + * Handle an incoming intra-cluster message + * + * @param data Message data + * @param len Message length (max: ZT_CLUSTER_MAX_MESSAGE_LENGTH) + */ + void handleIncomingStateMessage(const void *msg,unsigned int len); + + /** + * Advertise to the cluster that we have this peer + * + * @param peerAddress Peer address that we have + */ + void replicateHavePeer(const Address &peerAddress); + + /** + * Advertise a multicast LIKE to the cluster + * + * @param nwid Network ID + * @param peerAddress Peer address that sent LIKE + * @param group Multicast group + */ + void replicateMulticastLike(uint64_t nwid,const Address &peerAddress,const MulticastGroup &group); + + /** + * Advertise a network COM to the cluster + * + * @param com Certificate of network membership (contains peer and network ID) + */ + void replicateCertificateOfNetworkMembership(const CertificateOfMembership &com); + + /** + * This should be called no less frequently than once every 10 seconds. + */ + void doPeriodicTasks(); + + /** + * Add a member ID to this cluster + * + * @param memberId Member ID + */ + void addMember(uint16_t memberId); + +private: + void _send(uint16_t memberId,const void *msg,unsigned int len); + void _flush(uint16_t memberId); + + // These are initialized in the constructor and remain static + uint16_t _masterSecret[ZT_SHA512_DIGEST_LEN / sizeof(uint16_t)]; + unsigned char _key[ZT_PEER_SECRET_KEY_LENGTH]; + const RuntimeEnvironment *RR; + void (*_sendFunction)(void *,uint16_t,const void *,unsigned int); + void *_arg; + const int32_t _x; + const int32_t _y; + const int32_t _z; + const DistanceAlgorithm _da; + const uint16_t _id; + + struct _Member + { + unsigned char key[ZT_PEER_SECRET_KEY_LENGTH]; + + uint64_t lastReceivedFrom; + uint64_t lastReceivedAliveAnnouncement; + uint64_t lastSentTo; + uint64_t lastAnnouncedAliveTo; + + uint64_t load; + int32_t x,y,z; + + InetAddress physicalAddresses[ZT_CLUSTER_MEMBER_MAX_PHYSICAL_ADDRS]; + unsigned int physicalAddressCount; + + Buffer q; + + Mutex lock; + + _Member() : + lastReceivedFrom(0), + lastReceivedAliveAnnouncement(0), + lastSentTo(0), + lastAnnouncedAliveTo(0), + load(0), + x(0), + y(0), + z(0), + physicalAddressCount(0) {} + + ~_Member() { Utils::burn(key,sizeof(key)); } + }; + + _Member _members[65536]; // cluster IDs can be from 0 to 65535 (16-bit) + + std::vector _memberIds; + Mutex _memberIds_m; + + // Record tracking which members have which peers and how recently they claimed this + struct _PeerAffinity + { + _PeerAffinity(const Address &a,uint16_t mid,uint64_t ts) : + key((a.toInt() << 16) | (uint64_t)mid), + timestamp(ts) {} + + uint64_t key; + uint64_t timestamp; + + inline Address address() const throw() { return Address(key >> 16); } + inline uint16_t clusterMemberId() const throw() { return (uint16_t)(key & 0xffff); } + + inline bool operator<(const _PeerAffinity &pi) const throw() { return (key < pi.key); } + }; + + // A memory-efficient packed map of _PeerAffinity records searchable with std::binary_search() and std::lower_bound() + std::vector<_PeerAffinity> _peerAffinities; + Mutex _peerAffinities_m; +}; + +} // namespace ZeroTier + +#endif // ZT_ENABLE_CLUSTER + +#endif diff --git a/node/Constants.hpp b/node/Constants.hpp index 9ab390eb..afcb4e30 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -182,7 +182,7 @@ #define ZT_FRAGMENTED_PACKET_RECEIVE_TIMEOUT 1000 /** - * Length of secret key in bytes -- 256-bit for Salsa20 + * Length of secret key in bytes -- 256-bit -- do not change */ #define ZT_PEER_SECRET_KEY_LENGTH 32 diff --git a/node/Identity.hpp b/node/Identity.hpp index 19bb2e1f..6c33e74f 100644 --- a/node/Identity.hpp +++ b/node/Identity.hpp @@ -38,6 +38,7 @@ #include "Address.hpp" #include "C25519.hpp" #include "Buffer.hpp" +#include "SHA512.hpp" namespace ZeroTier { @@ -91,8 +92,7 @@ public: } template - Identity(const Buffer &b,unsigned int startAt = 0) - throw(std::out_of_range,std::invalid_argument) : + Identity(const Buffer &b,unsigned int startAt = 0) : _privateKey((C25519::Private *)0) { deserialize(b,startAt); @@ -137,6 +137,21 @@ public: */ inline bool hasPrivate() const throw() { return (_privateKey != (C25519::Private *)0); } + /** + * Compute the SHA512 hash of our private key (if we have one) + * + * @param sha Buffer to receive SHA512 (MUST be ZT_SHA512_DIGEST_LEN (64) bytes in length) + * @return True on success, false if no private key + */ + inline bool sha512PrivateKey(void *sha) const + { + if (_privateKey) { + SHA512::hash(sha,_privateKey->data,ZT_C25519_PRIVATE_KEY_LEN); + return true; + } + return false; + } + /** * Sign a message with this identity (private key required) * diff --git a/node/Packet.hpp b/node/Packet.hpp index e03190a2..0e8426b6 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -623,6 +623,10 @@ public: * may also ignore these messages if a peer is not known or is not being * actively communicated with. * + * Unfortunately the physical address format in this message pre-dates + * InetAddress's serialization format. :( ZeroTier is four years old and + * yes we've accumulated a tiny bit of cruft here and there. + * * No OK or ERROR is generated. */ VERB_RENDEZVOUS = 5, diff --git a/node/Peer.hpp b/node/Peer.hpp index cabf0793..0139f386 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -560,7 +560,8 @@ private: void _sortPaths(const uint64_t now); RemotePath *_getBestPath(const uint64_t now); - unsigned char _key[ZT_PEER_SECRET_KEY_LENGTH]; + unsigned char _key[ZT_PEER_SECRET_KEY_LENGTH]; // computed with key agreement, not serialized + uint64_t _lastUsed; uint64_t _lastReceive; // direct or indirect uint64_t _lastUnicastFrame; diff --git a/node/Topology.cpp b/node/Topology.cpp index 8b5deffc..2b973386 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -136,7 +136,7 @@ SharedPtr Topology::addPeer(const SharedPtr &peer) SharedPtr &p = _peers.set(peer->address(),peer); p->use(now); - _saveIdentity(p->identity()); + saveIdentity(p->identity()); return p; } @@ -172,6 +172,26 @@ SharedPtr Topology::getPeer(const Address &zta) return SharedPtr(); } +Identity Topology::getIdentity(const Address &zta) +{ + { + Mutex::Lock _l(_lock); + SharedPtr &ap = _peers[zta]; + if (ap) + return ap->identity(); + } + return _getIdentity(zta); +} + +void saveIdentity(const Identity &id) +{ + if (id) { + char p[128]; + Utils::snprintf(p,sizeof(p),"iddb.d/%.10llx",(unsigned long long)id.address().toInt()); + RR->node->dataStorePut(p,id.toString(false),false); + } +} + SharedPtr Topology::getBestRoot(const Address *avoid,unsigned int avoidCount,bool strictAvoid) { SharedPtr bestRoot; @@ -315,15 +335,6 @@ Identity Topology::_getIdentity(const Address &zta) return Identity(); } -void Topology::_saveIdentity(const Identity &id) -{ - if (id) { - char p[128]; - Utils::snprintf(p,sizeof(p),"iddb.d/%.10llx",(unsigned long long)id.address().toInt()); - RR->node->dataStorePut(p,id.toString(false),false); - } -} - void Topology::_setWorld(const World &newWorld) { // assumed _lock is locked (or in constructor) diff --git a/node/Topology.hpp b/node/Topology.hpp index 6f0170f0..9e9ccc01 100644 --- a/node/Topology.hpp +++ b/node/Topology.hpp @@ -78,6 +78,24 @@ public: */ SharedPtr getPeer(const Address &zta); + /** + * Get the identity of a peer + * + * @param zta ZeroTier address of peer + * @return Identity or NULL Identity if not found + */ + Identity getIdentity(const Address &zta); + + /** + * Cache an identity + * + * This is done automatically on addPeer(), and so is only useful for + * cluster identity replication. + * + * @param id Identity to cache + */ + void saveIdentity(const Identity &id); + /** * @return Vector of peers that are root servers */ @@ -210,7 +228,6 @@ public: private: Identity _getIdentity(const Address &zta); - void _saveIdentity(const Identity &id); void _setWorld(const World &newWorld); const RuntimeEnvironment *RR; diff --git a/objects.mk b/objects.mk index 64e5cfa7..10e0c334 100644 --- a/objects.mk +++ b/objects.mk @@ -4,6 +4,7 @@ OBJS=\ ext/http-parser/http_parser.o \ node/C25519.o \ node/CertificateOfMembership.o \ + node/Cluster.o \ node/Dictionary.o \ node/Identity.o \ node/IncomingPacket.o \ -- cgit v1.2.3 From 2debde3451838f62ed9b53d9b0086f7112416636 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 15 Oct 2015 07:22:17 -0700 Subject: GitHub issue #235, and I also see no reason not to communicate with people from other Worlds. --- node/Constants.hpp | 5 +++++ node/IncomingPacket.cpp | 31 +++++++++++++++---------------- 2 files changed, 20 insertions(+), 16 deletions(-) (limited to 'node/Constants.hpp') diff --git a/node/Constants.hpp b/node/Constants.hpp index afcb4e30..a60b76eb 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -347,6 +347,11 @@ */ #define ZT_MAX_BRIDGE_SPAM 16 +/** + * Maximum number of endpoints to contact per address type (to limit pushes like GitHub issue #235) + */ +#define ZT_PUSH_DIRECT_PATHS_MAX_ENDPOINTS_PER_TYPE 8 + /** * A test pseudo-network-ID that can be joined * diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 944e3043..d444258d 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -296,22 +296,18 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR) outp.append((uint16_t)ZEROTIER_ONE_VERSION_REVISION); _remoteAddress.serialize(outp); - if ((worldId != ZT_WORLD_ID_NULL)&&(worldId == RR->topology->worldId())) { - if (RR->topology->worldTimestamp() > worldTimestamp) { - World w(RR->topology->world()); - const unsigned int sizeAt = outp.size(); - outp.addSize(2); // make room for 16-bit size field - w.serialize(outp,false); - outp.setAt(sizeAt,(uint16_t)(outp.size() - (sizeAt + 2))); - } else { - outp.append((uint16_t)0); // no world update needed - } - - outp.armor(peer->key(),true); - RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size()); + if ((worldId != ZT_WORLD_ID_NULL)&&(RR->topology->worldTimestamp() > worldTimestamp)&&(worldId == RR->topology->worldId())) { + World w(RR->topology->world()); + const unsigned int sizeAt = outp.size(); + outp.addSize(2); // make room for 16-bit size field + w.serialize(outp,false); + outp.setAt(sizeAt,(uint16_t)(outp.size() - (sizeAt + 2))); } else { - TRACE("dropped HELLO from %s(%s): world ID mismatch: peer is %llu and we are %llu",source().toString().c_str(),_remoteAddress.toString().c_str(),worldId,RR->topology->worldId()); + outp.append((uint16_t)0); // no world update needed } + + outp.armor(peer->key(),true); + RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size()); } catch ( ... ) { TRACE("dropped HELLO from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str()); } @@ -867,6 +863,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha try { unsigned int count = at(ZT_PACKET_IDX_PAYLOAD); unsigned int ptr = ZT_PACKET_IDX_PAYLOAD + 2; + unsigned int v4Count = 0,v6Count = 0; while (count--) { // if ptr overflows Buffer will throw // TODO: some flags are not yet implemented @@ -882,14 +879,16 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha InetAddress a(field(ptr,4),4,at(ptr + 4)); if ( ((flags & 0x01) == 0) && (Path::isAddressValidForPath(a)) ) { TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str()); - peer->attemptToContactAt(RR,_localAddress,a,RR->node->now()); + if (v4Count++ < ZT_PUSH_DIRECT_PATHS_MAX_ENDPOINTS_PER_TYPE) + peer->attemptToContactAt(RR,_localAddress,a,RR->node->now()); } } break; case 6: { InetAddress a(field(ptr,16),16,at(ptr + 16)); if ( ((flags & 0x01) == 0) && (Path::isAddressValidForPath(a)) ) { TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str()); - peer->attemptToContactAt(RR,_localAddress,a,RR->node->now()); + if (v6Count++ < ZT_PUSH_DIRECT_PATHS_MAX_ENDPOINTS_PER_TYPE) + peer->attemptToContactAt(RR,_localAddress,a,RR->node->now()); } } break; } -- cgit v1.2.3 From 5ce3aac929ef217f3e813b5bc948dd28d021835f Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 16 Oct 2015 10:28:09 -0700 Subject: Add rate limit on receive of DIRECT_PATH_PUSH to prevent DOS exploitation. --- node/Constants.hpp | 7 ++++++- node/IncomingPacket.cpp | 7 +++++++ node/Peer.cpp | 7 ++++--- node/Peer.hpp | 32 ++++++++++++++++++++++---------- 4 files changed, 39 insertions(+), 14 deletions(-) (limited to 'node/Constants.hpp') diff --git a/node/Constants.hpp b/node/Constants.hpp index a60b76eb..e45602f7 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -322,7 +322,12 @@ /** * Interval between direct path pushes in milliseconds */ -#define ZT_DIRECT_PATH_PUSH_INTERVAL 300000 +#define ZT_DIRECT_PATH_PUSH_INTERVAL 120000 + +/** + * Minimum interval between direct path pushes from a given peer or we will ignore them + */ +#define ZT_DIRECT_PATH_PUSH_MIN_RECEIVE_INTERVAL 2500 /** * How long (max) to remember network certificates of membership? diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index d444258d..4386e370 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -861,6 +861,13 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const SharedPtr &peer) { try { + const uint64_t now = RR->node->now(); + if ((now - peer->lastDirectPathPushReceived()) >= ZT_DIRECT_PATH_PUSH_MIN_RECEIVE_INTERVAL) { + TRACE("dropped PUSH_DIRECT_PATHS from %s(%s): too frequent!",source().toString().c_str(),_remoteAddress.toString().c_str()); + return true; + } + peer->setLastDirectPathPushReceived(now); + unsigned int count = at(ZT_PACKET_IDX_PAYLOAD); unsigned int ptr = ZT_PACKET_IDX_PAYLOAD + 2; unsigned int v4Count = 0,v6Count = 0; diff --git a/node/Peer.cpp b/node/Peer.cpp index becc77f9..8c5c5783 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -52,7 +52,8 @@ Peer::Peer(const Identity &myIdentity,const Identity &peerIdentity) _lastMulticastFrame(0), _lastAnnouncedTo(0), _lastPathConfirmationSent(0), - _lastDirectPathPush(0), + _lastDirectPathPushSent(0), + _lastDirectPathPushReceived(0), _lastPathSort(0), _vMajor(0), _vMinor(0), @@ -210,8 +211,8 @@ void Peer::pushDirectPaths(const RuntimeEnvironment *RR,RemotePath *path,uint64_ { Mutex::Lock _l(_lock); - if (((now - _lastDirectPathPush) >= ZT_DIRECT_PATH_PUSH_INTERVAL)||(force)) { - _lastDirectPathPush = now; + if (((now - _lastDirectPathPushSent) >= ZT_DIRECT_PATH_PUSH_INTERVAL)||(force)) { + _lastDirectPathPushSent = now; std::vector dps(RR->node->directPaths()); if (dps.empty()) diff --git a/node/Peer.hpp b/node/Peer.hpp index 4fb399c5..043519d4 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -408,6 +408,16 @@ public: */ bool needsOurNetworkMembershipCertificate(uint64_t nwid,uint64_t now,bool updateLastPushedTime); + /** + * @return Time last direct path push was received + */ + inline uint64_t lastDirectPathPushReceived() const throw() { return _lastDirectPathPushReceived; } + + /** + * @param t New time of last direct path push received + */ + inline void setLastDirectPathPushReceived(uint64_t t) throw() { _lastDirectPathPushReceived = t; } + /** * Perform periodic cleaning operations */ @@ -438,10 +448,10 @@ public: { Mutex::Lock _l(_lock); - const unsigned int atPos = b.size(); + const unsigned int recSizePos = b.size(); b.addSize(4); // space for uint32_t field length - b.append((uint32_t)1); // version of serialized Peer data + b.append((uint16_t)1); // version of serialized Peer data _id.serialize(b,false); @@ -451,7 +461,8 @@ public: b.append((uint64_t)_lastMulticastFrame); b.append((uint64_t)_lastAnnouncedTo); b.append((uint64_t)_lastPathConfirmationSent); - b.append((uint64_t)_lastDirectPathPush); + b.append((uint64_t)_lastDirectPathPushSent); + b.append((uint64_t)_lastDirectPathPushReceived); b.append((uint64_t)_lastPathSort); b.append((uint16_t)_vProto); b.append((uint16_t)_vMajor); @@ -486,7 +497,7 @@ public: } } - b.setAt(atPos,(uint32_t)(b.size() - atPos)); // set size + b.setAt(recSizePos,(uint32_t)((b.size() - 4) - recSizePos)); // set size } /** @@ -500,13 +511,12 @@ public: template static inline SharedPtr deserializeNew(const Identity &myIdentity,const Buffer &b,unsigned int &p) { - const uint32_t recSize = b.template at(p); + const uint32_t recSize = b.template at(p); p += 4; if ((p + recSize) > b.size()) return SharedPtr(); // size invalid - p += 4; - if (b.template at(p) != 1) + if (b.template at(p) != 1) return SharedPtr(); // version mismatch - p += 4; + p += 2; Identity npid; p += npid.deserialize(b,p); @@ -521,7 +531,8 @@ public: np->_lastMulticastFrame = b.template at(p); p += 8; np->_lastAnnouncedTo = b.template at(p); p += 8; np->_lastPathConfirmationSent = b.template at(p); p += 8; - np->_lastDirectPathPush = b.template at(p); p += 8; + np->_lastDirectPathPushSent = b.template at(p); p += 8; + np->_lastDirectPathPushReceived = b.template at(p); p += 8; np->_lastPathSort = b.template at(p); p += 8; np->_vProto = b.template at(p); p += 2; np->_vMajor = b.template at(p); p += 2; @@ -570,7 +581,8 @@ private: uint64_t _lastMulticastFrame; uint64_t _lastAnnouncedTo; uint64_t _lastPathConfirmationSent; - uint64_t _lastDirectPathPush; + uint64_t _lastDirectPathPushSent; + uint64_t _lastDirectPathPushReceived; uint64_t _lastPathSort; uint16_t _vProto; uint16_t _vMajor; -- cgit v1.2.3 From 62db18b6dd0002520d4828b0091995a40b4eb2e9 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 27 Oct 2015 11:01:58 -0700 Subject: Lessen this limit just a bit to make cluster settle faster. --- node/Constants.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'node/Constants.hpp') diff --git a/node/Constants.hpp b/node/Constants.hpp index e45602f7..3ad07e4c 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -327,7 +327,7 @@ /** * Minimum interval between direct path pushes from a given peer or we will ignore them */ -#define ZT_DIRECT_PATH_PUSH_MIN_RECEIVE_INTERVAL 2500 +#define ZT_DIRECT_PATH_PUSH_MIN_RECEIVE_INTERVAL 2000 /** * How long (max) to remember network certificates of membership? -- cgit v1.2.3 From a1a0ee4edb0933c9b82abad8715def6a63049658 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 27 Oct 2015 12:01:00 -0700 Subject: Fix infinite loop in Cluster, clean up some stuff elsewhere, and back out rate limiting in PUSH_DIRECT_PATHS for now (but we will do something else to mitigate amplification attacks) --- node/Cluster.cpp | 5 ++-- node/Constants.hpp | 7 +----- node/IncomingPacket.cpp | 12 +++------- node/Peer.cpp | 1 - node/Peer.hpp | 17 ++------------ node/SelfAwareness.cpp | 12 +++++----- node/Topology.cpp | 62 +------------------------------------------------ 7 files changed, 16 insertions(+), 100 deletions(-) (limited to 'node/Constants.hpp') diff --git a/node/Cluster.cpp b/node/Cluster.cpp index bd455933..eef02bc7 100644 --- a/node/Cluster.cpp +++ b/node/Cluster.cpp @@ -239,7 +239,7 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len) const MAC mac(dmsg.field(ptr,6),6); ptr += 6; const uint32_t adi = dmsg.at(ptr); ptr += 4; RR->mc->add(RR->node->now(),nwid,MulticastGroup(mac,adi),address); - TRACE("[%u] %s likes %s/%.8x on %.16llu",(unsigned int)fromMemberId,address.toString().c_str(),mac.toString().c_str(),(unsigned int)adi,nwid); + TRACE("[%u] %s likes %s/%.8x on %.16llx",(unsigned int)fromMemberId,address.toString().c_str(),mac.toString().c_str(),(unsigned int)adi,nwid); } break; case STATE_MESSAGE_COM: { @@ -376,11 +376,12 @@ bool Cluster::sendViaCluster(const Address &fromPeerAddress,const Address &toPee Mutex::Lock _l2(_peerAffinities_m); std::vector<_PeerAffinity>::iterator i(std::lower_bound(_peerAffinities.begin(),_peerAffinities.end(),_PeerAffinity(toPeerAddress,0,0))); // O(log(n)) while ((i != _peerAffinities.end())&&(i->address() == toPeerAddress)) { - uint16_t mid = i->clusterMemberId(); + const uint16_t mid = i->clusterMemberId(); if ((mid != _id)&&(i->timestamp > mostRecentTimestamp)) { mostRecentTimestamp = i->timestamp; canHasPeer = mid; } + ++i; } } diff --git a/node/Constants.hpp b/node/Constants.hpp index 3ad07e4c..bef1183a 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -324,11 +324,6 @@ */ #define ZT_DIRECT_PATH_PUSH_INTERVAL 120000 -/** - * Minimum interval between direct path pushes from a given peer or we will ignore them - */ -#define ZT_DIRECT_PATH_PUSH_MIN_RECEIVE_INTERVAL 2000 - /** * How long (max) to remember network certificates of membership? * @@ -355,7 +350,7 @@ /** * Maximum number of endpoints to contact per address type (to limit pushes like GitHub issue #235) */ -#define ZT_PUSH_DIRECT_PATHS_MAX_ENDPOINTS_PER_TYPE 8 +#define ZT_PUSH_DIRECT_PATHS_MAX_ENDPOINTS_PER_TYPE 2 /** * A test pseudo-network-ID that can be joined diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 51e88364..2514cd64 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -622,7 +622,7 @@ bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,const Shared // Iterate through 18-byte network,MAC,ADI tuples for(unsigned int ptr=ZT_PACKET_IDX_PAYLOAD;ptr(ptr)); + const uint64_t nwid = at(ptr); const MulticastGroup group(MAC(field(ptr + 8,6),6),at(ptr + 14)); RR->mc->add(now,nwid,group,peer->address()); #ifdef ZT_ENABLE_CLUSTER @@ -888,12 +888,6 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha { try { const uint64_t now = RR->node->now(); - if ((now - peer->lastDirectPathPushReceived()) < ZT_DIRECT_PATH_PUSH_MIN_RECEIVE_INTERVAL) { - TRACE("dropped PUSH_DIRECT_PATHS from %s(%s): too frequent!",source().toString().c_str(),_remoteAddress.toString().c_str()); - return true; - } - peer->setLastDirectPathPushReceived(now); - const RemotePath *currentBest = peer->getBestPath(now); unsigned int count = at(ZT_PACKET_IDX_PAYLOAD); @@ -916,7 +910,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str()); if (v4Count++ < ZT_PUSH_DIRECT_PATHS_MAX_ENDPOINTS_PER_TYPE) { if ((!currentBest)||(currentBest->address() != a)) - peer->attemptToContactAt(RR,_localAddress,a,RR->node->now()); + peer->attemptToContactAt(RR,_localAddress,a,now); } } } break; @@ -926,7 +920,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str()); if (v6Count++ < ZT_PUSH_DIRECT_PATHS_MAX_ENDPOINTS_PER_TYPE) { if ((!currentBest)||(currentBest->address() != a)) - peer->attemptToContactAt(RR,_localAddress,a,RR->node->now()); + peer->attemptToContactAt(RR,_localAddress,a,now); } } } break; diff --git a/node/Peer.cpp b/node/Peer.cpp index f16f1421..d5367b17 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -55,7 +55,6 @@ Peer::Peer(const Identity &myIdentity,const Identity &peerIdentity) _lastAnnouncedTo(0), _lastPathConfirmationSent(0), _lastDirectPathPushSent(0), - _lastDirectPathPushReceived(0), _lastPathSort(0), _vProto(0), _vMajor(0), diff --git a/node/Peer.hpp b/node/Peer.hpp index 0aca70b4..04b541af 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -407,16 +407,6 @@ public: */ bool needsOurNetworkMembershipCertificate(uint64_t nwid,uint64_t now,bool updateLastPushedTime); - /** - * @return Time last direct path push was received - */ - inline uint64_t lastDirectPathPushReceived() const throw() { return _lastDirectPathPushReceived; } - - /** - * @param t New time of last direct path push received - */ - inline void setLastDirectPathPushReceived(uint64_t t) throw() { _lastDirectPathPushReceived = t; } - /** * Perform periodic cleaning operations */ @@ -450,7 +440,7 @@ public: const unsigned int recSizePos = b.size(); b.addSize(4); // space for uint32_t field length - b.append((uint16_t)1); // version of serialized Peer data + b.append((uint16_t)0); // version of serialized Peer data _id.serialize(b,false); @@ -461,7 +451,6 @@ public: b.append((uint64_t)_lastAnnouncedTo); b.append((uint64_t)_lastPathConfirmationSent); b.append((uint64_t)_lastDirectPathPushSent); - b.append((uint64_t)_lastDirectPathPushReceived); b.append((uint64_t)_lastPathSort); b.append((uint16_t)_vProto); b.append((uint16_t)_vMajor); @@ -513,7 +502,7 @@ public: const unsigned int recSize = b.template at(p); p += 4; if ((p + recSize) > b.size()) return SharedPtr(); // size invalid - if (b.template at(p) != 1) + if (b.template at(p) != 0) return SharedPtr(); // version mismatch p += 2; @@ -531,7 +520,6 @@ public: np->_lastAnnouncedTo = b.template at(p); p += 8; np->_lastPathConfirmationSent = b.template at(p); p += 8; np->_lastDirectPathPushSent = b.template at(p); p += 8; - np->_lastDirectPathPushReceived = b.template at(p); p += 8; np->_lastPathSort = b.template at(p); p += 8; np->_vProto = b.template at(p); p += 2; np->_vMajor = b.template at(p); p += 2; @@ -581,7 +569,6 @@ private: uint64_t _lastAnnouncedTo; uint64_t _lastPathConfirmationSent; uint64_t _lastDirectPathPushSent; - uint64_t _lastDirectPathPushReceived; uint64_t _lastPathSort; uint16_t _vProto; uint16_t _vMajor; diff --git a/node/SelfAwareness.cpp b/node/SelfAwareness.cpp index 1b70f17c..81d19369 100644 --- a/node/SelfAwareness.cpp +++ b/node/SelfAwareness.cpp @@ -123,16 +123,16 @@ void SelfAwareness::iam(const Address &reporter,const InetAddress &reporterPhysi // For all peers for whom we forgot an address, send a packet indirectly if // they are still considered alive so that we will re-establish direct links. - SharedPtr sn(RR->topology->getBestRoot()); - if (sn) { - RemotePath *snp = sn->getBestPath(now); - if (snp) { + SharedPtr r(RR->topology->getBestRoot()); + if (r) { + RemotePath *rp = r->getBestPath(now); + if (rp) { for(std::vector< SharedPtr >::const_iterator p(rset.peersReset.begin());p!=rset.peersReset.end();++p) { if ((*p)->alive(now)) { - TRACE("sending indirect NOP to %s via %s(%s) to re-establish link",(*p)->address().toString().c_str(),sn->address().toString().c_str(),snp->address().toString().c_str()); + TRACE("sending indirect NOP to %s via %s to re-establish link",(*p)->address().toString().c_str(),r->address().toString().c_str()); Packet outp((*p)->address(),RR->identity.address(),Packet::VERB_NOP); outp.armor((*p)->key(),true); - snp->send(RR,outp.data(),outp.size(),now); + rp->send(RR,outp.data(),outp.size(),now); } } } diff --git a/node/Topology.cpp b/node/Topology.cpp index 7bb4b449..09668ef5 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -254,68 +254,8 @@ SharedPtr Topology::getBestRoot(const Address *avoid,unsigned int avoidCou return *bestOverall; } - /* - unsigned int l,bestLatency = 65536; - uint64_t lds,ldr; - - // First look for a best root by comparing latencies, but exclude - // root servers that have not responded to direct messages in order to - // try to exclude any that are dead or unreachable. - for(std::vector< SharedPtr >::const_iterator sn(_rootPeers.begin());sn!=_rootPeers.end();) { - // Skip explicitly avoided relays - for(unsigned int i=0;iaddress()) - goto keep_searching_for_roots; - } - - // Skip possibly comatose or unreachable relays - lds = (*sn)->lastDirectSend(); - ldr = (*sn)->lastDirectReceive(); - if ((lds)&&(lds > ldr)&&((lds - ldr) > ZT_PEER_RELAY_CONVERSATION_LATENCY_THRESHOLD)) - goto keep_searching_for_roots; - - if ((*sn)->hasActiveDirectPath(now)) { - l = (*sn)->latency(); - if (bestRoot) { - if ((l)&&(l < bestLatency)) { - bestLatency = l; - bestRoot = *sn; - } - } else { - if (l) - bestLatency = l; - bestRoot = *sn; - } - } - -keep_searching_for_roots: - ++sn; - } - - if (bestRoot) { - bestRoot->use(now); - return bestRoot; - } else if (strictAvoid) - return SharedPtr(); - - // If we have nothing from above, just pick one without avoidance criteria. - for(std::vector< SharedPtr >::const_iterator sn=_rootPeers.begin();sn!=_rootPeers.end();++sn) { - if ((*sn)->hasActiveDirectPath(now)) { - unsigned int l = (*sn)->latency(); - if (bestRoot) { - if ((l)&&(l < bestLatency)) { - bestLatency = l; - bestRoot = *sn; - } - } else { - if (l) - bestLatency = l; - bestRoot = *sn; - } - } - } - */ } + return SharedPtr(); } -- cgit v1.2.3 From cc1b275ad97bf186f21b487aa57d7893bee3c956 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 27 Oct 2015 16:47:13 -0700 Subject: Replicate peer endpoints and forget paths if we have them -- this allows two clusters to talk to each other, whereas forgetting all paths does not. --- node/Cluster.cpp | 45 +++++++++++++++++++++++++++------------------ node/Cluster.hpp | 7 ++++++- node/Constants.hpp | 4 ++-- node/Peer.cpp | 11 ++++------- node/Peer.hpp | 19 +++++++++++++++++++ 5 files changed, 58 insertions(+), 28 deletions(-) (limited to 'node/Constants.hpp') diff --git a/node/Cluster.cpp b/node/Cluster.cpp index b2f3d585..0797d83d 100644 --- a/node/Cluster.cpp +++ b/node/Cluster.cpp @@ -210,22 +210,30 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len) } break; case STATE_MESSAGE_HAVE_PEER: { - try { - Identity id; - ptr += id.deserialize(dmsg,ptr); - if (id) { - RR->topology->saveIdentity(id); - { - Mutex::Lock _l2(_peerAffinities_m); - _PA &pa = _peerAffinities[id.address()]; - pa.ts = RR->node->now(); - pa.mid = fromMemberId; - } - TRACE("[%u] has %s",(unsigned int)fromMemberId,id.address().toString().c_str()); - } - } catch ( ... ) { - // ignore invalid identities - } + Identity id; + InetAddress physicalAddress; + ptr += id.deserialize(dmsg,ptr); + ptr += physicalAddress.deserialize(dmsg,ptr); + if (id) { + // Forget any paths that we have to this peer at its address + if (physicalAddress) { + SharedPtr myPeerRecord(RR->topology->getPeer(id.address())); + if (myPeerRecord) + myPeerRecord->removePathByAddress(physicalAddress); + } + + // Always save identity to update file time + RR->topology->saveIdentity(id); + + // Set peer affinity to its new home + { + Mutex::Lock _l2(_peerAffinities_m); + _PA &pa = _peerAffinities[id.address()]; + pa.ts = RR->node->now(); + pa.mid = fromMemberId; + } + TRACE("[%u] has %s @ %s",(unsigned int)fromMemberId,id.address().toString().c_str(),physicalAddress.toString().c_str()); + } } break; case STATE_MESSAGE_MULTICAST_LIKE: { @@ -396,7 +404,7 @@ bool Cluster::sendViaCluster(const Address &fromPeerAddress,const Address &toPee return true; } -void Cluster::replicateHavePeer(const Identity &peerId) +void Cluster::replicateHavePeer(const Identity &peerId,const InetAddress &physicalAddress) { const uint64_t now = RR->node->now(); { // Use peer affinity table to track our own last announce time for peers @@ -405,7 +413,7 @@ void Cluster::replicateHavePeer(const Identity &peerId) if (pa.mid != _id) { pa.ts = now; pa.mid = _id; - } else if ((now - pa.ts) >= ZT_CLUSTER_HAVE_PEER_ANNOUNCE_PERIOD) { + } else if ((now - pa.ts) < ZT_CLUSTER_HAVE_PEER_ANNOUNCE_PERIOD) { return; } else { pa.ts = now; @@ -415,6 +423,7 @@ void Cluster::replicateHavePeer(const Identity &peerId) // announcement Buffer<4096> buf; peerId.serialize(buf,false); + physicalAddress.serialize(buf); { Mutex::Lock _l(_memberIds_m); for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { diff --git a/node/Cluster.hpp b/node/Cluster.hpp index 42a26c7f..c3367d57 100644 --- a/node/Cluster.hpp +++ b/node/Cluster.hpp @@ -117,6 +117,10 @@ public: /** * Cluster member has this peer: * <[...] binary serialized peer identity> + * <[...] binary serialized peer remote physical address> + * + * Clusters send this message when they learn a path to a peer. The + * replicated physical address is the one learned. */ STATE_MESSAGE_HAVE_PEER = 2, @@ -225,8 +229,9 @@ public: * Advertise to the cluster that we have this peer * * @param peerId Identity of peer that we have + * @param physicalAddress Physical address of peer (from our POV) */ - void replicateHavePeer(const Identity &peerId); + void replicateHavePeer(const Identity &peerId,const InetAddress &physicalAddress); /** * Advertise a multicast LIKE to the cluster diff --git a/node/Constants.hpp b/node/Constants.hpp index bef1183a..4b06db44 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -317,7 +317,7 @@ /** * Minimum delay between attempts to confirm new paths to peers (to avoid HELLO flooding) */ -#define ZT_MIN_PATH_CONFIRMATION_INTERVAL 5000 +#define ZT_MIN_PATH_CONFIRMATION_INTERVAL 1000 /** * Interval between direct path pushes in milliseconds @@ -350,7 +350,7 @@ /** * Maximum number of endpoints to contact per address type (to limit pushes like GitHub issue #235) */ -#define ZT_PUSH_DIRECT_PATHS_MAX_ENDPOINTS_PER_TYPE 2 +#define ZT_PUSH_DIRECT_PATHS_MAX_ENDPOINTS_PER_TYPE 4 /** * A test pseudo-network-ID that can be joined diff --git a/node/Peer.cpp b/node/Peer.cpp index 99e2156e..99eb32c7 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -140,13 +140,10 @@ void Peer::received( _lastMulticastFrame = now; #ifdef ZT_ENABLE_CLUSTER - // If we're in cluster mode and there's a better endpoint, stop here and don't - // learn or confirm paths. Also reset any existing paths, since they should - // go there and no longer talk to us here. - if (redirectTo) { - _numPaths = 0; + // If we think this peer belongs elsewhere, don't learn this path or + // do other connection init stuff. + if (redirectTo) return; - } #endif if ((now - _lastAnnouncedTo) >= ((ZT_MULTICAST_LIKE_EXPIRE / 2) - 1000)) { @@ -206,7 +203,7 @@ void Peer::received( #ifdef ZT_ENABLE_CLUSTER if ((RR->cluster)&&(pathIsConfirmed)) - RR->cluster->replicateHavePeer(_id); + RR->cluster->replicateHavePeer(_id,remoteAddr); #endif if (needMulticastGroupAnnounce) { diff --git a/node/Peer.hpp b/node/Peer.hpp index aa75b3f4..69343f20 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -412,6 +412,25 @@ public: */ void clean(const RuntimeEnvironment *RR,uint64_t now); + /** + * Remove all paths with this remote address + * + * @param addr Remote address to remove + */ + inline void removePathByAddress(const InetAddress &addr) + { + Mutex::Lock _l(_lock); + unsigned int np = _numPaths; + unsigned int x = 0; + unsigned int y = 0; + while (x < np) { + if (_paths[x].address() != addr) + _paths[y++] = _paths[x]; + ++x; + } + _numPaths = y; + } + /** * Find a common set of addresses by which two peers can link, if any * -- cgit v1.2.3 From cdc99bfee10ac58a8dab1aabcb85e69f3862b7ad Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 27 Oct 2015 18:18:26 -0700 Subject: Add a circuit breaker for VERB_PUSH_DIRECT_PATHS. --- node/Constants.hpp | 26 ++++++++++++++++++++------ node/IncomingPacket.cpp | 5 +++++ node/Peer.cpp | 2 ++ node/Peer.hpp | 31 +++++++++++++++++++++++++++++-- 4 files changed, 56 insertions(+), 8 deletions(-) (limited to 'node/Constants.hpp') diff --git a/node/Constants.hpp b/node/Constants.hpp index 4b06db44..53cc64c7 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -319,11 +319,6 @@ */ #define ZT_MIN_PATH_CONFIRMATION_INTERVAL 1000 -/** - * Interval between direct path pushes in milliseconds - */ -#define ZT_DIRECT_PATH_PUSH_INTERVAL 120000 - /** * How long (max) to remember network certificates of membership? * @@ -347,10 +342,29 @@ */ #define ZT_MAX_BRIDGE_SPAM 16 +/** + * Interval between direct path pushes in milliseconds + */ +#define ZT_DIRECT_PATH_PUSH_INTERVAL 120000 + /** * Maximum number of endpoints to contact per address type (to limit pushes like GitHub issue #235) */ -#define ZT_PUSH_DIRECT_PATHS_MAX_ENDPOINTS_PER_TYPE 4 +#define ZT_PUSH_DIRECT_PATHS_MAX_ENDPOINTS_PER_TYPE 5 + +/** + * Time horizon for push direct paths cutoff + */ +#define ZT_PUSH_DIRECT_PATHS_CUTOFF_TIME 60000 + +/** + * Maximum number of direct path pushes within cutoff time + * + * This limits response to PUSH_DIRECT_PATHS to CUTOFF_LIMIT responses + * per CUTOFF_TIME milliseconds per peer to prevent this from being + * useful for DOS amplification attacks. + */ +#define ZT_PUSH_DIRECT_PATHS_CUTOFF_LIMIT 5 /** * A test pseudo-network-ID that can be joined diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index b1a9af3b..e985b34c 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -901,6 +901,11 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha { try { const uint64_t now = RR->node->now(); + if (!peer->shouldRespondToDirectPathPush(now)) { + TRACE("dropped PUSH_DIRECT_PATHS from %s(%s): circuit breaker tripped",source().toString().c_str(),_remoteAddress.toString().c_str()); + return true; + } + const Path *currentBest = peer->getBestPath(now); unsigned int count = at(ZT_PACKET_IDX_PAYLOAD); diff --git a/node/Peer.cpp b/node/Peer.cpp index 99eb32c7..976c7c44 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -55,6 +55,7 @@ Peer::Peer(const Identity &myIdentity,const Identity &peerIdentity) _lastAnnouncedTo(0), _lastPathConfirmationSent(0), _lastDirectPathPushSent(0), + _lastDirectPathPushReceive(0), _lastPathSort(0), _vProto(0), _vMajor(0), @@ -63,6 +64,7 @@ Peer::Peer(const Identity &myIdentity,const Identity &peerIdentity) _id(peerIdentity), _numPaths(0), _latency(0), + _directPathPushCutoffCount(0), _networkComs(4), _lastPushedComs(4) { diff --git a/node/Peer.hpp b/node/Peer.hpp index 69343f20..d9ef5fcb 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -431,6 +431,27 @@ public: _numPaths = y; } + /** + * Update direct path push stats and return true if we should respond + * + * This is a circuit breaker to make VERB_PUSH_DIRECT_PATHS not particularly + * useful as a DDOS amplification attack vector. Otherwise a malicious peer + * could send loads of these and cause others to bombard arbitrary IPs with + * traffic. + * + * @param now Current time + * @return True if we should respond + */ + inline bool shouldRespondToDirectPathPush(const uint64_t now) + { + Mutex::Lock _l(_lock); + if ((now - _lastDirectPathPushReceive) <= ZT_PUSH_DIRECT_PATHS_CUTOFF_TIME) + ++_directPathPushCutoffCount; + else _directPathPushCutoffCount = 0; + _lastDirectPathPushReceive = now; + return (_directPathPushCutoffCount >= ZT_PUSH_DIRECT_PATHS_CUTOFF_LIMIT); + } + /** * Find a common set of addresses by which two peers can link, if any * @@ -459,7 +480,7 @@ public: const unsigned int recSizePos = b.size(); b.addSize(4); // space for uint32_t field length - b.append((uint16_t)0); // version of serialized Peer data + b.append((uint16_t)1); // version of serialized Peer data _id.serialize(b,false); @@ -470,12 +491,14 @@ public: b.append((uint64_t)_lastAnnouncedTo); b.append((uint64_t)_lastPathConfirmationSent); b.append((uint64_t)_lastDirectPathPushSent); + b.append((uint64_t)_lastDirectPathPushReceive); b.append((uint64_t)_lastPathSort); b.append((uint16_t)_vProto); b.append((uint16_t)_vMajor); b.append((uint16_t)_vMinor); b.append((uint16_t)_vRevision); b.append((uint32_t)_latency); + b.append((uint16_t)_directPathPushCutoffCount); b.append((uint16_t)_numPaths); for(unsigned int i=0;i<_numPaths;++i) @@ -521,7 +544,7 @@ public: const unsigned int recSize = b.template at(p); p += 4; if ((p + recSize) > b.size()) return SharedPtr(); // size invalid - if (b.template at(p) != 0) + if (b.template at(p) != 1) return SharedPtr(); // version mismatch p += 2; @@ -539,12 +562,14 @@ public: np->_lastAnnouncedTo = b.template at(p); p += 8; np->_lastPathConfirmationSent = b.template at(p); p += 8; np->_lastDirectPathPushSent = b.template at(p); p += 8; + np->_lastDirectPathPushReceive = b.template at(p); p += 8; np->_lastPathSort = b.template at(p); p += 8; np->_vProto = b.template at(p); p += 2; np->_vMajor = b.template at(p); p += 2; np->_vMinor = b.template at(p); p += 2; np->_vRevision = b.template at(p); p += 2; np->_latency = b.template at(p); p += 4; + np->_directPathPushCutoffCount = b.template at(p); p += 2; const unsigned int numPaths = b.template at(p); p += 2; for(unsigned int i=0;i Date: Wed, 28 Oct 2015 09:11:30 -0700 Subject: Clean up PUSH_DIRECT_PATH limits a bit more and make them a bit smarter. --- node/Constants.hpp | 10 +++++----- node/IncomingPacket.cpp | 25 +++++++++++++++---------- node/InetAddress.hpp | 8 +++++++- 3 files changed, 27 insertions(+), 16 deletions(-) (limited to 'node/Constants.hpp') diff --git a/node/Constants.hpp b/node/Constants.hpp index 53cc64c7..5a4d4a4d 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -347,11 +347,6 @@ */ #define ZT_DIRECT_PATH_PUSH_INTERVAL 120000 -/** - * Maximum number of endpoints to contact per address type (to limit pushes like GitHub issue #235) - */ -#define ZT_PUSH_DIRECT_PATHS_MAX_ENDPOINTS_PER_TYPE 5 - /** * Time horizon for push direct paths cutoff */ @@ -366,6 +361,11 @@ */ #define ZT_PUSH_DIRECT_PATHS_CUTOFF_LIMIT 5 +/** + * Maximum number of paths per IP scope (e.g. global, link-local) and family (e.g. v4/v6) + */ +#define ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY 1 + /** * A test pseudo-network-ID that can be joined * diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index e985b34c..f06eb30c 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -901,16 +901,19 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha { try { const uint64_t now = RR->node->now(); + + // First, subject this to a rate limit if (!peer->shouldRespondToDirectPathPush(now)) { TRACE("dropped PUSH_DIRECT_PATHS from %s(%s): circuit breaker tripped",source().toString().c_str(),_remoteAddress.toString().c_str()); return true; } - const Path *currentBest = peer->getBestPath(now); + // Second, limit addresses by scope and type + uint8_t countPerScope[ZT_INETADDRESS_MAX_SCOPE+1][2]; // [][0] is v4, [][1] is v6 + memset(countPerScope,0,sizeof(countPerScope)); unsigned int count = at(ZT_PACKET_IDX_PAYLOAD); unsigned int ptr = ZT_PACKET_IDX_PAYLOAD + 2; - unsigned int v4Count = 0,v6Count = 0; while (count--) { // if ptr overflows Buffer will throw // TODO: some flags are not yet implemented @@ -925,20 +928,22 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha case 4: { InetAddress a(field(ptr,4),4,at(ptr + 4)); if ( ((flags & 0x01) == 0) && (Path::isAddressValidForPath(a)) ) { - TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str()); - if (v4Count++ < ZT_PUSH_DIRECT_PATHS_MAX_ENDPOINTS_PER_TYPE) { - if ((!currentBest)||(currentBest->address() != a)) - peer->attemptToContactAt(RR,_localAddress,a,now); + if (++countPerScope[(int)a.ipScope()][0] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) { + TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str()); + peer->attemptToContactAt(RR,_localAddress,a,now); + } else { + TRACE("ignoring contact for %s at %s -- too many per scope",peer->address().toString().c_str(),a.toString().c_str()); } } } break; case 6: { InetAddress a(field(ptr,16),16,at(ptr + 16)); if ( ((flags & 0x01) == 0) && (Path::isAddressValidForPath(a)) ) { - TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str()); - if (v6Count++ < ZT_PUSH_DIRECT_PATHS_MAX_ENDPOINTS_PER_TYPE) { - if ((!currentBest)||(currentBest->address() != a)) - peer->attemptToContactAt(RR,_localAddress,a,now); + if (++countPerScope[(int)a.ipScope()][1] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) { + TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str()); + peer->attemptToContactAt(RR,_localAddress,a,now); + } else { + TRACE("ignoring contact for %s at %s -- too many per scope",peer->address().toString().c_str(),a.toString().c_str()); } } } break; diff --git a/node/InetAddress.hpp b/node/InetAddress.hpp index fcbed4b1..5e5eb06e 100644 --- a/node/InetAddress.hpp +++ b/node/InetAddress.hpp @@ -42,6 +42,11 @@ namespace ZeroTier { +/** + * Maximum integer value of enum IpScope + */ +#define ZT_INETADDRESS_MAX_SCOPE 7 + /** * Extends sockaddr_storage with friendly C++ methods * @@ -66,7 +71,8 @@ struct InetAddress : public sockaddr_storage * IP address scope * * Note that these values are in ascending order of path preference and - * MUST remain that way or Path must be changed to reflect. + * MUST remain that way or Path must be changed to reflect. Also be sure + * to change ZT_INETADDRESS_MAX_SCOPE if the max changes. */ enum IpScope { -- cgit v1.2.3 From b6725c44150af306130d38a2875ab31a640607c8 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 30 Oct 2015 11:48:33 -0700 Subject: Optimize AntiRecursion. --- node/AntiRecursion.hpp | 66 ++++++++++++++++++++++++++++++++------------------ node/Constants.hpp | 5 ---- 2 files changed, 42 insertions(+), 29 deletions(-) (limited to 'node/Constants.hpp') diff --git a/node/AntiRecursion.hpp b/node/AntiRecursion.hpp index 4bb24bf3..8629d19a 100644 --- a/node/AntiRecursion.hpp +++ b/node/AntiRecursion.hpp @@ -35,28 +35,28 @@ namespace ZeroTier { -#define ZT_ANTIRECURSION_TAIL_LEN 128 +/** + * Size of anti-recursion history + */ +#define ZT_ANTIRECURSION_HISTORY_SIZE 16 /** * Filter to prevent recursion (ZeroTier-over-ZeroTier) * * This works by logging ZeroTier packets that we send. It's then invoked - * again against packets read from local Ethernet taps. If the last N + * again against packets read from local Ethernet taps. If the last 32 * bytes representing the ZeroTier packet match in the tap frame, then * the frame is a re-injection of a frame that we sent and is rejected. * * This means that ZeroTier packets simply will not traverse ZeroTier * networks, which would cause all sorts of weird problems. * - * NOTE: this is applied to low-level packets before they are sent to - * SocketManager and/or sockets, not to fully assembled packets before - * (possible) fragmentation. + * This is highly optimized code since it's checked for every packet. */ class AntiRecursion { public: AntiRecursion() - throw() { memset(_history,0,sizeof(_history)); _ptr = 0; @@ -68,13 +68,20 @@ public: * @param data ZT packet data * @param len Length of packet */ - inline void logOutgoingZT(const void *data,unsigned int len) - throw() + inline void logOutgoingZT(const void *const data,const unsigned int len) { - ArItem *i = &(_history[_ptr++ % ZT_ANTIRECURSION_HISTORY_SIZE]); - const unsigned int tl = (len > ZT_ANTIRECURSION_TAIL_LEN) ? ZT_ANTIRECURSION_TAIL_LEN : len; - memcpy(i->tail,((const unsigned char *)data) + (len - tl),tl); - i->len = tl; + if (len < 32) + return; +#ifdef ZT_NO_TYPE_PUNNING + memcpy(_history[++_ptr % ZT_ANTIRECURSION_HISTORY_SIZE].tail,reinterpret_cast(data) + (len - 32),32); +#else + uint64_t *t = _history[++_ptr % ZT_ANTIRECURSION_HISTORY_SIZE].tail; + const uint64_t *p = reinterpret_cast(reinterpret_cast(data) + (len - 32)); + *(t++) = *(p++); + *(t++) = *(p++); + *(t++) = *(p++); + *t = *p; +#endif } /** @@ -84,25 +91,36 @@ public: * @param len Length of frame * @return True if frame is OK to be passed, false if it's a ZT frame that we sent */ - inline bool checkEthernetFrame(const void *data,unsigned int len) - throw() + inline bool checkEthernetFrame(const void *const data,const unsigned int len) const { - for(unsigned int h=0;hlen > 0)&&(len >= i->len)&&(!memcmp(((const unsigned char *)data) + (len - i->len),i->tail,i->len))) + if (len < 32) + return true; + const uint8_t *const pp = reinterpret_cast(data) + (len - 32); + const _ArItem *i = _history; + const _ArItem *const end = i + ZT_ANTIRECURSION_HISTORY_SIZE; + while (i != end) { +#ifdef ZT_NO_TYPE_PUNNING + if (!memcmp(pp,i->tail,32)) return false; +#else + const uint64_t *t = i->tail; + const uint64_t *p = reinterpret_cast(pp); + uint64_t bits = *(t++) ^ *(p++); + bits |= *(t++) ^ *(p++); + bits |= *(t++) ^ *(p++); + bits |= *t ^ *p; + if (!bits) + return false; +#endif + ++i; } return true; } private: - struct ArItem - { - unsigned char tail[ZT_ANTIRECURSION_TAIL_LEN]; - unsigned int len; - }; - ArItem _history[ZT_ANTIRECURSION_HISTORY_SIZE]; - volatile unsigned int _ptr; + struct _ArItem { uint64_t tail[4]; }; + _ArItem _history[ZT_ANTIRECURSION_HISTORY_SIZE]; + volatile unsigned long _ptr; }; } // namespace ZeroTier diff --git a/node/Constants.hpp b/node/Constants.hpp index 5a4d4a4d..1d5fa6f4 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -309,11 +309,6 @@ */ #define ZT_NAT_T_TACTICAL_ESCALATION_DELAY 1000 -/** - * Size of anti-recursion history (see AntiRecursion.hpp) - */ -#define ZT_ANTIRECURSION_HISTORY_SIZE 16 - /** * Minimum delay between attempts to confirm new paths to peers (to avoid HELLO flooding) */ -- cgit v1.2.3 From 60ce886605c0298fc22dbce48beb106a96bd35e2 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 2 Nov 2015 15:15:20 -0800 Subject: Tweak some timings for better reliability. --- node/Cluster.cpp | 6 +- node/Cluster.hpp | 8 +- node/Constants.hpp | 24 +---- node/Multicaster.cpp | 220 ++++++++++++++++++++++--------------------- node/Node.cpp | 13 +-- tests/http/big-test-kill.sh | 2 +- tests/http/big-test-ready.sh | 2 +- tests/http/big-test-start.sh | 4 +- 8 files changed, 131 insertions(+), 148 deletions(-) (limited to 'node/Constants.hpp') diff --git a/node/Cluster.cpp b/node/Cluster.cpp index d0daae43..e9e31ede 100644 --- a/node/Cluster.cpp +++ b/node/Cluster.cpp @@ -85,7 +85,8 @@ Cluster::Cluster( _members(new _Member[ZT_CLUSTER_MAX_MEMBERS]), _peerAffinities(65536), _lastCleanedPeerAffinities(0), - _lastCheckedPeersForAnnounce(0) + _lastCheckedPeersForAnnounce(0), + _lastFlushed(0) { uint16_t stmp[ZT_SHA512_DIGEST_LEN / sizeof(uint16_t)]; @@ -510,7 +511,8 @@ void Cluster::doPeriodicTasks() } // Flush outgoing packet send queue every doPeriodicTasks() - { + if ((now - _lastFlushed) >= ZT_CLUSTER_FLUSH_PERIOD) { + _lastFlushed = now; Mutex::Lock _l(_memberIds_m); for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { Mutex::Lock _l2(_members[*mid].lock); diff --git a/node/Cluster.hpp b/node/Cluster.hpp index 7d7a1ced..f1caa436 100644 --- a/node/Cluster.hpp +++ b/node/Cluster.hpp @@ -55,13 +55,18 @@ /** * How often should we announce that we have a peer? */ -#define ZT_CLUSTER_HAVE_PEER_ANNOUNCE_PERIOD ((ZT_PEER_ACTIVITY_TIMEOUT / 2) - 1000) +#define ZT_CLUSTER_HAVE_PEER_ANNOUNCE_PERIOD (ZT_PEER_DIRECT_PING_DELAY / 2) /** * Desired period between doPeriodicTasks() in milliseconds */ #define ZT_CLUSTER_PERIODIC_TASK_PERIOD 250 +/** + * How often to flush outgoing message queues (maximum interval) + */ +#define ZT_CLUSTER_FLUSH_PERIOD 500 + namespace ZeroTier { class RuntimeEnvironment; @@ -355,6 +360,7 @@ private: uint64_t _lastCleanedPeerAffinities; uint64_t _lastCheckedPeersForAnnounce; + uint64_t _lastFlushed; }; } // namespace ZeroTier diff --git a/node/Constants.hpp b/node/Constants.hpp index 1d5fa6f4..bb62484d 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -173,13 +173,8 @@ /** * Timeout for receipt of fragmented packets in ms - * - * Since there's no retransmits, this is just a really bad case scenario for - * transit time. It's short enough that a DOS attack from exhausing buffers is - * very unlikely, as the transfer rate would have to be fast enough to fill - * system memory in this time. */ -#define ZT_FRAGMENTED_PACKET_RECEIVE_TIMEOUT 1000 +#define ZT_FRAGMENTED_PACKET_RECEIVE_TIMEOUT 500 /** * Length of secret key in bytes -- 256-bit -- do not change @@ -194,7 +189,7 @@ /** * Overriding granularity for timer tasks to prevent CPU-intensive thrashing on every packet */ -#define ZT_CORE_TIMER_TASK_GRANULARITY 1000 +#define ZT_CORE_TIMER_TASK_GRANULARITY 500 /** * How long to remember peer records in RAM if they haven't been used @@ -269,7 +264,7 @@ /** * Delay between ordinary case pings of direct links */ -#define ZT_PEER_DIRECT_PING_DELAY 120000 +#define ZT_PEER_DIRECT_PING_DELAY 60000 /** * Delay between requests for updated network autoconf information @@ -279,18 +274,7 @@ /** * Timeout for overall peer activity (measured from last receive) */ -#define ZT_PEER_ACTIVITY_TIMEOUT (ZT_PEER_DIRECT_PING_DELAY + (ZT_PING_CHECK_INVERVAL * 3)) - -/** - * Stop relaying via peers that have not responded to direct sends - * - * When we send something (including frames), we generally expect a response. - * Switching relays if no response in a short period of time causes more - * rapid failover if a root server goes down or becomes unreachable. In the - * mistaken case, little harm is done as it'll pick the next-fastest - * root server and will switch back eventually. - */ -#define ZT_PEER_RELAY_CONVERSATION_LATENCY_THRESHOLD 10000 +#define ZT_PEER_ACTIVITY_TIMEOUT ((ZT_PEER_DIRECT_PING_DELAY * 3) + (ZT_PING_CHECK_INVERVAL * 2)) /** * Minimum interval between attempts by relays to unite peers diff --git a/node/Multicaster.cpp b/node/Multicaster.cpp index e43d7d88..01e6b799 100644 --- a/node/Multicaster.cpp +++ b/node/Multicaster.cpp @@ -175,128 +175,130 @@ void Multicaster::send( unsigned long idxbuf[8194]; unsigned long *indexes = idxbuf; - Mutex::Lock _l(_groups_m); - MulticastGroupStatus &gs = _groups[Multicaster::Key(nwid,mg)]; - - if (!gs.members.empty()) { - // Allocate a memory buffer if group is monstrous - if (gs.members.size() > (sizeof(idxbuf) / sizeof(unsigned long))) - indexes = new unsigned long[gs.members.size()]; - - // Generate a random permutation of member indexes - for(unsigned long i=0;i0;--i) { - unsigned long j = (unsigned long)RR->node->prng() % (i + 1); - unsigned long tmp = indexes[j]; - indexes[j] = indexes[i]; - indexes[i] = tmp; + try { + Mutex::Lock _l(_groups_m); + MulticastGroupStatus &gs = _groups[Multicaster::Key(nwid,mg)]; + + if (!gs.members.empty()) { + // Allocate a memory buffer if group is monstrous + if (gs.members.size() > (sizeof(idxbuf) / sizeof(unsigned long))) + indexes = new unsigned long[gs.members.size()]; + + // Generate a random permutation of member indexes + for(unsigned long i=0;i0;--i) { + unsigned long j = (unsigned long)RR->node->prng() % (i + 1); + unsigned long tmp = indexes[j]; + indexes[j] = indexes[i]; + indexes[i] = tmp; + } } - } - if (gs.members.size() >= limit) { - // Skip queue if we already have enough members to complete the send operation - OutboundMulticast out; - - out.init( - RR, - now, - nwid, - com, - limit, - 1, // we'll still gather a little from peers to keep multicast list fresh - src, - mg, - etherType, - data, - len); - - unsigned int count = 0; - - for(std::vector
::const_iterator ast(alwaysSendTo.begin());ast!=alwaysSendTo.end();++ast) { - if (*ast != RR->identity.address()) { - out.sendOnly(RR,*ast); - if (++count >= limit) - break; + if (gs.members.size() >= limit) { + // Skip queue if we already have enough members to complete the send operation + OutboundMulticast out; + + out.init( + RR, + now, + nwid, + com, + limit, + 1, // we'll still gather a little from peers to keep multicast list fresh + src, + mg, + etherType, + data, + len); + + unsigned int count = 0; + + for(std::vector
::const_iterator ast(alwaysSendTo.begin());ast!=alwaysSendTo.end();++ast) { + if (*ast != RR->identity.address()) { + out.sendOnly(RR,*ast); // optimization: don't use dedup log if it's a one-pass send + if (++count >= limit) + break; + } } - } - unsigned long idx = 0; - while ((count < limit)&&(idx < gs.members.size())) { - Address ma(gs.members[indexes[idx++]].address); - if (std::find(alwaysSendTo.begin(),alwaysSendTo.end(),ma) == alwaysSendTo.end()) { - out.sendOnly(RR,ma); - ++count; + unsigned long idx = 0; + while ((count < limit)&&(idx < gs.members.size())) { + Address ma(gs.members[indexes[idx++]].address); + if (std::find(alwaysSendTo.begin(),alwaysSendTo.end(),ma) == alwaysSendTo.end()) { + out.sendOnly(RR,ma); // optimization: don't use dedup log if it's a one-pass send + ++count; + } } - } - } else { - unsigned int gatherLimit = (limit - (unsigned int)gs.members.size()) + 1; - - if ((now - gs.lastExplicitGather) >= ZT_MULTICAST_EXPLICIT_GATHER_DELAY) { - gs.lastExplicitGather = now; - SharedPtr r(RR->topology->getBestRoot()); - if (r) { - TRACE(">>MC upstream GATHER up to %u for group %.16llx/%s",gatherLimit,nwid,mg.toString().c_str()); - - const CertificateOfMembership *com = (CertificateOfMembership *)0; - { - SharedPtr nw(RR->node->network(nwid)); - if (nw) { - SharedPtr nconf(nw->config2()); - if ((nconf)&&(nconf->com())&&(nconf->isPrivate())&&(r->needsOurNetworkMembershipCertificate(nwid,now,true))) - com = &(nconf->com()); + } else { + unsigned int gatherLimit = (limit - (unsigned int)gs.members.size()) + 1; + + if ((gs.members.empty())||((now - gs.lastExplicitGather) >= ZT_MULTICAST_EXPLICIT_GATHER_DELAY)) { + gs.lastExplicitGather = now; + SharedPtr r(RR->topology->getBestRoot()); + if (r) { + TRACE(">>MC upstream GATHER up to %u for group %.16llx/%s",gatherLimit,nwid,mg.toString().c_str()); + + const CertificateOfMembership *com = (CertificateOfMembership *)0; + { + SharedPtr nw(RR->node->network(nwid)); + if (nw) { + SharedPtr nconf(nw->config2()); + if ((nconf)&&(nconf->com())&&(nconf->isPrivate())&&(r->needsOurNetworkMembershipCertificate(nwid,now,true))) + com = &(nconf->com()); + } } - } - Packet outp(r->address(),RR->identity.address(),Packet::VERB_MULTICAST_GATHER); - outp.append(nwid); - outp.append((uint8_t)(com ? 0x01 : 0x00)); - mg.mac().appendTo(outp); - outp.append((uint32_t)mg.adi()); - outp.append((uint32_t)gatherLimit); - if (com) - com->serialize(outp); - outp.armor(r->key(),true); - r->send(RR,outp.data(),outp.size(),now); + Packet outp(r->address(),RR->identity.address(),Packet::VERB_MULTICAST_GATHER); + outp.append(nwid); + outp.append((uint8_t)(com ? 0x01 : 0x00)); + mg.mac().appendTo(outp); + outp.append((uint32_t)mg.adi()); + outp.append((uint32_t)gatherLimit); + if (com) + com->serialize(outp); + outp.armor(r->key(),true); + r->send(RR,outp.data(),outp.size(),now); + } + gatherLimit = 0; } - gatherLimit = 0; - } - gs.txQueue.push_back(OutboundMulticast()); - OutboundMulticast &out = gs.txQueue.back(); - - out.init( - RR, - now, - nwid, - com, - limit, - gatherLimit, - src, - mg, - etherType, - data, - len); - - unsigned int count = 0; - - for(std::vector
::const_iterator ast(alwaysSendTo.begin());ast!=alwaysSendTo.end();++ast) { - if (*ast != RR->identity.address()) { - out.sendAndLog(RR,*ast); - if (++count >= limit) - break; + gs.txQueue.push_back(OutboundMulticast()); + OutboundMulticast &out = gs.txQueue.back(); + + out.init( + RR, + now, + nwid, + com, + limit, + gatherLimit, + src, + mg, + etherType, + data, + len); + + unsigned int count = 0; + + for(std::vector
::const_iterator ast(alwaysSendTo.begin());ast!=alwaysSendTo.end();++ast) { + if (*ast != RR->identity.address()) { + out.sendAndLog(RR,*ast); + if (++count >= limit) + break; + } } - } - unsigned long idx = 0; - while ((count < limit)&&(idx < gs.members.size())) { - Address ma(gs.members[indexes[idx++]].address); - if (std::find(alwaysSendTo.begin(),alwaysSendTo.end(),ma) == alwaysSendTo.end()) { - out.sendAndLog(RR,ma); - ++count; + unsigned long idx = 0; + while ((count < limit)&&(idx < gs.members.size())) { + Address ma(gs.members[indexes[idx++]].address); + if (std::find(alwaysSendTo.begin(),alwaysSendTo.end(),ma) == alwaysSendTo.end()) { + out.sendAndLog(RR,ma); + ++count; + } } } - } + } catch ( ... ) {} // this is a sanity check to catch any failures and make sure indexes[] still gets deleted // Free allocated memory buffer if any if (indexes != idxbuf) diff --git a/node/Node.cpp b/node/Node.cpp index 42180e99..74acc869 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -305,18 +305,7 @@ ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextB for(std::vector< SharedPtr >::const_iterator n(needConfig.begin());n!=needConfig.end();++n) (*n)->requestConfiguration(); - // Attempt to contact network preferred relays that we don't have direct links to - std::sort(networkRelays.begin(),networkRelays.end()); - networkRelays.erase(std::unique(networkRelays.begin(),networkRelays.end()),networkRelays.end()); - for(std::vector< std::pair >::const_iterator nr(networkRelays.begin());nr!=networkRelays.end();++nr) { - if (nr->second) { - SharedPtr rp(RR->topology->getPeer(nr->first)); - if ((rp)&&(!rp->hasActiveDirectPath(now))) - rp->attemptToContactAt(RR,InetAddress(),nr->second,now); - } - } - - // Ping living or root server/relay peers + // Do pings and keepalives _PingPeersThatNeedPing pfunc(RR,now,networkRelays); RR->topology->eachPeer<_PingPeersThatNeedPing &>(pfunc); diff --git a/tests/http/big-test-kill.sh b/tests/http/big-test-kill.sh index 4a764d1f..59f36788 100755 --- a/tests/http/big-test-kill.sh +++ b/tests/http/big-test-kill.sh @@ -13,6 +13,6 @@ CONTAINER_IMAGE=zerotier/http-test export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin -pssh -h big-test-hosts -i -t 128 -p 256 "docker ps -aq | xargs -r docker rm -f" +pssh -h big-test-hosts -i -t 0 -p 256 "docker ps -aq | xargs -r docker rm -f" exit 0 diff --git a/tests/http/big-test-ready.sh b/tests/http/big-test-ready.sh index 391ca2a1..aa540bba 100755 --- a/tests/http/big-test-ready.sh +++ b/tests/http/big-test-ready.sh @@ -25,6 +25,6 @@ export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin # docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE #done -pssh -h big-test-hosts -i -t 128 -p 256 "docker pull $CONTAINER_IMAGE" +pssh -h big-test-hosts -i -t 0 -p 256 "docker pull $CONTAINER_IMAGE" exit 0 diff --git a/tests/http/big-test-start.sh b/tests/http/big-test-start.sh index a5e71ef1..43166c6e 100755 --- a/tests/http/big-test-start.sh +++ b/tests/http/big-test-start.sh @@ -1,7 +1,7 @@ #!/bin/bash # Edit as needed -- note that >1000 per host is likely problematic due to Linux kernel limits -NUM_CONTAINERS=100 +NUM_CONTAINERS=25 CONTAINER_IMAGE=zerotier/http-test # @@ -25,6 +25,6 @@ export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin # docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE #done -pssh -h big-test-hosts -i -t 128 -p 256 "for ((n=0;n<$NUM_CONTAINERS;n++)); do docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE; sleep 0.25; done" +pssh -h big-test-hosts -i -t 0 -p 256 "for ((n=0;n<$NUM_CONTAINERS;n++)); do docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE; sleep 0.25; done" exit 0 -- cgit v1.2.3 From 7fbe2f7adf3575f3a21fc1ab3a5a2a036e18e6e2 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 2 Nov 2015 15:38:53 -0800 Subject: Tweak some more timings for better reliability. --- node/Cluster.hpp | 2 +- node/Constants.hpp | 10 +++++----- node/Node.cpp | 2 +- node/Peer.hpp | 4 ++-- node/SelfAwareness.cpp | 2 +- node/Switch.cpp | 6 +++--- node/Topology.hpp | 9 ++++++--- tests/http/big-test-start.sh | 4 ++-- 8 files changed, 21 insertions(+), 18 deletions(-) (limited to 'node/Constants.hpp') diff --git a/node/Cluster.hpp b/node/Cluster.hpp index f1caa436..ee220999 100644 --- a/node/Cluster.hpp +++ b/node/Cluster.hpp @@ -55,7 +55,7 @@ /** * How often should we announce that we have a peer? */ -#define ZT_CLUSTER_HAVE_PEER_ANNOUNCE_PERIOD (ZT_PEER_DIRECT_PING_DELAY / 2) +#define ZT_CLUSTER_HAVE_PEER_ANNOUNCE_PERIOD ZT_PEER_DIRECT_PING_DELAY /** * Desired period between doPeriodicTasks() in milliseconds diff --git a/node/Constants.hpp b/node/Constants.hpp index bb62484d..552688a6 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -267,14 +267,14 @@ #define ZT_PEER_DIRECT_PING_DELAY 60000 /** - * Delay between requests for updated network autoconf information + * Timeout for overall peer activity (measured from last receive) */ -#define ZT_NETWORK_AUTOCONF_DELAY 60000 +#define ZT_PEER_ACTIVITY_TIMEOUT ((ZT_PEER_DIRECT_PING_DELAY * 4) + ZT_PING_CHECK_INVERVAL) /** - * Timeout for overall peer activity (measured from last receive) + * Delay between requests for updated network autoconf information */ -#define ZT_PEER_ACTIVITY_TIMEOUT ((ZT_PEER_DIRECT_PING_DELAY * 3) + (ZT_PING_CHECK_INVERVAL * 2)) +#define ZT_NETWORK_AUTOCONF_DELAY 60000 /** * Minimum interval between attempts by relays to unite peers @@ -283,7 +283,7 @@ * a RENDEZVOUS message no more than this often. This instructs the peers * to attempt NAT-t and gives each the other's corresponding IP:port pair. */ -#define ZT_MIN_UNITE_INTERVAL 60000 +#define ZT_MIN_UNITE_INTERVAL 30000 /** * Delay between initial direct NAT-t packet and more aggressive techniques diff --git a/node/Node.cpp b/node/Node.cpp index 74acc869..82cb7ddb 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -263,7 +263,7 @@ public: } lastReceiveFromUpstream = std::max(p->lastReceive(),lastReceiveFromUpstream); - } else if (p->alive(_now)) { + } else if (p->activelyTransferringFrames(_now)) { // Normal nodes get their preferred link kept alive if the node has generated frame traffic recently p->doPingAndKeepalive(RR,_now,0); } diff --git a/node/Peer.hpp b/node/Peer.hpp index e5db3bde..ad4c6746 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -231,9 +231,9 @@ public: inline uint64_t lastAnnouncedTo() const throw() { return _lastAnnouncedTo; } /** - * @return True if peer has received an actual data frame within ZT_PEER_ACTIVITY_TIMEOUT milliseconds + * @return True if this peer is actively sending real network frames */ - inline uint64_t alive(uint64_t now) const throw() { return ((now - lastFrame()) < ZT_PEER_ACTIVITY_TIMEOUT); } + inline uint64_t activelyTransferringFrames(uint64_t now) const throw() { return ((now - lastFrame()) < ZT_PEER_ACTIVITY_TIMEOUT); } /** * @return Current latency or 0 if unknown (max: 65535) diff --git a/node/SelfAwareness.cpp b/node/SelfAwareness.cpp index d8eca071..ce75eb03 100644 --- a/node/SelfAwareness.cpp +++ b/node/SelfAwareness.cpp @@ -128,7 +128,7 @@ void SelfAwareness::iam(const Address &reporter,const InetAddress &reporterPhysi // links to be re-established if possible, possibly using a root server or some // other relay. for(std::vector< SharedPtr >::const_iterator p(rset.peersReset.begin());p!=rset.peersReset.end();++p) { - if ((*p)->alive(now)) { + if ((*p)->activelyTransferringFrames(now)) { Packet outp((*p)->address(),RR->identity.address(),Packet::VERB_NOP); RR->sw->send(outp,true,0); } diff --git a/node/Switch.cpp b/node/Switch.cpp index 2f72f57a..120ce7a4 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -442,8 +442,8 @@ unsigned long Switch::doTimerTasks(uint64_t now) Mutex::Lock _l(_contactQueue_m); for(std::list::iterator qi(_contactQueue.begin());qi!=_contactQueue.end();) { if (now >= qi->fireAtTime) { - if ((!qi->peer->alive(now))||(qi->peer->hasActiveDirectPath(now))) { - // Cancel attempt if we've already connected or peer is no longer "alive" + if (qi->peer->hasActiveDirectPath(now)) { + // Cancel if connection has succeeded _contactQueue.erase(qi++); continue; } else { @@ -539,7 +539,7 @@ unsigned long Switch::doTimerTasks(uint64_t now) _LastUniteKey *k = (_LastUniteKey *)0; uint64_t *v = (uint64_t *)0; while (i.next(k,v)) { - if ((now - *v) >= (ZT_MIN_UNITE_INTERVAL * 16)) + if ((now - *v) >= (ZT_MIN_UNITE_INTERVAL * 8)) _lastUniteAttempt.erase(*k); } } diff --git a/node/Topology.hpp b/node/Topology.hpp index 4c1a2ab3..a0c28b0f 100644 --- a/node/Topology.hpp +++ b/node/Topology.hpp @@ -81,6 +81,11 @@ public: /** * Get a peer only if it is presently in memory (no disk cache) * + * This also does not update the lastUsed() time for peers, which means + * that it won't prevent them from falling out of RAM. This is currently + * used in the Cluster code to update peer info without forcing all peers + * across the entire cluster to remain in memory cache. + * * @param zta ZeroTier address * @param now Current time */ @@ -88,10 +93,8 @@ public: { Mutex::Lock _l(_lock); const SharedPtr *const ap = _peers.get(zta); - if (ap) { - (*ap)->use(now); + if (ap) return *ap; - } return SharedPtr(); } diff --git a/tests/http/big-test-start.sh b/tests/http/big-test-start.sh index 43166c6e..f300ac61 100755 --- a/tests/http/big-test-start.sh +++ b/tests/http/big-test-start.sh @@ -1,7 +1,7 @@ #!/bin/bash # Edit as needed -- note that >1000 per host is likely problematic due to Linux kernel limits -NUM_CONTAINERS=25 +NUM_CONTAINERS=50 CONTAINER_IMAGE=zerotier/http-test # @@ -25,6 +25,6 @@ export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin # docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE #done -pssh -h big-test-hosts -i -t 0 -p 256 "for ((n=0;n<$NUM_CONTAINERS;n++)); do docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE; sleep 0.25; done" +pssh -h big-test-hosts -o big-test-out -t 0 -p 256 "for ((n=0;n<$NUM_CONTAINERS;n++)); do docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE; sleep 0.25; done" exit 0 -- cgit v1.2.3