From 0655a1fcbe3aaae3ed5dee94ebe05edf10823b07 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 13 Jul 2017 16:42:43 -0700 Subject: Move old cluster code into attic. --- attic/Cluster.cpp | 1042 +++++++++++++++++++++++++++++++++++++++++++++++++++++ attic/Cluster.hpp | 463 ++++++++++++++++++++++++ 2 files changed, 1505 insertions(+) create mode 100644 attic/Cluster.cpp create mode 100644 attic/Cluster.hpp (limited to 'attic') diff --git a/attic/Cluster.cpp b/attic/Cluster.cpp new file mode 100644 index 00000000..119aec29 --- /dev/null +++ b/attic/Cluster.cpp @@ -0,0 +1,1042 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. + */ + +#ifdef ZT_ENABLE_CLUSTER + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "../version.h" + +#include "Cluster.hpp" +#include "RuntimeEnvironment.hpp" +#include "MulticastGroup.hpp" +#include "CertificateOfMembership.hpp" +#include "Salsa20.hpp" +#include "Poly1305.hpp" +#include "Identity.hpp" +#include "Topology.hpp" +#include "Packet.hpp" +#include "Switch.hpp" +#include "Node.hpp" +#include "Network.hpp" +#include "Array.hpp" + +namespace ZeroTier { + +static inline double _dist3d(int x1,int y1,int z1,int x2,int y2,int z2) + throw() +{ + double dx = ((double)x2 - (double)x1); + double dy = ((double)y2 - (double)y1); + double dz = ((double)z2 - (double)z1); + return sqrt((dx * dx) + (dy * dy) + (dz * dz)); +} + +// An entry in _ClusterSendQueue +struct _ClusterSendQueueEntry +{ + uint64_t timestamp; + Address fromPeerAddress; + Address toPeerAddress; + // if we ever support larger transport MTUs this must be increased + unsigned char data[ZT_CLUSTER_SEND_QUEUE_DATA_MAX]; + unsigned int len; + bool unite; +}; + +// A multi-index map with entry memory pooling -- this allows our queue to +// be O(log(N)) and is complex enough that it makes the code a lot cleaner +// to break it out from Cluster. +class _ClusterSendQueue +{ +public: + _ClusterSendQueue() : + _poolCount(0) {} + ~_ClusterSendQueue() {} // memory is automatically freed when _chunks is destroyed + + inline void enqueue(uint64_t now,const Address &from,const Address &to,const void *data,unsigned int len,bool unite) + { + if (len > ZT_CLUSTER_SEND_QUEUE_DATA_MAX) + return; + + Mutex::Lock _l(_lock); + + // Delete oldest queue entry for this sender if this enqueue() would take them over the per-sender limit + { + std::set< std::pair >::iterator qi(_bySrc.lower_bound(std::pair(from,(_ClusterSendQueueEntry *)0))); + std::set< std::pair >::iterator oldest(qi); + unsigned long countForSender = 0; + while ((qi != _bySrc.end())&&(qi->first == from)) { + if (qi->second->timestamp < oldest->second->timestamp) + oldest = qi; + ++countForSender; + ++qi; + } + if (countForSender >= ZT_CLUSTER_MAX_QUEUE_PER_SENDER) { + _byDest.erase(std::pair(oldest->second->toPeerAddress,oldest->second)); + _pool[_poolCount++] = oldest->second; + _bySrc.erase(oldest); + } + } + + _ClusterSendQueueEntry *e; + if (_poolCount > 0) { + e = _pool[--_poolCount]; + } else { + if (_chunks.size() >= ZT_CLUSTER_MAX_QUEUE_CHUNKS) + return; // queue is totally full! + _chunks.push_back(Array<_ClusterSendQueueEntry,ZT_CLUSTER_QUEUE_CHUNK_SIZE>()); + e = &(_chunks.back().data[0]); + for(unsigned int i=1;itimestamp = now; + e->fromPeerAddress = from; + e->toPeerAddress = to; + memcpy(e->data,data,len); + e->len = len; + e->unite = unite; + + _bySrc.insert(std::pair(from,e)); + _byDest.insert(std::pair(to,e)); + } + + inline void expire(uint64_t now) + { + Mutex::Lock _l(_lock); + for(std::set< std::pair >::iterator qi(_bySrc.begin());qi!=_bySrc.end();) { + if ((now - qi->second->timestamp) > ZT_CLUSTER_QUEUE_EXPIRATION) { + _byDest.erase(std::pair(qi->second->toPeerAddress,qi->second)); + _pool[_poolCount++] = qi->second; + _bySrc.erase(qi++); + } else ++qi; + } + } + + /** + * Get and dequeue entries for a given destination address + * + * After use these entries must be returned with returnToPool()! + * + * @param dest Destination address + * @param results Array to fill with results + * @param maxResults Size of results[] in pointers + * @return Number of actual results returned + */ + inline unsigned int getByDest(const Address &dest,_ClusterSendQueueEntry **results,unsigned int maxResults) + { + unsigned int count = 0; + Mutex::Lock _l(_lock); + std::set< std::pair >::iterator qi(_byDest.lower_bound(std::pair(dest,(_ClusterSendQueueEntry *)0))); + while ((qi != _byDest.end())&&(qi->first == dest)) { + _bySrc.erase(std::pair(qi->second->fromPeerAddress,qi->second)); + results[count++] = qi->second; + if (count == maxResults) + break; + _byDest.erase(qi++); + } + return count; + } + + /** + * Return entries to pool after use + * + * @param entries Array of entries + * @param count Number of entries + */ + inline void returnToPool(_ClusterSendQueueEntry **entries,unsigned int count) + { + Mutex::Lock _l(_lock); + for(unsigned int i=0;i > _chunks; + _ClusterSendQueueEntry *_pool[ZT_CLUSTER_QUEUE_CHUNK_SIZE * ZT_CLUSTER_MAX_QUEUE_CHUNKS]; + unsigned long _poolCount; + std::set< std::pair > _bySrc; + std::set< std::pair > _byDest; + Mutex _lock; +}; + +Cluster::Cluster( + const RuntimeEnvironment *renv, + uint16_t id, + const std::vector &zeroTierPhysicalEndpoints, + int32_t x, + int32_t y, + int32_t z, + void (*sendFunction)(void *,unsigned int,const void *,unsigned int), + void *sendFunctionArg, + int (*addressToLocationFunction)(void *,const struct sockaddr_storage *,int *,int *,int *), + void *addressToLocationFunctionArg) : + RR(renv), + _sendQueue(new _ClusterSendQueue()), + _sendFunction(sendFunction), + _sendFunctionArg(sendFunctionArg), + _addressToLocationFunction(addressToLocationFunction), + _addressToLocationFunctionArg(addressToLocationFunctionArg), + _x(x), + _y(y), + _z(z), + _id(id), + _zeroTierPhysicalEndpoints(zeroTierPhysicalEndpoints), + _members(new _Member[ZT_CLUSTER_MAX_MEMBERS]), + _lastFlushed(0), + _lastCleanedRemotePeers(0), + _lastCleanedQueue(0) +{ + uint16_t stmp[ZT_SHA512_DIGEST_LEN / sizeof(uint16_t)]; + + // Generate master secret by hashing the secret from our Identity key pair + RR->identity.sha512PrivateKey(_masterSecret); + + // Generate our inbound message key, which is the master secret XORed with our ID and hashed twice + memcpy(stmp,_masterSecret,sizeof(stmp)); + stmp[0] ^= Utils::hton(id); + SHA512::hash(stmp,stmp,sizeof(stmp)); + SHA512::hash(stmp,stmp,sizeof(stmp)); + memcpy(_key,stmp,sizeof(_key)); + Utils::burn(stmp,sizeof(stmp)); +} + +Cluster::~Cluster() +{ + Utils::burn(_masterSecret,sizeof(_masterSecret)); + Utils::burn(_key,sizeof(_key)); + delete [] _members; + delete _sendQueue; +} + +void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len) +{ + Buffer dmsg; + { + // FORMAT: <[16] iv><[8] MAC><... data> + if ((len < 24)||(len > ZT_CLUSTER_MAX_MESSAGE_LENGTH)) + return; + + // 16-byte IV: first 8 bytes XORed with key, last 8 bytes used as Salsa20 64-bit IV + char keytmp[32]; + memcpy(keytmp,_key,32); + for(int i=0;i<8;++i) + keytmp[i] ^= reinterpret_cast(msg)[i]; + Salsa20 s20(keytmp,reinterpret_cast(msg) + 8); + Utils::burn(keytmp,sizeof(keytmp)); + + // One-time-use Poly1305 key from first 32 bytes of Salsa20 keystream (as per DJB/NaCl "standard") + char polykey[ZT_POLY1305_KEY_LEN]; + memset(polykey,0,sizeof(polykey)); + s20.crypt12(polykey,polykey,sizeof(polykey)); + + // Compute 16-byte MAC + char mac[ZT_POLY1305_MAC_LEN]; + Poly1305::compute(mac,reinterpret_cast(msg) + 24,len - 24,polykey); + + // Check first 8 bytes of MAC against 64-bit MAC in stream + if (!Utils::secureEq(mac,reinterpret_cast(msg) + 16,8)) + return; + + // Decrypt! + dmsg.setSize(len - 24); + s20.crypt12(reinterpret_cast(msg) + 24,const_cast(dmsg.data()),dmsg.size()); + } + + if (dmsg.size() < 4) + return; + const uint16_t fromMemberId = dmsg.at(0); + unsigned int ptr = 2; + if (fromMemberId == _id) // sanity check: we don't talk to ourselves + return; + const uint16_t toMemberId = dmsg.at(ptr); + ptr += 2; + if (toMemberId != _id) // sanity check: message not for us? + return; + + { // make sure sender is actually considered a member + Mutex::Lock _l3(_memberIds_m); + if (std::find(_memberIds.begin(),_memberIds.end(),fromMemberId) == _memberIds.end()) + return; + } + + try { + while (ptr < dmsg.size()) { + const unsigned int mlen = dmsg.at(ptr); ptr += 2; + const unsigned int nextPtr = ptr + mlen; + if (nextPtr > dmsg.size()) + break; + + int mtype = -1; + try { + switch((StateMessageType)(mtype = (int)dmsg[ptr++])) { + default: + break; + + case CLUSTER_MESSAGE_ALIVE: { + _Member &m = _members[fromMemberId]; + Mutex::Lock mlck(m.lock); + ptr += 7; // skip version stuff, not used yet + m.x = dmsg.at(ptr); ptr += 4; + m.y = dmsg.at(ptr); ptr += 4; + m.z = dmsg.at(ptr); ptr += 4; + ptr += 8; // skip local clock, not used + m.load = dmsg.at(ptr); ptr += 8; + m.peers = dmsg.at(ptr); ptr += 8; + ptr += 8; // skip flags, unused +#ifdef ZT_TRACE + std::string addrs; +#endif + unsigned int physicalAddressCount = dmsg[ptr++]; + m.zeroTierPhysicalEndpoints.clear(); + for(unsigned int i=0;i 0) + addrs.push_back(','); + addrs.append(m.zeroTierPhysicalEndpoints.back().toString()); + } +#endif + } +#ifdef ZT_TRACE + if ((RR->node->now() - m.lastReceivedAliveAnnouncement) >= ZT_CLUSTER_TIMEOUT) { + TRACE("[%u] I'm alive! peers close to %d,%d,%d can be redirected to: %s",(unsigned int)fromMemberId,m.x,m.y,m.z,addrs.c_str()); + } +#endif + m.lastReceivedAliveAnnouncement = RR->node->now(); + } break; + + case CLUSTER_MESSAGE_HAVE_PEER: { + Identity id; + ptr += id.deserialize(dmsg,ptr); + if (id) { + { + Mutex::Lock _l(_remotePeers_m); + _RemotePeer &rp = _remotePeers[std::pair(id.address(),(unsigned int)fromMemberId)]; + if (!rp.lastHavePeerReceived) { + RR->topology->saveIdentity((void *)0,id); + RR->identity.agree(id,rp.key,ZT_PEER_SECRET_KEY_LENGTH); + } + rp.lastHavePeerReceived = RR->node->now(); + } + + _ClusterSendQueueEntry *q[16384]; // 16384 is "tons" + unsigned int qc = _sendQueue->getByDest(id.address(),q,16384); + for(unsigned int i=0;irelayViaCluster(q[i]->fromPeerAddress,q[i]->toPeerAddress,q[i]->data,q[i]->len,q[i]->unite); + _sendQueue->returnToPool(q,qc); + + TRACE("[%u] has %s (retried %u queued sends)",(unsigned int)fromMemberId,id.address().toString().c_str(),qc); + } + } break; + + case CLUSTER_MESSAGE_WANT_PEER: { + const Address zeroTierAddress(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); ptr += ZT_ADDRESS_LENGTH; + SharedPtr peer(RR->topology->getPeerNoCache(zeroTierAddress)); + if ( (peer) && (peer->hasLocalClusterOptimalPath(RR->node->now())) ) { + Buffer<1024> buf; + peer->identity().serialize(buf); + Mutex::Lock _l2(_members[fromMemberId].lock); + _send(fromMemberId,CLUSTER_MESSAGE_HAVE_PEER,buf.data(),buf.size()); + } + } break; + + case CLUSTER_MESSAGE_REMOTE_PACKET: { + const unsigned int plen = dmsg.at(ptr); ptr += 2; + if (plen) { + Packet remotep(dmsg.field(ptr,plen),plen); ptr += plen; + //TRACE("remote %s from %s via %u (%u bytes)",Packet::verbString(remotep.verb()),remotep.source().toString().c_str(),fromMemberId,plen); + switch(remotep.verb()) { + case Packet::VERB_WHOIS: _doREMOTE_WHOIS(fromMemberId,remotep); break; + case Packet::VERB_MULTICAST_GATHER: _doREMOTE_MULTICAST_GATHER(fromMemberId,remotep); break; + default: break; // ignore things we don't care about across cluster + } + } + } break; + + case CLUSTER_MESSAGE_PROXY_UNITE: { + const Address localPeerAddress(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); ptr += ZT_ADDRESS_LENGTH; + const Address remotePeerAddress(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); ptr += ZT_ADDRESS_LENGTH; + const unsigned int numRemotePeerPaths = dmsg[ptr++]; + InetAddress remotePeerPaths[256]; // size is 8-bit, so 256 is max + for(unsigned int i=0;inode->now(); + SharedPtr localPeer(RR->topology->getPeerNoCache(localPeerAddress)); + if ((localPeer)&&(numRemotePeerPaths > 0)) { + InetAddress bestLocalV4,bestLocalV6; + localPeer->getRendezvousAddresses(now,bestLocalV4,bestLocalV6); + + InetAddress bestRemoteV4,bestRemoteV6; + for(unsigned int i=0;iidentity.address(),Packet::VERB_RENDEZVOUS); + rendezvousForLocal.append((uint8_t)0); + remotePeerAddress.appendTo(rendezvousForLocal); + + Buffer<2048> rendezvousForRemote; + remotePeerAddress.appendTo(rendezvousForRemote); + rendezvousForRemote.append((uint8_t)Packet::VERB_RENDEZVOUS); + rendezvousForRemote.addSize(2); // space for actual packet payload length + rendezvousForRemote.append((uint8_t)0); // flags == 0 + localPeerAddress.appendTo(rendezvousForRemote); + + bool haveMatch = false; + if ((bestLocalV6)&&(bestRemoteV6)) { + haveMatch = true; + + rendezvousForLocal.append((uint16_t)bestRemoteV6.port()); + rendezvousForLocal.append((uint8_t)16); + rendezvousForLocal.append(bestRemoteV6.rawIpData(),16); + + rendezvousForRemote.append((uint16_t)bestLocalV6.port()); + rendezvousForRemote.append((uint8_t)16); + rendezvousForRemote.append(bestLocalV6.rawIpData(),16); + rendezvousForRemote.setAt(ZT_ADDRESS_LENGTH + 1,(uint16_t)(9 + 16)); + } else if ((bestLocalV4)&&(bestRemoteV4)) { + haveMatch = true; + + rendezvousForLocal.append((uint16_t)bestRemoteV4.port()); + rendezvousForLocal.append((uint8_t)4); + rendezvousForLocal.append(bestRemoteV4.rawIpData(),4); + + rendezvousForRemote.append((uint16_t)bestLocalV4.port()); + rendezvousForRemote.append((uint8_t)4); + rendezvousForRemote.append(bestLocalV4.rawIpData(),4); + rendezvousForRemote.setAt(ZT_ADDRESS_LENGTH + 1,(uint16_t)(9 + 4)); + } + + if (haveMatch) { + { + Mutex::Lock _l2(_members[fromMemberId].lock); + _send(fromMemberId,CLUSTER_MESSAGE_PROXY_SEND,rendezvousForRemote.data(),rendezvousForRemote.size()); + } + RR->sw->send((void *)0,rendezvousForLocal,true); + } + } + } break; + + case CLUSTER_MESSAGE_PROXY_SEND: { + const Address rcpt(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); ptr += ZT_ADDRESS_LENGTH; + const Packet::Verb verb = (Packet::Verb)dmsg[ptr++]; + const unsigned int len = dmsg.at(ptr); ptr += 2; + Packet outp(rcpt,RR->identity.address(),verb); + outp.append(dmsg.field(ptr,len),len); ptr += len; + RR->sw->send((void *)0,outp,true); + //TRACE("[%u] proxy send %s to %s length %u",(unsigned int)fromMemberId,Packet::verbString(verb),rcpt.toString().c_str(),len); + } break; + + case CLUSTER_MESSAGE_NETWORK_CONFIG: { + const SharedPtr network(RR->node->network(dmsg.at(ptr))); + if (network) { + // Copy into a Packet just to conform to Network API. Eventually + // will want to refactor. + network->handleConfigChunk((void *)0,0,Address(),Buffer(dmsg),ptr); + } + } break; + } + } catch ( ... ) { + TRACE("invalid message of size %u type %d (inner decode), discarding",mlen,mtype); + // drop invalids + } + + ptr = nextPtr; + } + } catch ( ... ) { + TRACE("invalid message (outer loop), discarding"); + // drop invalids + } +} + +void Cluster::broadcastHavePeer(const Identity &id) +{ + Buffer<1024> buf; + id.serialize(buf); + Mutex::Lock _l(_memberIds_m); + for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { + Mutex::Lock _l2(_members[*mid].lock); + _send(*mid,CLUSTER_MESSAGE_HAVE_PEER,buf.data(),buf.size()); + } +} + +void Cluster::broadcastNetworkConfigChunk(const void *chunk,unsigned int len) +{ + Mutex::Lock _l(_memberIds_m); + for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { + Mutex::Lock _l2(_members[*mid].lock); + _send(*mid,CLUSTER_MESSAGE_NETWORK_CONFIG,chunk,len); + } +} + +int Cluster::checkSendViaCluster(const Address &toPeerAddress,uint64_t &mostRecentTs,void *peerSecret) +{ + const uint64_t now = RR->node->now(); + mostRecentTs = 0; + int mostRecentMemberId = -1; + { + Mutex::Lock _l2(_remotePeers_m); + std::map< std::pair,_RemotePeer >::const_iterator rpe(_remotePeers.lower_bound(std::pair(toPeerAddress,0))); + for(;;) { + if ((rpe == _remotePeers.end())||(rpe->first.first != toPeerAddress)) + break; + else if (rpe->second.lastHavePeerReceived > mostRecentTs) { + mostRecentTs = rpe->second.lastHavePeerReceived; + memcpy(peerSecret,rpe->second.key,ZT_PEER_SECRET_KEY_LENGTH); + mostRecentMemberId = (int)rpe->first.second; + } + ++rpe; + } + } + + const uint64_t ageOfMostRecentHavePeerAnnouncement = now - mostRecentTs; + if (ageOfMostRecentHavePeerAnnouncement >= (ZT_PEER_ACTIVITY_TIMEOUT / 3)) { + if (ageOfMostRecentHavePeerAnnouncement >= ZT_PEER_ACTIVITY_TIMEOUT) + mostRecentMemberId = -1; + + bool sendWantPeer = true; + { + Mutex::Lock _l(_remotePeers_m); + _RemotePeer &rp = _remotePeers[std::pair(toPeerAddress,(unsigned int)_id)]; + if ((now - rp.lastSentWantPeer) >= ZT_CLUSTER_WANT_PEER_EVERY) { + rp.lastSentWantPeer = now; + } else { + sendWantPeer = false; // don't flood WANT_PEER + } + } + if (sendWantPeer) { + char tmp[ZT_ADDRESS_LENGTH]; + toPeerAddress.copyTo(tmp,ZT_ADDRESS_LENGTH); + { + Mutex::Lock _l(_memberIds_m); + for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { + Mutex::Lock _l2(_members[*mid].lock); + _send(*mid,CLUSTER_MESSAGE_WANT_PEER,tmp,ZT_ADDRESS_LENGTH); + } + } + } + } + + return mostRecentMemberId; +} + +bool Cluster::sendViaCluster(int mostRecentMemberId,const Address &toPeerAddress,const void *data,unsigned int len) +{ + if ((mostRecentMemberId < 0)||(mostRecentMemberId >= ZT_CLUSTER_MAX_MEMBERS)) // sanity check + return false; + Mutex::Lock _l2(_members[mostRecentMemberId].lock); + for(std::vector::const_iterator i1(_zeroTierPhysicalEndpoints.begin());i1!=_zeroTierPhysicalEndpoints.end();++i1) { + for(std::vector::const_iterator i2(_members[mostRecentMemberId].zeroTierPhysicalEndpoints.begin());i2!=_members[mostRecentMemberId].zeroTierPhysicalEndpoints.end();++i2) { + if (i1->ss_family == i2->ss_family) { + TRACE("sendViaCluster sending %u bytes to %s by way of %u (%s->%s)",len,toPeerAddress.toString().c_str(),(unsigned int)mostRecentMemberId,i1->toString().c_str(),i2->toString().c_str()); + RR->node->putPacket((void *)0,*i1,*i2,data,len); + return true; + } + } + } + return false; +} + +void Cluster::relayViaCluster(const Address &fromPeerAddress,const Address &toPeerAddress,const void *data,unsigned int len,bool unite) +{ + if (len > ZT_PROTO_MAX_PACKET_LENGTH) // sanity check + return; + + const uint64_t now = RR->node->now(); + + uint64_t mostRecentTs = 0; + int mostRecentMemberId = -1; + { + Mutex::Lock _l2(_remotePeers_m); + std::map< std::pair,_RemotePeer >::const_iterator rpe(_remotePeers.lower_bound(std::pair(toPeerAddress,0))); + for(;;) { + if ((rpe == _remotePeers.end())||(rpe->first.first != toPeerAddress)) + break; + else if (rpe->second.lastHavePeerReceived > mostRecentTs) { + mostRecentTs = rpe->second.lastHavePeerReceived; + mostRecentMemberId = (int)rpe->first.second; + } + ++rpe; + } + } + + const uint64_t ageOfMostRecentHavePeerAnnouncement = now - mostRecentTs; + if (ageOfMostRecentHavePeerAnnouncement >= (ZT_PEER_ACTIVITY_TIMEOUT / 3)) { + // Enqueue and wait if peer seems alive, but do WANT_PEER to refresh homing + const bool enqueueAndWait = ((ageOfMostRecentHavePeerAnnouncement >= ZT_PEER_ACTIVITY_TIMEOUT)||(mostRecentMemberId < 0)); + + // Poll everyone with WANT_PEER if the age of our most recent entry is + // approaching expiration (or has expired, or does not exist). + bool sendWantPeer = true; + { + Mutex::Lock _l(_remotePeers_m); + _RemotePeer &rp = _remotePeers[std::pair(toPeerAddress,(unsigned int)_id)]; + if ((now - rp.lastSentWantPeer) >= ZT_CLUSTER_WANT_PEER_EVERY) { + rp.lastSentWantPeer = now; + } else { + sendWantPeer = false; // don't flood WANT_PEER + } + } + if (sendWantPeer) { + char tmp[ZT_ADDRESS_LENGTH]; + toPeerAddress.copyTo(tmp,ZT_ADDRESS_LENGTH); + { + Mutex::Lock _l(_memberIds_m); + for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { + Mutex::Lock _l2(_members[*mid].lock); + _send(*mid,CLUSTER_MESSAGE_WANT_PEER,tmp,ZT_ADDRESS_LENGTH); + } + } + } + + // If there isn't a good place to send via, then enqueue this for retrying + // later and return after having broadcasted a WANT_PEER. + if (enqueueAndWait) { + TRACE("relayViaCluster %s -> %s enqueueing to wait for HAVE_PEER",fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str()); + _sendQueue->enqueue(now,fromPeerAddress,toPeerAddress,data,len,unite); + return; + } + } + + if (mostRecentMemberId >= 0) { + Buffer<1024> buf; + if (unite) { + InetAddress v4,v6; + if (fromPeerAddress) { + SharedPtr fromPeer(RR->topology->getPeerNoCache(fromPeerAddress)); + if (fromPeer) + fromPeer->getRendezvousAddresses(now,v4,v6); + } + uint8_t addrCount = 0; + if (v4) + ++addrCount; + if (v6) + ++addrCount; + if (addrCount) { + toPeerAddress.appendTo(buf); + fromPeerAddress.appendTo(buf); + buf.append(addrCount); + if (v4) + v4.serialize(buf); + if (v6) + v6.serialize(buf); + } + } + + { + Mutex::Lock _l2(_members[mostRecentMemberId].lock); + if (buf.size() > 0) + _send(mostRecentMemberId,CLUSTER_MESSAGE_PROXY_UNITE,buf.data(),buf.size()); + + for(std::vector::const_iterator i1(_zeroTierPhysicalEndpoints.begin());i1!=_zeroTierPhysicalEndpoints.end();++i1) { + for(std::vector::const_iterator i2(_members[mostRecentMemberId].zeroTierPhysicalEndpoints.begin());i2!=_members[mostRecentMemberId].zeroTierPhysicalEndpoints.end();++i2) { + if (i1->ss_family == i2->ss_family) { + TRACE("relayViaCluster relaying %u bytes from %s to %s by way of %u (%s->%s)",len,fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str(),(unsigned int)mostRecentMemberId,i1->toString().c_str(),i2->toString().c_str()); + RR->node->putPacket((void *)0,*i1,*i2,data,len); + return; + } + } + } + + TRACE("relayViaCluster relaying %u bytes from %s to %s by way of %u failed: no common endpoints with the same address family!",len,fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str(),(unsigned int)mostRecentMemberId); + } + } +} + +void Cluster::sendDistributedQuery(const Packet &pkt) +{ + Buffer<4096> buf; + buf.append((uint16_t)pkt.size()); + buf.append(pkt.data(),pkt.size()); + Mutex::Lock _l(_memberIds_m); + for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { + Mutex::Lock _l2(_members[*mid].lock); + _send(*mid,CLUSTER_MESSAGE_REMOTE_PACKET,buf.data(),buf.size()); + } +} + +void Cluster::doPeriodicTasks() +{ + const uint64_t now = RR->node->now(); + + if ((now - _lastFlushed) >= ZT_CLUSTER_FLUSH_PERIOD) { + _lastFlushed = now; + + Mutex::Lock _l(_memberIds_m); + for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { + Mutex::Lock _l2(_members[*mid].lock); + + if ((now - _members[*mid].lastAnnouncedAliveTo) >= ((ZT_CLUSTER_TIMEOUT / 2) - 1000)) { + _members[*mid].lastAnnouncedAliveTo = now; + + Buffer<2048> alive; + alive.append((uint16_t)ZEROTIER_ONE_VERSION_MAJOR); + alive.append((uint16_t)ZEROTIER_ONE_VERSION_MINOR); + alive.append((uint16_t)ZEROTIER_ONE_VERSION_REVISION); + alive.append((uint8_t)ZT_PROTO_VERSION); + if (_addressToLocationFunction) { + alive.append((int32_t)_x); + alive.append((int32_t)_y); + alive.append((int32_t)_z); + } else { + alive.append((int32_t)0); + alive.append((int32_t)0); + alive.append((int32_t)0); + } + alive.append((uint64_t)now); + alive.append((uint64_t)0); // TODO: compute and send load average + alive.append((uint64_t)RR->topology->countActive(now)); + alive.append((uint64_t)0); // unused/reserved flags + alive.append((uint8_t)_zeroTierPhysicalEndpoints.size()); + for(std::vector::const_iterator pe(_zeroTierPhysicalEndpoints.begin());pe!=_zeroTierPhysicalEndpoints.end();++pe) + pe->serialize(alive); + _send(*mid,CLUSTER_MESSAGE_ALIVE,alive.data(),alive.size()); + } + + _flush(*mid); + } + } + + if ((now - _lastCleanedRemotePeers) >= (ZT_PEER_ACTIVITY_TIMEOUT * 2)) { + _lastCleanedRemotePeers = now; + + Mutex::Lock _l(_remotePeers_m); + for(std::map< std::pair,_RemotePeer >::iterator rp(_remotePeers.begin());rp!=_remotePeers.end();) { + if ((now - rp->second.lastHavePeerReceived) >= ZT_PEER_ACTIVITY_TIMEOUT) + _remotePeers.erase(rp++); + else ++rp; + } + } + + if ((now - _lastCleanedQueue) >= ZT_CLUSTER_QUEUE_EXPIRATION) { + _lastCleanedQueue = now; + _sendQueue->expire(now); + } +} + +void Cluster::addMember(uint16_t memberId) +{ + if ((memberId >= ZT_CLUSTER_MAX_MEMBERS)||(memberId == _id)) + return; + + Mutex::Lock _l2(_members[memberId].lock); + + { + Mutex::Lock _l(_memberIds_m); + if (std::find(_memberIds.begin(),_memberIds.end(),memberId) != _memberIds.end()) + return; + _memberIds.push_back(memberId); + std::sort(_memberIds.begin(),_memberIds.end()); + } + + _members[memberId].clear(); + + // Generate this member's message key from the master and its ID + uint16_t stmp[ZT_SHA512_DIGEST_LEN / sizeof(uint16_t)]; + memcpy(stmp,_masterSecret,sizeof(stmp)); + stmp[0] ^= Utils::hton(memberId); + SHA512::hash(stmp,stmp,sizeof(stmp)); + SHA512::hash(stmp,stmp,sizeof(stmp)); + memcpy(_members[memberId].key,stmp,sizeof(_members[memberId].key)); + Utils::burn(stmp,sizeof(stmp)); + + // Prepare q + _members[memberId].q.clear(); + char iv[16]; + Utils::getSecureRandom(iv,16); + _members[memberId].q.append(iv,16); + _members[memberId].q.addSize(8); // room for MAC + _members[memberId].q.append((uint16_t)_id); + _members[memberId].q.append((uint16_t)memberId); +} + +void Cluster::removeMember(uint16_t memberId) +{ + Mutex::Lock _l(_memberIds_m); + std::vector newMemberIds; + for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { + if (*mid != memberId) + newMemberIds.push_back(*mid); + } + _memberIds = newMemberIds; +} + +bool Cluster::findBetterEndpoint(InetAddress &redirectTo,const Address &peerAddress,const InetAddress &peerPhysicalAddress,bool offload) +{ + if (_addressToLocationFunction) { + // Pick based on location if it can be determined + int px = 0,py = 0,pz = 0; + if (_addressToLocationFunction(_addressToLocationFunctionArg,reinterpret_cast(&peerPhysicalAddress),&px,&py,&pz) == 0) { + TRACE("no geolocation data for %s",peerPhysicalAddress.toIpString().c_str()); + return false; + } + + // Find member closest to this peer + const uint64_t now = RR->node->now(); + std::vector best; + const double currentDistance = _dist3d(_x,_y,_z,px,py,pz); + double bestDistance = (offload ? 2147483648.0 : currentDistance); +#ifdef ZT_TRACE + unsigned int bestMember = _id; +#endif + { + Mutex::Lock _l(_memberIds_m); + for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { + _Member &m = _members[*mid]; + Mutex::Lock _ml(m.lock); + + // Consider member if it's alive and has sent us a location and one or more physical endpoints to send peers to + if ( ((now - m.lastReceivedAliveAnnouncement) < ZT_CLUSTER_TIMEOUT) && ((m.x != 0)||(m.y != 0)||(m.z != 0)) && (m.zeroTierPhysicalEndpoints.size() > 0) ) { + const double mdist = _dist3d(m.x,m.y,m.z,px,py,pz); + if (mdist < bestDistance) { + bestDistance = mdist; +#ifdef ZT_TRACE + bestMember = *mid; +#endif + best = m.zeroTierPhysicalEndpoints; + } + } + } + } + + // Redirect to a closer member if it has a ZeroTier endpoint address in the same ss_family + for(std::vector::const_iterator a(best.begin());a!=best.end();++a) { + if (a->ss_family == peerPhysicalAddress.ss_family) { + TRACE("%s at [%d,%d,%d] is %f from us but %f from %u, can redirect to %s",peerAddress.toString().c_str(),px,py,pz,currentDistance,bestDistance,bestMember,a->toString().c_str()); + redirectTo = *a; + return true; + } + } + TRACE("%s at [%d,%d,%d] is %f from us, no better endpoints found",peerAddress.toString().c_str(),px,py,pz,currentDistance); + return false; + } else { + // TODO: pick based on load if no location info? + return false; + } +} + +bool Cluster::isClusterPeerFrontplane(const InetAddress &ip) const +{ + Mutex::Lock _l(_memberIds_m); + for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { + Mutex::Lock _l2(_members[*mid].lock); + for(std::vector::const_iterator i2(_members[*mid].zeroTierPhysicalEndpoints.begin());i2!=_members[*mid].zeroTierPhysicalEndpoints.end();++i2) { + if (ip == *i2) + return true; + } + } + return false; +} + +void Cluster::status(ZT_ClusterStatus &status) const +{ + const uint64_t now = RR->node->now(); + memset(&status,0,sizeof(ZT_ClusterStatus)); + + status.myId = _id; + + { + ZT_ClusterMemberStatus *const s = &(status.members[status.clusterSize++]); + s->id = _id; + s->alive = 1; + s->x = _x; + s->y = _y; + s->z = _z; + s->load = 0; // TODO + s->peers = RR->topology->countActive(now); + for(std::vector::const_iterator ep(_zeroTierPhysicalEndpoints.begin());ep!=_zeroTierPhysicalEndpoints.end();++ep) { + if (s->numZeroTierPhysicalEndpoints >= ZT_CLUSTER_MAX_ZT_PHYSICAL_ADDRESSES) // sanity check + break; + memcpy(&(s->zeroTierPhysicalEndpoints[s->numZeroTierPhysicalEndpoints++]),&(*ep),sizeof(struct sockaddr_storage)); + } + } + + { + Mutex::Lock _l1(_memberIds_m); + for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { + if (status.clusterSize >= ZT_CLUSTER_MAX_MEMBERS) // sanity check + break; + + _Member &m = _members[*mid]; + Mutex::Lock ml(m.lock); + + ZT_ClusterMemberStatus *const s = &(status.members[status.clusterSize++]); + s->id = *mid; + s->msSinceLastHeartbeat = (unsigned int)std::min((uint64_t)(~((unsigned int)0)),(now - m.lastReceivedAliveAnnouncement)); + s->alive = (s->msSinceLastHeartbeat < ZT_CLUSTER_TIMEOUT) ? 1 : 0; + s->x = m.x; + s->y = m.y; + s->z = m.z; + s->load = m.load; + s->peers = m.peers; + for(std::vector::const_iterator ep(m.zeroTierPhysicalEndpoints.begin());ep!=m.zeroTierPhysicalEndpoints.end();++ep) { + if (s->numZeroTierPhysicalEndpoints >= ZT_CLUSTER_MAX_ZT_PHYSICAL_ADDRESSES) // sanity check + break; + memcpy(&(s->zeroTierPhysicalEndpoints[s->numZeroTierPhysicalEndpoints++]),&(*ep),sizeof(struct sockaddr_storage)); + } + } + } +} + +void Cluster::_send(uint16_t memberId,StateMessageType type,const void *msg,unsigned int len) +{ + if ((len + 3) > (ZT_CLUSTER_MAX_MESSAGE_LENGTH - (24 + 2 + 2))) // sanity check + return; + _Member &m = _members[memberId]; + // assumes m.lock is locked! + if ((m.q.size() + len + 3) > ZT_CLUSTER_MAX_MESSAGE_LENGTH) + _flush(memberId); + m.q.append((uint16_t)(len + 1)); + m.q.append((uint8_t)type); + m.q.append(msg,len); +} + +void Cluster::_flush(uint16_t memberId) +{ + _Member &m = _members[memberId]; + // assumes m.lock is locked! + if (m.q.size() > (24 + 2 + 2)) { // 16-byte IV + 8-byte MAC + 2 byte from-member-ID + 2 byte to-member-ID + // Create key from member's key and IV + char keytmp[32]; + memcpy(keytmp,m.key,32); + for(int i=0;i<8;++i) + keytmp[i] ^= m.q[i]; + Salsa20 s20(keytmp,m.q.field(8,8)); + Utils::burn(keytmp,sizeof(keytmp)); + + // One-time-use Poly1305 key from first 32 bytes of Salsa20 keystream (as per DJB/NaCl "standard") + char polykey[ZT_POLY1305_KEY_LEN]; + memset(polykey,0,sizeof(polykey)); + s20.crypt12(polykey,polykey,sizeof(polykey)); + + // Encrypt m.q in place + s20.crypt12(reinterpret_cast(m.q.data()) + 24,const_cast(reinterpret_cast(m.q.data())) + 24,m.q.size() - 24); + + // Add MAC for authentication (encrypt-then-MAC) + char mac[ZT_POLY1305_MAC_LEN]; + Poly1305::compute(mac,reinterpret_cast(m.q.data()) + 24,m.q.size() - 24,polykey); + memcpy(m.q.field(16,8),mac,8); + + // Send! + _sendFunction(_sendFunctionArg,memberId,m.q.data(),m.q.size()); + + // Prepare for more + m.q.clear(); + char iv[16]; + Utils::getSecureRandom(iv,16); + m.q.append(iv,16); + m.q.addSize(8); // room for MAC + m.q.append((uint16_t)_id); // from member ID + m.q.append((uint16_t)memberId); // to member ID + } +} + +void Cluster::_doREMOTE_WHOIS(uint64_t fromMemberId,const Packet &remotep) +{ + if (remotep.payloadLength() >= ZT_ADDRESS_LENGTH) { + Identity queried(RR->topology->getIdentity((void *)0,Address(remotep.payload(),ZT_ADDRESS_LENGTH))); + if (queried) { + Buffer<1024> routp; + remotep.source().appendTo(routp); + routp.append((uint8_t)Packet::VERB_OK); + routp.addSize(2); // space for length + routp.append((uint8_t)Packet::VERB_WHOIS); + routp.append(remotep.packetId()); + queried.serialize(routp); + routp.setAt(ZT_ADDRESS_LENGTH + 1,(uint16_t)(routp.size() - ZT_ADDRESS_LENGTH - 3)); + + TRACE("responding to remote WHOIS from %s @ %u with identity of %s",remotep.source().toString().c_str(),(unsigned int)fromMemberId,queried.address().toString().c_str()); + Mutex::Lock _l2(_members[fromMemberId].lock); + _send(fromMemberId,CLUSTER_MESSAGE_PROXY_SEND,routp.data(),routp.size()); + } + } +} + +void Cluster::_doREMOTE_MULTICAST_GATHER(uint64_t fromMemberId,const Packet &remotep) +{ + const uint64_t nwid = remotep.at(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_NETWORK_ID); + const MulticastGroup mg(MAC(remotep.field(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_MAC,6),6),remotep.at(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_ADI)); + unsigned int gatherLimit = remotep.at(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_GATHER_LIMIT); + const Address remotePeerAddress(remotep.source()); + + if (gatherLimit) { + Buffer routp; + remotePeerAddress.appendTo(routp); + routp.append((uint8_t)Packet::VERB_OK); + routp.addSize(2); // space for length + routp.append((uint8_t)Packet::VERB_MULTICAST_GATHER); + routp.append(remotep.packetId()); + routp.append(nwid); + mg.mac().appendTo(routp); + routp.append((uint32_t)mg.adi()); + + if (gatherLimit > ((ZT_CLUSTER_MAX_MESSAGE_LENGTH - 80) / 5)) + gatherLimit = ((ZT_CLUSTER_MAX_MESSAGE_LENGTH - 80) / 5); + if (RR->mc->gather(remotePeerAddress,nwid,mg,routp,gatherLimit)) { + routp.setAt(ZT_ADDRESS_LENGTH + 1,(uint16_t)(routp.size() - ZT_ADDRESS_LENGTH - 3)); + + TRACE("responding to remote MULTICAST_GATHER from %s @ %u with %u bytes",remotePeerAddress.toString().c_str(),(unsigned int)fromMemberId,routp.size()); + Mutex::Lock _l2(_members[fromMemberId].lock); + _send(fromMemberId,CLUSTER_MESSAGE_PROXY_SEND,routp.data(),routp.size()); + } + } +} + +} // namespace ZeroTier + +#endif // ZT_ENABLE_CLUSTER diff --git a/attic/Cluster.hpp b/attic/Cluster.hpp new file mode 100644 index 00000000..74b091f5 --- /dev/null +++ b/attic/Cluster.hpp @@ -0,0 +1,463 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. + */ + +#ifndef ZT_CLUSTER_HPP +#define ZT_CLUSTER_HPP + +#ifdef ZT_ENABLE_CLUSTER + +#include + +#include "Constants.hpp" +#include "../include/ZeroTierOne.h" +#include "Address.hpp" +#include "InetAddress.hpp" +#include "SHA512.hpp" +#include "Utils.hpp" +#include "Buffer.hpp" +#include "Mutex.hpp" +#include "SharedPtr.hpp" +#include "Hashtable.hpp" +#include "Packet.hpp" +#include "SharedPtr.hpp" + +/** + * Timeout for cluster members being considered "alive" + * + * A cluster member is considered dead and will no longer have peers + * redirected to it if we have not heard a heartbeat in this long. + */ +#define ZT_CLUSTER_TIMEOUT 5000 + +/** + * Desired period between doPeriodicTasks() in milliseconds + */ +#define ZT_CLUSTER_PERIODIC_TASK_PERIOD 20 + +/** + * How often to flush outgoing message queues (maximum interval) + */ +#define ZT_CLUSTER_FLUSH_PERIOD ZT_CLUSTER_PERIODIC_TASK_PERIOD + +/** + * Maximum number of queued outgoing packets per sender address + */ +#define ZT_CLUSTER_MAX_QUEUE_PER_SENDER 16 + +/** + * Expiration time for send queue entries + */ +#define ZT_CLUSTER_QUEUE_EXPIRATION 3000 + +/** + * Chunk size for allocating queue entries + * + * Queue entries are allocated in chunks of this many and are added to a pool. + * ZT_CLUSTER_MAX_QUEUE_GLOBAL must be evenly divisible by this. + */ +#define ZT_CLUSTER_QUEUE_CHUNK_SIZE 32 + +/** + * Maximum number of chunks to ever allocate + * + * This is a global sanity limit to prevent resource exhaustion attacks. It + * works out to about 600mb of RAM. You'll never see this on a normal edge + * node. We're unlikely to see this on a root server unless someone is DOSing + * us. In that case cluster relaying will be affected but other functions + * should continue to operate normally. + */ +#define ZT_CLUSTER_MAX_QUEUE_CHUNKS 8194 + +/** + * Max data per queue entry + */ +#define ZT_CLUSTER_SEND_QUEUE_DATA_MAX 1500 + +/** + * We won't send WANT_PEER to other members more than every (ms) per recipient + */ +#define ZT_CLUSTER_WANT_PEER_EVERY 1000 + +namespace ZeroTier { + +class RuntimeEnvironment; +class MulticastGroup; +class Peer; +class Identity; + +// Internal class implemented inside Cluster.cpp +class _ClusterSendQueue; + +/** + * Multi-homing cluster state replication and packet relaying + * + * Multi-homing means more than one node sharing the same ZeroTier identity. + * There is nothing in the protocol to prevent this, but to make it work well + * requires the devices sharing an identity to cooperate and share some + * information. + * + * There are three use cases we want to fulfill: + * + * (1) Multi-homing of root servers with handoff for efficient routing, + * HA, and load balancing across many commodity nodes. + * (2) Multi-homing of network controllers for the same reason. + * (3) Multi-homing of nodes on virtual networks, such as domain servers + * and other important endpoints. + * + * These use cases are in order of escalating difficulty. The initial + * version of Cluster is aimed at satisfying the first, though you are + * free to try #2 and #3. + */ +class Cluster +{ +public: + /** + * State message types + */ + enum StateMessageType + { + CLUSTER_MESSAGE_NOP = 0, + + /** + * This cluster member is alive: + * <[2] version minor> + * <[2] version major> + * <[2] version revision> + * <[1] protocol version> + * <[4] X location (signed 32-bit)> + * <[4] Y location (signed 32-bit)> + * <[4] Z location (signed 32-bit)> + * <[8] local clock at this member> + * <[8] load average> + * <[8] number of peers> + * <[8] flags (currently unused, must be zero)> + * <[1] number of preferred ZeroTier endpoints> + * <[...] InetAddress(es) of preferred ZeroTier endpoint(s)> + * + * Cluster members constantly broadcast an alive heartbeat and will only + * receive peer redirects if they've done so within the timeout. + */ + CLUSTER_MESSAGE_ALIVE = 1, + + /** + * Cluster member has this peer: + * <[...] serialized identity of peer> + * + * This is typically sent in response to WANT_PEER but can also be pushed + * to prepopulate if this makes sense. + */ + CLUSTER_MESSAGE_HAVE_PEER = 2, + + /** + * Cluster member wants this peer: + * <[5] ZeroTier address of peer> + * + * Members that have a direct link to this peer will respond with + * HAVE_PEER. + */ + CLUSTER_MESSAGE_WANT_PEER = 3, + + /** + * A remote packet that we should also possibly respond to: + * <[2] 16-bit length of remote packet> + * <[...] remote packet payload> + * + * Cluster members may relay requests by relaying the request packet. + * These may include requests such as WHOIS and MULTICAST_GATHER. The + * packet must be already decrypted, decompressed, and authenticated. + * + * This can only be used for small request packets as per the cluster + * message size limit, but since these are the only ones in question + * this is fine. + * + * If a response is generated it is sent via PROXY_SEND. + */ + CLUSTER_MESSAGE_REMOTE_PACKET = 4, + + /** + * Request that VERB_RENDEZVOUS be sent to a peer that we have: + * <[5] ZeroTier address of peer on recipient's side> + * <[5] ZeroTier address of peer on sender's side> + * <[1] 8-bit number of sender's peer's active path addresses> + * <[...] series of serialized InetAddresses of sender's peer's paths> + * + * This requests that we perform NAT-t introduction between a peer that + * we have and one on the sender's side. The sender furnishes contact + * info for its peer, and we send VERB_RENDEZVOUS to both sides: to ours + * directly and with PROXY_SEND to theirs. + */ + CLUSTER_MESSAGE_PROXY_UNITE = 5, + + /** + * Request that a cluster member send a packet to a locally-known peer: + * <[5] ZeroTier address of recipient> + * <[1] packet verb> + * <[2] length of packet payload> + * <[...] packet payload> + * + * This differs from RELAY in that it requests the receiving cluster + * member to actually compose a ZeroTier Packet from itself to the + * provided recipient. RELAY simply says "please forward this blob." + * RELAY is used to implement peer-to-peer relaying with RENDEZVOUS, + * while PROXY_SEND is used to implement proxy sending (which right + * now is only used to send RENDEZVOUS). + */ + CLUSTER_MESSAGE_PROXY_SEND = 6, + + /** + * Replicate a network config for a network we belong to: + * <[...] network config chunk> + * + * This is used by clusters to avoid every member having to query + * for the same netconf for networks all members belong to. + * + * The first field of a network config chunk is the network ID, + * so this can be checked to look up the network on receipt. + */ + CLUSTER_MESSAGE_NETWORK_CONFIG = 7 + }; + + /** + * Construct a new cluster + */ + Cluster( + const RuntimeEnvironment *renv, + uint16_t id, + const std::vector &zeroTierPhysicalEndpoints, + int32_t x, + int32_t y, + int32_t z, + void (*sendFunction)(void *,unsigned int,const void *,unsigned int), + void *sendFunctionArg, + int (*addressToLocationFunction)(void *,const struct sockaddr_storage *,int *,int *,int *), + void *addressToLocationFunctionArg); + + ~Cluster(); + + /** + * @return This cluster member's ID + */ + inline uint16_t id() const throw() { return _id; } + + /** + * Handle an incoming intra-cluster message + * + * @param data Message data + * @param len Message length (max: ZT_CLUSTER_MAX_MESSAGE_LENGTH) + */ + void handleIncomingStateMessage(const void *msg,unsigned int len); + + /** + * Broadcast that we have a given peer + * + * This should be done when new peers are first contacted. + * + * @param id Identity of peer + */ + void broadcastHavePeer(const Identity &id); + + /** + * Broadcast a network config chunk to other members of cluster + * + * @param chunk Chunk data + * @param len Length of chunk + */ + void broadcastNetworkConfigChunk(const void *chunk,unsigned int len); + + /** + * If the cluster has this peer, prepare the packet to send via cluster + * + * Note that outp is only armored (or modified at all) if the return value is a member ID. + * + * @param toPeerAddress Value of outp.destination(), simply to save additional lookup + * @param ts Result: set to time of last HAVE_PEER from the cluster + * @param peerSecret Result: Buffer to fill with peer secret on valid return value, must be at least ZT_PEER_SECRET_KEY_LENGTH bytes + * @return -1 if cluster does not know this peer, or a member ID to pass to sendViaCluster() + */ + int checkSendViaCluster(const Address &toPeerAddress,uint64_t &mostRecentTs,void *peerSecret); + + /** + * Send data via cluster front plane (packet head or fragment) + * + * @param haveMemberId Member ID that has this peer as returned by prepSendviaCluster() + * @param toPeerAddress Destination peer address + * @param data Packet or packet fragment data + * @param len Length of packet or fragment + * @return True if packet was sent (and outp was modified via armoring) + */ + bool sendViaCluster(int haveMemberId,const Address &toPeerAddress,const void *data,unsigned int len); + + /** + * Relay a packet via the cluster + * + * This is used in the outgoing packet and relaying logic in Switch to + * relay packets to other cluster members. It isn't PROXY_SEND-- that is + * used internally in Cluster to send responses to peer queries. + * + * @param fromPeerAddress Source peer address (if known, should be NULL for fragments) + * @param toPeerAddress Destination peer address + * @param data Packet or packet fragment data + * @param len Length of packet or fragment + * @param unite If true, also request proxy unite across cluster + */ + void relayViaCluster(const Address &fromPeerAddress,const Address &toPeerAddress,const void *data,unsigned int len,bool unite); + + /** + * Send a distributed query to other cluster members + * + * Some queries such as WHOIS or MULTICAST_GATHER need a response from other + * cluster members. Replies (if any) will be sent back to the peer via + * PROXY_SEND across the cluster. + * + * @param pkt Packet to distribute + */ + void sendDistributedQuery(const Packet &pkt); + + /** + * Call every ~ZT_CLUSTER_PERIODIC_TASK_PERIOD milliseconds. + */ + void doPeriodicTasks(); + + /** + * Add a member ID to this cluster + * + * @param memberId Member ID + */ + void addMember(uint16_t memberId); + + /** + * Remove a member ID from this cluster + * + * @param memberId Member ID to remove + */ + void removeMember(uint16_t memberId); + + /** + * Find a better cluster endpoint for this peer (if any) + * + * @param redirectTo InetAddress to be set to a better endpoint (if there is one) + * @param peerAddress Address of peer to (possibly) redirect + * @param peerPhysicalAddress Physical address of peer's current best path (where packet was most recently received or getBestPath()->address()) + * @param offload Always redirect if possible -- can be used to offload peers during shutdown + * @return True if redirectTo was set to a new address, false if redirectTo was not modified + */ + bool findBetterEndpoint(InetAddress &redirectTo,const Address &peerAddress,const InetAddress &peerPhysicalAddress,bool offload); + + /** + * @param ip Address to check + * @return True if this is a cluster frontplane address (excluding our addresses) + */ + bool isClusterPeerFrontplane(const InetAddress &ip) const; + + /** + * Fill out ZT_ClusterStatus structure (from core API) + * + * @param status Reference to structure to hold result (anything there is replaced) + */ + void status(ZT_ClusterStatus &status) const; + +private: + void _send(uint16_t memberId,StateMessageType type,const void *msg,unsigned int len); + void _flush(uint16_t memberId); + + void _doREMOTE_WHOIS(uint64_t fromMemberId,const Packet &remotep); + void _doREMOTE_MULTICAST_GATHER(uint64_t fromMemberId,const Packet &remotep); + + // These are initialized in the constructor and remain immutable ------------ + uint16_t _masterSecret[ZT_SHA512_DIGEST_LEN / sizeof(uint16_t)]; + unsigned char _key[ZT_PEER_SECRET_KEY_LENGTH]; + const RuntimeEnvironment *RR; + _ClusterSendQueue *const _sendQueue; + void (*_sendFunction)(void *,unsigned int,const void *,unsigned int); + void *_sendFunctionArg; + int (*_addressToLocationFunction)(void *,const struct sockaddr_storage *,int *,int *,int *); + void *_addressToLocationFunctionArg; + const int32_t _x; + const int32_t _y; + const int32_t _z; + const uint16_t _id; + const std::vector _zeroTierPhysicalEndpoints; + // end immutable fields ----------------------------------------------------- + + struct _Member + { + unsigned char key[ZT_PEER_SECRET_KEY_LENGTH]; + + uint64_t lastReceivedAliveAnnouncement; + uint64_t lastAnnouncedAliveTo; + + uint64_t load; + uint64_t peers; + int32_t x,y,z; + + std::vector zeroTierPhysicalEndpoints; + + Buffer q; + + Mutex lock; + + inline void clear() + { + lastReceivedAliveAnnouncement = 0; + lastAnnouncedAliveTo = 0; + load = 0; + peers = 0; + x = 0; + y = 0; + z = 0; + zeroTierPhysicalEndpoints.clear(); + q.clear(); + } + + _Member() { this->clear(); } + ~_Member() { Utils::burn(key,sizeof(key)); } + }; + _Member *const _members; + + std::vector _memberIds; + Mutex _memberIds_m; + + struct _RemotePeer + { + _RemotePeer() : lastHavePeerReceived(0),lastSentWantPeer(0) {} + ~_RemotePeer() { Utils::burn(key,ZT_PEER_SECRET_KEY_LENGTH); } + uint64_t lastHavePeerReceived; + uint64_t lastSentWantPeer; + uint8_t key[ZT_PEER_SECRET_KEY_LENGTH]; // secret key from identity agreement + }; + std::map< std::pair,_RemotePeer > _remotePeers; // we need ordered behavior and lower_bound here + Mutex _remotePeers_m; + + uint64_t _lastFlushed; + uint64_t _lastCleanedRemotePeers; + uint64_t _lastCleanedQueue; +}; + +} // namespace ZeroTier + +#endif // ZT_ENABLE_CLUSTER + +#endif -- cgit v1.2.3